def registry(region_name, framework=None): """ Return docker registry for the given AWS region for the given framework. This is only used for SparkML and Scikit-learn for now. """ try: account_id = image_registry_map[region_name][framework] return get_ecr_image_uri_prefix(account_id, region_name) except KeyError: logging.error("The specific image or region does not exist") raise
def container_image(sagemaker_session): """ Create a Multi-Model container image for use with integration testcases since 1P containers supporting multiple models are not available yet""" region = sagemaker_session.boto_region_name ecr_client = sagemaker_session.boto_session.client("ecr", region_name=region) sts_client = sagemaker_session.boto_session.client( "sts", region_name=region, endpoint_url=utils.sts_regional_endpoint(region)) account_id = sts_client.get_caller_identity()["Account"] algorithm_name = "sagemaker-multimodel-integ-test-{}".format( sagemaker_timestamp()) ecr_image_uri_prefix = get_ecr_image_uri_prefix(account=account_id, region=region) ecr_image = "{prefix}/{algorithm_name}:latest".format( prefix=ecr_image_uri_prefix, algorithm_name=algorithm_name) # Build and tag docker image locally docker_client = docker.from_env() image, build_log = docker_client.images.build(path=os.path.join( DATA_DIR, "multimodel", "container"), tag=algorithm_name, rm=True) image.tag(ecr_image, tag="latest") # Create AWS ECR and push the local docker image to it _create_repository(ecr_client, algorithm_name) username, password = _ecr_login(ecr_client) # Retry docker image push for _ in retries(3, "Upload docker image to ECR repo", seconds_to_sleep=10): try: docker_client.images.push(ecr_image, auth_config={ "username": username, "password": password }) break except requests.exceptions.ConnectionError: # This can happen when we try to create multiple repositories in parallel, so we retry pass yield ecr_image # Delete repository after the multi model integration tests complete _delete_repository(ecr_client, algorithm_name)
def registry(region_name, algorithm=None): """Return docker registry for the given AWS region Note: Not all the algorithms listed below have an Amazon Estimator implemented. For full list of pre-implemented Estimators, look at: https://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/amazon """ if algorithm in [ None, "pca", "kmeans", "linear-learner", "factorization-machines", "ntm", "randomcutforest", "knn", "object2vec", "ipinsights", ]: account_id = { "us-east-1": "382416733822", "us-east-2": "404615174143", "us-west-2": "174872318107", "eu-west-1": "438346466558", "eu-central-1": "664544806723", "ap-northeast-1": "351501993468", "ap-northeast-2": "835164637446", "ap-southeast-2": "712309505854", "us-gov-west-1": "226302683700", "ap-southeast-1": "475088953585", "ap-south-1": "991648021394", "ca-central-1": "469771592824", "eu-west-2": "644912444149", "us-west-1": "632365934929", "us-iso-east-1": "490574956308", }[region_name] elif algorithm in ["lda"]: account_id = { "us-east-1": "766337827248", "us-east-2": "999911452149", "us-west-2": "266724342769", "eu-west-1": "999678624901", "eu-central-1": "353608530281", "ap-northeast-1": "258307448986", "ap-northeast-2": "293181348795", "ap-southeast-2": "297031611018", "us-gov-west-1": "226302683700", "ap-southeast-1": "475088953585", "ap-south-1": "991648021394", "ca-central-1": "469771592824", "eu-west-2": "644912444149", "us-west-1": "632365934929", "us-iso-east-1": "490574956308", }[region_name] elif algorithm in ["forecasting-deepar"]: account_id = { "us-east-1": "522234722520", "us-east-2": "566113047672", "us-west-2": "156387875391", "eu-west-1": "224300973850", "eu-central-1": "495149712605", "ap-northeast-1": "633353088612", "ap-northeast-2": "204372634319", "ap-southeast-2": "514117268639", "us-gov-west-1": "226302683700", "ap-southeast-1": "475088953585", "ap-south-1": "991648021394", "ca-central-1": "469771592824", "eu-west-2": "644912444149", "us-west-1": "632365934929", "us-iso-east-1": "490574956308", }[region_name] elif algorithm in [ "xgboost", "seq2seq", "image-classification", "blazingtext", "object-detection", "semantic-segmentation", ]: account_id = { "us-east-1": "811284229777", "us-east-2": "825641698319", "us-west-2": "433757028032", "eu-west-1": "685385470294", "eu-central-1": "813361260812", "ap-northeast-1": "501404015308", "ap-northeast-2": "306986355934", "ap-southeast-2": "544295431143", "us-gov-west-1": "226302683700", "ap-southeast-1": "475088953585", "ap-south-1": "991648021394", "ca-central-1": "469771592824", "eu-west-2": "644912444149", "us-west-1": "632365934929", "us-iso-east-1": "490574956308", }[region_name] elif algorithm in ["image-classification-neo", "xgboost-neo"]: account_id = { "us-west-2": "301217895009", "us-east-1": "785573368785", "eu-west-1": "802834080501", "us-east-2": "007439368137", }[region_name] else: raise ValueError( "Algorithm class:{} does not have mapping to account_id with images" .format(algorithm)) return get_ecr_image_uri_prefix(account_id, region_name)
def create_image_uri( region, framework, instance_type, framework_version, py_version=None, account=None, accelerator_type=None, optimized_families=None, ): """Return the ECR URI of an image. Args: region (str): AWS region where the image is uploaded. framework (str): framework used by the image. instance_type (str): SageMaker instance type. Used to determine device type (cpu/gpu/family-specific optimized). framework_version (str): The version of the framework. py_version (str): Optional. Python version. If specified, should be one of 'py2' or 'py3'. If not specified, image uri will not include a python component. account (str): AWS account that contains the image. (default: '520713654638') accelerator_type (str): SageMaker Elastic Inference accelerator type. optimized_families (str): Instance families for which there exist specific optimized images. Returns: str: The appropriate image URI based on the given parameters. """ optimized_families = optimized_families or [] if py_version and py_version not in VALID_PY_VERSIONS: raise ValueError("invalid py_version argument: {}".format(py_version)) if _accelerator_type_valid_for_framework( framework=framework, accelerator_type=accelerator_type, optimized_families=optimized_families, ): framework += "-eia" # Handle Account Number for Gov Cloud and frameworks with DLC merged images if account is None: account = _registry_id( region=region, framework=framework, py_version=py_version, account=DEFAULT_ACCOUNT, framework_version=framework_version, ) # Handle Local Mode if instance_type.startswith("local"): device_type = "cpu" if instance_type == "local" else "gpu" elif not instance_type.startswith("ml."): raise ValueError( "{} is not a valid SageMaker instance type. See: " "https://aws.amazon.com/sagemaker/pricing/instance-types/".format( instance_type)) else: family = instance_type.split(".")[1] # For some frameworks, we have optimized images for specific families, e.g c5 or p3. # In those cases, we use the family name in the image tag. In other cases, we use # 'cpu' or 'gpu'. if family in optimized_families: device_type = family elif family[0] in ["g", "p"]: device_type = "gpu" else: device_type = "cpu" using_merged_images = _using_merged_images(region, framework, py_version, framework_version) if not py_version or (using_merged_images and framework == "tensorflow-serving-eia"): tag = "{}-{}".format(framework_version, device_type) else: tag = "{}-{}-{}".format(framework_version, device_type, py_version) if using_merged_images: return "{}/{}:{}".format(get_ecr_image_uri_prefix(account, region), MERGED_FRAMEWORKS_REPO_MAP[framework], tag) return "{}/sagemaker-{}:{}".format( get_ecr_image_uri_prefix(account, region), framework, tag)
def create_image_uri( region, framework, instance_type, framework_version, py_version=None, account=None, accelerator_type=None, optimized_families=None, ): """Return the ECR URI of an image. Args: region (str): AWS region where the image is uploaded. framework (str): framework used by the image. instance_type (str): SageMaker instance type. Used to determine device type (cpu/gpu/family-specific optimized). framework_version (str): The version of the framework. py_version (str): Optional. Python version. If specified, should be one of 'py2' or 'py3'. If not specified, image uri will not include a python component. account (str): AWS account that contains the image. (default: '520713654638') accelerator_type (str): SageMaker Elastic Inference accelerator type. optimized_families (str): Instance families for which there exist specific optimized images. Returns: str: The appropriate image URI based on the given parameters. """ logger.warning( "'create_image_uri' will be deprecated in favor of 'ImageURIProvider' class " "in SageMaker Python SDK v2." ) optimized_families = optimized_families or [] if py_version and py_version not in VALID_PY_VERSIONS: raise ValueError("invalid py_version argument: {}".format(py_version)) if py_version == "py37" and framework not in PY37_SUPPORTED_FRAMEWORKS: raise ValueError("{} does not support Python 3.7 at this time.".format(framework)) if _accelerator_type_valid_for_framework( framework=framework, py_version=py_version, accelerator_type=accelerator_type, optimized_families=optimized_families, ): framework += "-eia" # Handle account number for specific cases (e.g. GovCloud, opt-in regions, DLC images etc.) if account is None: account = _registry_id( region=region, framework=framework, py_version=py_version, account=DEFAULT_ACCOUNT, framework_version=framework_version, ) # Handle Local Mode if instance_type.startswith("local"): device_type = "cpu" if instance_type == "local" else "gpu" elif not instance_type.startswith("ml."): raise ValueError( "{} is not a valid SageMaker instance type. See: " "https://aws.amazon.com/sagemaker/pricing/instance-types/".format(instance_type) ) else: family = instance_type.split(".")[1] # For some frameworks, we have optimized images for specific families, e.g c5 or p3. # In those cases, we use the family name in the image tag. In other cases, we use # 'cpu' or 'gpu'. if family in optimized_families: device_type = family elif family.startswith("inf"): device_type = "inf" elif family[0] in ["g", "p"]: device_type = "gpu" else: device_type = "cpu" if device_type == "inf": if region not in INFERENTIA_SUPPORTED_REGIONS: raise ValueError( "Inferentia is not supported in region {}. Supported regions are {}".format( region, ", ".join(INFERENTIA_SUPPORTED_REGIONS) ) ) if framework not in INFERENTIA_VERSION_RANGES: raise ValueError( "Inferentia does not support {}. Currently it supports " "MXNet and TensorFlow with more frameworks coming soon.".format( framework.split("-")[-1] ) ) if not _is_inferentia_supported(framework, framework_version): raise ValueError( "Inferentia is not supported with {} version {}.".format( framework.split("-")[-1], framework_version ) ) use_dlc_image = _is_dlc_version(framework, framework_version, py_version) if not py_version or (use_dlc_image and framework == "tensorflow-serving-eia"): tag = "{}-{}".format(framework_version, device_type) else: tag = "{}-{}-{}".format(framework_version, device_type, py_version) if use_dlc_image: ecr_repo = MERGED_FRAMEWORKS_REPO_MAP[framework] else: ecr_repo = "sagemaker-{}".format(framework) return "{}/{}:{}".format(get_ecr_image_uri_prefix(account, region), ecr_repo, tag)
def registry(region_name, algorithm=None): """Return docker registry for the given AWS region Note: Not all the algorithms listed below have an Amazon Estimator implemented. For full list of pre-implemented Estimators, look at: https://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/amazon """ if algorithm in [ None, 'pca', 'kmeans', 'linear-learner', 'factorization-machines', 'ntm', 'randomcutforest', 'knn', 'object2vec', 'ipinsights' ]: account_id = { 'us-east-1': '382416733822', 'us-east-2': '404615174143', 'us-west-2': '174872318107', 'eu-west-1': '438346466558', 'eu-central-1': '664544806723', 'ap-northeast-1': '351501993468', 'ap-northeast-2': '835164637446', 'ap-southeast-2': '712309505854', 'us-gov-west-1': '226302683700', 'ap-southeast-1': '475088953585', 'ap-south-1': '991648021394', 'ca-central-1': '469771592824', 'eu-west-2': '644912444149', 'us-west-1': '632365934929', 'us-iso-east-1': '490574956308', }[region_name] elif algorithm in ['lda']: account_id = { 'us-east-1': '766337827248', 'us-east-2': '999911452149', 'us-west-2': '266724342769', 'eu-west-1': '999678624901', 'eu-central-1': '353608530281', 'ap-northeast-1': '258307448986', 'ap-northeast-2': '293181348795', 'ap-southeast-2': '297031611018', 'us-gov-west-1': '226302683700', 'ap-southeast-1': '475088953585', 'ap-south-1': '991648021394', 'ca-central-1': '469771592824', 'eu-west-2': '644912444149', 'us-west-1': '632365934929', 'us-iso-east-1': '490574956308', }[region_name] elif algorithm in ['forecasting-deepar']: account_id = { 'us-east-1': '522234722520', 'us-east-2': '566113047672', 'us-west-2': '156387875391', 'eu-west-1': '224300973850', 'eu-central-1': '495149712605', 'ap-northeast-1': '633353088612', 'ap-northeast-2': '204372634319', 'ap-southeast-2': '514117268639', 'us-gov-west-1': '226302683700', 'ap-southeast-1': '475088953585', 'ap-south-1': '991648021394', 'ca-central-1': '469771592824', 'eu-west-2': '644912444149', 'us-west-1': '632365934929', 'us-iso-east-1': '490574956308', }[region_name] elif algorithm in [ 'xgboost', 'seq2seq', 'image-classification', 'blazingtext', 'object-detection', 'semantic-segmentation' ]: account_id = { 'us-east-1': '811284229777', 'us-east-2': '825641698319', 'us-west-2': '433757028032', 'eu-west-1': '685385470294', 'eu-central-1': '813361260812', 'ap-northeast-1': '501404015308', 'ap-northeast-2': '306986355934', 'ap-southeast-2': '544295431143', 'us-gov-west-1': '226302683700', 'ap-southeast-1': '475088953585', 'ap-south-1': '991648021394', 'ca-central-1': '469771592824', 'eu-west-2': '644912444149', 'us-west-1': '632365934929', 'us-iso-east-1': '490574956308', }[region_name] elif algorithm in ['image-classification-neo', 'xgboost-neo']: account_id = { 'us-west-2': '301217895009', 'us-east-1': '785573368785', 'eu-west-1': '802834080501', 'us-east-2': '007439368137', }[region_name] else: raise ValueError( 'Algorithm class:{} does not have mapping to account_id with images' .format(algorithm)) return get_ecr_image_uri_prefix(account_id, region_name)
def registry(region_name, algorithm=None): """Return docker registry for the given AWS region Note: Not all the algorithms listed below have an Amazon Estimator implemented. For full list of pre-implemented Estimators, look at: https://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/amazon Args: region_name (str): The region name for the account. algorithm (str): The algorithm for the account. Raises: ValueError: If invalid algorithm passed in or if mapping does not exist for given algorithm and region. """ region_to_accounts = {} if algorithm in [ None, "pca", "kmeans", "linear-learner", "factorization-machines", "ntm", "randomcutforest", "knn", "object2vec", "ipinsights", ]: region_to_accounts = { "us-east-1": "382416733822", "us-east-2": "404615174143", "us-west-2": "174872318107", "eu-west-1": "438346466558", "eu-central-1": "664544806723", "ap-northeast-1": "351501993468", "ap-northeast-2": "835164637446", "ap-southeast-2": "712309505854", "us-gov-west-1": "226302683700", "ap-southeast-1": "475088953585", "ap-south-1": "991648021394", "ca-central-1": "469771592824", "eu-west-2": "644912444149", "us-west-1": "632365934929", "us-iso-east-1": "490574956308", "ap-east-1": "286214385809", "eu-north-1": "669576153137", "eu-west-3": "749696950732", "sa-east-1": "855470959533", "me-south-1": "249704162688", "cn-north-1": "390948362332", "cn-northwest-1": "387376663083", } elif algorithm in ["lda"]: region_to_accounts = { "us-east-1": "766337827248", "us-east-2": "999911452149", "us-west-2": "266724342769", "eu-west-1": "999678624901", "eu-central-1": "353608530281", "ap-northeast-1": "258307448986", "ap-northeast-2": "293181348795", "ap-southeast-2": "297031611018", "us-gov-west-1": "226302683700", "ap-southeast-1": "475088953585", "ap-south-1": "991648021394", "ca-central-1": "469771592824", "eu-west-2": "644912444149", "us-west-1": "632365934929", "us-iso-east-1": "490574956308", } elif algorithm in ["forecasting-deepar"]: region_to_accounts = { "us-east-1": "522234722520", "us-east-2": "566113047672", "us-west-2": "156387875391", "eu-west-1": "224300973850", "eu-central-1": "495149712605", "ap-northeast-1": "633353088612", "ap-northeast-2": "204372634319", "ap-southeast-2": "514117268639", "us-gov-west-1": "226302683700", "ap-southeast-1": "475088953585", "ap-south-1": "991648021394", "ca-central-1": "469771592824", "eu-west-2": "644912444149", "us-west-1": "632365934929", "us-iso-east-1": "490574956308", "ap-east-1": "286214385809", "eu-north-1": "669576153137", "eu-west-3": "749696950732", "sa-east-1": "855470959533", "me-south-1": "249704162688", "cn-north-1": "390948362332", "cn-northwest-1": "387376663083", } elif algorithm in [ "xgboost", "seq2seq", "image-classification", "blazingtext", "object-detection", "semantic-segmentation", ]: region_to_accounts = { "us-east-1": "811284229777", "us-east-2": "825641698319", "us-west-2": "433757028032", "eu-west-1": "685385470294", "eu-central-1": "813361260812", "ap-northeast-1": "501404015308", "ap-northeast-2": "306986355934", "ap-southeast-2": "544295431143", "us-gov-west-1": "226302683700", "ap-southeast-1": "475088953585", "ap-south-1": "991648021394", "ca-central-1": "469771592824", "eu-west-2": "644912444149", "us-west-1": "632365934929", "us-iso-east-1": "490574956308", "ap-east-1": "286214385809", "eu-north-1": "669576153137", "eu-west-3": "749696950732", "sa-east-1": "855470959533", "me-south-1": "249704162688", "cn-north-1": "390948362332", "cn-northwest-1": "387376663083", } elif algorithm in ["image-classification-neo", "xgboost-neo"]: region_to_accounts = NEO_IMAGE_ACCOUNT else: raise ValueError( "Algorithm class:{} does not have mapping to account_id with images" .format(algorithm)) if region_name in region_to_accounts: account_id = region_to_accounts[region_name] return get_ecr_image_uri_prefix(account_id, region_name) raise ValueError( "Algorithm ({algorithm}) is unsupported for region ({region_name}).". format(algorithm=algorithm, region_name=region_name))
def create_image_uri(region, framework, instance_type, framework_version, py_version=None, account='520713654638', accelerator_type=None, optimized_families=None): """Return the ECR URI of an image. Args: region (str): AWS region where the image is uploaded. framework (str): framework used by the image. instance_type (str): SageMaker instance type. Used to determine device type (cpu/gpu/family-specific optimized). framework_version (str): The version of the framework. py_version (str): Optional. Python version. If specified, should be one of 'py2' or 'py3'. If not specified, image uri will not include a python component. account (str): AWS account that contains the image. (default: '520713654638') accelerator_type (str): SageMaker Elastic Inference accelerator type. optimized_families (str): Instance families for which there exist specific optimized images. Returns: str: The appropriate image URI based on the given parameters. """ optimized_families = optimized_families or [] if py_version and py_version not in VALID_PY_VERSIONS: raise ValueError('invalid py_version argument: {}'.format(py_version)) # Handle Account Number for Gov Cloud account = VALID_ACCOUNTS_BY_REGION.get(region, account) # Handle Local Mode if instance_type.startswith('local'): device_type = 'cpu' if instance_type == 'local' else 'gpu' elif not instance_type.startswith('ml.'): raise ValueError( '{} is not a valid SageMaker instance type. See: ' 'https://aws.amazon.com/sagemaker/pricing/instance-types/'.format( instance_type)) else: family = instance_type.split('.')[1] # For some frameworks, we have optimized images for specific families, e.g c5 or p3. In those cases, # we use the family name in the image tag. In other cases, we use 'cpu' or 'gpu'. if family in optimized_families: device_type = family elif family[0] in ['g', 'p']: device_type = 'gpu' else: device_type = 'cpu' if py_version: tag = "{}-{}-{}".format(framework_version, device_type, py_version) else: tag = "{}-{}".format(framework_version, device_type) if _accelerator_type_valid_for_framework( framework=framework, accelerator_type=accelerator_type, optimized_families=optimized_families): framework += '-eia' return "{}/sagemaker-{}:{}" \ .format(get_ecr_image_uri_prefix(account, region), framework, tag)