def _prepare_init_params_from_job_description(cls, job_details): """Convert the job description to init params that can be handled by the class constructor Args: job_details: the returned job details from a describe_training_job API call. Returns: dictionary: The transformed init_params """ init_params = super(PyTorch, cls)._prepare_init_params_from_job_description(job_details) image_name = init_params.pop('image') framework, py_version, tag = framework_name_from_image(image_name) if not framework: # If we were unable to parse the framework name from the image it is not one of our # officially supported images, in this case just add the image to the init params. init_params['image_name'] = image_name return init_params init_params['py_version'] = py_version init_params['framework_version'] = framework_version_from_tag(tag) training_job_name = init_params['base_job_name'] if framework != cls.__framework_name__: raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name)) return init_params
def _prepare_init_params_from_job_description(cls, job_details): """Convert the job description to init params that can be handled by the class constructor Args: job_details: the returned job details from a describe_training_job API call. Returns: dictionary: The transformed init_params """ init_params = super(MXNet, cls)._prepare_init_params_from_job_description(job_details) image_name = init_params.pop('image') framework, py_version, tag = framework_name_from_image(image_name) if not framework: # If we were unable to parse the framework name from the image it is not one of our # officially supported images, in this case just add the image to the init params. init_params['image_name'] = image_name return init_params init_params['py_version'] = py_version # We switched image tagging scheme from regular image version (e.g. '1.0') to more expressive # containing framework version, device type and python version (e.g. '0.12-gpu-py2'). # For backward compatibility map deprecated image tag '1.0' to a '0.12' framework version # otherwise extract framework version from the tag itself. init_params['framework_version'] = '0.12' if tag == '1.0' else framework_version_from_tag(tag) training_job_name = init_params['base_job_name'] if framework != cls.__framework_name__: raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name)) return init_params
def _prepare_init_params_from_job_description(cls, job_details): """Convert the job description to init params that can be handled by the class constructor Args: job_details: the returned job details from a describe_training_job API call. Returns: dictionary: The transformed init_params """ init_params = super(Chainer, cls)._prepare_init_params_from_job_description(job_details) for argument in [Chainer._use_mpi, Chainer._num_processes, Chainer._process_slots_per_host, Chainer._additional_mpi_options]: value = init_params['hyperparameters'].pop(argument, None) if value: init_params[argument[len('sagemaker_'):]] = value image_name = init_params.pop('image') framework, py_version, tag = framework_name_from_image(image_name) if not framework: # If we were unable to parse the framework name from the image it is not one of our # officially supported images, in this case just add the image to the init params. init_params['image_name'] = image_name return init_params init_params['py_version'] = py_version init_params['framework_version'] = framework_version_from_tag(tag) training_job_name = init_params['base_job_name'] if framework != cls.__framework_name__: raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name)) return init_params
def test_legacy_name_from_image_any_tag(): image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-py2-cpu:any-tag' framework, py_ver, tag = framework_name_from_image(image_name) assert framework == 'tensorflow' assert py_ver == 'py2' assert tag == 'any-tag'
def test_legacy_name_from_wrong_device(): framework, py_ver, tag = framework_name_from_image('123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-myown-py4-gpu:1') assert framework is None assert py_ver is None assert tag is None
def test_legacy_name_from_wrong_python(): framework, py_ver, tag = framework_name_from_image( '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-myown-py4-gpu:1') assert framework is None assert py_ver is None assert tag is None
def test_framework_name_from_image_mxnet(): image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet:1.1-gpu-py3' assert ('mxnet', 'py3', '1.1-gpu-py3') == framework_name_from_image(image_name)
def test_legacy_name_from_wrong_device(): framework, py_ver, tag, _ = fw_utils.framework_name_from_image( "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-myown-py4-gpu:1") assert framework is None assert py_ver is None assert tag is None
def test_framework_name_from_image_rl(): image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-rl-mxnet:toolkit1.1-gpu-py3" assert ("mxnet", "py3", "toolkit1.1-gpu-py3", None) == fw_utils.framework_name_from_image(image_uri)
def test_legacy_name_from_image_any_tag(): image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-py2-cpu:any-tag' framework, py_ver, tag = fw_utils.framework_name_from_image(image_name) assert framework == 'tensorflow' assert py_ver == 'py2' assert tag == 'any-tag'
def test_legacy_name_from_framework_image(): image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet-py3-gpu:2.5.6-gpu-py2' framework, py_ver, tag = fw_utils.framework_name_from_image(image_name) assert framework == 'mxnet' assert py_ver == 'py3' assert tag == '2.5.6-gpu-py2'
def test_framework_name_from_image_rl(): image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-rl-mxnet:toolkit1.1-gpu-py3' assert ( 'mxnet', 'py3', 'toolkit1.1-gpu-py3') == fw_utils.framework_name_from_image(image_name)
def test_framework_name_from_image_tf(): image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6-cpu-py2' assert ('tensorflow', 'py2', '1.6-cpu-py2') == fw_utils.framework_name_from_image(image_name)
def test_framework_name_from_other(): framework, py_ver = framework_name_from_image( '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-myown-py2-gpu:1') assert framework is None assert py_ver is None
def test_framework_name_from_framework_image(): framework, py_ver = framework_name_from_image( '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet-py2-gpu:1') assert framework == 'mxnet' assert py_ver == 'py2'
def test_framework_name_from_image_mxnet_in_gov(): image_uri = "123.dkr.ecr.region-name.c2s.ic.gov/sagemaker-mxnet:1.1-gpu-py3" assert ("mxnet", "py3", "1.1-gpu-py3", None) == fw_utils.framework_name_from_image(image_uri)
def test_framework_name_from_image_tf(): image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6-cpu-py2" assert ("tensorflow", "py2", "1.6-cpu-py2", None) == fw_utils.framework_name_from_image(image_uri)
def test_framework_name_from_image_mxnet(): image_name = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet:1.1-gpu-py3" assert ("mxnet", "py3", "1.1-gpu-py3", None) == fw_utils.framework_name_from_image(image_name)
def test_legacy_name_from_framework_image(): image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet-py3-gpu:2.5.6-gpu-py2" framework, py_ver, tag, _ = fw_utils.framework_name_from_image(image_uri) assert framework == "mxnet" assert py_ver == "py3" assert tag == "2.5.6-gpu-py2"
def test_framework_name_from_image_mxnet(): image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet:1.1-gpu-py3' assert ('mxnet', 'py3', '1.1-gpu-py3') == framework_name_from_image(image_name)
def test_legacy_name_from_image_any_tag(): image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-py2-cpu:any-tag" framework, py_ver, tag, _ = fw_utils.framework_name_from_image(image_uri) assert framework == "tensorflow" assert py_ver == "py2" assert tag == "any-tag"
def test_framework_name_from_image_tf(): image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6-cpu-py2' assert ('tensorflow', 'py2', '1.6-cpu-py2') == framework_name_from_image(image_name)
def test_legacy_name_from_framework_image(): image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet-py3-gpu:2.5.6-gpu-py2' framework, py_ver, tag = framework_name_from_image(image_name) assert framework == 'mxnet' assert py_ver == 'py3' assert tag == '2.5.6-gpu-py2'
def test_framework_name_from_image_tf_scriptmode(): image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-scriptmode:1.12-cpu-py3' assert ('tensorflow', 'py3', '1.12-cpu-py3', 'scriptmode') == fw_utils.framework_name_from_image(image_name)