def _prepare_init_params_from_job_description(cls, job_details):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(PyTorch, cls)._prepare_init_params_from_job_description(job_details)
image_name = init_params.pop('image')
framework, py_version, tag = framework_name_from_image(image_name)
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params['image_name'] = image_name
return init_params
init_params['py_version'] = py_version
init_params['framework_version'] = framework_version_from_tag(tag)
training_job_name = init_params['base_job_name']
if framework != cls.__framework_name__:
raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name))
return init_params
def _prepare_init_params_from_job_description(cls, job_details):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(MXNet, cls)._prepare_init_params_from_job_description(job_details)
image_name = init_params.pop('image')
framework, py_version, tag = framework_name_from_image(image_name)
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params['image_name'] = image_name
return init_params
init_params['py_version'] = py_version
# We switched image tagging scheme from regular image version (e.g. '1.0') to more expressive
# containing framework version, device type and python version (e.g. '0.12-gpu-py2').
# For backward compatibility map deprecated image tag '1.0' to a '0.12' framework version
# otherwise extract framework version from the tag itself.
init_params['framework_version'] = '0.12' if tag == '1.0' else framework_version_from_tag(tag)
training_job_name = init_params['base_job_name']
if framework != cls.__framework_name__:
raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name))
return init_params
def _prepare_init_params_from_job_description(cls, job_details):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(Chainer, cls)._prepare_init_params_from_job_description(job_details)
for argument in [Chainer._use_mpi, Chainer._num_processes, Chainer._process_slots_per_host,
Chainer._additional_mpi_options]:
value = init_params['hyperparameters'].pop(argument, None)
if value:
init_params[argument[len('sagemaker_'):]] = value
image_name = init_params.pop('image')
framework, py_version, tag = framework_name_from_image(image_name)
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params['image_name'] = image_name
return init_params
init_params['py_version'] = py_version
init_params['framework_version'] = framework_version_from_tag(tag)
training_job_name = init_params['base_job_name']
if framework != cls.__framework_name__:
raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name))
return init_params
def test_legacy_name_from_image_any_tag():
image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-py2-cpu:any-tag'
framework, py_ver, tag = framework_name_from_image(image_name)
assert framework == 'tensorflow'
assert py_ver == 'py2'
assert tag == 'any-tag'
def test_legacy_name_from_wrong_device():
framework, py_ver, tag = framework_name_from_image('123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-myown-py4-gpu:1')
assert framework is None
assert py_ver is None
assert tag is None
def test_legacy_name_from_wrong_python():
framework, py_ver, tag = framework_name_from_image(
'123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-myown-py4-gpu:1')
assert framework is None
assert py_ver is None
assert tag is None
def test_framework_name_from_image_mxnet():
image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet:1.1-gpu-py3'
assert ('mxnet', 'py3', '1.1-gpu-py3') == framework_name_from_image(image_name)
def test_legacy_name_from_wrong_device():
framework, py_ver, tag, _ = fw_utils.framework_name_from_image(
"123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-myown-py4-gpu:1")
assert framework is None
assert py_ver is None
assert tag is None
def test_framework_name_from_image_rl():
image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-rl-mxnet:toolkit1.1-gpu-py3"
assert ("mxnet", "py3", "toolkit1.1-gpu-py3",
None) == fw_utils.framework_name_from_image(image_uri)
def test_legacy_name_from_image_any_tag():
image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-py2-cpu:any-tag'
framework, py_ver, tag = fw_utils.framework_name_from_image(image_name)
assert framework == 'tensorflow'
assert py_ver == 'py2'
assert tag == 'any-tag'
def test_legacy_name_from_framework_image():
image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet-py3-gpu:2.5.6-gpu-py2'
framework, py_ver, tag = fw_utils.framework_name_from_image(image_name)
assert framework == 'mxnet'
assert py_ver == 'py3'
assert tag == '2.5.6-gpu-py2'
def test_framework_name_from_image_rl():
image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-rl-mxnet:toolkit1.1-gpu-py3'
assert (
'mxnet', 'py3',
'toolkit1.1-gpu-py3') == fw_utils.framework_name_from_image(image_name)
def test_framework_name_from_image_tf():
image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6-cpu-py2'
assert ('tensorflow', 'py2',
'1.6-cpu-py2') == fw_utils.framework_name_from_image(image_name)
def test_framework_name_from_other():
framework, py_ver = framework_name_from_image(
'123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-myown-py2-gpu:1')
assert framework is None
assert py_ver is None
def test_framework_name_from_framework_image():
framework, py_ver = framework_name_from_image(
'123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet-py2-gpu:1')
assert framework == 'mxnet'
assert py_ver == 'py2'
def test_framework_name_from_image_mxnet_in_gov():
image_uri = "123.dkr.ecr.region-name.c2s.ic.gov/sagemaker-mxnet:1.1-gpu-py3"
assert ("mxnet", "py3", "1.1-gpu-py3",
None) == fw_utils.framework_name_from_image(image_uri)
def test_framework_name_from_image_tf():
image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6-cpu-py2"
assert ("tensorflow", "py2", "1.6-cpu-py2",
None) == fw_utils.framework_name_from_image(image_uri)
def test_framework_name_from_image_mxnet():
image_name = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet:1.1-gpu-py3"
assert ("mxnet", "py3", "1.1-gpu-py3",
None) == fw_utils.framework_name_from_image(image_name)
def test_legacy_name_from_framework_image():
image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet-py3-gpu:2.5.6-gpu-py2"
framework, py_ver, tag, _ = fw_utils.framework_name_from_image(image_uri)
assert framework == "mxnet"
assert py_ver == "py3"
assert tag == "2.5.6-gpu-py2"
def test_framework_name_from_image_mxnet():
image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet:1.1-gpu-py3'
assert ('mxnet', 'py3', '1.1-gpu-py3') == framework_name_from_image(image_name)
def test_legacy_name_from_image_any_tag():
image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-py2-cpu:any-tag"
framework, py_ver, tag, _ = fw_utils.framework_name_from_image(image_uri)
assert framework == "tensorflow"
assert py_ver == "py2"
assert tag == "any-tag"
def test_framework_name_from_image_tf():
image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6-cpu-py2'
assert ('tensorflow', 'py2', '1.6-cpu-py2') == framework_name_from_image(image_name)
def test_legacy_name_from_framework_image():
image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet-py3-gpu:2.5.6-gpu-py2'
framework, py_ver, tag = framework_name_from_image(image_name)
assert framework == 'mxnet'
assert py_ver == 'py3'
assert tag == '2.5.6-gpu-py2'
def test_framework_name_from_image_tf_scriptmode():
image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-scriptmode:1.12-cpu-py3'
assert ('tensorflow', 'py3', '1.12-cpu-py3',
'scriptmode') == fw_utils.framework_name_from_image(image_name)