def _prepare_init_params_from_job_description(cls,
job_details,
model_channel_name=None):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(
XGBoost,
cls)._prepare_init_params_from_job_description(job_details)
image_name = init_params.pop("image")
framework, py_version, tag, _ = framework_name_from_image(image_name)
init_params["py_version"] = py_version
if framework and framework != cls.__framework_name__:
training_job_name = init_params["base_job_name"]
raise ValueError(
"Training job: {} didn't use image for requested framework".
format(training_job_name))
init_params["framework_version"] = framework_version_from_tag(tag)
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params["image_name"] = image_name
return init_params
def _prepare_init_params_from_job_description(cls, job_details, model_channel_name=None):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
model_channel_name (str): Name of the channel where pre-trained model data will be downloaded.
Returns:
dictionary: The transformed init_params
"""
init_params = super(PyTorch, cls)._prepare_init_params_from_job_description(job_details, model_channel_name)
image_name = init_params.pop('image')
framework, py_version, tag, _ = framework_name_from_image(image_name)
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params['image_name'] = image_name
return init_params
init_params['py_version'] = py_version
init_params['framework_version'] = framework_version_from_tag(tag)
training_job_name = init_params['base_job_name']
if framework != cls.__framework_name__:
raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name))
return init_params
def _prepare_init_params_from_job_description(cls, job_details):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(PyTorch, cls)._prepare_init_params_from_job_description(job_details)
image_name = init_params.pop('image')
framework, py_version, tag = framework_name_from_image(image_name)
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params['image_name'] = image_name
return init_params
init_params['py_version'] = py_version
init_params['framework_version'] = framework_version_from_tag(tag)
training_job_name = init_params['base_job_name']
if framework != cls.__framework_name__:
raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name))
return init_params
def _prepare_init_params_from_job_description(cls, job_details):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(
MXNet, cls)._prepare_init_params_from_job_description(job_details)
framework, py_version, tag = framework_name_from_image(
init_params.pop('image'))
init_params['py_version'] = py_version
# We switched image tagging scheme from regular image version (e.g. '1.0') to more expressive
# containing framework version, device type and python version (e.g. '0.12-gpu-py2').
# For backward compatibility map deprecated image tag '1.0' to a '0.12' framework version
# otherwise extract framework version from the tag itself.
init_params[
'framework_version'] = '0.12' if tag == '1.0' else framework_version_from_tag(
tag)
training_job_name = init_params['base_job_name']
if framework != cls.__framework_name__:
raise ValueError(
"Training job: {} didn't use image for requested framework".
format(training_job_name))
return init_params
def _prepare_init_params_from_job_description(cls, job_details):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(
PyTorch,
cls)._prepare_init_params_from_job_description(job_details)
framework, py_version, tag = framework_name_from_image(
init_params.pop('image'))
init_params['py_version'] = py_version
init_params['framework_version'] = framework_version_from_tag(tag)
training_job_name = init_params['base_job_name']
if framework != cls.__framework_name__:
raise ValueError(
"Training job: {} didn't use image for requested framework".
format(training_job_name))
return init_params
def _prepare_init_params_from_job_description(cls, job_details, model_channel_name=None):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
model_channel_name (str): Name of the channel where pre-trained model data will be downloaded.
Returns:
dictionary: The transformed init_params
"""
init_params = super(MXNet, cls)._prepare_init_params_from_job_description(job_details, model_channel_name)
image_name = init_params.pop('image')
framework, py_version, tag = framework_name_from_image(image_name)
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params['image_name'] = image_name
return init_params
init_params['py_version'] = py_version
# We switched image tagging scheme from regular image version (e.g. '1.0') to more expressive
# containing framework version, device type and python version (e.g. '0.12-gpu-py2').
# For backward compatibility map deprecated image tag '1.0' to a '0.12' framework version
# otherwise extract framework version from the tag itself.
init_params['framework_version'] = '0.12' if tag == '1.0' else framework_version_from_tag(tag)
training_job_name = init_params['base_job_name']
if framework != cls.__framework_name__:
raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name))
return init_params
def _prepare_init_params_from_job_description(cls, job_details):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(MXNet, cls)._prepare_init_params_from_job_description(job_details)
image_name = init_params.pop('image')
framework, py_version, tag = framework_name_from_image(image_name)
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params['image_name'] = image_name
return init_params
init_params['py_version'] = py_version
# We switched image tagging scheme from regular image version (e.g. '1.0') to more expressive
# containing framework version, device type and python version (e.g. '0.12-gpu-py2').
# For backward compatibility map deprecated image tag '1.0' to a '0.12' framework version
# otherwise extract framework version from the tag itself.
init_params['framework_version'] = '0.12' if tag == '1.0' else framework_version_from_tag(tag)
training_job_name = init_params['base_job_name']
if framework != cls.__framework_name__:
raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name))
return init_params
def test_framework_version_from_tag():
tags = (
"1.5rc-keras-cpu-py2",
"1.5rc-keras-gpu-py2",
"1.5rc-keras-cpu-py3",
"1.5rc-keras-gpu-py36",
"1.5rc-keras-gpu-py37",
)
for tag in tags:
version = fw_utils.framework_version_from_tag(tag)
assert "1.5rc-keras" == version
def _prepare_init_params_from_job_description(cls,
job_details,
model_channel_name=None):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(TensorFlow,
cls)._prepare_init_params_from_job_description(
job_details, model_channel_name)
# Move some of the tensorflow specific init params from hyperparameters into the main init
# params.
for argument in ("checkpoint_path", "training_steps",
"evaluation_steps", "model_dir"):
value = init_params["hyperparameters"].pop(argument, None)
if value is not None:
init_params[argument] = value
image_name = init_params.pop("image")
framework, py_version, tag, script_mode = fw.framework_name_from_image(
image_name)
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params["image_name"] = image_name
return init_params
if script_mode:
init_params["script_mode"] = True
init_params["py_version"] = py_version
# We switched image tagging scheme from regular image version (e.g. '1.0') to more
# expressive containing framework version, device type and python version
# (e.g. '1.5-gpu-py2'). For backward compatibility map deprecated image tag '1.0' to a
# '1.4' framework version otherwise extract framework version from the tag itself.
init_params["framework_version"] = ("1.4" if tag == "1.0" else
fw.framework_version_from_tag(tag))
training_job_name = init_params["base_job_name"]
if framework != cls.__framework_name__:
raise ValueError(
"Training job: {} didn't use image for requested framework".
format(training_job_name))
return init_params
def _prepare_init_params_from_job_description(cls,
job_details,
model_channel_name=None):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(TensorFlow,
cls)._prepare_init_params_from_job_description(
job_details, model_channel_name)
image_uri = init_params.pop("image_uri")
framework, py_version, tag, script_mode = fw.framework_name_from_image(
image_uri)
if not framework:
# If we were unable to parse the framework name from the image, it is not one of our
# officially supported images, so just add the image to the init params.
init_params["image_uri"] = image_uri
return init_params
model_dir = init_params["hyperparameters"].pop("model_dir", None)
if model_dir:
init_params["model_dir"] = model_dir
elif script_mode is None:
init_params["model_dir"] = False
init_params["py_version"] = py_version
# We switched image tagging scheme from regular image version (e.g. '1.0') to more
# expressive containing framework version, device type and python version
# (e.g. '1.5-gpu-py2'). For backward compatibility map deprecated image tag '1.0' to a
# '1.4' framework version otherwise extract framework version from the tag itself.
init_params["framework_version"] = ("1.4" if tag == "1.0" else
fw.framework_version_from_tag(tag))
# Legacy images are required to be passed in explicitly.
if not script_mode:
init_params["image_uri"] = image_uri
if framework != cls._framework_name:
raise ValueError(
"Training job: {} didn't use image for requested framework".
format(job_details["TrainingJobName"]))
return init_params
def _prepare_init_params_from_job_description(cls,
job_details,
model_channel_name=None):
"""Convert the job description to init params that can be handled by the
class constructor
Args:
job_details: the returned job details from a describe_training_job
API call.
model_channel_name (str): Name of the channel where pre-trained
model data will be downloaded.
Returns:
dictionary: The transformed init_params
"""
init_params = super(Chainer,
cls)._prepare_init_params_from_job_description(
job_details, model_channel_name)
for argument in [
Chainer._use_mpi,
Chainer._num_processes,
Chainer._process_slots_per_host,
Chainer._additional_mpi_options,
]:
value = init_params["hyperparameters"].pop(argument, None)
if value:
init_params[argument[len("sagemaker_"):]] = value
image_uri = init_params.pop("image_uri")
framework, py_version, tag, _ = framework_name_from_image(image_uri)
if tag is None:
framework_version = None
else:
framework_version = framework_version_from_tag(tag)
init_params["framework_version"] = framework_version
init_params["py_version"] = py_version
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params["image_uri"] = image_uri
return init_params
if framework != cls._framework_name:
raise ValueError(
"Training job: {} didn't use image for requested framework".
format(job_details["TrainingJobName"]))
return init_params
def _prepare_init_params_from_job_description(cls, job_details):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(
TensorFlow,
cls)._prepare_init_params_from_job_description(job_details)
# Move some of the tensorflow specific init params from hyperparameters into the main init params.
for argument in [
'checkpoint_path', 'training_steps', 'evaluation_steps'
]:
value = init_params['hyperparameters'].pop(argument, None)
if value is not None:
init_params[argument] = value
image_name = init_params.pop('image')
framework, py_version, tag = framework_name_from_image(image_name)
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params['image_name'] = image_name
return init_params
init_params['py_version'] = py_version
# We switched image tagging scheme from regular image version (e.g. '1.0') to more expressive
# containing framework version, device type and python version (e.g. '1.5-gpu-py2').
# For backward compatibility map deprecated image tag '1.0' to a '1.4' framework version
# otherwise extract framework version from the tag itself.
init_params[
'framework_version'] = '1.4' if tag == '1.0' else framework_version_from_tag(
tag)
training_job_name = init_params['base_job_name']
if framework != cls.__framework_name__:
raise ValueError(
"Training job: {} didn't use image for requested framework".
format(training_job_name))
return init_params
def _prepare_init_params_from_job_description(cls, job_details):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(
Chainer,
cls)._prepare_init_params_from_job_description(job_details)
for argument in [
Chainer._use_mpi, Chainer._num_processes,
Chainer._process_slots_per_host,
Chainer._additional_mpi_options
]:
value = init_params['hyperparameters'].pop(argument, None)
if value:
init_params[argument[len('sagemaker_'):]] = value
image_name = init_params.pop('image')
framework, py_version, tag = framework_name_from_image(image_name)
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params['image_name'] = image_name
return init_params
init_params['py_version'] = py_version
init_params['framework_version'] = framework_version_from_tag(tag)
training_job_name = init_params['base_job_name']
if framework != cls.__framework_name__:
raise ValueError(
"Training job: {} didn't use image for requested framework".
format(training_job_name))
return init_params
def _prepare_init_params_from_job_description(cls,
job_details,
model_channel_name=None):
"""Convert the job description to init params that can be handled by the
class constructor
Args:
job_details: the returned job details from a describe_training_job
API call.
model_channel_name (str): Name of the channel where pre-trained
model data will be downloaded (default: None).
Returns:
dictionary: The transformed init_params
"""
init_params = super(SKLearn,
cls)._prepare_init_params_from_job_description(
job_details, model_channel_name)
image_uri = init_params.pop("image_uri")
framework, py_version, tag, _ = framework_name_from_image(image_uri)
if tag is None:
framework_version = None
else:
framework_version = framework_version_from_tag(tag)
init_params["framework_version"] = framework_version
init_params["py_version"] = py_version
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params["image_uri"] = image_uri
return init_params
if framework and framework != "scikit-learn":
raise ValueError(
"Training job: {} didn't use image for requested framework".
format(job_details["TrainingJobName"]))
return init_params
def _prepare_init_params_from_job_description(cls, job_details):
"""Convert the job description to init params that can be handled by the class constructor
Args:
job_details: the returned job details from a describe_training_job API call.
Returns:
dictionary: The transformed init_params
"""
init_params = super(Chainer, cls)._prepare_init_params_from_job_description(job_details)
for argument in [Chainer._use_mpi, Chainer._num_processes, Chainer._process_slots_per_host,
Chainer._additional_mpi_options]:
value = init_params['hyperparameters'].pop(argument, None)
if value:
init_params[argument[len('sagemaker_'):]] = value
image_name = init_params.pop('image')
framework, py_version, tag = framework_name_from_image(image_name)
if not framework:
# If we were unable to parse the framework name from the image it is not one of our
# officially supported images, in this case just add the image to the init params.
init_params['image_name'] = image_name
return init_params
init_params['py_version'] = py_version
init_params['framework_version'] = framework_version_from_tag(tag)
training_job_name = init_params['base_job_name']
if framework != cls.__framework_name__:
raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name))
return init_params
def test_framework_version_from_tag_other():
version = fw_utils.framework_version_from_tag("weird-tag-py2")
assert version is None
def test_framework_version_from_tag():
version = fw_utils.framework_version_from_tag("1.5rc-keras-gpu-py2")
assert version == "1.5rc-keras"
def test_framework_version_from_tag():
version = framework_version_from_tag('1.5rc-keras-gpu-py2')
assert version == '1.5rc-keras'
def test_framework_version_from_tag_other():
version = framework_version_from_tag('weird-tag-py2')
assert version is None
def test_framework_version_from_tag_other():
version = framework_version_from_tag('weird-tag-py2')
assert version is None
def test_framework_version_from_tag():
version = framework_version_from_tag('1.5rc-keras-gpu-py2')
assert version == '1.5rc-keras'
