def test_tf(m_tar, e_tar, time, strftime, sagemaker_session, tf_version):
tf = TensorFlow(entry_point=SCRIPT_FILE,
role=ROLE,
sagemaker_session=sagemaker_session,
training_steps=1000,
evaluation_steps=10,
train_instance_count=INSTANCE_COUNT,
train_instance_type=INSTANCE_TYPE,
framework_version=tf_version,
requirements_file=REQUIREMENTS_FILE,
source_dir=DATA_DIR)
inputs = 's3://mybucket/train'
s3_prefix = 's3://{}/{}/source/sourcedir.tar.gz'.format(
BUCKET_NAME, JOB_NAME)
e_tar.return_value = UploadedCode(s3_prefix=s3_prefix,
script_name=SCRIPT_FILE)
s3_prefix = 's3://{}/{}/sourcedir.tar.gz'.format(BUCKET_NAME, JOB_NAME)
m_tar.return_value = UploadedCode(s3_prefix=s3_prefix,
script_name=SCRIPT_FILE)
tf.fit(inputs=inputs)
call_names = [c[0] for c in sagemaker_session.method_calls]
assert call_names == ['train', 'logs_for_job']
expected_train_args = _create_train_job(tf_version)
expected_train_args['input_config'][0]['DataSource']['S3DataSource'][
'S3Uri'] = inputs
actual_train_args = sagemaker_session.method_calls[0][2]
assert actual_train_args == expected_train_args
model = tf.create_model()
environment = {
'Environment': {
'SAGEMAKER_SUBMIT_DIRECTORY':
's3://{}/{}/sourcedir.tar.gz'.format(BUCKET_NAME, JOB_NAME),
'SAGEMAKER_PROGRAM':
'dummy_script.py',
'SAGEMAKER_REQUIREMENTS':
'dummy_requirements.txt',
'SAGEMAKER_ENABLE_CLOUDWATCH_METRICS':
'false',
'SAGEMAKER_REGION':
'us-west-2',
'SAGEMAKER_CONTAINER_LOG_LEVEL':
'20'
},
'Image':
create_image_uri('us-west-2', "tensorflow", INSTANCE_TYPE, tf_version,
"py2"),
'ModelDataUrl':
's3://m/m.tar.gz'
}
assert environment == model.prepare_container_def(INSTANCE_TYPE)
assert 'cpu' in model.prepare_container_def(INSTANCE_TYPE)['Image']
predictor = tf.deploy(1, INSTANCE_TYPE)
assert isinstance(predictor, TensorFlowPredictor)
def _upload_code(self, key_prefix, repack=False):
local_code = utils.get_config_value("local.local_code",
self.sagemaker_session.config)
if self.sagemaker_session.local_mode and local_code:
self.uploaded_code = None
elif not repack:
bucket = self.bucket or self.sagemaker_session.default_bucket()
self.uploaded_code = fw_utils.tar_and_upload_dir(
session=self.sagemaker_session.boto_session,
bucket=bucket,
s3_key_prefix=key_prefix,
script=self.entry_point,
directory=self.source_dir,
dependencies=self.dependencies,
)
if repack:
bucket = self.bucket or self.sagemaker_session.default_bucket()
repacked_model_data = "s3://" + os.path.join(
bucket, key_prefix, "model.tar.gz")
utils.repack_model(
inference_script=self.entry_point,
source_directory=self.source_dir,
dependencies=self.dependencies,
model_uri=self.model_data,
repacked_model_uri=repacked_model_data,
sagemaker_session=self.sagemaker_session,
)
self.repacked_model_data = repacked_model_data
self.uploaded_code = UploadedCode(
s3_prefix=self.repacked_model_data,
script_name=os.path.basename(self.entry_point))
def test_model_mms_version(repack_model, sagemaker_session):
model_kms_key = "kms-key"
model = MXNetModel(
MODEL_DATA,
role=ROLE,
entry_point=SCRIPT_PATH,
framework_version=MXNetModel._LOWEST_MMS_VERSION,
sagemaker_session=sagemaker_session,
name="test-mxnet-model",
model_kms_key=model_kms_key,
)
predictor = model.deploy(1, GPU)
repack_model.assert_called_once_with(
inference_script=SCRIPT_PATH,
source_directory=None,
dependencies=[],
model_uri=MODEL_DATA,
repacked_model_uri="s3://mybucket/test-mxnet-model/model.tar.gz",
sagemaker_session=sagemaker_session,
kms_key=model_kms_key,
)
assert model.model_data == MODEL_DATA
assert model.repacked_model_data == "s3://mybucket/test-mxnet-model/model.tar.gz"
assert model.uploaded_code == UploadedCode(
s3_prefix="s3://mybucket/test-mxnet-model/model.tar.gz",
script_name=os.path.basename(SCRIPT_PATH),
)
assert isinstance(predictor, MXNetPredictor)
def attach(cls, training_job_name, sagemaker_session=None):
"""Attach to an existing training job.
Create an Estimator bound to an existing training job, each subclass is responsible to implement
``_prepare_init_params_from_job_description()`` as this method delegates the actual conversion of a training
job description to the arguments that the class constructor expects. After attaching, if the training job has a
Complete status, it can be ``deploy()`` ed to create a SageMaker Endpoint and return a ``Predictor``.
If the training job is in progress, attach will block and display log messages
from the training job, until the training job completes.
Args:
training_job_name (str): The name of the training job to attach to.
sagemaker_session (sagemaker.session.Session): Session object which manages interactions with
Amazon SageMaker APIs and any other AWS services needed. If not specified, the estimator creates one
using the default AWS configuration chain.
Examples:
>>> my_estimator.fit(wait=False)
>>> training_job_name = my_estimator.latest_training_job.name
Later on:
>>> attached_estimator = Estimator.attach(training_job_name)
>>> attached_estimator.deploy()
Returns:
Instance of the calling ``Estimator`` Class with the attached training job.
"""
estimator = super(Framework, cls).attach(training_job_name, sagemaker_session)
estimator.uploaded_code = UploadedCode(estimator.source_dir, estimator.entry_point)
return estimator
def test_tar_and_upload_dir_s3(sagemaker_session):
bucket = 'mybucker'
s3_key_prefix = 'something/source'
script = 'mnist.py'
directory = 's3://m'
result = tar_and_upload_dir(sagemaker_session, bucket, s3_key_prefix, script, directory)
assert result == UploadedCode('s3://m', 'mnist.py')
def test_tar_and_upload_dir_not_s3(sagemaker_session):
bucket = 'mybucker'
s3_key_prefix = 'something/source'
script = os.path.basename(__file__)
directory = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
result = tar_and_upload_dir(sagemaker_session, bucket, s3_key_prefix, script, directory)
assert result == UploadedCode('s3://{}/{}/sourcedir.tar.gz'.format(bucket, s3_key_prefix), script)
def attach(cls, training_job_name, sagemaker_session=None, **kwargs):
"""Attach to an existing training job.
Create an Estimator bound to an existing training job. After attaching, if
the training job has a Complete status, it can be ``deploy()`` ed to create
a SageMaker Endpoint and return a ``Predictor``.
If the training job is in progress, attach will block and display log messages
from the training job, until the training job completes.
Args:
training_job_name (str): The name of the training job to attach to.
sagemaker_session (sagemaker.session.Session): Session object which manages interactions with
Amazon SageMaker APIs and any other AWS services needed. If not specified, the estimator creates one
using the default AWS configuration chain.
**kwargs: Additional kwargs passed to the :class:`~sagemaker.estimator.Estimator` constructor.
Returns:
sagemaker.estimator.Framework: ``Estimator`` with the attached training job.
"""
sagemaker_session = sagemaker_session or Session()
if training_job_name is not None:
job_details = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=training_job_name)
init_params, hp, _ = cls._prepare_estimator_params_from_job_description(
job_details)
else:
# this case is only valid when called from inheriting class and then the class must declare framework
if not hasattr(cls, '__framework_name__'):
raise ValueError('must specify training_job name')
init_params = dict(kwargs)
hp = init_params.pop('hyperparameters')
# parameters for framework classes
framework_init_params = dict()
framework_init_params['entry_point'] = json.loads(
hp.get(SCRIPT_PARAM_NAME))
framework_init_params['source_dir'] = json.loads(
hp.get(DIR_PARAM_NAME))
framework_init_params['enable_cloudwatch_metrics'] = json.loads(
hp.get(CLOUDWATCH_METRICS_PARAM_NAME))
framework_init_params['container_log_level'] = json.loads(
hp.get(CONTAINER_LOG_LEVEL_PARAM_NAME))
# drop json and remove other SageMaker specific additions
hyperparameters = {entry: json.loads(hp[entry]) for entry in hp}
framework_init_params['hyperparameters'] = hyperparameters
init_params.update(framework_init_params)
estimator = cls(sagemaker_session=sagemaker_session, **init_params)
estimator.latest_training_job = _TrainingJob(
sagemaker_session=sagemaker_session,
training_job_name=init_params['base_job_name'])
estimator.latest_training_job.wait()
estimator.uploaded_code = UploadedCode(estimator.source_dir,
estimator.entry_point)
return estimator
def attach(cls, training_job_name, sagemaker_session=None, model_channel_name="model"):
"""Attach to an existing training job.
Create an Estimator bound to an existing training job, each subclass
is responsible to implement
``_prepare_init_params_from_job_description()`` as this method delegates
the actual conversion of a training job description to the arguments
that the class constructor expects. After attaching, if the training job
has a Complete status, it can be ``deploy()`` ed to create a SageMaker
Endpoint and return a ``Predictor``.
If the training job is in progress, attach will block and display log
messages from the training job, until the training job completes.
Examples:
>>> my_estimator.fit(wait=False)
>>> training_job_name = my_estimator.latest_training_job.name
Later on:
>>> attached_estimator = Estimator.attach(training_job_name)
>>> attached_estimator.deploy()
Args:
training_job_name (str): The name of the training job to attach to.
sagemaker_session (sagemaker.session.Session): Session object which
manages interactions with Amazon SageMaker APIs and any other
AWS services needed. If not specified, the estimator creates one
using the default AWS configuration chain.
model_channel_name (str): Name of the channel where pre-trained
model data will be downloaded (default: 'model'). If no channel
with the same name exists in the training job, this option will
be ignored.
Returns:
Instance of the calling ``Estimator`` Class with the attached
training job.
"""
sagemaker_session = sagemaker_session or Session()
job_details = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=training_job_name
)
init_params = cls._prepare_init_params_from_job_description(job_details, model_channel_name)
tags = sagemaker_session.sagemaker_client.list_tags(
ResourceArn=job_details["TrainingJobArn"]
)["Tags"]
init_params.update(tags=tags)
estimator = cls(sagemaker_session=sagemaker_session, **init_params)
estimator.latest_training_job = _TrainingJob(
sagemaker_session=sagemaker_session, job_name=training_job_name
)
estimator._current_job_name = estimator.latest_training_job.name
estimator.latest_training_job.wait()
# pylint gets confused thinking that estimator is an EstimatorBase instance, but it actually
# is a Framework or any of its derived classes. We can safely ignore the no-member errors.
estimator.uploaded_code = UploadedCode(
estimator.source_dir, estimator.entry_point # pylint: disable=no-member
)
return estimator
def test_create_model_with_network_isolation(upload, sagemaker_session):
source_dir = "s3://mybucket/source"
repacked_model_data = "s3://mybucket/prefix/model.tar.gz"
sklearn_model = SKLearnModel(
model_data=source_dir,
role=ROLE,
sagemaker_session=sagemaker_session,
entry_point=SCRIPT_PATH,
enable_network_isolation=True,
)
sklearn_model.uploaded_code = UploadedCode(s3_prefix=repacked_model_data, script_name="script")
sklearn_model.repacked_model_data = repacked_model_data
model_values = sklearn_model.prepare_container_def(CPU)
assert model_values["Environment"]["SAGEMAKER_SUBMIT_DIRECTORY"] == "/opt/ml/model/code"
assert model_values["ModelDataUrl"] == repacked_model_data
def test_model_mms_version(repack_model, sagemaker_session):
model = MXNetModel(MODEL_DATA,
role=ROLE,
entry_point=SCRIPT_PATH,
framework_version=MXNetModel._LOWEST_MMS_VERSION,
sagemaker_session=sagemaker_session)
predictor = model.deploy(1, GPU)
repack_model.assert_called_once_with(inference_script=SCRIPT_PATH,
source_directory=None,
model_uri=MODEL_DATA,
sagemaker_session=sagemaker_session)
assert model.model_data == MODEL_DATA
assert model.repacked_model_data == REPACKED_MODEL_DATA
assert model.uploaded_code == UploadedCode(
s3_prefix=REPACKED_MODEL_DATA,
script_name=os.path.basename(SCRIPT_PATH))
assert isinstance(predictor, MXNetPredictor)
def uploaded_code(
s3_prefix="s3://mocked_s3_uri_from_upload_data/my_job_name/source/sourcedir.tar.gz",
script_name="processing_code.py",
):
return UploadedCode(s3_prefix=s3_prefix, script_name=script_name)
