def test_dict_of_mixed_input_types():
input_list = _Job._format_inputs_to_input_config({
'a': 's3://foo/bar',
'b': s3_input('s3://whizz/bang')})
expected = [
{'ChannelName': 'a',
'DataSource': {
'S3DataSource': {
'S3DataDistributionType': 'FullyReplicated',
'S3DataType': 'S3Prefix',
'S3Uri': 's3://foo/bar'
}
}
},
{
'ChannelName': 'b',
'DataSource': {
'S3DataSource': {
'S3DataDistributionType': 'FullyReplicated',
'S3DataType': 'S3Prefix',
'S3Uri': 's3://whizz/bang'
}
}
}]
# convert back into map for comparison so list order (which is arbitrary) is ignored
assert {c['ChannelName']: c for c in input_list} == {c['ChannelName']: c for c in expected}
def test_format_string_uri_input():
inputs = s3_input(BUCKET_NAME)
s3_uri_input = _Job._format_string_uri_input(inputs)
assert s3_uri_input.config['DataSource']['S3DataSource']['S3Uri'] == inputs.config[
'DataSource']['S3DataSource']['S3Uri']
def test_tf_deploy_model_server_workers_unset(sagemaker_session):
tf = _build_tf(sagemaker_session)
tf.fit(inputs=s3_input('s3://mybucket/train'))
tf.deploy(initial_instance_count=1, instance_type='ml.c2.2xlarge')
assert MODEL_SERVER_WORKERS_PARAM_NAME.upper() not in sagemaker_session.method_calls[3][1][2]['Environment']
def test_format_inputs_to_input_config_s3_input():
inputs = s3_input(BUCKET_NAME)
channels = _Job._format_inputs_to_input_config(inputs)
assert channels[0]['DataSource']['S3DataSource']['S3Uri'] == inputs.config['DataSource'][
'S3DataSource']['S3Uri']
def test_tf_training_and_evaluation_steps(sagemaker_session):
job_name = "sagemaker-tensorflow-py2-gpu-2017-10-24-14-12-09"
output_path = "s3://{}/output/{}/".format(sagemaker_session.default_bucket(), job_name)
tf = _build_tf(sagemaker_session, training_steps=123, evaluation_steps=456, output_path=output_path)
tf.fit(inputs=s3_input('s3://mybucket/train'))
assert tf.hyperparameters()['training_steps'] == '123'
assert tf.hyperparameters()['evaluation_steps'] == '456'
def test_tf_checkpoint_not_set(sagemaker_session):
job_name = "sagemaker-tensorflow-py2-gpu-2017-10-24-14-12-09"
tf = _build_tf(sagemaker_session, checkpoint_path=None, base_job_name=job_name,
output_path="s3://{}/".format(sagemaker_session.default_bucket()))
tf.fit(inputs=s3_input('s3://mybucket/train'), job_name=job_name)
expected_result = '"s3://{}/{}/checkpoints"'.format(sagemaker_session.default_bucket(), job_name)
assert tf.hyperparameters()['checkpoint_path'] == expected_result
def test_container_log_level(sagemaker_session):
fw = DummyFramework(entry_point=SCRIPT_PATH, role='DummyRole', sagemaker_session=sagemaker_session,
train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE,
container_log_level=logging.DEBUG)
fw.fit(inputs=s3_input('s3://mybucket/train'))
_, _, train_kwargs = sagemaker_session.train.mock_calls[0]
assert train_kwargs['hyperparameters']['sagemaker_container_log_level'] == '10'
def test_enable_cloudwatch_metrics(sagemaker_session):
fw = DummyFramework(entry_point=SCRIPT_PATH, role='DummyRole', sagemaker_session=sagemaker_session,
train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE,
enable_cloudwatch_metrics=True)
fw.fit(inputs=s3_input('s3://mybucket/train'))
_, _, train_kwargs = sagemaker_session.train.mock_calls[0]
assert train_kwargs['hyperparameters']['sagemaker_enable_cloudwatch_metrics']
def test_fit_force_generation(strftime, sagemaker_session):
fw = DummyFramework(entry_point=SCRIPT_PATH,
role=ROLE,
sagemaker_session=sagemaker_session,
train_instance_count=INSTANCE_COUNT,
train_instance_type=INSTANCE_TYPE,
base_job_name='some',
enable_cloudwatch_metrics=True)
fw.base_job_name = None
fw.fit(inputs=s3_input('s3://mybucket/train'))
assert JOB_NAME == fw.latest_training_job.name
def test_tf_deploy_model_server_workers(sagemaker_session):
tf = _build_tf(sagemaker_session)
tf.fit(inputs=s3_input("s3://mybucket/train"))
tf.deploy(initial_instance_count=1, instance_type="ml.c2.2xlarge", model_server_workers=2)
assert (
"2"
== sagemaker_session.method_calls[3][1][2]["Environment"][
MODEL_SERVER_WORKERS_PARAM_NAME.upper()
]
)
def test_tf_checkpoint_not_set(sagemaker_session):
job_name = "sagemaker-tensorflow-py2-gpu-2017-10-24-14-12-09"
tf = _build_tf(sagemaker_session,
checkpoint_path=None,
base_job_name=job_name,
output_path="s3://{}/".format(
sagemaker_session.default_bucket()))
tf.fit(inputs=s3_input('s3://mybucket/train'), job_name=job_name)
expected_result = '"s3://{}/{}/checkpoints"'.format(
sagemaker_session.default_bucket(), job_name)
assert tf.hyperparameters()['checkpoint_path'] == expected_result
def test_tf_training_and_evaluation_steps(sagemaker_session):
job_name = "sagemaker-tensorflow-py2-gpu-2017-10-24-14-12-09"
output_path = "s3://{}/output/{}/".format(
sagemaker_session.default_bucket(), job_name)
tf = _build_tf(sagemaker_session,
training_steps=123,
evaluation_steps=456,
output_path=output_path)
tf.fit(inputs=s3_input('s3://mybucket/train'))
assert tf.hyperparameters()['training_steps'] == '123'
assert tf.hyperparameters()['evaluation_steps'] == '456'
def test_tf_training_and_evaluation_steps_not_set(sagemaker_session):
job_name = "sagemaker-tensorflow-py2-gpu-2017-10-24-14-12-09"
output_path = "s3://{}/output/{}/".format(
sagemaker_session.default_bucket(), job_name)
tf = _build_tf(sagemaker_session,
training_steps=None,
evaluation_steps=None,
output_path=output_path)
tf.fit(inputs=s3_input("s3://mybucket/train"))
assert tf.hyperparameters()["training_steps"] == "null"
assert tf.hyperparameters()["evaluation_steps"] == "null"
def test_enable_cloudwatch_metrics(sagemaker_session):
fw = DummyFramework(entry_point=SCRIPT_PATH,
role='DummyRole',
sagemaker_session=sagemaker_session,
train_instance_count=INSTANCE_COUNT,
train_instance_type=INSTANCE_TYPE,
enable_cloudwatch_metrics=True)
fw.fit(inputs=s3_input('s3://mybucket/train'))
_, _, train_kwargs = sagemaker_session.train.mock_calls[0]
assert train_kwargs['hyperparameters'][
'sagemaker_enable_cloudwatch_metrics']
def test_container_log_level(sagemaker_session):
fw = DummyFramework(entry_point=SCRIPT_PATH,
role='DummyRole',
sagemaker_session=sagemaker_session,
train_instance_count=INSTANCE_COUNT,
train_instance_type=INSTANCE_TYPE,
container_log_level=logging.DEBUG)
fw.fit(inputs=s3_input('s3://mybucket/train'))
_, _, train_kwargs = sagemaker_session.train.mock_calls[0]
assert train_kwargs['hyperparameters'][
'sagemaker_container_log_level'] == '10'
def test_load_config(estimator):
inputs = s3_input(BUCKET_NAME)
config = _Job._load_config(inputs, estimator)
assert config['input_config'][0]['DataSource']['S3DataSource']['S3Uri'] == BUCKET_NAME
assert config['role'] == ROLE
assert config['output_config']['S3OutputPath'] == S3_OUTPUT_PATH
assert 'KmsKeyId' not in config['output_config']
assert config['resource_config']['InstanceCount'] == INSTANCE_COUNT
assert config['resource_config']['InstanceType'] == INSTANCE_TYPE
assert config['resource_config']['VolumeSizeInGB'] == VOLUME_SIZE
assert config['stop_condition']['MaxRuntimeInSeconds'] == MAX_RUNTIME
def _format_string_uri_input(uri_input, validate_uri=True):
if isinstance(uri_input,
str) and validate_uri and uri_input.startswith('s3://'):
return s3_input(uri_input)
elif isinstance(
uri_input,
str) and validate_uri and uri_input.startswith('file://'):
return file_input(uri_input)
elif isinstance(uri_input, str) and validate_uri:
raise ValueError(
'Training input data must be a valid S3 or FILE URI: must start with "s3://" or '
'"file://"')
elif isinstance(uri_input, str):
return s3_input(uri_input)
elif isinstance(uri_input, s3_input):
return uri_input
elif isinstance(uri_input, file_input):
return uri_input
else:
raise ValueError(
'Cannot format input {}. Expecting one of str, s3_input, or file_input'
.format(uri_input))
def _format_model_uri_input(model_uri, validate_uri=True):
if isinstance(model_uri, string_types
) and validate_uri and model_uri.startswith('s3://'):
return s3_input(model_uri,
input_mode='File',
distribution='FullyReplicated',
content_type='application/x-sagemaker-model')
elif isinstance(model_uri, string_types
) and validate_uri and model_uri.startswith('file://'):
return file_input(model_uri)
elif isinstance(model_uri, string_types) and validate_uri:
raise ValueError(
'Model URI must be a valid S3 or FILE URI: must start with "s3://" or '
'"file://')
elif isinstance(model_uri, string_types):
return s3_input(model_uri,
input_mode='File',
distribution='FullyReplicated',
content_type='application/x-sagemaker-model')
else:
raise ValueError(
'Cannot format model URI {}. Expecting str'.format(model_uri))
def test_load_config(estimator):
inputs = s3_input(BUCKET_NAME)
config = _Job._load_config(inputs, estimator)
assert config["input_config"][0]["DataSource"]["S3DataSource"]["S3Uri"] == BUCKET_NAME
assert config["role"] == ROLE
assert config["output_config"]["S3OutputPath"] == S3_OUTPUT_PATH
assert "KmsKeyId" not in config["output_config"]
assert config["resource_config"]["InstanceCount"] == INSTANCE_COUNT
assert config["resource_config"]["InstanceType"] == INSTANCE_TYPE
assert config["resource_config"]["VolumeSizeInGB"] == VOLUME_SIZE
assert config["stop_condition"]["MaxRuntimeInSeconds"] == MAX_RUNTIME
def _format_s3_uri_input(input):
if isinstance(input, str):
if not input.startswith('s3://'):
raise ValueError(
'Training input data must be a valid S3 URI and must start with "s3://"'
)
return s3_input(input)
if isinstance(input, s3_input):
return input
else:
raise ValueError(
'Cannot format input {}. Expecting one of str or s3_input'.
format(input))
def test_fit_verify_job_name(strftime, sagemaker_session):
fw = DummyFramework(entry_point=SCRIPT_PATH, role='DummyRole', sagemaker_session=sagemaker_session,
train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE,
enable_cloudwatch_metrics=True, tags=TAGS)
fw.fit(inputs=s3_input('s3://mybucket/train'))
_, _, train_kwargs = sagemaker_session.train.mock_calls[0]
assert train_kwargs['hyperparameters']['sagemaker_enable_cloudwatch_metrics']
assert train_kwargs['image'] == IMAGE_NAME
assert train_kwargs['input_mode'] == 'File'
assert train_kwargs['tags'] == TAGS
assert train_kwargs['job_name'] == JOB_NAME
assert fw.latest_training_job.name == JOB_NAME
def test_format_input_s3_input():
input_dict = _Job._format_inputs_to_input_config(s3_input('s3://foo/bar', distribution='ShardedByS3Key',
compression='gzip', content_type='whizz',
record_wrapping='bang'))
assert input_dict == [{
'CompressionType': 'gzip',
'ChannelName': 'training',
'ContentType': 'whizz',
'DataSource': {
'S3DataSource': {
'S3DataType': 'S3Prefix',
'S3DataDistributionType': 'ShardedByS3Key',
'S3Uri': 's3://foo/bar'}},
'RecordWrapperType': 'bang'}]
def test_load_config_with_code_channel_no_code_uri(framework):
inputs = s3_input(BUCKET_NAME)
framework.model_uri = MODEL_URI
framework.model_channel_name = MODEL_CHANNEL_NAME
framework._enable_network_isolation = True
config = _Job._load_config(inputs, framework)
assert len(config["input_config"]) == 2
assert config["input_config"][0]["DataSource"]["S3DataSource"]["S3Uri"] == BUCKET_NAME
assert config["role"] == ROLE
assert config["output_config"]["S3OutputPath"] == S3_OUTPUT_PATH
assert "KmsKeyId" not in config["output_config"]
assert config["resource_config"]["InstanceCount"] == INSTANCE_COUNT
assert config["resource_config"]["InstanceType"] == INSTANCE_TYPE
def _format_string_uri_input(input):
if isinstance(input, str):
if input.startswith('s3://'):
return s3_input(input)
elif input.startswith('file://'):
return file_input(input)
else:
raise ValueError('Training input data must be a valid S3 or FILE URI: must start with "s3://" or '
'"file://"')
elif isinstance(input, s3_input):
return input
elif isinstance(input, file_input):
return input
else:
raise ValueError('Cannot format input {}. Expecting one of str, s3_input, or file_input'.format(input))
def test_load_config_with_model_channel(estimator):
inputs = s3_input(BUCKET_NAME)
estimator.model_uri = MODEL_URI
estimator.model_channel_name = MODEL_CHANNEL_NAME
config = _Job._load_config(inputs, estimator)
assert config["input_config"][0]["DataSource"]["S3DataSource"]["S3Uri"] == BUCKET_NAME
assert config["input_config"][1]["DataSource"]["S3DataSource"]["S3Uri"] == MODEL_URI
assert config["input_config"][1]["ChannelName"] == MODEL_CHANNEL_NAME
assert config["role"] == ROLE
assert config["output_config"]["S3OutputPath"] == S3_OUTPUT_PATH
assert "KmsKeyId" not in config["output_config"]
assert config["resource_config"]["InstanceCount"] == INSTANCE_COUNT
assert config["resource_config"]["InstanceType"] == INSTANCE_TYPE
assert config["resource_config"]["VolumeSizeInGB"] == VOLUME_SIZE
assert config["stop_condition"]["MaxRuntimeInSeconds"] == MAX_RUNTIME
def test_load_config_with_model_channel(estimator):
inputs = s3_input(BUCKET_NAME)
estimator.model_uri = MODEL_URI
estimator.model_channel_name = CHANNEL_NAME
config = _Job._load_config(inputs, estimator)
assert config['input_config'][0]['DataSource']['S3DataSource']['S3Uri'] == BUCKET_NAME
assert config['input_config'][1]['DataSource']['S3DataSource']['S3Uri'] == MODEL_URI
assert config['input_config'][1]['ChannelName'] == CHANNEL_NAME
assert config['role'] == ROLE
assert config['output_config']['S3OutputPath'] == S3_OUTPUT_PATH
assert 'KmsKeyId' not in config['output_config']
assert config['resource_config']['InstanceCount'] == INSTANCE_COUNT
assert config['resource_config']['InstanceType'] == INSTANCE_TYPE
assert config['resource_config']['VolumeSizeInGB'] == VOLUME_SIZE
assert config['stop_condition']['MaxRuntimeInSeconds'] == MAX_RUNTIME
def test_format_input_s3_input():
input_dict = _Job._format_inputs_to_input_config(
s3_input(
"s3://foo/bar",
distribution="ShardedByS3Key",
compression="gzip",
content_type="whizz",
record_wrapping="bang",
))
assert input_dict == [{
"CompressionType": "gzip",
"ChannelName": "training",
"ContentType": "whizz",
"DataSource": {
"S3DataSource": {
"S3DataType": "S3Prefix",
"S3DataDistributionType": "ShardedByS3Key",
"S3Uri": "s3://foo/bar",
}
},
"RecordWrapperType": "bang",
}]
def test_s3_input_mode(sagemaker_session, tuner):
expected_input_mode = 'Pipe'
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'failure_script.py')
mxnet = MXNet(entry_point=script_path,
role=ROLE,
framework_version=FRAMEWORK_VERSION,
train_instance_count=TRAIN_INSTANCE_COUNT,
train_instance_type=TRAIN_INSTANCE_TYPE,
sagemaker_session=sagemaker_session)
tuner.estimator = mxnet
tags = [{'Name': 'some-tag-without-a-value'}]
tuner.tags = tags
hyperparameter_ranges = {'num_components': IntegerParameter(2, 4),
'algorithm_mode': CategoricalParameter(['regular', 'randomized'])}
tuner._hyperparameter_ranges = hyperparameter_ranges
tuner.fit(inputs=s3_input('s3://mybucket/train_manifest', input_mode=expected_input_mode))
actual_input_mode = sagemaker_session.method_calls[1][2]['input_mode']
assert actual_input_mode == expected_input_mode
def records_s3_input(self):
"""Return a s3_input to represent the training data"""
return s3_input(self.s3_data, distribution='ShardedByS3Key', s3_data_type=self.s3_data_type)
}
# create estimator
xgb_estimator = Estimator(image_name=container,
hyperparameters=hyperparameters,
role=sagemaker_role,
sagemaker_session=sagemaker.session.Session(sess),
train_instance_count=1,
train_instance_type='ml.m5.4xlarge',
train_volume_size=5,
output_path=config.SAGEMAKER_MODEL_S3_DEST)
# create training inputs
sagemaker_taining_job_name = config.SAGEMAKER_TRAINING_JOB_NAME_PREFIX + '-{}'.format(
guid)
sagemaker_training_data = s3_input(config.SAGEMAKER_TRAINING_DATA_S3_SOURCE,
content_type=config.SAGEMAKER_CONTENT_TYPE)
sagemaker_validation_data = s3_input(
config.SAGEMAKER_VALIDATION_DATA_S3_SOURCE,
content_type=config.SAGEMAKER_CONTENT_TYPE)
sagemaker_training_inputs = {
'train': sagemaker_training_data,
'validation': sagemaker_validation_data
}
# train_config specifies SageMaker training configuration
training_config = training_config(estimator=xgb_estimator,
inputs=sagemaker_training_inputs,
job_name=sagemaker_taining_job_name)
sagemaker_model_name = config.SAGEMAKER_MODEL_NAME_PREFIX + '-{}'.format(guid)
sagemaker_endpoint_name = config.SAGEMAKER_ENDPOINT_NAME_PREFIX + '-{}'.format(
sess = sagemaker.Session()
role = config['role']
bucket = config['bucket']
key_prefix = config['key-prefix']
config['job-name'] = job_name
if __name__ == '__main__':
input_train = sess.upload_data(path='train.csv',
bucket=bucket,
key_prefix=key_prefix)
input_validation = sess.upload_data(path='validation.csv',
bucket=bucket,
key_prefix=key_prefix)
s3_input_train = s3_input(s3_data=input_train, content_type='text/csv')
s3_input_validation = s3_input(s3_data=input_validation,
content_type='text/csv')
container = get_image_uri(boto3.Session().region_name, 'xgboost')
xgb = sagemaker.estimator.Estimator(
container,
role,
train_instance_count=1,
train_instance_type='ml.m4.xlarge',
sagemaker_session=sess,
output_path=config['s3_output_location'])
xgb.set_hyperparameters(max_depth=5,
'Extract, Transform, Load',
parameters={"JobName": job_name,
"Arguments":{
'--S3_SOURCE': data_source,
'--S3_DEST': 's3a://{}/{}/'.format(bucket, project_name),
'--TRAIN_KEY': train_prefix + '/',
'--VAL_KEY': val_prefix +'/'}
}
)
training_step = steps.TrainingStep(
'Model Training',
estimator=xgb,
data={
'train': s3_input(train_data, content_type='csv'),
'validation': s3_input(validation_data, content_type='csv')
},
job_name=training_job_name,
wait_for_completion=True
)
model_step = steps.ModelStep(
'Save Model',
model=training_step.get_expected_model(),
model_name=execution_input['ModelName'],
result_path='$.ModelStepResults'
)
lambda_step = steps.compute.LambdaStep(
'Query Training Results',
def records_s3_input(self):
"""Return a s3_input to represent the training data"""
return s3_input(self.s3_data,
distribution='ShardedByS3Key',
s3_data_type=self.s3_data_type)
train_instance_type='ml.c4.xlarge',
output_path=output_path,
sagemaker_session=sess)
num_topics = 6
ntm.set_hyperparameters(num_topics=num_topics,
feature_dim=vocab_size,
mini_batch_size=128,
epochs=100,
num_patience_epochs=5,
tolerance=0.001)
# In[83]:
from sagemaker.session import s3_input
s3_train = s3_input(s3_train_data, distribution='ShardedByS3Key')
# In[84]:
ntm.fit({'train': s3_train, 'test': s3_val_data})
# In[59]:
ntm_predictor = ntm.deploy(initial_instance_count=1,
instance_type='ml.m4.xlarge')
# In[65]:
from sagemaker.predictor import csv_serializer, json_deserializer
ntm_predictor.content_type = 'text/csv'
