def test_inference_pipeline_batch_transform(sagemaker_session):
sparkml_model_data = sagemaker_session.upload_data(
path=os.path.join(SPARKML_DATA_PATH, 'mleap_model.tar.gz'),
key_prefix='integ-test-data/sparkml/model')
xgb_model_data = sagemaker_session.upload_data(
path=os.path.join(XGBOOST_DATA_PATH, 'xgb_model.tar.gz'),
key_prefix='integ-test-data/xgboost/model')
batch_job_name = 'test-inference-pipeline-batch-{}'.format(
sagemaker_timestamp())
sparkml_model = SparkMLModel(model_data=sparkml_model_data,
env={'SAGEMAKER_SPARKML_SCHEMA': SCHEMA},
sagemaker_session=sagemaker_session)
xgb_image = get_image_uri(sagemaker_session.boto_region_name, 'xgboost')
xgb_model = Model(model_data=xgb_model_data,
image=xgb_image,
sagemaker_session=sagemaker_session)
model = PipelineModel(models=[sparkml_model, xgb_model],
role='SageMakerRole',
sagemaker_session=sagemaker_session,
name=batch_job_name)
transformer = model.transformer(1, 'ml.m4.xlarge')
transform_input_key_prefix = 'integ-test-data/sparkml_xgboost/transform'
transform_input = transformer.sagemaker_session.upload_data(
path=VALID_DATA_PATH, key_prefix=transform_input_key_prefix)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
transformer.transform(transform_input,
content_type=CONTENT_TYPE_CSV,
job_name=batch_job_name)
transformer.wait()
def test_transform_byo_estimator(sagemaker_session, cpu_instance_type):
data_path = os.path.join(DATA_DIR, "one_p_mnist")
pickle_args = {} if sys.version_info.major == 2 else {"encoding": "latin1"}
tags = [{"Key": "some-tag", "Value": "value-for-tag"}]
# Load the data into memory as numpy arrays
train_set_path = os.path.join(data_path, "mnist.pkl.gz")
with gzip.open(train_set_path, "rb") as f:
train_set, _, _ = pickle.load(f, **pickle_args)
kmeans = KMeans(
role="SageMakerRole",
train_instance_count=1,
train_instance_type=cpu_instance_type,
k=10,
sagemaker_session=sagemaker_session,
output_path="s3://{}/".format(sagemaker_session.default_bucket()),
)
# set kmeans specific hp
kmeans.init_method = "random"
kmeans.max_iterators = 1
kmeans.tol = 1
kmeans.num_trials = 1
kmeans.local_init_method = "kmeans++"
kmeans.half_life_time_size = 1
kmeans.epochs = 1
records = kmeans.record_set(train_set[0][:100])
job_name = unique_name_from_base("test-kmeans-attach")
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
kmeans.fit(records, job_name=job_name)
estimator = Estimator.attach(training_job_name=job_name, sagemaker_session=sagemaker_session)
estimator._enable_network_isolation = True
transform_input_path = os.path.join(data_path, "transform_input.csv")
transform_input_key_prefix = "integ-test-data/one_p_mnist/transform"
transform_input = kmeans.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix
)
transformer = estimator.transformer(1, cpu_instance_type, tags=tags)
transformer.transform(transform_input, content_type="text/csv")
with timeout_and_delete_model_with_transformer(
transformer, sagemaker_session, minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES
):
transformer.wait()
model_desc = sagemaker_session.sagemaker_client.describe_model(
ModelName=transformer.model_name
)
assert model_desc["EnableNetworkIsolation"]
model_tags = sagemaker_session.sagemaker_client.list_tags(
ResourceArn=model_desc["ModelArn"]
)["Tags"]
assert tags == model_tags
def test_transform_mxnet(mxnet_estimator, mxnet_transform_input,
sagemaker_session, cpu_instance_type):
kms_key_arn = get_or_create_kms_key(sagemaker_session)
output_filter = "$"
input_filter = "$"
transformer = _create_transformer_and_transform_job(
mxnet_estimator,
mxnet_transform_input,
cpu_instance_type,
kms_key_arn,
input_filter=input_filter,
output_filter=output_filter,
join_source=None,
)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
transformer.wait()
job_desc = transformer.sagemaker_session.describe_transform_job(
job_name=transformer.latest_transform_job.name)
assert kms_key_arn == job_desc["TransformResources"]["VolumeKmsKeyId"]
assert output_filter == job_desc["DataProcessing"]["OutputFilter"]
assert input_filter == job_desc["DataProcessing"]["InputFilter"]
def test_transform_pytorch_vpc_custom_model_bucket(
sagemaker_session,
pytorch_inference_latest_version,
pytorch_inference_latest_py_version,
cpu_instance_type,
custom_bucket_name,
):
data_dir = os.path.join(DATA_DIR, "pytorch_mnist")
ec2_client = sagemaker_session.boto_session.client("ec2")
subnet_ids, security_group_id = get_or_create_vpc_resources(ec2_client)
model_data = sagemaker_session.upload_data(
path=os.path.join(data_dir, "model.tar.gz"),
bucket=custom_bucket_name,
key_prefix="integ-test-data/pytorch_mnist/model",
)
model = PyTorchModel(
model_data=model_data,
entry_point=os.path.join(data_dir, "mnist.py"),
role="SageMakerRole",
framework_version=pytorch_inference_latest_version,
py_version=pytorch_inference_latest_py_version,
sagemaker_session=sagemaker_session,
vpc_config={
"Subnets": subnet_ids,
"SecurityGroupIds": [security_group_id]
},
code_location="s3://{}".format(custom_bucket_name),
)
transform_input = sagemaker_session.upload_data(
path=os.path.join(data_dir, "transform", "data.npy"),
key_prefix="integ-test-data/pytorch_mnist/transform",
)
transformer = model.transformer(1, cpu_instance_type)
transformer.transform(
transform_input,
content_type="application/x-npy",
job_name=unique_name_from_base("test-transform-vpc"),
)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
transformer.wait()
model_desc = sagemaker_session.sagemaker_client.describe_model(
ModelName=transformer.model_name)
assert set(subnet_ids) == set(model_desc["VpcConfig"]["Subnets"])
assert [security_group_id
] == model_desc["VpcConfig"]["SecurityGroupIds"]
model_bucket, _ = s3.parse_s3_url(
model_desc["PrimaryContainer"]["ModelDataUrl"])
assert custom_bucket_name == model_bucket
def test_transform_byo_estimator(sagemaker_session):
data_path = os.path.join(DATA_DIR, 'one_p_mnist')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
tags = [{'Key': 'some-tag', 'Value': 'value-for-tag'}]
# Load the data into memory as numpy arrays
train_set_path = os.path.join(data_path, 'mnist.pkl.gz')
with gzip.open(train_set_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
kmeans = KMeans(role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
k=10,
sagemaker_session=sagemaker_session,
output_path='s3://{}/'.format(
sagemaker_session.default_bucket()))
# set kmeans specific hp
kmeans.init_method = 'random'
kmeans.max_iterators = 1
kmeans.tol = 1
kmeans.num_trials = 1
kmeans.local_init_method = 'kmeans++'
kmeans.half_life_time_size = 1
kmeans.epochs = 1
records = kmeans.record_set(train_set[0][:100])
job_name = unique_name_from_base('test-kmeans-attach')
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
kmeans.fit(records, job_name=job_name)
transform_input_path = os.path.join(data_path, 'transform_input.csv')
transform_input_key_prefix = 'integ-test-data/one_p_mnist/transform'
transform_input = kmeans.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix)
estimator = Estimator.attach(training_job_name=job_name,
sagemaker_session=sagemaker_session)
transformer = estimator.transformer(1, 'ml.m4.xlarge', tags=tags)
transformer.transform(transform_input, content_type='text/csv')
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
transformer.wait()
model_desc = sagemaker_session.sagemaker_client.describe_model(
ModelName=transformer.model_name)
model_tags = sagemaker_session.sagemaker_client.list_tags(
ResourceArn=model_desc['ModelArn'])['Tags']
assert tags == model_tags
def test_transform_mxnet(sagemaker_session, mxnet_full_version, cpu_instance_type):
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
script_path = os.path.join(data_path, "mnist.py")
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
train_instance_count=1,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version,
)
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "train"), key_prefix="integ-test-data/mxnet_mnist/train"
)
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "test"), key_prefix="integ-test-data/mxnet_mnist/test"
)
job_name = unique_name_from_base("test-mxnet-transform")
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
mx.fit({"train": train_input, "test": test_input}, job_name=job_name)
transform_input_path = os.path.join(data_path, "transform", "data.csv")
transform_input_key_prefix = "integ-test-data/mxnet_mnist/transform"
transform_input = mx.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix
)
kms_key_arn = get_or_create_kms_key(sagemaker_session)
output_filter = "$"
input_filter = "$"
transformer = _create_transformer_and_transform_job(
mx,
transform_input,
cpu_instance_type,
kms_key_arn,
input_filter=input_filter,
output_filter=output_filter,
join_source=None,
)
with timeout_and_delete_model_with_transformer(
transformer, sagemaker_session, minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES
):
transformer.wait()
job_desc = transformer.sagemaker_session.sagemaker_client.describe_transform_job(
TransformJobName=transformer.latest_transform_job.name
)
assert kms_key_arn == job_desc["TransformResources"]["VolumeKmsKeyId"]
assert output_filter == job_desc["DataProcessing"]["OutputFilter"]
assert input_filter == job_desc["DataProcessing"]["InputFilter"]
def test_transform_mxnet_logs(
mxnet_estimator, mxnet_transform_input, sagemaker_session, cpu_instance_type
):
with timeout(minutes=45):
transformer = _create_transformer_and_transform_job(
mxnet_estimator, mxnet_transform_input, cpu_instance_type, wait=True, logs=True
)
with timeout_and_delete_model_with_transformer(
transformer, sagemaker_session, minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES
):
transformer.wait()
def test_timeout_and_delete_model_with_transformer_does_not_throw_when_method_ends_gracefully(
_show_logs, _cleanup_logs, session, transformer):
with timeout_and_delete_model_with_transformer(
sagemaker_session=session,
transformer=transformer,
hours=0,
minutes=0,
seconds=LONG_TIMEOUT_THAT_WILL_NEVER_BE_EXCEEDED,
sleep_between_cleanup_attempts=0,
):
pass
assert transformer.delete_model.call_count == 1
def test_timeout_and_delete_model_with_transformer_throws_timeout_exception_when_method_times_out(
_show_logs, _cleanup_logs, session, transformer):
with pytest.raises(stopit.utils.TimeoutException):
with timeout_and_delete_model_with_transformer(
sagemaker_session=session,
transformer=transformer,
hours=0,
minutes=0,
seconds=SHORT_TIMEOUT_TO_FORCE_TIMEOUT_TO_OCCUR,
sleep_between_cleanup_attempts=0,
):
time.sleep(LONG_DURATION_TO_EXCEED_TIMEOUT)
def test_transform_mxnet_vpc(sagemaker_session, mxnet_full_version):
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
script_path = os.path.join(data_path, "mnist.py")
ec2_client = sagemaker_session.boto_session.client("ec2")
subnet_ids, security_group_id = get_or_create_vpc_resources(
ec2_client, sagemaker_session.boto_session.region_name
)
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
train_instance_count=1,
train_instance_type="ml.c4.xlarge",
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version,
subnets=subnet_ids,
security_group_ids=[security_group_id],
)
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "train"), key_prefix="integ-test-data/mxnet_mnist/train"
)
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "test"), key_prefix="integ-test-data/mxnet_mnist/test"
)
job_name = unique_name_from_base("test-mxnet-vpc")
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
mx.fit({"train": train_input, "test": test_input}, job_name=job_name)
job_desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mx.latest_training_job.name
)
assert set(subnet_ids) == set(job_desc["VpcConfig"]["Subnets"])
assert [security_group_id] == job_desc["VpcConfig"]["SecurityGroupIds"]
transform_input_path = os.path.join(data_path, "transform", "data.csv")
transform_input_key_prefix = "integ-test-data/mxnet_mnist/transform"
transform_input = mx.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix
)
transformer = _create_transformer_and_transform_job(mx, transform_input)
with timeout_and_delete_model_with_transformer(
transformer, sagemaker_session, minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES
):
transformer.wait()
model_desc = sagemaker_session.sagemaker_client.describe_model(
ModelName=transformer.model_name
)
assert set(subnet_ids) == set(model_desc["VpcConfig"]["Subnets"])
assert [security_group_id] == model_desc["VpcConfig"]["SecurityGroupIds"]
def test_timeout_and_delete_model_with_transformer_fails_when_method_throws_exception(
_show_logs, _cleanup_logs, session, transformer):
with pytest.raises(ValueError) as exception:
with timeout_and_delete_model_with_transformer(
sagemaker_session=session,
transformer=transformer,
hours=0,
minutes=1,
sleep_between_cleanup_attempts=0,
):
raise ValueError(EXCEPTION_MESSAGE)
assert EXCEPTION_MESSAGE in str(exception.value)
assert transformer.delete_model.call_count == 1
def test_attach_transform_kmeans(sagemaker_session, cpu_instance_type):
data_path = os.path.join(DATA_DIR, "one_p_mnist")
pickle_args = {} if sys.version_info.major == 2 else {"encoding": "latin1"}
# Load the data into memory as numpy arrays
train_set_path = os.path.join(data_path, "mnist.pkl.gz")
with gzip.open(train_set_path, "rb") as f:
train_set, _, _ = pickle.load(f, **pickle_args)
kmeans = KMeans(
role="SageMakerRole",
train_instance_count=1,
train_instance_type=cpu_instance_type,
k=10,
sagemaker_session=sagemaker_session,
output_path="s3://{}/".format(sagemaker_session.default_bucket()),
)
# set kmeans specific hp
kmeans.init_method = "random"
kmeans.max_iterators = 1
kmeans.tol = 1
kmeans.num_trials = 1
kmeans.local_init_method = "kmeans++"
kmeans.half_life_time_size = 1
kmeans.epochs = 1
records = kmeans.record_set(train_set[0][:100])
job_name = unique_name_from_base("test-kmeans-attach")
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
kmeans.fit(records, job_name=job_name)
transform_input_path = os.path.join(data_path, "transform_input.csv")
transform_input_key_prefix = "integ-test-data/one_p_mnist/transform"
transform_input = kmeans.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix)
transformer = _create_transformer_and_transform_job(
kmeans, transform_input, cpu_instance_type)
attached_transformer = Transformer.attach(
transformer.latest_transform_job.name,
sagemaker_session=sagemaker_session)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
attached_transformer.wait()
def test_single_transformer_multiple_jobs(sagemaker_session,
mxnet_full_version,
cpu_instance_type):
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
script_path = os.path.join(data_path, "mnist.py")
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
train_instance_count=1,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version,
)
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "train"),
key_prefix="integ-test-data/mxnet_mnist/train")
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "test"),
key_prefix="integ-test-data/mxnet_mnist/test")
job_name = unique_name_from_base("test-mxnet-transform")
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
mx.fit({"train": train_input, "test": test_input}, job_name=job_name)
transform_input_path = os.path.join(data_path, "transform", "data.csv")
transform_input_key_prefix = "integ-test-data/mxnet_mnist/transform"
transform_input = mx.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix)
transformer = mx.transformer(1, cpu_instance_type)
job_name = unique_name_from_base("test-mxnet-transform")
transformer.transform(transform_input,
content_type="text/csv",
job_name=job_name)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
assert transformer.output_path == "s3://{}/{}".format(
sagemaker_session.default_bucket(), job_name)
job_name = unique_name_from_base("test-mxnet-transform")
transformer.transform(transform_input,
content_type="text/csv",
job_name=job_name)
assert transformer.output_path == "s3://{}/{}".format(
sagemaker_session.default_bucket(), job_name)
def test_transform_mxnet_vpc(sagemaker_session, mxnet_full_version):
data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
script_path = os.path.join(data_path, 'mnist.py')
ec2_client = sagemaker_session.boto_session.client('ec2')
subnet_ids, security_group_id = get_or_create_vpc_resources(
ec2_client, sagemaker_session.boto_session.region_name)
mx = MXNet(entry_point=script_path,
role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version,
subnets=subnet_ids,
security_group_ids=[security_group_id])
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, 'train'),
key_prefix='integ-test-data/mxnet_mnist/train')
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, 'test'),
key_prefix='integ-test-data/mxnet_mnist/test')
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
mx.fit({'train': train_input, 'test': test_input})
job_desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mx.latest_training_job.name)
assert set(subnet_ids) == set(job_desc['VpcConfig']['Subnets'])
assert [security_group_id] == job_desc['VpcConfig']['SecurityGroupIds']
transform_input_path = os.path.join(data_path, 'transform', 'data.csv')
transform_input_key_prefix = 'integ-test-data/mxnet_mnist/transform'
transform_input = mx.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix)
transformer = _create_transformer_and_transform_job(mx, transform_input)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
transformer.wait()
model_desc = sagemaker_session.sagemaker_client.describe_model(
ModelName=transformer.model_name)
assert set(subnet_ids) == set(model_desc['VpcConfig']['Subnets'])
assert [security_group_id
] == model_desc['VpcConfig']['SecurityGroupIds']
def test_transform_mxnet(sagemaker_session, mxnet_full_version):
data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
script_path = os.path.join(data_path, 'mnist.py')
mx = MXNet(entry_point=script_path,
role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version)
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, 'train'),
key_prefix='integ-test-data/mxnet_mnist/train')
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, 'test'),
key_prefix='integ-test-data/mxnet_mnist/test')
job_name = unique_name_from_base('test-mxnet-transform')
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
mx.fit({'train': train_input, 'test': test_input}, job_name=job_name)
transform_input_path = os.path.join(data_path, 'transform', 'data.csv')
transform_input_key_prefix = 'integ-test-data/mxnet_mnist/transform'
transform_input = mx.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix)
kms_key_arn = get_or_create_kms_key(sagemaker_session)
output_filter = "$"
transformer = _create_transformer_and_transform_job(
mx,
transform_input,
kms_key_arn,
input_filter=None,
output_filter=output_filter,
join_source=None)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
transformer.wait()
job_desc = transformer.sagemaker_session.sagemaker_client.describe_transform_job(
TransformJobName=transformer.latest_transform_job.name)
assert kms_key_arn == job_desc['TransformResources']['VolumeKmsKeyId']
assert output_filter == job_desc['DataProcessing']['OutputFilter']
def test_attach_transform_kmeans(sagemaker_session):
data_path = os.path.join(DATA_DIR, 'one_p_mnist')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
# Load the data into memory as numpy arrays
train_set_path = os.path.join(data_path, 'mnist.pkl.gz')
with gzip.open(train_set_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
kmeans = KMeans(role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
k=10,
sagemaker_session=sagemaker_session,
output_path='s3://{}/'.format(
sagemaker_session.default_bucket()))
# set kmeans specific hp
kmeans.init_method = 'random'
kmeans.max_iterators = 1
kmeans.tol = 1
kmeans.num_trials = 1
kmeans.local_init_method = 'kmeans++'
kmeans.half_life_time_size = 1
kmeans.epochs = 1
records = kmeans.record_set(train_set[0][:100])
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
kmeans.fit(records)
transform_input_path = os.path.join(data_path, 'transform_input.csv')
transform_input_key_prefix = 'integ-test-data/one_p_mnist/transform'
transform_input = kmeans.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix)
transformer = _create_transformer_and_transform_job(
kmeans, transform_input)
attached_transformer = Transformer.attach(
transformer.latest_transform_job.name,
sagemaker_session=sagemaker_session)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
attached_transformer.wait()
def test_transform_mxnet_tags(mxnet_estimator, mxnet_transform_input,
sagemaker_session, cpu_instance_type):
tags = [{"Key": "some-tag", "Value": "value-for-tag"}]
transformer = mxnet_estimator.transformer(1, cpu_instance_type, tags=tags)
transformer.transform(mxnet_transform_input, content_type="text/csv")
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
transformer.wait()
model_desc = sagemaker_session.sagemaker_client.describe_model(
ModelName=transformer.model_name)
model_tags = sagemaker_session.sagemaker_client.list_tags(
ResourceArn=model_desc["ModelArn"])["Tags"]
assert tags == model_tags
def test_transform_mxnet_tags(sagemaker_session, mxnet_full_version, cpu_instance_type):
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
script_path = os.path.join(data_path, "mnist.py")
tags = [{"Key": "some-tag", "Value": "value-for-tag"}]
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
train_instance_count=1,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version,
)
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "train"), key_prefix="integ-test-data/mxnet_mnist/train"
)
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "test"), key_prefix="integ-test-data/mxnet_mnist/test"
)
job_name = unique_name_from_base("test-mxnet-transform")
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
mx.fit({"train": train_input, "test": test_input}, job_name=job_name)
transform_input_path = os.path.join(data_path, "transform", "data.csv")
transform_input_key_prefix = "integ-test-data/mxnet_mnist/transform"
transform_input = mx.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix
)
transformer = mx.transformer(1, cpu_instance_type, tags=tags)
transformer.transform(transform_input, content_type="text/csv")
with timeout_and_delete_model_with_transformer(
transformer, sagemaker_session, minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES
):
transformer.wait()
model_desc = sagemaker_session.sagemaker_client.describe_model(
ModelName=transformer.model_name
)
model_tags = sagemaker_session.sagemaker_client.list_tags(
ResourceArn=model_desc["ModelArn"]
)["Tags"]
assert tags == model_tags
def test_transform_model_client_config(
mxnet_estimator, mxnet_transform_input, sagemaker_session, cpu_instance_type
):
model_client_config = {"InvocationsTimeoutInSeconds": 60, "InvocationsMaxRetries": 2}
transformer = mxnet_estimator.transformer(1, cpu_instance_type)
transformer.transform(
mxnet_transform_input, content_type="text/csv", model_client_config=model_client_config
)
with timeout_and_delete_model_with_transformer(
transformer, sagemaker_session, minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES
):
transformer.wait()
transform_job_desc = sagemaker_session.sagemaker_client.describe_transform_job(
TransformJobName=transformer.latest_transform_job.name
)
assert model_client_config == transform_job_desc["ModelClientConfig"]
def test_single_transformer_multiple_jobs(
mxnet_estimator, mxnet_transform_input, sagemaker_session, cpu_instance_type
):
transformer = mxnet_estimator.transformer(1, cpu_instance_type)
job_name = unique_name_from_base("test-mxnet-transform")
transformer.transform(mxnet_transform_input, content_type="text/csv", job_name=job_name)
with timeout_and_delete_model_with_transformer(
transformer, sagemaker_session, minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES
):
assert transformer.output_path == "s3://{}/{}".format(
sagemaker_session.default_bucket(), job_name
)
job_name = unique_name_from_base("test-mxnet-transform")
transformer.transform(mxnet_transform_input, content_type="text/csv", job_name=job_name)
assert transformer.output_path == "s3://{}/{}".format(
sagemaker_session.default_bucket(), job_name
)
def test_inference_pipeline_batch_transform(sagemaker_session,
cpu_instance_type):
sparkml_model_data = sagemaker_session.upload_data(
path=os.path.join(SPARKML_DATA_PATH, "mleap_model.tar.gz"),
key_prefix="integ-test-data/sparkml/model",
)
xgb_model_data = sagemaker_session.upload_data(
path=os.path.join(XGBOOST_DATA_PATH, "xgb_model.tar.gz"),
key_prefix="integ-test-data/xgboost/model",
)
batch_job_name = "test-inference-pipeline-batch-{}".format(
sagemaker_timestamp())
sparkml_model = SparkMLModel(
model_data=sparkml_model_data,
env={"SAGEMAKER_SPARKML_SCHEMA": SCHEMA},
sagemaker_session=sagemaker_session,
)
xgb_image = image_uris.retrieve("xgboost",
sagemaker_session.boto_region_name,
version="1",
image_scope="inference")
xgb_model = Model(model_data=xgb_model_data,
image_uri=xgb_image,
sagemaker_session=sagemaker_session)
model = PipelineModel(
models=[sparkml_model, xgb_model],
role="SageMakerRole",
sagemaker_session=sagemaker_session,
name=batch_job_name,
)
transformer = model.transformer(1, cpu_instance_type)
transform_input_key_prefix = "integ-test-data/sparkml_xgboost/transform"
transform_input = transformer.sagemaker_session.upload_data(
path=VALID_DATA_PATH, key_prefix=transform_input_key_prefix)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
transformer.transform(transform_input,
content_type="text/csv",
job_name=batch_job_name)
transformer.wait()
def test_attach_transform_kmeans(sagemaker_session, cpu_instance_type):
kmeans = KMeans(
role="SageMakerRole",
instance_count=1,
instance_type=cpu_instance_type,
k=10,
sagemaker_session=sagemaker_session,
output_path="s3://{}/".format(sagemaker_session.default_bucket()),
)
# set kmeans specific hp
kmeans.init_method = "random"
kmeans.max_iterators = 1
kmeans.tol = 1
kmeans.num_trials = 1
kmeans.local_init_method = "kmeans++"
kmeans.half_life_time_size = 1
kmeans.epochs = 1
records = kmeans.record_set(datasets.one_p_mnist()[0][:100])
job_name = unique_name_from_base("test-kmeans-attach")
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
kmeans.fit(records, job_name=job_name)
transform_input_path = os.path.join(DATA_DIR, "one_p_mnist",
"transform_input.csv")
transform_input_key_prefix = "integ-test-data/one_p_mnist/transform"
transform_input = kmeans.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix)
transformer = _create_transformer_and_transform_job(
kmeans, transform_input, cpu_instance_type)
attached_transformer = Transformer.attach(
transformer.latest_transform_job.name,
sagemaker_session=sagemaker_session)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
attached_transformer.wait()
def test_transform_mxnet_logs(sagemaker_session, mxnet_full_version,
cpu_instance_type):
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
script_path = os.path.join(data_path, "mnist.py")
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
train_instance_count=1,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version,
)
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "train"),
key_prefix="integ-test-data/mxnet_mnist/train")
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "test"),
key_prefix="integ-test-data/mxnet_mnist/test")
job_name = unique_name_from_base("test-mxnet-transform")
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
mx.fit({"train": train_input, "test": test_input}, job_name=job_name)
transform_input_path = os.path.join(data_path, "transform", "data.csv")
transform_input_key_prefix = "integ-test-data/mxnet_mnist/transform"
transform_input = mx.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix)
with timeout(minutes=45):
transformer = _create_transformer_and_transform_job(mx,
transform_input,
cpu_instance_type,
wait=True,
logs=True)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
transformer.wait()
def test_transform_mxnet(sagemaker_session, mxnet_full_version):
data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
script_path = os.path.join(data_path, 'mnist.py')
mx = MXNet(entry_point=script_path,
role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version)
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, 'train'),
key_prefix='integ-test-data/mxnet_mnist/train')
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, 'test'),
key_prefix='integ-test-data/mxnet_mnist/test')
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
mx.fit({'train': train_input, 'test': test_input})
transform_input_path = os.path.join(data_path, 'transform', 'data.csv')
transform_input_key_prefix = 'integ-test-data/mxnet_mnist/transform'
transform_input = mx.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix)
sts_client = sagemaker_session.boto_session.client('sts')
account_id = sts_client.get_caller_identity()['Account']
kms_client = sagemaker_session.boto_session.client('kms')
kms_key_arn = get_or_create_kms_key(kms_client, account_id)
transformer = _create_transformer_and_transform_job(
mx, transform_input, kms_key_arn)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
transformer.wait()
job_desc = transformer.sagemaker_session.sagemaker_client.describe_transform_job(
TransformJobName=transformer.latest_transform_job.name)
assert kms_key_arn == job_desc['TransformResources']['VolumeKmsKeyId']
def test_timeout_and_delete_model_with_transformer_retries_resource_deletion_on_failure(
_show_logs, _cleanup_logs, _delete_schedules_associated_with_endpoint, session, transformer
):
transformer.delete_model = Mock(
side_effect=ClientError(
error_response={"Error": {"Code": 403, "Message": "ValidationException"}},
operation_name="Unit Test",
)
)
with timeout_and_delete_model_with_transformer(
sagemaker_session=session,
transformer=transformer,
hours=0,
minutes=0,
seconds=LONG_TIMEOUT_THAT_WILL_NEVER_BE_EXCEEDED,
sleep_between_cleanup_attempts=0,
):
pass
assert transformer.delete_model.call_count == 3
def test_transform_mxnet_tags(sagemaker_session, mxnet_full_version):
data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
script_path = os.path.join(data_path, 'mnist.py')
tags = [{'Key': 'some-tag', 'Value': 'value-for-tag'}]
mx = MXNet(entry_point=script_path,
role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version)
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, 'train'),
key_prefix='integ-test-data/mxnet_mnist/train')
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, 'test'),
key_prefix='integ-test-data/mxnet_mnist/test')
job_name = unique_name_from_base('test-mxnet-transform')
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
mx.fit({'train': train_input, 'test': test_input}, job_name=job_name)
transform_input_path = os.path.join(data_path, 'transform', 'data.csv')
transform_input_key_prefix = 'integ-test-data/mxnet_mnist/transform'
transform_input = mx.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix)
transformer = mx.transformer(1, 'ml.m4.xlarge', tags=tags)
transformer.transform(transform_input, content_type='text/csv')
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
transformer.wait()
model_desc = sagemaker_session.sagemaker_client.describe_model(
ModelName=transformer.model_name)
model_tags = sagemaker_session.sagemaker_client.list_tags(
ResourceArn=model_desc['ModelArn'])['Tags']
assert tags == model_tags
def test_transform_tf_kms_network_isolation(sagemaker_session,
cpu_instance_type, tmpdir,
tf_full_version, py_version):
data_path = os.path.join(DATA_DIR, "tensorflow_mnist")
tf = TensorFlow(
entry_point=os.path.join(data_path, "mnist.py"),
role="SageMakerRole",
train_instance_count=1,
train_instance_type=cpu_instance_type,
framework_version=tf_full_version,
script_mode=True,
py_version=py_version,
sagemaker_session=sagemaker_session,
)
s3_prefix = "integ-test-data/tf-scriptmode/mnist"
training_input = sagemaker_session.upload_data(
path=os.path.join(data_path, "data"),
key_prefix="{}/training".format(s3_prefix))
job_name = unique_name_from_base("test-tf-transform")
tf.fit(inputs=training_input, job_name=job_name)
transform_input = sagemaker_session.upload_data(
path=os.path.join(data_path, "transform"),
key_prefix="{}/transform".format(s3_prefix))
with bucket_with_encryption(sagemaker_session,
"SageMakerRole") as (bucket_with_kms, kms_key):
output_path = "{}/{}/output".format(bucket_with_kms, job_name)
transformer = tf.transformer(
instance_count=1,
instance_type=cpu_instance_type,
output_path=output_path,
output_kms_key=kms_key,
volume_kms_key=kms_key,
enable_network_isolation=True,
)
with timeout_and_delete_model_with_transformer(
transformer,
sagemaker_session,
minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES):
transformer.transform(transform_input,
job_name=job_name,
content_type="text/csv",
wait=True)
model_desc = sagemaker_session.sagemaker_client.describe_model(
ModelName=transformer.model_name)
assert model_desc["EnableNetworkIsolation"]
job_desc = sagemaker_session.describe_transform_job(job_name=job_name)
assert job_desc["TransformOutput"]["S3OutputPath"] == output_path
assert job_desc["TransformOutput"]["KmsKeyId"] == kms_key
assert job_desc["TransformResources"]["VolumeKmsKeyId"] == kms_key
s3.S3Downloader.download(
s3_uri=output_path,
local_path=os.path.join(tmpdir, "tf-batch-output"),
session=sagemaker_session,
)
with open(os.path.join(tmpdir, "tf-batch-output",
"data.csv.out")) as f:
result = json.load(f)
assert len(result["predictions"][0]["probabilities"]) == 10
assert result["predictions"][0]["classes"] == 1
