def test_tfs_model_image_accelerator_not_supported(sagemaker_session):
model = TensorFlowModel(
"s3://some/data.tar.gz",
role=ROLE,
framework_version="1.13.0",
sagemaker_session=sagemaker_session,
)
# assert error is not raised
model.deploy(instance_type="ml.c4.xlarge",
initial_instance_count=1,
accelerator_type="ml.eia1.medium")
model = TensorFlowModel(
"s3://some/data.tar.gz",
role=ROLE,
framework_version="2.1",
sagemaker_session=sagemaker_session,
)
# assert error is not raised
model.deploy(instance_type="ml.c4.xlarge", initial_instance_count=1)
with pytest.raises(AttributeError) as e:
model.deploy(
instance_type="ml.c4.xlarge",
accelerator_type="ml.eia1.medium",
initial_instance_count=1,
)
assert str(e.value) == "The TensorFlow version 2.1 doesn't support EIA."
def test_model(sagemaker_session, tf_version):
model = TensorFlowModel("s3://some/data.tar.gz",
role=ROLE,
entry_point=SCRIPT_PATH,
sagemaker_session=sagemaker_session)
predictor = model.deploy(1, GPU_IMAGE_NAME)
assert isinstance(predictor, TensorFlowPredictor)
def test_deploy_model_with_accelerator(sagemaker_session, tf_training_job,
ei_tf_full_version):
endpoint_name = 'test-tf-deploy-model-ei-{}'.format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=tf_training_job)
model_data = desc['ModelArtifacts']['S3ModelArtifacts']
script_path = os.path.join(DATA_DIR, 'iris', 'iris-dnn-classifier.py')
model = TensorFlowModel(model_data,
'SageMakerRole',
entry_point=script_path,
framework_version=ei_tf_full_version,
sagemaker_session=sagemaker_session)
json_predictor = model.deploy(initial_instance_count=1,
instance_type='ml.c4.xlarge',
endpoint_name=endpoint_name,
accelerator_type='ml.eia1.medium')
features = [6.4, 3.2, 4.5, 1.5]
dict_result = json_predictor.predict({'inputs': features})
print('predict result: {}'.format(dict_result))
list_result = json_predictor.predict(features)
print('predict result: {}'.format(list_result))
assert dict_result == list_result
def test_tfs_model_with_dependencies(repack_model, model_code_key_prefix,
sagemaker_session,
tensorflow_inference_version):
model = TensorFlowModel(
"s3://some/data.tar.gz",
entry_point="train.py",
dependencies=["src", "lib"],
role=ROLE,
framework_version=tensorflow_inference_version,
image_uri="my-image",
sagemaker_session=sagemaker_session,
)
model.prepare_container_def(INSTANCE_TYPE)
model_code_key_prefix.assert_called_with(model.key_prefix, model.name,
model.image_uri)
repack_model.assert_called_with(
"train.py",
None,
["src", "lib"],
"s3://some/data.tar.gz",
"s3://my_bucket/key-prefix/model.tar.gz",
sagemaker_session,
kms_key=None,
)
def main():
session = LocalSession()
session.config = {'local': {'local_code': True}}
role = DUMMY_IAM_ROLE
model_dir = 's3://tensorflow-script-mode-local-model-inference/model.tar.gz'
model = TensorFlowModel(
entry_point='inference.py',
source_dir='./code',
role=role,
model_data=model_dir,
framework_version='2.3.0',
)
print('Deploying endpoint in local mode')
predictor = model.deploy(
initial_instance_count=1,
instance_type='local',
)
dummy_inputs = {
'bucket_name': 'tensorflow-script-mode-local-model-inference',
'object_name': 'instances.json'
}
predictions = predictor.predict(dummy_inputs)
print("predictions: {}".format(predictions))
print('About to delete the endpoint')
predictor.delete_endpoint(predictor.endpoint_name)
predictor.delete_model()
def sagemaker_deploy(model_artifacts_s3, sagemaker_role, initial_instance_count, instance_type, endpoint_name):
tf_model = TensorFlowModel(model_data=model_artifacts_s3, role=sagemaker_role, framework_version='2.2')
predictor = tf_model.deploy(initial_instance_count=initial_instance_count,
instance_type=instance_type,
endpoint_name=endpoint_name,
wait=True)
return predictor
def test_model(sagemaker_session, tf_version):
model = TensorFlowModel(MODEL_DATA,
role=ROLE,
entry_point=SCRIPT_PATH,
sagemaker_session=sagemaker_session)
predictor = model.deploy(1, INSTANCE_TYPE)
assert isinstance(predictor, TensorFlowPredictor)
def deploy(self):
"""
Use SageMaker to deploy the model as a stand-alone, PaaS endpoint, with our choice of the underlying
Docker image and hardware capabilities.
Available images for inferences can be chosen from AWS official list:
https://github.com/aws/deep-learning-containers/blob/master/available_images.md
Once the endpoint is deployed, you can add a further step with for example behavioral testing, to
ensure model robustness (e.g. see https://arxiv.org/pdf/2005.04118.pdf). Here, we just "prove" that
the endpoint is up and running!
"""
from sagemaker.tensorflow import TensorFlowModel
# generate a signature for the endpoint, using learning rate and timestamp as a convention
ENDPOINT_NAME = 'regression-{}-endpoint'.format(
int(round(time.time() * 1000)))
# print out the name, so that we can use it when deploying our lambda
print("\n\n================\nEndpoint name is: {}\n\n".format(
ENDPOINT_NAME))
model = TensorFlowModel(model_data=self.best_s3_model_path,
image_uri=self.DOCKER_IMAGE_URI,
role=self.IAM_SAGEMAKER_ROLE)
predictor = model.deploy(initial_instance_count=1,
instance_type=self.SAGEMAKER_INSTANCE,
endpoint_name=ENDPOINT_NAME)
# run a small test against the endpoint
# pick a number for X and check the predicted Y is sensible
input = {'instances': np.array([[0.57457947234]])}
# output is on the form {'predictions': [[10.879798]]}
result = predictor.predict(input)
print(input, result)
assert result['predictions'][0][0] > 0
self.next(self.end)
def test_model_deploy_with_serverless_inference_config(sagemaker_session,
tf_full_version,
tf_full_py_version):
endpoint_name = unique_name_from_base("sagemaker-tensorflow-serverless")
model_data = sagemaker_session.upload_data(
path=os.path.join(tests.integ.DATA_DIR,
"tensorflow-serving-test-model.tar.gz"),
key_prefix="tensorflow-serving/models",
)
with tests.integ.timeout.timeout_and_delete_endpoint_by_name(
endpoint_name=endpoint_name,
sagemaker_session=sagemaker_session,
hours=2,
sleep_between_cleanup_attempts=20,
exponential_sleep=True,
):
model = TensorFlowModel(
model_data=model_data,
role=ROLE,
framework_version=tf_full_version,
sagemaker_session=sagemaker_session,
)
predictor = model.deploy(
serverless_inference_config=ServerlessInferenceConfig(),
endpoint_name=endpoint_name,
)
input_data = {"instances": [1.0, 2.0, 5.0]}
expected_result = {"predictions": [3.5, 4.0, 5.5]}
result = predictor.predict(input_data)
assert expected_result == result
def test_model_image_accelerator(sagemaker_session):
model = TensorFlowModel(MODEL_DATA,
role=ROLE,
entry_point=SCRIPT_PATH,
sagemaker_session=sagemaker_session)
container_def = model.prepare_container_def(
INSTANCE_TYPE, accelerator_type=ACCELERATOR_TYPE)
assert container_def['Image'] == _get_full_cpu_image_uri_with_ei(
defaults.TF_VERSION)
def test_tfs_model_with_custom_image(sagemaker_session, tensorflow_inference_version):
model = TensorFlowModel(
"s3://some/data.tar.gz",
role=ROLE,
framework_version=tensorflow_inference_version,
image_uri="my-image",
sagemaker_session=sagemaker_session,
)
cdef = model.prepare_container_def(INSTANCE_TYPE)
assert cdef["Image"] == "my-image"
def test_tfs_model_with_log_level(sagemaker_session, tensorflow_inference_version):
model = TensorFlowModel(
"s3://some/data.tar.gz",
role=ROLE,
framework_version=tensorflow_inference_version,
container_log_level=logging.INFO,
sagemaker_session=sagemaker_session,
)
cdef = model.prepare_container_def(INSTANCE_TYPE)
assert cdef["Environment"] == {TensorFlowModel.LOG_LEVEL_PARAM_NAME: "info"}
def test_model_prepare_container_def_no_instance_type_or_image(tensorflow_inference_version):
model = TensorFlowModel(
"s3://some/data.tar.gz", role=ROLE, framework_version=tensorflow_inference_version
)
with pytest.raises(ValueError) as e:
model.prepare_container_def()
expected_msg = "Must supply either an instance type (for choosing CPU vs GPU) or an image URI."
assert expected_msg in str(e)
def test_tfs_model_accelerator(retrieve_image_uri, sagemaker_session, tensorflow_eia_version):
model = TensorFlowModel(
"s3://some/data.tar.gz",
role=ROLE,
framework_version=tensorflow_eia_version,
sagemaker_session=sagemaker_session,
)
cdef = model.prepare_container_def(INSTANCE_TYPE, accelerator_type=ACCELERATOR_TYPE)
retrieve_image_uri.assert_called_with(
"tensorflow",
REGION,
version=tensorflow_eia_version,
instance_type=INSTANCE_TYPE,
accelerator_type=ACCELERATOR_TYPE,
image_scope="inference",
)
assert IMAGE == cdef["Image"]
predictor = model.deploy(INSTANCE_COUNT, INSTANCE_TYPE)
assert isinstance(predictor, TensorFlowPredictor)
def main():
session = LocalSession()
session.config = {'local': {'local_code': True}}
role = DUMMY_IAM_ROLE
model_dir = 's3://aws-ml-blog/artifacts/tensorflow-script-mode-local-model-inference/model.tar.gz'
model = TensorFlowModel(
entry_point='inference.py',
source_dir='./code',
role=role,
model_data=model_dir,
framework_version='2.3.0',
)
print('Deploying endpoint in local mode')
print(
'Note: if launching for the first time in local mode, container image download might take a few minutes to complete.'
)
predictor = model.deploy(
initial_instance_count=1,
instance_type='local',
)
print('Endpoint deployed in local mode')
dummy_inputs = {
'bucket_name':
'aws-ml-blog',
'object_name':
'artifacts/tensorflow-script-mode-local-model-inference/instances.json'
}
predictions = predictor.predict(dummy_inputs)
print("predictions: {}".format(predictions))
print('About to delete the endpoint')
predictor.delete_endpoint(predictor.endpoint_name)
def test_deploy_elastic_inference_with_pretrained_model(pretrained_model_data, docker_image_uri, sagemaker_session, instance_type, accelerator_type):
resource_path = os.path.join(SCRIPT_PATH, '../resources')
endpoint_name = 'test-tf-ei-deploy-model-{}'.format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name=endpoint_name, sagemaker_session=sagemaker_session,
minutes=20):
tensorflow_model = TensorFlowModel(model_data=pretrained_model_data,
entry_point='default_entry_point.py',
source_dir=resource_path,
role='SageMakerRole',
image=docker_image_uri,
sagemaker_session=sagemaker_session)
logger.info('deploying model to endpoint: {}'.format(endpoint_name))
predictor = tensorflow_model.deploy(initial_instance_count=1,
instance_type=instance_type,
accelerator_type=accelerator_type,
endpoint_name=endpoint_name)
random_input = np.random.rand(1, 1, 3, 3)
predict_response = predictor.predict({'input': random_input.tolist()})
assert predict_response['outputs']['probabilities']
def test_estimator_py2_deprecation_warning(warning, sagemaker_session):
estimator = TensorFlow(
entry_point=SCRIPT_PATH,
role=ROLE,
sagemaker_session=sagemaker_session,
train_instance_count=INSTANCE_COUNT,
train_instance_type=INSTANCE_TYPE,
py_version="py2",
)
assert estimator.py_version == "py2"
warning.assert_called_with(estimator.__framework_name__, defaults.LATEST_PY2_VERSION)
model = TensorFlowModel(
MODEL_DATA,
role=ROLE,
entry_point=SCRIPT_PATH,
sagemaker_session=sagemaker_session,
py_version="py2",
)
assert model.py_version == "py2"
warning.assert_called_with(model.__framework_name__, defaults.LATEST_PY2_VERSION)
def test_empty_framework_version(warning, sagemaker_session):
estimator = TensorFlow(
entry_point=SCRIPT_PATH,
role=ROLE,
sagemaker_session=sagemaker_session,
train_instance_count=INSTANCE_COUNT,
train_instance_type=INSTANCE_TYPE,
framework_version=None,
)
assert estimator.framework_version == defaults.TF_VERSION
warning.assert_called_with(defaults.TF_VERSION, estimator.LATEST_VERSION)
model = TensorFlowModel(
MODEL_DATA,
role=ROLE,
entry_point=SCRIPT_PATH,
sagemaker_session=sagemaker_session,
framework_version=None,
)
assert model.framework_version == defaults.TF_VERSION
warning.assert_called_with(defaults.TF_VERSION, defaults.LATEST_VERSION)
def test_model(sagemaker_session, tf_version):
model = TensorFlowModel("s3://some/data.tar.gz", role=ROLE, entry_point=SCRIPT_PATH,
sagemaker_session=sagemaker_session)
predictor = model.deploy(1, INSTANCE_TYPE)
assert isinstance(predictor, TensorFlowPredictor)
def test_model_registration_with_tensorflow_model_with_pipeline_model(
sagemaker_session, role, tf_full_version, tf_full_py_version,
pipeline_name, region_name):
base_dir = os.path.join(DATA_DIR, "tensorflow_mnist")
entry_point = os.path.join(base_dir, "mnist_v2.py")
input_path = sagemaker_session.upload_data(
path=os.path.join(base_dir, "data"),
key_prefix="integ-test-data/tf-scriptmode/mnist/training",
)
inputs = TrainingInput(s3_data=input_path)
instance_count = ParameterInteger(name="InstanceCount", default_value=1)
instance_type = ParameterString(name="InstanceType",
default_value="ml.m5.xlarge")
tensorflow_estimator = TensorFlow(
entry_point=entry_point,
role=role,
instance_count=instance_count,
instance_type=instance_type,
framework_version=tf_full_version,
py_version=tf_full_py_version,
sagemaker_session=sagemaker_session,
)
step_train = TrainingStep(
name="MyTrain",
estimator=tensorflow_estimator,
inputs=inputs,
)
model = TensorFlowModel(
entry_point=entry_point,
framework_version="2.4",
model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,
role=role,
sagemaker_session=sagemaker_session,
)
pipeline_model = PipelineModel(name="MyModelPipeline",
models=[model],
role=role,
sagemaker_session=sagemaker_session)
step_register_model = RegisterModel(
name="MyRegisterModel",
model=pipeline_model,
model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,
content_types=["application/json"],
response_types=["application/json"],
inference_instances=["ml.t2.medium", "ml.m5.large"],
transform_instances=["ml.m5.large"],
model_package_group_name=f"{pipeline_name}TestModelPackageGroup",
)
pipeline = Pipeline(
name=pipeline_name,
parameters=[
instance_count,
instance_type,
],
steps=[step_train, step_register_model],
sagemaker_session=sagemaker_session,
)
try:
response = pipeline.create(role)
create_arn = response["PipelineArn"]
assert re.match(
rf"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}",
create_arn,
)
for _ in retries(
max_retry_count=5,
exception_message_prefix=
"Waiting for a successful execution of pipeline",
seconds_to_sleep=10,
):
execution = pipeline.start(parameters={})
assert re.match(
rf"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}/execution/",
execution.arn,
)
try:
execution.wait(delay=30, max_attempts=60)
except WaiterError:
pass
execution_steps = execution.list_steps()
assert len(execution_steps) == 3
for step in execution_steps:
assert step["StepStatus"] == "Succeeded"
break
finally:
try:
pipeline.delete()
except Exception:
pass
