def test_model(sagemaker_session, tf_version):
model = TensorFlowModel(MODEL_DATA,
role=ROLE,
entry_point=SCRIPT_PATH,
sagemaker_session=sagemaker_session)
predictor = model.deploy(1, INSTANCE_TYPE)
assert isinstance(predictor, TensorFlowPredictor)
def main():
session = LocalSession()
session.config = {'local': {'local_code': True}}
role = DUMMY_IAM_ROLE
model_dir = 's3://tensorflow-script-mode-local-model-inference/model.tar.gz'
model = TensorFlowModel(
entry_point='inference.py',
source_dir='./code',
role=role,
model_data=model_dir,
framework_version='2.3.0',
)
print('Deploying endpoint in local mode')
predictor = model.deploy(
initial_instance_count=1,
instance_type='local',
)
dummy_inputs = {
'bucket_name': 'tensorflow-script-mode-local-model-inference',
'object_name': 'instances.json'
}
predictions = predictor.predict(dummy_inputs)
print("predictions: {}".format(predictions))
print('About to delete the endpoint')
predictor.delete_endpoint(predictor.endpoint_name)
predictor.delete_model()
def sagemaker_deploy(model_artifacts_s3, sagemaker_role, initial_instance_count, instance_type, endpoint_name):
tf_model = TensorFlowModel(model_data=model_artifacts_s3, role=sagemaker_role, framework_version='2.2')
predictor = tf_model.deploy(initial_instance_count=initial_instance_count,
instance_type=instance_type,
endpoint_name=endpoint_name,
wait=True)
return predictor
def test_model_deploy_with_serverless_inference_config(sagemaker_session,
tf_full_version,
tf_full_py_version):
endpoint_name = unique_name_from_base("sagemaker-tensorflow-serverless")
model_data = sagemaker_session.upload_data(
path=os.path.join(tests.integ.DATA_DIR,
"tensorflow-serving-test-model.tar.gz"),
key_prefix="tensorflow-serving/models",
)
with tests.integ.timeout.timeout_and_delete_endpoint_by_name(
endpoint_name=endpoint_name,
sagemaker_session=sagemaker_session,
hours=2,
sleep_between_cleanup_attempts=20,
exponential_sleep=True,
):
model = TensorFlowModel(
model_data=model_data,
role=ROLE,
framework_version=tf_full_version,
sagemaker_session=sagemaker_session,
)
predictor = model.deploy(
serverless_inference_config=ServerlessInferenceConfig(),
endpoint_name=endpoint_name,
)
input_data = {"instances": [1.0, 2.0, 5.0]}
expected_result = {"predictions": [3.5, 4.0, 5.5]}
result = predictor.predict(input_data)
assert expected_result == result
def test_deploy_model_with_accelerator(sagemaker_session, tf_training_job,
ei_tf_full_version):
endpoint_name = 'test-tf-deploy-model-ei-{}'.format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=tf_training_job)
model_data = desc['ModelArtifacts']['S3ModelArtifacts']
script_path = os.path.join(DATA_DIR, 'iris', 'iris-dnn-classifier.py')
model = TensorFlowModel(model_data,
'SageMakerRole',
entry_point=script_path,
framework_version=ei_tf_full_version,
sagemaker_session=sagemaker_session)
json_predictor = model.deploy(initial_instance_count=1,
instance_type='ml.c4.xlarge',
endpoint_name=endpoint_name,
accelerator_type='ml.eia1.medium')
features = [6.4, 3.2, 4.5, 1.5]
dict_result = json_predictor.predict({'inputs': features})
print('predict result: {}'.format(dict_result))
list_result = json_predictor.predict(features)
print('predict result: {}'.format(list_result))
assert dict_result == list_result
def test_model(sagemaker_session, tf_version):
model = TensorFlowModel("s3://some/data.tar.gz",
role=ROLE,
entry_point=SCRIPT_PATH,
sagemaker_session=sagemaker_session)
predictor = model.deploy(1, GPU_IMAGE_NAME)
assert isinstance(predictor, TensorFlowPredictor)
def deploy(self):
"""
Use SageMaker to deploy the model as a stand-alone, PaaS endpoint, with our choice of the underlying
Docker image and hardware capabilities.
Available images for inferences can be chosen from AWS official list:
https://github.com/aws/deep-learning-containers/blob/master/available_images.md
Once the endpoint is deployed, you can add a further step with for example behavioral testing, to
ensure model robustness (e.g. see https://arxiv.org/pdf/2005.04118.pdf). Here, we just "prove" that
the endpoint is up and running!
"""
from sagemaker.tensorflow import TensorFlowModel
# generate a signature for the endpoint, using learning rate and timestamp as a convention
ENDPOINT_NAME = 'regression-{}-endpoint'.format(
int(round(time.time() * 1000)))
# print out the name, so that we can use it when deploying our lambda
print("\n\n================\nEndpoint name is: {}\n\n".format(
ENDPOINT_NAME))
model = TensorFlowModel(model_data=self.best_s3_model_path,
image_uri=self.DOCKER_IMAGE_URI,
role=self.IAM_SAGEMAKER_ROLE)
predictor = model.deploy(initial_instance_count=1,
instance_type=self.SAGEMAKER_INSTANCE,
endpoint_name=ENDPOINT_NAME)
# run a small test against the endpoint
# pick a number for X and check the predicted Y is sensible
input = {'instances': np.array([[0.57457947234]])}
# output is on the form {'predictions': [[10.879798]]}
result = predictor.predict(input)
print(input, result)
assert result['predictions'][0][0] > 0
self.next(self.end)
def test_tfs_model_image_accelerator_not_supported(sagemaker_session):
model = TensorFlowModel(
"s3://some/data.tar.gz",
role=ROLE,
framework_version="1.13.0",
sagemaker_session=sagemaker_session,
)
# assert error is not raised
model.deploy(instance_type="ml.c4.xlarge",
initial_instance_count=1,
accelerator_type="ml.eia1.medium")
model = TensorFlowModel(
"s3://some/data.tar.gz",
role=ROLE,
framework_version="2.1",
sagemaker_session=sagemaker_session,
)
# assert error is not raised
model.deploy(instance_type="ml.c4.xlarge", initial_instance_count=1)
with pytest.raises(AttributeError) as e:
model.deploy(
instance_type="ml.c4.xlarge",
accelerator_type="ml.eia1.medium",
initial_instance_count=1,
)
assert str(e.value) == "The TensorFlow version 2.1 doesn't support EIA."
def test_tfs_model_accelerator(retrieve_image_uri, sagemaker_session, tensorflow_eia_version):
model = TensorFlowModel(
"s3://some/data.tar.gz",
role=ROLE,
framework_version=tensorflow_eia_version,
sagemaker_session=sagemaker_session,
)
cdef = model.prepare_container_def(INSTANCE_TYPE, accelerator_type=ACCELERATOR_TYPE)
retrieve_image_uri.assert_called_with(
"tensorflow",
REGION,
version=tensorflow_eia_version,
instance_type=INSTANCE_TYPE,
accelerator_type=ACCELERATOR_TYPE,
image_scope="inference",
)
assert IMAGE == cdef["Image"]
predictor = model.deploy(INSTANCE_COUNT, INSTANCE_TYPE)
assert isinstance(predictor, TensorFlowPredictor)
def main():
session = LocalSession()
session.config = {'local': {'local_code': True}}
role = DUMMY_IAM_ROLE
model_dir = 's3://aws-ml-blog/artifacts/tensorflow-script-mode-local-model-inference/model.tar.gz'
model = TensorFlowModel(
entry_point='inference.py',
source_dir='./code',
role=role,
model_data=model_dir,
framework_version='2.3.0',
)
print('Deploying endpoint in local mode')
print(
'Note: if launching for the first time in local mode, container image download might take a few minutes to complete.'
)
predictor = model.deploy(
initial_instance_count=1,
instance_type='local',
)
print('Endpoint deployed in local mode')
dummy_inputs = {
'bucket_name':
'aws-ml-blog',
'object_name':
'artifacts/tensorflow-script-mode-local-model-inference/instances.json'
}
predictions = predictor.predict(dummy_inputs)
print("predictions: {}".format(predictions))
print('About to delete the endpoint')
predictor.delete_endpoint(predictor.endpoint_name)
def test_deploy_elastic_inference_with_pretrained_model(pretrained_model_data, docker_image_uri, sagemaker_session, instance_type, accelerator_type):
resource_path = os.path.join(SCRIPT_PATH, '../resources')
endpoint_name = 'test-tf-ei-deploy-model-{}'.format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name=endpoint_name, sagemaker_session=sagemaker_session,
minutes=20):
tensorflow_model = TensorFlowModel(model_data=pretrained_model_data,
entry_point='default_entry_point.py',
source_dir=resource_path,
role='SageMakerRole',
image=docker_image_uri,
sagemaker_session=sagemaker_session)
logger.info('deploying model to endpoint: {}'.format(endpoint_name))
predictor = tensorflow_model.deploy(initial_instance_count=1,
instance_type=instance_type,
accelerator_type=accelerator_type,
endpoint_name=endpoint_name)
random_input = np.random.rand(1, 1, 3, 3)
predict_response = predictor.predict({'input': random_input.tolist()})
assert predict_response['outputs']['probabilities']
def test_model(sagemaker_session, tf_version):
model = TensorFlowModel("s3://some/data.tar.gz", role=ROLE, entry_point=SCRIPT_PATH,
sagemaker_session=sagemaker_session)
predictor = model.deploy(1, INSTANCE_TYPE)
assert isinstance(predictor, TensorFlowPredictor)