def test_model(sagemaker_session, tf_version): model = TensorFlowModel(MODEL_DATA, role=ROLE, entry_point=SCRIPT_PATH, sagemaker_session=sagemaker_session) predictor = model.deploy(1, INSTANCE_TYPE) assert isinstance(predictor, TensorFlowPredictor)
def main(): session = LocalSession() session.config = {'local': {'local_code': True}} role = DUMMY_IAM_ROLE model_dir = 's3://tensorflow-script-mode-local-model-inference/model.tar.gz' model = TensorFlowModel( entry_point='inference.py', source_dir='./code', role=role, model_data=model_dir, framework_version='2.3.0', ) print('Deploying endpoint in local mode') predictor = model.deploy( initial_instance_count=1, instance_type='local', ) dummy_inputs = { 'bucket_name': 'tensorflow-script-mode-local-model-inference', 'object_name': 'instances.json' } predictions = predictor.predict(dummy_inputs) print("predictions: {}".format(predictions)) print('About to delete the endpoint') predictor.delete_endpoint(predictor.endpoint_name) predictor.delete_model()
def sagemaker_deploy(model_artifacts_s3, sagemaker_role, initial_instance_count, instance_type, endpoint_name): tf_model = TensorFlowModel(model_data=model_artifacts_s3, role=sagemaker_role, framework_version='2.2') predictor = tf_model.deploy(initial_instance_count=initial_instance_count, instance_type=instance_type, endpoint_name=endpoint_name, wait=True) return predictor
def test_model_deploy_with_serverless_inference_config(sagemaker_session, tf_full_version, tf_full_py_version): endpoint_name = unique_name_from_base("sagemaker-tensorflow-serverless") model_data = sagemaker_session.upload_data( path=os.path.join(tests.integ.DATA_DIR, "tensorflow-serving-test-model.tar.gz"), key_prefix="tensorflow-serving/models", ) with tests.integ.timeout.timeout_and_delete_endpoint_by_name( endpoint_name=endpoint_name, sagemaker_session=sagemaker_session, hours=2, sleep_between_cleanup_attempts=20, exponential_sleep=True, ): model = TensorFlowModel( model_data=model_data, role=ROLE, framework_version=tf_full_version, sagemaker_session=sagemaker_session, ) predictor = model.deploy( serverless_inference_config=ServerlessInferenceConfig(), endpoint_name=endpoint_name, ) input_data = {"instances": [1.0, 2.0, 5.0]} expected_result = {"predictions": [3.5, 4.0, 5.5]} result = predictor.predict(input_data) assert expected_result == result
def test_deploy_model_with_accelerator(sagemaker_session, tf_training_job, ei_tf_full_version): endpoint_name = 'test-tf-deploy-model-ei-{}'.format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=tf_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'iris', 'iris-dnn-classifier.py') model = TensorFlowModel(model_data, 'SageMakerRole', entry_point=script_path, framework_version=ei_tf_full_version, sagemaker_session=sagemaker_session) json_predictor = model.deploy(initial_instance_count=1, instance_type='ml.c4.xlarge', endpoint_name=endpoint_name, accelerator_type='ml.eia1.medium') features = [6.4, 3.2, 4.5, 1.5] dict_result = json_predictor.predict({'inputs': features}) print('predict result: {}'.format(dict_result)) list_result = json_predictor.predict(features) print('predict result: {}'.format(list_result)) assert dict_result == list_result
def test_model(sagemaker_session, tf_version): model = TensorFlowModel("s3://some/data.tar.gz", role=ROLE, entry_point=SCRIPT_PATH, sagemaker_session=sagemaker_session) predictor = model.deploy(1, GPU_IMAGE_NAME) assert isinstance(predictor, TensorFlowPredictor)
def deploy(self): """ Use SageMaker to deploy the model as a stand-alone, PaaS endpoint, with our choice of the underlying Docker image and hardware capabilities. Available images for inferences can be chosen from AWS official list: https://github.com/aws/deep-learning-containers/blob/master/available_images.md Once the endpoint is deployed, you can add a further step with for example behavioral testing, to ensure model robustness (e.g. see https://arxiv.org/pdf/2005.04118.pdf). Here, we just "prove" that the endpoint is up and running! """ from sagemaker.tensorflow import TensorFlowModel # generate a signature for the endpoint, using learning rate and timestamp as a convention ENDPOINT_NAME = 'regression-{}-endpoint'.format( int(round(time.time() * 1000))) # print out the name, so that we can use it when deploying our lambda print("\n\n================\nEndpoint name is: {}\n\n".format( ENDPOINT_NAME)) model = TensorFlowModel(model_data=self.best_s3_model_path, image_uri=self.DOCKER_IMAGE_URI, role=self.IAM_SAGEMAKER_ROLE) predictor = model.deploy(initial_instance_count=1, instance_type=self.SAGEMAKER_INSTANCE, endpoint_name=ENDPOINT_NAME) # run a small test against the endpoint # pick a number for X and check the predicted Y is sensible input = {'instances': np.array([[0.57457947234]])} # output is on the form {'predictions': [[10.879798]]} result = predictor.predict(input) print(input, result) assert result['predictions'][0][0] > 0 self.next(self.end)
def test_tfs_model_image_accelerator_not_supported(sagemaker_session): model = TensorFlowModel( "s3://some/data.tar.gz", role=ROLE, framework_version="1.13.0", sagemaker_session=sagemaker_session, ) # assert error is not raised model.deploy(instance_type="ml.c4.xlarge", initial_instance_count=1, accelerator_type="ml.eia1.medium") model = TensorFlowModel( "s3://some/data.tar.gz", role=ROLE, framework_version="2.1", sagemaker_session=sagemaker_session, ) # assert error is not raised model.deploy(instance_type="ml.c4.xlarge", initial_instance_count=1) with pytest.raises(AttributeError) as e: model.deploy( instance_type="ml.c4.xlarge", accelerator_type="ml.eia1.medium", initial_instance_count=1, ) assert str(e.value) == "The TensorFlow version 2.1 doesn't support EIA."
def test_tfs_model_accelerator(retrieve_image_uri, sagemaker_session, tensorflow_eia_version): model = TensorFlowModel( "s3://some/data.tar.gz", role=ROLE, framework_version=tensorflow_eia_version, sagemaker_session=sagemaker_session, ) cdef = model.prepare_container_def(INSTANCE_TYPE, accelerator_type=ACCELERATOR_TYPE) retrieve_image_uri.assert_called_with( "tensorflow", REGION, version=tensorflow_eia_version, instance_type=INSTANCE_TYPE, accelerator_type=ACCELERATOR_TYPE, image_scope="inference", ) assert IMAGE == cdef["Image"] predictor = model.deploy(INSTANCE_COUNT, INSTANCE_TYPE) assert isinstance(predictor, TensorFlowPredictor)
def main(): session = LocalSession() session.config = {'local': {'local_code': True}} role = DUMMY_IAM_ROLE model_dir = 's3://aws-ml-blog/artifacts/tensorflow-script-mode-local-model-inference/model.tar.gz' model = TensorFlowModel( entry_point='inference.py', source_dir='./code', role=role, model_data=model_dir, framework_version='2.3.0', ) print('Deploying endpoint in local mode') print( 'Note: if launching for the first time in local mode, container image download might take a few minutes to complete.' ) predictor = model.deploy( initial_instance_count=1, instance_type='local', ) print('Endpoint deployed in local mode') dummy_inputs = { 'bucket_name': 'aws-ml-blog', 'object_name': 'artifacts/tensorflow-script-mode-local-model-inference/instances.json' } predictions = predictor.predict(dummy_inputs) print("predictions: {}".format(predictions)) print('About to delete the endpoint') predictor.delete_endpoint(predictor.endpoint_name)
def test_deploy_elastic_inference_with_pretrained_model(pretrained_model_data, docker_image_uri, sagemaker_session, instance_type, accelerator_type): resource_path = os.path.join(SCRIPT_PATH, '../resources') endpoint_name = 'test-tf-ei-deploy-model-{}'.format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name=endpoint_name, sagemaker_session=sagemaker_session, minutes=20): tensorflow_model = TensorFlowModel(model_data=pretrained_model_data, entry_point='default_entry_point.py', source_dir=resource_path, role='SageMakerRole', image=docker_image_uri, sagemaker_session=sagemaker_session) logger.info('deploying model to endpoint: {}'.format(endpoint_name)) predictor = tensorflow_model.deploy(initial_instance_count=1, instance_type=instance_type, accelerator_type=accelerator_type, endpoint_name=endpoint_name) random_input = np.random.rand(1, 1, 3, 3) predict_response = predictor.predict({'input': random_input.tolist()}) assert predict_response['outputs']['probabilities']
def test_model(sagemaker_session, tf_version): model = TensorFlowModel("s3://some/data.tar.gz", role=ROLE, entry_point=SCRIPT_PATH, sagemaker_session=sagemaker_session) predictor = model.deploy(1, INSTANCE_TYPE) assert isinstance(predictor, TensorFlowPredictor)