def test_tfs_model_image_accelerator_not_supported(sagemaker_session): model = TensorFlowModel( "s3://some/data.tar.gz", role=ROLE, framework_version="1.13.0", sagemaker_session=sagemaker_session, ) # assert error is not raised model.deploy(instance_type="ml.c4.xlarge", initial_instance_count=1, accelerator_type="ml.eia1.medium") model = TensorFlowModel( "s3://some/data.tar.gz", role=ROLE, framework_version="2.1", sagemaker_session=sagemaker_session, ) # assert error is not raised model.deploy(instance_type="ml.c4.xlarge", initial_instance_count=1) with pytest.raises(AttributeError) as e: model.deploy( instance_type="ml.c4.xlarge", accelerator_type="ml.eia1.medium", initial_instance_count=1, ) assert str(e.value) == "The TensorFlow version 2.1 doesn't support EIA."
def test_model(sagemaker_session, tf_version): model = TensorFlowModel("s3://some/data.tar.gz", role=ROLE, entry_point=SCRIPT_PATH, sagemaker_session=sagemaker_session) predictor = model.deploy(1, GPU_IMAGE_NAME) assert isinstance(predictor, TensorFlowPredictor)
def test_deploy_model_with_accelerator(sagemaker_session, tf_training_job, ei_tf_full_version): endpoint_name = 'test-tf-deploy-model-ei-{}'.format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=tf_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'iris', 'iris-dnn-classifier.py') model = TensorFlowModel(model_data, 'SageMakerRole', entry_point=script_path, framework_version=ei_tf_full_version, sagemaker_session=sagemaker_session) json_predictor = model.deploy(initial_instance_count=1, instance_type='ml.c4.xlarge', endpoint_name=endpoint_name, accelerator_type='ml.eia1.medium') features = [6.4, 3.2, 4.5, 1.5] dict_result = json_predictor.predict({'inputs': features}) print('predict result: {}'.format(dict_result)) list_result = json_predictor.predict(features) print('predict result: {}'.format(list_result)) assert dict_result == list_result
def test_tfs_model_with_dependencies(repack_model, model_code_key_prefix, sagemaker_session, tensorflow_inference_version): model = TensorFlowModel( "s3://some/data.tar.gz", entry_point="train.py", dependencies=["src", "lib"], role=ROLE, framework_version=tensorflow_inference_version, image_uri="my-image", sagemaker_session=sagemaker_session, ) model.prepare_container_def(INSTANCE_TYPE) model_code_key_prefix.assert_called_with(model.key_prefix, model.name, model.image_uri) repack_model.assert_called_with( "train.py", None, ["src", "lib"], "s3://some/data.tar.gz", "s3://my_bucket/key-prefix/model.tar.gz", sagemaker_session, kms_key=None, )
def main(): session = LocalSession() session.config = {'local': {'local_code': True}} role = DUMMY_IAM_ROLE model_dir = 's3://tensorflow-script-mode-local-model-inference/model.tar.gz' model = TensorFlowModel( entry_point='inference.py', source_dir='./code', role=role, model_data=model_dir, framework_version='2.3.0', ) print('Deploying endpoint in local mode') predictor = model.deploy( initial_instance_count=1, instance_type='local', ) dummy_inputs = { 'bucket_name': 'tensorflow-script-mode-local-model-inference', 'object_name': 'instances.json' } predictions = predictor.predict(dummy_inputs) print("predictions: {}".format(predictions)) print('About to delete the endpoint') predictor.delete_endpoint(predictor.endpoint_name) predictor.delete_model()
def sagemaker_deploy(model_artifacts_s3, sagemaker_role, initial_instance_count, instance_type, endpoint_name): tf_model = TensorFlowModel(model_data=model_artifacts_s3, role=sagemaker_role, framework_version='2.2') predictor = tf_model.deploy(initial_instance_count=initial_instance_count, instance_type=instance_type, endpoint_name=endpoint_name, wait=True) return predictor
def test_model(sagemaker_session, tf_version): model = TensorFlowModel(MODEL_DATA, role=ROLE, entry_point=SCRIPT_PATH, sagemaker_session=sagemaker_session) predictor = model.deploy(1, INSTANCE_TYPE) assert isinstance(predictor, TensorFlowPredictor)
def deploy(self): """ Use SageMaker to deploy the model as a stand-alone, PaaS endpoint, with our choice of the underlying Docker image and hardware capabilities. Available images for inferences can be chosen from AWS official list: https://github.com/aws/deep-learning-containers/blob/master/available_images.md Once the endpoint is deployed, you can add a further step with for example behavioral testing, to ensure model robustness (e.g. see https://arxiv.org/pdf/2005.04118.pdf). Here, we just "prove" that the endpoint is up and running! """ from sagemaker.tensorflow import TensorFlowModel # generate a signature for the endpoint, using learning rate and timestamp as a convention ENDPOINT_NAME = 'regression-{}-endpoint'.format( int(round(time.time() * 1000))) # print out the name, so that we can use it when deploying our lambda print("\n\n================\nEndpoint name is: {}\n\n".format( ENDPOINT_NAME)) model = TensorFlowModel(model_data=self.best_s3_model_path, image_uri=self.DOCKER_IMAGE_URI, role=self.IAM_SAGEMAKER_ROLE) predictor = model.deploy(initial_instance_count=1, instance_type=self.SAGEMAKER_INSTANCE, endpoint_name=ENDPOINT_NAME) # run a small test against the endpoint # pick a number for X and check the predicted Y is sensible input = {'instances': np.array([[0.57457947234]])} # output is on the form {'predictions': [[10.879798]]} result = predictor.predict(input) print(input, result) assert result['predictions'][0][0] > 0 self.next(self.end)
def test_model_deploy_with_serverless_inference_config(sagemaker_session, tf_full_version, tf_full_py_version): endpoint_name = unique_name_from_base("sagemaker-tensorflow-serverless") model_data = sagemaker_session.upload_data( path=os.path.join(tests.integ.DATA_DIR, "tensorflow-serving-test-model.tar.gz"), key_prefix="tensorflow-serving/models", ) with tests.integ.timeout.timeout_and_delete_endpoint_by_name( endpoint_name=endpoint_name, sagemaker_session=sagemaker_session, hours=2, sleep_between_cleanup_attempts=20, exponential_sleep=True, ): model = TensorFlowModel( model_data=model_data, role=ROLE, framework_version=tf_full_version, sagemaker_session=sagemaker_session, ) predictor = model.deploy( serverless_inference_config=ServerlessInferenceConfig(), endpoint_name=endpoint_name, ) input_data = {"instances": [1.0, 2.0, 5.0]} expected_result = {"predictions": [3.5, 4.0, 5.5]} result = predictor.predict(input_data) assert expected_result == result
def test_model_image_accelerator(sagemaker_session): model = TensorFlowModel(MODEL_DATA, role=ROLE, entry_point=SCRIPT_PATH, sagemaker_session=sagemaker_session) container_def = model.prepare_container_def( INSTANCE_TYPE, accelerator_type=ACCELERATOR_TYPE) assert container_def['Image'] == _get_full_cpu_image_uri_with_ei( defaults.TF_VERSION)
def test_tfs_model_with_custom_image(sagemaker_session, tensorflow_inference_version): model = TensorFlowModel( "s3://some/data.tar.gz", role=ROLE, framework_version=tensorflow_inference_version, image_uri="my-image", sagemaker_session=sagemaker_session, ) cdef = model.prepare_container_def(INSTANCE_TYPE) assert cdef["Image"] == "my-image"
def test_tfs_model_with_log_level(sagemaker_session, tensorflow_inference_version): model = TensorFlowModel( "s3://some/data.tar.gz", role=ROLE, framework_version=tensorflow_inference_version, container_log_level=logging.INFO, sagemaker_session=sagemaker_session, ) cdef = model.prepare_container_def(INSTANCE_TYPE) assert cdef["Environment"] == {TensorFlowModel.LOG_LEVEL_PARAM_NAME: "info"}
def test_model_prepare_container_def_no_instance_type_or_image(tensorflow_inference_version): model = TensorFlowModel( "s3://some/data.tar.gz", role=ROLE, framework_version=tensorflow_inference_version ) with pytest.raises(ValueError) as e: model.prepare_container_def() expected_msg = "Must supply either an instance type (for choosing CPU vs GPU) or an image URI." assert expected_msg in str(e)
def test_tfs_model_accelerator(retrieve_image_uri, sagemaker_session, tensorflow_eia_version): model = TensorFlowModel( "s3://some/data.tar.gz", role=ROLE, framework_version=tensorflow_eia_version, sagemaker_session=sagemaker_session, ) cdef = model.prepare_container_def(INSTANCE_TYPE, accelerator_type=ACCELERATOR_TYPE) retrieve_image_uri.assert_called_with( "tensorflow", REGION, version=tensorflow_eia_version, instance_type=INSTANCE_TYPE, accelerator_type=ACCELERATOR_TYPE, image_scope="inference", ) assert IMAGE == cdef["Image"] predictor = model.deploy(INSTANCE_COUNT, INSTANCE_TYPE) assert isinstance(predictor, TensorFlowPredictor)
def main(): session = LocalSession() session.config = {'local': {'local_code': True}} role = DUMMY_IAM_ROLE model_dir = 's3://aws-ml-blog/artifacts/tensorflow-script-mode-local-model-inference/model.tar.gz' model = TensorFlowModel( entry_point='inference.py', source_dir='./code', role=role, model_data=model_dir, framework_version='2.3.0', ) print('Deploying endpoint in local mode') print( 'Note: if launching for the first time in local mode, container image download might take a few minutes to complete.' ) predictor = model.deploy( initial_instance_count=1, instance_type='local', ) print('Endpoint deployed in local mode') dummy_inputs = { 'bucket_name': 'aws-ml-blog', 'object_name': 'artifacts/tensorflow-script-mode-local-model-inference/instances.json' } predictions = predictor.predict(dummy_inputs) print("predictions: {}".format(predictions)) print('About to delete the endpoint') predictor.delete_endpoint(predictor.endpoint_name)
def test_deploy_elastic_inference_with_pretrained_model(pretrained_model_data, docker_image_uri, sagemaker_session, instance_type, accelerator_type): resource_path = os.path.join(SCRIPT_PATH, '../resources') endpoint_name = 'test-tf-ei-deploy-model-{}'.format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name=endpoint_name, sagemaker_session=sagemaker_session, minutes=20): tensorflow_model = TensorFlowModel(model_data=pretrained_model_data, entry_point='default_entry_point.py', source_dir=resource_path, role='SageMakerRole', image=docker_image_uri, sagemaker_session=sagemaker_session) logger.info('deploying model to endpoint: {}'.format(endpoint_name)) predictor = tensorflow_model.deploy(initial_instance_count=1, instance_type=instance_type, accelerator_type=accelerator_type, endpoint_name=endpoint_name) random_input = np.random.rand(1, 1, 3, 3) predict_response = predictor.predict({'input': random_input.tolist()}) assert predict_response['outputs']['probabilities']
def test_estimator_py2_deprecation_warning(warning, sagemaker_session): estimator = TensorFlow( entry_point=SCRIPT_PATH, role=ROLE, sagemaker_session=sagemaker_session, train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, py_version="py2", ) assert estimator.py_version == "py2" warning.assert_called_with(estimator.__framework_name__, defaults.LATEST_PY2_VERSION) model = TensorFlowModel( MODEL_DATA, role=ROLE, entry_point=SCRIPT_PATH, sagemaker_session=sagemaker_session, py_version="py2", ) assert model.py_version == "py2" warning.assert_called_with(model.__framework_name__, defaults.LATEST_PY2_VERSION)
def test_empty_framework_version(warning, sagemaker_session): estimator = TensorFlow( entry_point=SCRIPT_PATH, role=ROLE, sagemaker_session=sagemaker_session, train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, framework_version=None, ) assert estimator.framework_version == defaults.TF_VERSION warning.assert_called_with(defaults.TF_VERSION, estimator.LATEST_VERSION) model = TensorFlowModel( MODEL_DATA, role=ROLE, entry_point=SCRIPT_PATH, sagemaker_session=sagemaker_session, framework_version=None, ) assert model.framework_version == defaults.TF_VERSION warning.assert_called_with(defaults.TF_VERSION, defaults.LATEST_VERSION)
def test_model(sagemaker_session, tf_version): model = TensorFlowModel("s3://some/data.tar.gz", role=ROLE, entry_point=SCRIPT_PATH, sagemaker_session=sagemaker_session) predictor = model.deploy(1, INSTANCE_TYPE) assert isinstance(predictor, TensorFlowPredictor)
def test_model_registration_with_tensorflow_model_with_pipeline_model( sagemaker_session, role, tf_full_version, tf_full_py_version, pipeline_name, region_name): base_dir = os.path.join(DATA_DIR, "tensorflow_mnist") entry_point = os.path.join(base_dir, "mnist_v2.py") input_path = sagemaker_session.upload_data( path=os.path.join(base_dir, "data"), key_prefix="integ-test-data/tf-scriptmode/mnist/training", ) inputs = TrainingInput(s3_data=input_path) instance_count = ParameterInteger(name="InstanceCount", default_value=1) instance_type = ParameterString(name="InstanceType", default_value="ml.m5.xlarge") tensorflow_estimator = TensorFlow( entry_point=entry_point, role=role, instance_count=instance_count, instance_type=instance_type, framework_version=tf_full_version, py_version=tf_full_py_version, sagemaker_session=sagemaker_session, ) step_train = TrainingStep( name="MyTrain", estimator=tensorflow_estimator, inputs=inputs, ) model = TensorFlowModel( entry_point=entry_point, framework_version="2.4", model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts, role=role, sagemaker_session=sagemaker_session, ) pipeline_model = PipelineModel(name="MyModelPipeline", models=[model], role=role, sagemaker_session=sagemaker_session) step_register_model = RegisterModel( name="MyRegisterModel", model=pipeline_model, model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts, content_types=["application/json"], response_types=["application/json"], inference_instances=["ml.t2.medium", "ml.m5.large"], transform_instances=["ml.m5.large"], model_package_group_name=f"{pipeline_name}TestModelPackageGroup", ) pipeline = Pipeline( name=pipeline_name, parameters=[ instance_count, instance_type, ], steps=[step_train, step_register_model], sagemaker_session=sagemaker_session, ) try: response = pipeline.create(role) create_arn = response["PipelineArn"] assert re.match( rf"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}", create_arn, ) for _ in retries( max_retry_count=5, exception_message_prefix= "Waiting for a successful execution of pipeline", seconds_to_sleep=10, ): execution = pipeline.start(parameters={}) assert re.match( rf"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}/execution/", execution.arn, ) try: execution.wait(delay=30, max_attempts=60) except WaiterError: pass execution_steps = execution.list_steps() assert len(execution_steps) == 3 for step in execution_steps: assert step["StepStatus"] == "Succeeded" break finally: try: pipeline.delete() except Exception: pass