def test_multi_data_model_deploy_pretrained_models_local_mode(container_image, sagemaker_session): timestamp = sagemaker_timestamp() endpoint_name = "test-multimodel-endpoint-{}".format(timestamp) model_name = "test-multimodel-{}".format(timestamp) # Define pretrained model local path pretrained_model_data_local_path = os.path.join(DATA_DIR, "sparkml_model", "mleap_model.tar.gz") with timeout(minutes=30): model_data_prefix = os.path.join( "s3://", sagemaker_session.default_bucket(), "multimodel-{}/".format(timestamp) ) multi_data_model = MultiDataModel( name=model_name, model_data_prefix=model_data_prefix, image_uri=container_image, role=ROLE, sagemaker_session=sagemaker_session, ) # Add model before deploy multi_data_model.add_model(pretrained_model_data_local_path, PRETRAINED_MODEL_PATH_1) # Deploy model to an endpoint multi_data_model.deploy(1, "local", endpoint_name=endpoint_name) # Add models after deploy multi_data_model.add_model(pretrained_model_data_local_path, PRETRAINED_MODEL_PATH_2) endpoint_models = [] for model_path in multi_data_model.list_models(): endpoint_models.append(model_path) assert PRETRAINED_MODEL_PATH_1 in endpoint_models assert PRETRAINED_MODEL_PATH_2 in endpoint_models predictor = Predictor( endpoint_name=endpoint_name, sagemaker_session=multi_data_model.sagemaker_session, serializer=NumpySerializer(), deserializer=string_deserializer, ) data = numpy.zeros(shape=(1, 1, 28, 28)) result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_1) assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_1) result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_2) assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_2) # Cleanup multi_data_model.sagemaker_session.sagemaker_client.delete_endpoint_config( EndpointConfigName=endpoint_name ) multi_data_model.sagemaker_session.delete_endpoint(endpoint_name) multi_data_model.delete_model() with pytest.raises(Exception) as exception: sagemaker_session.sagemaker_client.describe_model(ModelName=multi_data_model.name) assert "Could not find model" in str(exception.value) sagemaker_session.sagemaker_client.describe_endpoint_config(name=endpoint_name) assert "Could not find endpoint" in str(exception.value)
def test_inference_pipeline_model_deploy(sagemaker_session, cpu_instance_type): sparkml_data_path = os.path.join(DATA_DIR, "sparkml_model") xgboost_data_path = os.path.join(DATA_DIR, "xgboost_model") endpoint_name = "test-inference-pipeline-deploy-{}".format( sagemaker_timestamp()) sparkml_model_data = sagemaker_session.upload_data( path=os.path.join(sparkml_data_path, "mleap_model.tar.gz"), key_prefix="integ-test-data/sparkml/model", ) xgb_model_data = sagemaker_session.upload_data( path=os.path.join(xgboost_data_path, "xgb_model.tar.gz"), key_prefix="integ-test-data/xgboost/model", ) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): sparkml_model = SparkMLModel( model_data=sparkml_model_data, env={"SAGEMAKER_SPARKML_SCHEMA": SCHEMA}, sagemaker_session=sagemaker_session, ) xgb_image = image_uris.retrieve("xgboost", sagemaker_session.boto_region_name, version="1", image_scope="inference") xgb_model = Model(model_data=xgb_model_data, image_uri=xgb_image, sagemaker_session=sagemaker_session) model = PipelineModel( models=[sparkml_model, xgb_model], role="SageMakerRole", sagemaker_session=sagemaker_session, name=endpoint_name, ) model.deploy(1, cpu_instance_type, endpoint_name=endpoint_name) predictor = Predictor( endpoint_name=endpoint_name, sagemaker_session=sagemaker_session, serializer=JSONSerializer, content_type="text/csv", accept="text/csv", ) with open(VALID_DATA_PATH, "r") as f: valid_data = f.read() assert predictor.predict(valid_data) == "0.714013934135" with open(INVALID_DATA_PATH, "r") as f: invalid_data = f.read() assert predictor.predict(invalid_data) is None model.delete_model() with pytest.raises(Exception) as exception: sagemaker_session.sagemaker_client.describe_model(ModelName=model.name) assert "Could not find model" in str(exception.value)
def test_predict_invocation_with_target_variant(sagemaker_session, multi_variant_endpoint): predictor = Predictor( endpoint_name=multi_variant_endpoint.endpoint_name, sagemaker_session=sagemaker_session, serializer=CSVSerializer(), ) # Validate that no exception is raised when the target_variant is specified. predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_1) predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_2)
def test_predict_invocation_with_target_variant_local_mode( sagemaker_session, multi_variant_endpoint): if sagemaker_session._region_name is None: sagemaker_session._region_name = DEFAULT_REGION predictor = Predictor( endpoint_name=multi_variant_endpoint.endpoint_name, sagemaker_session=sagemaker_session, serializer=CSVSerializer(), deserializer=CSVDeserializer(), ) # Validate that no exception is raised when the target_variant is specified. predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_1) predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_2)
def test_predict_with_inference_id(sagemaker_session, endpoint_name): predictor = Predictor( endpoint_name=endpoint_name, sagemaker_session=sagemaker_session, serializer=CSVSerializer(), ) # Validate that no exception is raised when the target_variant is specified. response = predictor.predict(TEST_CSV_DATA, inference_id="foo") assert response
def test_predict_call_with_multiple_accept_types(): sagemaker_session = ret_csv_sagemaker_session() predictor = Predictor(ENDPOINT, sagemaker_session, serializer=CSVSerializer(), deserializer=PandasDeserializer()) data = [1, 2] predictor.predict(data) assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called expected_request_args = { "Accept": "text/csv, application/json", "Body": "1,2", "ContentType": CSV_CONTENT_TYPE, "EndpointName": ENDPOINT, } call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args assert kwargs == expected_request_args
def main(): image_name = "sagemaker-sklearn-rf-regressor-local" # Prepare data for model inference - we use the Boston housing dataset print('Preparing data for model inference') data = fetch_california_housing() X_train, X_test, y_train, y_test = train_test_split(data.data, data.target, test_size=0.25, random_state=42) # we don't train a model, so we will need only the testing data testX = pd.DataFrame(X_test, columns=data.feature_names) # Download a pre-trained model file print('Downloading a pre-trained model file') s3.download_file( 'aws-ml-blog', 'artifacts/scikit_learn_bring_your_own_model/model.joblib', 'model.joblib') # Creating a model.tar.gz file tar = tarfile.open('model.tar.gz', 'w:gz') tar.add('model.joblib') tar.close() model = Model(image_uri=image_name, role=DUMMY_IAM_ROLE, model_data='file://./model.tar.gz') print('Deploying endpoint in local mode') endpoint = model.deploy(initial_instance_count=1, instance_type='local', endpoint_name="my-local-endpoint") predictor = Predictor(endpoint_name="my-local-endpoint", sagemaker_session=sagemaker_session, serializer=CSVSerializer(), deserializer=CSVDeserializer()) predictions = predictor.predict(testX[data.feature_names].head(5).to_csv( header=False, index=False)) print(f"Predictions: {predictions}") print('About to delete the endpoint to stop paying (if in cloud mode).') predictor.delete_endpoint(predictor.endpoint_name)
class Predictor: def __init__(self, endpoint_name): self.endpoint_name = endpoint_name self.session = get_sagemaker_session() self.attach_predictor() def attach_predictor(self): self.predictor = SagemakerPredictor(endpoint_name=self.endpoint_name, sagemaker_session=self.session, serializer=NumpySerializer(), deserializer=NumpyDeserializer()) def predict(self, data): return self.predictor.predict(data) def undeploy(self): self.predictor.delete_endpoint()
def test_predict_call_pass_through(): sagemaker_session = empty_sagemaker_session() predictor = Predictor(ENDPOINT, sagemaker_session) data = "untouched" result = predictor.predict(data) assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called expected_request_args = { "Accept": DEFAULT_ACCEPT, "Body": data, "ContentType": DEFAULT_CONTENT_TYPE, "EndpointName": ENDPOINT, } call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args assert kwargs == expected_request_args assert result == RETURN_VALUE
def test_predict_call_with_target_variant(): sagemaker_session = empty_sagemaker_session() predictor = Predictor(ENDPOINT, sagemaker_session) data = "untouched" result = predictor.predict(data, target_variant=PRODUCTION_VARIANT_1) assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called expected_request_args = { "Accept": DEFAULT_ACCEPT, "Body": data, "ContentType": DEFAULT_CONTENT_TYPE, "EndpointName": ENDPOINT, "TargetVariant": PRODUCTION_VARIANT_1, } call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args assert kwargs == expected_request_args assert result == RETURN_VALUE
def test_predict_call_with_json(): sagemaker_session = json_sagemaker_session() predictor = Predictor(ENDPOINT, sagemaker_session, serializer=JSONSerializer()) data = [1, 2] result = predictor.predict(data) assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called expected_request_args = { "Accept": DEFAULT_ACCEPT, "Body": json.dumps(data), "ContentType": "application/json", "EndpointName": ENDPOINT, } call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args assert kwargs == expected_request_args assert result == json.dumps([RETURN_VALUE])
def test_predict_call_with_csv(): sagemaker_session = ret_csv_sagemaker_session() predictor = Predictor(ENDPOINT, sagemaker_session, serializer=CSVSerializer(), deserializer=CSVDeserializer()) data = [1, 2] result = predictor.predict(data) assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called expected_request_args = { "Accept": CSV_CONTENT_TYPE, "Body": "1,2", "ContentType": CSV_CONTENT_TYPE, "EndpointName": ENDPOINT, } call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args assert kwargs == expected_request_args assert result == [["1", "2", "3"]]
def test_multi_data_model_deploy_pretrained_models_update_endpoint( container_image, sagemaker_session, cpu_instance_type, alternative_cpu_instance_type ): timestamp = sagemaker_timestamp() endpoint_name = "test-multimodel-endpoint-{}".format(timestamp) model_name = "test-multimodel-{}".format(timestamp) # Define pretrained model local path pretrained_model_data_local_path = os.path.join(DATA_DIR, "sparkml_model", "mleap_model.tar.gz") with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): model_data_prefix = os.path.join( "s3://", sagemaker_session.default_bucket(), "multimodel-{}/".format(timestamp) ) multi_data_model = MultiDataModel( name=model_name, model_data_prefix=model_data_prefix, image_uri=container_image, role=ROLE, sagemaker_session=sagemaker_session, ) # Add model before deploy multi_data_model.add_model(pretrained_model_data_local_path, PRETRAINED_MODEL_PATH_1) # Deploy model to an endpoint multi_data_model.deploy(1, cpu_instance_type, endpoint_name=endpoint_name) # Add model after deploy multi_data_model.add_model(pretrained_model_data_local_path, PRETRAINED_MODEL_PATH_2) # List model assertions endpoint_models = [] for model_path in multi_data_model.list_models(): endpoint_models.append(model_path) assert PRETRAINED_MODEL_PATH_1 in endpoint_models assert PRETRAINED_MODEL_PATH_2 in endpoint_models predictor = Predictor( endpoint_name=endpoint_name, sagemaker_session=sagemaker_session, serializer=NumpySerializer(), deserializer=string_deserializer, ) data = numpy.zeros(shape=(1, 1, 28, 28)) result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_1) assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_1) result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_2) assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_2) endpoint_desc = sagemaker_session.sagemaker_client.describe_endpoint( EndpointName=endpoint_name ) old_config_name = endpoint_desc["EndpointConfigName"] # Update endpoint predictor.update_endpoint( initial_instance_count=1, instance_type=alternative_cpu_instance_type ) endpoint_desc = sagemaker_session.sagemaker_client.describe_endpoint( EndpointName=endpoint_name ) new_config_name = endpoint_desc["EndpointConfigName"] new_config = sagemaker_session.sagemaker_client.describe_endpoint_config( EndpointConfigName=new_config_name ) assert old_config_name != new_config_name assert new_config["ProductionVariants"][0]["InstanceType"] == alternative_cpu_instance_type assert new_config["ProductionVariants"][0]["InitialInstanceCount"] == 1 # Cleanup sagemaker_session.sagemaker_client.delete_endpoint_config( EndpointConfigName=old_config_name ) sagemaker_session.sagemaker_client.delete_endpoint_config( EndpointConfigName=new_config_name ) multi_data_model.delete_model() with pytest.raises(Exception) as exception: sagemaker_session.sagemaker_client.describe_model(ModelName=model_name) assert "Could not find model" in str(exception.value) sagemaker_session.sagemaker_client.describe_endpoint_config(name=old_config_name) assert "Could not find endpoint" in str(exception.value) sagemaker_session.sagemaker_client.describe_endpoint_config(name=new_config_name) assert "Could not find endpoint" in str(exception.value)
def test_multi_data_model_deploy_train_model_from_amazon_first_party_estimator( container_image, sagemaker_session, cpu_instance_type ): timestamp = sagemaker_timestamp() endpoint_name = "test-multimodel-endpoint-{}".format(timestamp) model_name = "test-multimodel-{}".format(timestamp) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): rcf_model_v1 = __rcf_training_job( sagemaker_session, container_image, cpu_instance_type, 50, 20 ) model_data_prefix = os.path.join( "s3://", sagemaker_session.default_bucket(), "multimodel-{}/".format(timestamp) ) multi_data_model = MultiDataModel( name=model_name, model_data_prefix=model_data_prefix, model=rcf_model_v1, sagemaker_session=sagemaker_session, ) # Add model before deploy multi_data_model.add_model(rcf_model_v1.model_data, PRETRAINED_MODEL_PATH_1) # Deploy model to an endpoint multi_data_model.deploy(1, cpu_instance_type, endpoint_name=endpoint_name) # Train another model rcf_model_v2 = __rcf_training_job( sagemaker_session, container_image, cpu_instance_type, 70, 20 ) # Deploy newly trained model multi_data_model.add_model(rcf_model_v2.model_data, PRETRAINED_MODEL_PATH_2) # List model assertions endpoint_models = [] for model_path in multi_data_model.list_models(): endpoint_models.append(model_path) assert PRETRAINED_MODEL_PATH_1 in endpoint_models assert PRETRAINED_MODEL_PATH_2 in endpoint_models # Define a predictor to set `serializer` parameter with `NumpySerializer` # instead of `JSONSerializer` in the default predictor returned by `MXNetPredictor` # Since we are using a placeholder container image the prediction results are not accurate. predictor = Predictor( endpoint_name=endpoint_name, sagemaker_session=sagemaker_session, serializer=NumpySerializer(), deserializer=string_deserializer, ) data = numpy.random.rand(1, 14) # Prediction result for the first model result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_1) assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_1) # Prediction result for the second model result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_2) assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_2) # Cleanup sagemaker_session.sagemaker_client.delete_endpoint_config(EndpointConfigName=endpoint_name) multi_data_model.delete_model() with pytest.raises(Exception) as exception: sagemaker_session.sagemaker_client.describe_model(ModelName=model_name) assert "Could not find model" in str(exception.value) sagemaker_session.sagemaker_client.describe_endpoint_config(name=endpoint_name) assert "Could not find endpoint" in str(exception.value)
from sagemaker.predictor import Predictor from sagemaker.serializers import CSVDeserializer, CSVSerializer import time endpoint_name = "xgb-churn-monitor" predictor = Predictor( endpoint_name=endpoint_name, deserializer=CSVDeserializer(), serializer=CSVSerializer(), ) # get a subset of test data for a quick test #!head -120 test_data/test-dataset-input-cols.csv > test_data/test_sample.csv print("Sending test traffic to the endpoint {}. \nPlease wait...".format( endpoint_name)) with open("test_data/test_sample.csv", "r") as f: for row in f: payload = row.rstrip("\n") response = predictor.predict(data=payload) time.sleep(0.5) print("Done!")