def test_multi_data_model_deploy_pretrained_models_local_mode(container_image, sagemaker_session):
timestamp = sagemaker_timestamp()
endpoint_name = "test-multimodel-endpoint-{}".format(timestamp)
model_name = "test-multimodel-{}".format(timestamp)
# Define pretrained model local path
pretrained_model_data_local_path = os.path.join(DATA_DIR, "sparkml_model", "mleap_model.tar.gz")
with timeout(minutes=30):
model_data_prefix = os.path.join(
"s3://", sagemaker_session.default_bucket(), "multimodel-{}/".format(timestamp)
)
multi_data_model = MultiDataModel(
name=model_name,
model_data_prefix=model_data_prefix,
image_uri=container_image,
role=ROLE,
sagemaker_session=sagemaker_session,
)
# Add model before deploy
multi_data_model.add_model(pretrained_model_data_local_path, PRETRAINED_MODEL_PATH_1)
# Deploy model to an endpoint
multi_data_model.deploy(1, "local", endpoint_name=endpoint_name)
# Add models after deploy
multi_data_model.add_model(pretrained_model_data_local_path, PRETRAINED_MODEL_PATH_2)
endpoint_models = []
for model_path in multi_data_model.list_models():
endpoint_models.append(model_path)
assert PRETRAINED_MODEL_PATH_1 in endpoint_models
assert PRETRAINED_MODEL_PATH_2 in endpoint_models
predictor = Predictor(
endpoint_name=endpoint_name,
sagemaker_session=multi_data_model.sagemaker_session,
serializer=NumpySerializer(),
deserializer=string_deserializer,
)
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_1)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_1)
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_2)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_2)
# Cleanup
multi_data_model.sagemaker_session.sagemaker_client.delete_endpoint_config(
EndpointConfigName=endpoint_name
)
multi_data_model.sagemaker_session.delete_endpoint(endpoint_name)
multi_data_model.delete_model()
with pytest.raises(Exception) as exception:
sagemaker_session.sagemaker_client.describe_model(ModelName=multi_data_model.name)
assert "Could not find model" in str(exception.value)
sagemaker_session.sagemaker_client.describe_endpoint_config(name=endpoint_name)
assert "Could not find endpoint" in str(exception.value)
def test_inference_pipeline_model_deploy(sagemaker_session, cpu_instance_type):
sparkml_data_path = os.path.join(DATA_DIR, "sparkml_model")
xgboost_data_path = os.path.join(DATA_DIR, "xgboost_model")
endpoint_name = "test-inference-pipeline-deploy-{}".format(
sagemaker_timestamp())
sparkml_model_data = sagemaker_session.upload_data(
path=os.path.join(sparkml_data_path, "mleap_model.tar.gz"),
key_prefix="integ-test-data/sparkml/model",
)
xgb_model_data = sagemaker_session.upload_data(
path=os.path.join(xgboost_data_path, "xgb_model.tar.gz"),
key_prefix="integ-test-data/xgboost/model",
)
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
sparkml_model = SparkMLModel(
model_data=sparkml_model_data,
env={"SAGEMAKER_SPARKML_SCHEMA": SCHEMA},
sagemaker_session=sagemaker_session,
)
xgb_image = image_uris.retrieve("xgboost",
sagemaker_session.boto_region_name,
version="1",
image_scope="inference")
xgb_model = Model(model_data=xgb_model_data,
image_uri=xgb_image,
sagemaker_session=sagemaker_session)
model = PipelineModel(
models=[sparkml_model, xgb_model],
role="SageMakerRole",
sagemaker_session=sagemaker_session,
name=endpoint_name,
)
model.deploy(1, cpu_instance_type, endpoint_name=endpoint_name)
predictor = Predictor(
endpoint_name=endpoint_name,
sagemaker_session=sagemaker_session,
serializer=JSONSerializer,
content_type="text/csv",
accept="text/csv",
)
with open(VALID_DATA_PATH, "r") as f:
valid_data = f.read()
assert predictor.predict(valid_data) == "0.714013934135"
with open(INVALID_DATA_PATH, "r") as f:
invalid_data = f.read()
assert predictor.predict(invalid_data) is None
model.delete_model()
with pytest.raises(Exception) as exception:
sagemaker_session.sagemaker_client.describe_model(ModelName=model.name)
assert "Could not find model" in str(exception.value)
def test_predict_invocation_with_target_variant(sagemaker_session,
multi_variant_endpoint):
predictor = Predictor(
endpoint_name=multi_variant_endpoint.endpoint_name,
sagemaker_session=sagemaker_session,
serializer=CSVSerializer(),
)
# Validate that no exception is raised when the target_variant is specified.
predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_1)
predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_2)
def test_predict_invocation_with_target_variant_local_mode(
sagemaker_session, multi_variant_endpoint):
if sagemaker_session._region_name is None:
sagemaker_session._region_name = DEFAULT_REGION
predictor = Predictor(
endpoint_name=multi_variant_endpoint.endpoint_name,
sagemaker_session=sagemaker_session,
serializer=CSVSerializer(),
deserializer=CSVDeserializer(),
)
# Validate that no exception is raised when the target_variant is specified.
predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_1)
predictor.predict(TEST_CSV_DATA, target_variant=TEST_VARIANT_2)
def test_predict_with_inference_id(sagemaker_session, endpoint_name):
predictor = Predictor(
endpoint_name=endpoint_name,
sagemaker_session=sagemaker_session,
serializer=CSVSerializer(),
)
# Validate that no exception is raised when the target_variant is specified.
response = predictor.predict(TEST_CSV_DATA, inference_id="foo")
assert response
def test_predict_call_with_multiple_accept_types():
sagemaker_session = ret_csv_sagemaker_session()
predictor = Predictor(ENDPOINT,
sagemaker_session,
serializer=CSVSerializer(),
deserializer=PandasDeserializer())
data = [1, 2]
predictor.predict(data)
assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called
expected_request_args = {
"Accept": "text/csv, application/json",
"Body": "1,2",
"ContentType": CSV_CONTENT_TYPE,
"EndpointName": ENDPOINT,
}
call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args
assert kwargs == expected_request_args
def main():
image_name = "sagemaker-sklearn-rf-regressor-local"
# Prepare data for model inference - we use the Boston housing dataset
print('Preparing data for model inference')
data = fetch_california_housing()
X_train, X_test, y_train, y_test = train_test_split(data.data,
data.target,
test_size=0.25,
random_state=42)
# we don't train a model, so we will need only the testing data
testX = pd.DataFrame(X_test, columns=data.feature_names)
# Download a pre-trained model file
print('Downloading a pre-trained model file')
s3.download_file(
'aws-ml-blog',
'artifacts/scikit_learn_bring_your_own_model/model.joblib',
'model.joblib')
# Creating a model.tar.gz file
tar = tarfile.open('model.tar.gz', 'w:gz')
tar.add('model.joblib')
tar.close()
model = Model(image_uri=image_name,
role=DUMMY_IAM_ROLE,
model_data='file://./model.tar.gz')
print('Deploying endpoint in local mode')
endpoint = model.deploy(initial_instance_count=1,
instance_type='local',
endpoint_name="my-local-endpoint")
predictor = Predictor(endpoint_name="my-local-endpoint",
sagemaker_session=sagemaker_session,
serializer=CSVSerializer(),
deserializer=CSVDeserializer())
predictions = predictor.predict(testX[data.feature_names].head(5).to_csv(
header=False, index=False))
print(f"Predictions: {predictions}")
print('About to delete the endpoint to stop paying (if in cloud mode).')
predictor.delete_endpoint(predictor.endpoint_name)
class Predictor:
def __init__(self, endpoint_name):
self.endpoint_name = endpoint_name
self.session = get_sagemaker_session()
self.attach_predictor()
def attach_predictor(self):
self.predictor = SagemakerPredictor(endpoint_name=self.endpoint_name,
sagemaker_session=self.session,
serializer=NumpySerializer(),
deserializer=NumpyDeserializer())
def predict(self, data):
return self.predictor.predict(data)
def undeploy(self):
self.predictor.delete_endpoint()
def test_predict_call_pass_through():
sagemaker_session = empty_sagemaker_session()
predictor = Predictor(ENDPOINT, sagemaker_session)
data = "untouched"
result = predictor.predict(data)
assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called
expected_request_args = {
"Accept": DEFAULT_ACCEPT,
"Body": data,
"ContentType": DEFAULT_CONTENT_TYPE,
"EndpointName": ENDPOINT,
}
call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args
assert kwargs == expected_request_args
assert result == RETURN_VALUE
def test_predict_call_with_target_variant():
sagemaker_session = empty_sagemaker_session()
predictor = Predictor(ENDPOINT, sagemaker_session)
data = "untouched"
result = predictor.predict(data, target_variant=PRODUCTION_VARIANT_1)
assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called
expected_request_args = {
"Accept": DEFAULT_ACCEPT,
"Body": data,
"ContentType": DEFAULT_CONTENT_TYPE,
"EndpointName": ENDPOINT,
"TargetVariant": PRODUCTION_VARIANT_1,
}
call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args
assert kwargs == expected_request_args
assert result == RETURN_VALUE
def test_predict_call_with_json():
sagemaker_session = json_sagemaker_session()
predictor = Predictor(ENDPOINT,
sagemaker_session,
serializer=JSONSerializer())
data = [1, 2]
result = predictor.predict(data)
assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called
expected_request_args = {
"Accept": DEFAULT_ACCEPT,
"Body": json.dumps(data),
"ContentType": "application/json",
"EndpointName": ENDPOINT,
}
call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args
assert kwargs == expected_request_args
assert result == json.dumps([RETURN_VALUE])
def test_predict_call_with_csv():
sagemaker_session = ret_csv_sagemaker_session()
predictor = Predictor(ENDPOINT,
sagemaker_session,
serializer=CSVSerializer(),
deserializer=CSVDeserializer())
data = [1, 2]
result = predictor.predict(data)
assert sagemaker_session.sagemaker_runtime_client.invoke_endpoint.called
expected_request_args = {
"Accept": CSV_CONTENT_TYPE,
"Body": "1,2",
"ContentType": CSV_CONTENT_TYPE,
"EndpointName": ENDPOINT,
}
call_args, kwargs = sagemaker_session.sagemaker_runtime_client.invoke_endpoint.call_args
assert kwargs == expected_request_args
assert result == [["1", "2", "3"]]
def test_multi_data_model_deploy_pretrained_models_update_endpoint(
container_image, sagemaker_session, cpu_instance_type, alternative_cpu_instance_type
):
timestamp = sagemaker_timestamp()
endpoint_name = "test-multimodel-endpoint-{}".format(timestamp)
model_name = "test-multimodel-{}".format(timestamp)
# Define pretrained model local path
pretrained_model_data_local_path = os.path.join(DATA_DIR, "sparkml_model", "mleap_model.tar.gz")
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model_data_prefix = os.path.join(
"s3://", sagemaker_session.default_bucket(), "multimodel-{}/".format(timestamp)
)
multi_data_model = MultiDataModel(
name=model_name,
model_data_prefix=model_data_prefix,
image_uri=container_image,
role=ROLE,
sagemaker_session=sagemaker_session,
)
# Add model before deploy
multi_data_model.add_model(pretrained_model_data_local_path, PRETRAINED_MODEL_PATH_1)
# Deploy model to an endpoint
multi_data_model.deploy(1, cpu_instance_type, endpoint_name=endpoint_name)
# Add model after deploy
multi_data_model.add_model(pretrained_model_data_local_path, PRETRAINED_MODEL_PATH_2)
# List model assertions
endpoint_models = []
for model_path in multi_data_model.list_models():
endpoint_models.append(model_path)
assert PRETRAINED_MODEL_PATH_1 in endpoint_models
assert PRETRAINED_MODEL_PATH_2 in endpoint_models
predictor = Predictor(
endpoint_name=endpoint_name,
sagemaker_session=sagemaker_session,
serializer=NumpySerializer(),
deserializer=string_deserializer,
)
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_1)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_1)
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_2)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_2)
endpoint_desc = sagemaker_session.sagemaker_client.describe_endpoint(
EndpointName=endpoint_name
)
old_config_name = endpoint_desc["EndpointConfigName"]
# Update endpoint
predictor.update_endpoint(
initial_instance_count=1, instance_type=alternative_cpu_instance_type
)
endpoint_desc = sagemaker_session.sagemaker_client.describe_endpoint(
EndpointName=endpoint_name
)
new_config_name = endpoint_desc["EndpointConfigName"]
new_config = sagemaker_session.sagemaker_client.describe_endpoint_config(
EndpointConfigName=new_config_name
)
assert old_config_name != new_config_name
assert new_config["ProductionVariants"][0]["InstanceType"] == alternative_cpu_instance_type
assert new_config["ProductionVariants"][0]["InitialInstanceCount"] == 1
# Cleanup
sagemaker_session.sagemaker_client.delete_endpoint_config(
EndpointConfigName=old_config_name
)
sagemaker_session.sagemaker_client.delete_endpoint_config(
EndpointConfigName=new_config_name
)
multi_data_model.delete_model()
with pytest.raises(Exception) as exception:
sagemaker_session.sagemaker_client.describe_model(ModelName=model_name)
assert "Could not find model" in str(exception.value)
sagemaker_session.sagemaker_client.describe_endpoint_config(name=old_config_name)
assert "Could not find endpoint" in str(exception.value)
sagemaker_session.sagemaker_client.describe_endpoint_config(name=new_config_name)
assert "Could not find endpoint" in str(exception.value)
def test_multi_data_model_deploy_train_model_from_amazon_first_party_estimator(
container_image, sagemaker_session, cpu_instance_type
):
timestamp = sagemaker_timestamp()
endpoint_name = "test-multimodel-endpoint-{}".format(timestamp)
model_name = "test-multimodel-{}".format(timestamp)
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
rcf_model_v1 = __rcf_training_job(
sagemaker_session, container_image, cpu_instance_type, 50, 20
)
model_data_prefix = os.path.join(
"s3://", sagemaker_session.default_bucket(), "multimodel-{}/".format(timestamp)
)
multi_data_model = MultiDataModel(
name=model_name,
model_data_prefix=model_data_prefix,
model=rcf_model_v1,
sagemaker_session=sagemaker_session,
)
# Add model before deploy
multi_data_model.add_model(rcf_model_v1.model_data, PRETRAINED_MODEL_PATH_1)
# Deploy model to an endpoint
multi_data_model.deploy(1, cpu_instance_type, endpoint_name=endpoint_name)
# Train another model
rcf_model_v2 = __rcf_training_job(
sagemaker_session, container_image, cpu_instance_type, 70, 20
)
# Deploy newly trained model
multi_data_model.add_model(rcf_model_v2.model_data, PRETRAINED_MODEL_PATH_2)
# List model assertions
endpoint_models = []
for model_path in multi_data_model.list_models():
endpoint_models.append(model_path)
assert PRETRAINED_MODEL_PATH_1 in endpoint_models
assert PRETRAINED_MODEL_PATH_2 in endpoint_models
# Define a predictor to set `serializer` parameter with `NumpySerializer`
# instead of `JSONSerializer` in the default predictor returned by `MXNetPredictor`
# Since we are using a placeholder container image the prediction results are not accurate.
predictor = Predictor(
endpoint_name=endpoint_name,
sagemaker_session=sagemaker_session,
serializer=NumpySerializer(),
deserializer=string_deserializer,
)
data = numpy.random.rand(1, 14)
# Prediction result for the first model
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_1)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_1)
# Prediction result for the second model
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_2)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_2)
# Cleanup
sagemaker_session.sagemaker_client.delete_endpoint_config(EndpointConfigName=endpoint_name)
multi_data_model.delete_model()
with pytest.raises(Exception) as exception:
sagemaker_session.sagemaker_client.describe_model(ModelName=model_name)
assert "Could not find model" in str(exception.value)
sagemaker_session.sagemaker_client.describe_endpoint_config(name=endpoint_name)
assert "Could not find endpoint" in str(exception.value)
from sagemaker.predictor import Predictor
from sagemaker.serializers import CSVDeserializer, CSVSerializer
import time
endpoint_name = "xgb-churn-monitor"
predictor = Predictor(
endpoint_name=endpoint_name,
deserializer=CSVDeserializer(),
serializer=CSVSerializer(),
)
# get a subset of test data for a quick test
#!head -120 test_data/test-dataset-input-cols.csv > test_data/test_sample.csv
print("Sending test traffic to the endpoint {}. \nPlease wait...".format(
endpoint_name))
with open("test_data/test_sample.csv", "r") as f:
for row in f:
payload = row.rstrip("\n")
response = predictor.predict(data=payload)
time.sleep(0.5)
print("Done!")
