def test_deploy_multi_data_framework_model(sagemaker_session, mxnet_model):
model = MultiDataModel(
name=MODEL_NAME,
model_data_prefix=VALID_MULTI_MODEL_DATA_PREFIX,
sagemaker_session=sagemaker_session,
model=mxnet_model,
)
predictor = model.deploy(
initial_instance_count=INSTANCE_COUNT,
instance_type=INSTANCE_TYPE,
endpoint_name=MULTI_MODEL_ENDPOINT_NAME,
)
# Assert if this is called with mxnet_model parameters
sagemaker_session.create_model.assert_called_with(
MODEL_NAME,
MXNET_ROLE,
model.prepare_container_def(INSTANCE_TYPE),
vpc_config=None,
enable_network_isolation=True,
tags=None,
)
sagemaker_session.endpoint_from_production_variants.assert_called_with(
name=MULTI_MODEL_ENDPOINT_NAME,
wait=True,
tags=None,
kms_key=None,
data_capture_config_dict=None,
production_variants=EXPECTED_PROD_VARIANT,
)
sagemaker_session.create_endpoint_config.assert_not_called()
assert isinstance(predictor, MXNetPredictor)
def create_model(self):
if not self.multimodel:
self.sagemakermodel = Model(
name="model-" + self.name,
model_data=self.modelpath[0],
image=self.image,
role=sagemaker.get_execution_role(),
# sagemaker_session=self.session,
predictor_cls=sagemaker.predictor.RealTimePredictor,
)
else:
self.sagemakermodel = MultiDataModel(
name="model-" + self.name,
model_data_prefix="/".join(self.modelpath[0].split("/")[:-1]) +
"/",
image=self.image,
role=sagemaker.get_execution_role(),
# sagemaker_session=self.session,
predictor_cls=sagemaker.predictor.RealTimePredictor,
)
for path in self.modelpath:
self.add_model(path, "serving/")
self.ei = False
def test_deploy_multi_data_model(sagemaker_session):
model = MultiDataModel(
name=MODEL_NAME,
model_data_prefix=VALID_MULTI_MODEL_DATA_PREFIX,
image=IMAGE,
role=ROLE,
sagemaker_session=sagemaker_session,
env={"EXTRA_ENV_MOCK": "MockValue"},
)
model.deploy(
initial_instance_count=INSTANCE_COUNT,
instance_type=INSTANCE_TYPE,
endpoint_name=MULTI_MODEL_ENDPOINT_NAME,
)
sagemaker_session.create_model.assert_called_with(
MODEL_NAME,
ROLE,
model.prepare_container_def(INSTANCE_TYPE),
vpc_config=None,
enable_network_isolation=False,
tags=None,
)
sagemaker_session.endpoint_from_production_variants.assert_called_with(
name=MULTI_MODEL_ENDPOINT_NAME,
wait=True,
tags=None,
kms_key=None,
data_capture_config_dict=None,
production_variants=EXPECTED_PROD_VARIANT,
)
def multi_data_model(sagemaker_session):
return MultiDataModel(
name=MODEL_NAME,
model_data_prefix=VALID_MULTI_MODEL_DATA_PREFIX,
image=IMAGE,
role=ROLE,
sagemaker_session=sagemaker_session,
)
def test_multi_data_model_deploy_pretrained_models_local_mode(container_image, sagemaker_session):
timestamp = sagemaker_timestamp()
endpoint_name = "test-multimodel-endpoint-{}".format(timestamp)
model_name = "test-multimodel-{}".format(timestamp)
# Define pretrained model local path
pretrained_model_data_local_path = os.path.join(DATA_DIR, "sparkml_model", "mleap_model.tar.gz")
with timeout(minutes=30):
model_data_prefix = os.path.join(
"s3://", sagemaker_session.default_bucket(), "multimodel-{}/".format(timestamp)
)
multi_data_model = MultiDataModel(
name=model_name,
model_data_prefix=model_data_prefix,
image_uri=container_image,
role=ROLE,
sagemaker_session=sagemaker_session,
)
# Add model before deploy
multi_data_model.add_model(pretrained_model_data_local_path, PRETRAINED_MODEL_PATH_1)
# Deploy model to an endpoint
multi_data_model.deploy(1, "local", endpoint_name=endpoint_name)
# Add models after deploy
multi_data_model.add_model(pretrained_model_data_local_path, PRETRAINED_MODEL_PATH_2)
endpoint_models = []
for model_path in multi_data_model.list_models():
endpoint_models.append(model_path)
assert PRETRAINED_MODEL_PATH_1 in endpoint_models
assert PRETRAINED_MODEL_PATH_2 in endpoint_models
predictor = Predictor(
endpoint_name=endpoint_name,
sagemaker_session=multi_data_model.sagemaker_session,
serializer=NumpySerializer(),
deserializer=string_deserializer,
)
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_1)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_1)
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_2)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_2)
# Cleanup
multi_data_model.sagemaker_session.sagemaker_client.delete_endpoint_config(
EndpointConfigName=endpoint_name
)
multi_data_model.sagemaker_session.delete_endpoint(endpoint_name)
multi_data_model.delete_model()
with pytest.raises(Exception) as exception:
sagemaker_session.sagemaker_client.describe_model(ModelName=multi_data_model.name)
assert "Could not find model" in str(exception.value)
sagemaker_session.sagemaker_client.describe_endpoint_config(name=endpoint_name)
assert "Could not find endpoint" in str(exception.value)
def test_multi_data_model_create_with_model_arg_only(mxnet_model):
model = MultiDataModel(
name=MODEL_NAME, model_data_prefix=VALID_MULTI_MODEL_DATA_PREFIX, model=mxnet_model
)
assert model.model_data_prefix == VALID_MULTI_MODEL_DATA_PREFIX
assert model.model == mxnet_model
assert hasattr(model, "role") is False
assert hasattr(model, "image") is False
def test_multi_data_model_create_with_invalid_model_data_prefix():
invalid_model_data_prefix = "https://mybucket/path/"
with pytest.raises(ValueError) as ex:
MultiDataModel(
name=MODEL_NAME, model_data_prefix=invalid_model_data_prefix, image=IMAGE, role=ROLE
)
err_msg = 'ValueError: Expecting S3 model prefix beginning with "s3://". Received: "{}"'.format(
invalid_model_data_prefix
)
assert err_msg in str(ex)
def test_multi_data_model_create_with_invalid_arguments(sagemaker_session, mxnet_model):
with pytest.raises(ValueError) as ex:
MultiDataModel(
name=MODEL_NAME,
model_data_prefix=VALID_MULTI_MODEL_DATA_PREFIX,
image=IMAGE,
role=ROLE,
sagemaker_session=sagemaker_session,
model=mxnet_model,
)
assert (
"Parameters image, role or kwargs are not permitted when model parameter is passed."
in str(ex)
)
def test_multi_data_model_create(sagemaker_session):
model = MultiDataModel(
name=MODEL_NAME,
model_data_prefix=VALID_MULTI_MODEL_DATA_PREFIX,
image=IMAGE,
role=ROLE,
sagemaker_session=sagemaker_session,
)
assert model.sagemaker_session == sagemaker_session
assert model.name == MODEL_NAME
assert model.model_data_prefix == VALID_MULTI_MODEL_DATA_PREFIX
assert model.role == ROLE
assert model.image == IMAGE
assert model.vpc_config is None
def test_deploy_model_update(sagemaker_session):
model = MultiDataModel(
name=MODEL_NAME,
model_data_prefix=VALID_MULTI_MODEL_DATA_PREFIX,
image=IMAGE,
role=ROLE,
sagemaker_session=sagemaker_session,
)
model.deploy(
initial_instance_count=INSTANCE_COUNT,
instance_type=INSTANCE_TYPE,
endpoint_name=MULTI_MODEL_ENDPOINT_NAME,
update_endpoint=True,
)
sagemaker_session.create_model.assert_called()
sagemaker_session.create_endpoint_config.assert_called_with(
name=model.name,
model_name=model.name,
initial_instance_count=INSTANCE_COUNT,
instance_type=INSTANCE_TYPE,
accelerator_type=None,
tags=None,
kms_key=None,
data_capture_config_dict=None,
)
config_name = sagemaker_session.create_endpoint_config(
name=model.name,
model_name=model.name,
initial_instance_count=INSTANCE_COUNT,
instance_type=INSTANCE_TYPE,
accelerator_type=None,
)
sagemaker_session.update_endpoint.assert_called_with(
MULTI_MODEL_ENDPOINT_NAME, config_name, wait=True
)
sagemaker_session.create_endpoint.assert_not_called()
def test_prepare_container_def_mxnet(sagemaker_session, mxnet_model):
expected_container_env_keys = [
"SAGEMAKER_CONTAINER_LOG_LEVEL",
"SAGEMAKER_ENABLE_CLOUDWATCH_METRICS",
"SAGEMAKER_PROGRAM",
"SAGEMAKER_REGION",
"SAGEMAKER_SUBMIT_DIRECTORY",
]
model = MultiDataModel(
name=MODEL_NAME,
model_data_prefix=VALID_MULTI_MODEL_DATA_PREFIX,
sagemaker_session=sagemaker_session,
model=mxnet_model,
)
container_def = model.prepare_container_def(INSTANCE_TYPE)
assert container_def["Image"] == MXNET_IMAGE
assert container_def["ModelDataUrl"] == VALID_MULTI_MODEL_DATA_PREFIX
assert container_def["Mode"] == MULTI_MODEL_CONTAINER_MODE
# Check if the environment variables defined only for MXNetModel
# are part of the MultiDataModel container definition
assert set(container_def["Environment"].keys()) == set(expected_container_env_keys)
def test_multi_data_model_deploy_pretrained_models_update_endpoint(
container_image, sagemaker_session, cpu_instance_type, alternative_cpu_instance_type
):
timestamp = sagemaker_timestamp()
endpoint_name = "test-multimodel-endpoint-{}".format(timestamp)
model_name = "test-multimodel-{}".format(timestamp)
# Define pretrained model local path
pretrained_model_data_local_path = os.path.join(DATA_DIR, "sparkml_model", "mleap_model.tar.gz")
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model_data_prefix = os.path.join(
"s3://", sagemaker_session.default_bucket(), "multimodel-{}/".format(timestamp)
)
multi_data_model = MultiDataModel(
name=model_name,
model_data_prefix=model_data_prefix,
image_uri=container_image,
role=ROLE,
sagemaker_session=sagemaker_session,
)
# Add model before deploy
multi_data_model.add_model(pretrained_model_data_local_path, PRETRAINED_MODEL_PATH_1)
# Deploy model to an endpoint
multi_data_model.deploy(1, cpu_instance_type, endpoint_name=endpoint_name)
# Add model after deploy
multi_data_model.add_model(pretrained_model_data_local_path, PRETRAINED_MODEL_PATH_2)
# List model assertions
endpoint_models = []
for model_path in multi_data_model.list_models():
endpoint_models.append(model_path)
assert PRETRAINED_MODEL_PATH_1 in endpoint_models
assert PRETRAINED_MODEL_PATH_2 in endpoint_models
predictor = Predictor(
endpoint_name=endpoint_name,
sagemaker_session=sagemaker_session,
serializer=NumpySerializer(),
deserializer=string_deserializer,
)
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_1)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_1)
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_2)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_2)
endpoint_desc = sagemaker_session.sagemaker_client.describe_endpoint(
EndpointName=endpoint_name
)
old_config_name = endpoint_desc["EndpointConfigName"]
# Update endpoint
predictor.update_endpoint(
initial_instance_count=1, instance_type=alternative_cpu_instance_type
)
endpoint_desc = sagemaker_session.sagemaker_client.describe_endpoint(
EndpointName=endpoint_name
)
new_config_name = endpoint_desc["EndpointConfigName"]
new_config = sagemaker_session.sagemaker_client.describe_endpoint_config(
EndpointConfigName=new_config_name
)
assert old_config_name != new_config_name
assert new_config["ProductionVariants"][0]["InstanceType"] == alternative_cpu_instance_type
assert new_config["ProductionVariants"][0]["InitialInstanceCount"] == 1
# Cleanup
sagemaker_session.sagemaker_client.delete_endpoint_config(
EndpointConfigName=old_config_name
)
sagemaker_session.sagemaker_client.delete_endpoint_config(
EndpointConfigName=new_config_name
)
multi_data_model.delete_model()
with pytest.raises(Exception) as exception:
sagemaker_session.sagemaker_client.describe_model(ModelName=model_name)
assert "Could not find model" in str(exception.value)
sagemaker_session.sagemaker_client.describe_endpoint_config(name=old_config_name)
assert "Could not find endpoint" in str(exception.value)
sagemaker_session.sagemaker_client.describe_endpoint_config(name=new_config_name)
assert "Could not find endpoint" in str(exception.value)
def test_multi_data_model_deploy_train_model_from_amazon_first_party_estimator(
container_image, sagemaker_session, cpu_instance_type
):
timestamp = sagemaker_timestamp()
endpoint_name = "test-multimodel-endpoint-{}".format(timestamp)
model_name = "test-multimodel-{}".format(timestamp)
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
rcf_model_v1 = __rcf_training_job(
sagemaker_session, container_image, cpu_instance_type, 50, 20
)
model_data_prefix = os.path.join(
"s3://", sagemaker_session.default_bucket(), "multimodel-{}/".format(timestamp)
)
multi_data_model = MultiDataModel(
name=model_name,
model_data_prefix=model_data_prefix,
model=rcf_model_v1,
sagemaker_session=sagemaker_session,
)
# Add model before deploy
multi_data_model.add_model(rcf_model_v1.model_data, PRETRAINED_MODEL_PATH_1)
# Deploy model to an endpoint
multi_data_model.deploy(1, cpu_instance_type, endpoint_name=endpoint_name)
# Train another model
rcf_model_v2 = __rcf_training_job(
sagemaker_session, container_image, cpu_instance_type, 70, 20
)
# Deploy newly trained model
multi_data_model.add_model(rcf_model_v2.model_data, PRETRAINED_MODEL_PATH_2)
# List model assertions
endpoint_models = []
for model_path in multi_data_model.list_models():
endpoint_models.append(model_path)
assert PRETRAINED_MODEL_PATH_1 in endpoint_models
assert PRETRAINED_MODEL_PATH_2 in endpoint_models
# Define a predictor to set `serializer` parameter with `NumpySerializer`
# instead of `JSONSerializer` in the default predictor returned by `MXNetPredictor`
# Since we are using a placeholder container image the prediction results are not accurate.
predictor = Predictor(
endpoint_name=endpoint_name,
sagemaker_session=sagemaker_session,
serializer=NumpySerializer(),
deserializer=string_deserializer,
)
data = numpy.random.rand(1, 14)
# Prediction result for the first model
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_1)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_1)
# Prediction result for the second model
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_2)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_2)
# Cleanup
sagemaker_session.sagemaker_client.delete_endpoint_config(EndpointConfigName=endpoint_name)
multi_data_model.delete_model()
with pytest.raises(Exception) as exception:
sagemaker_session.sagemaker_client.describe_model(ModelName=model_name)
assert "Could not find model" in str(exception.value)
sagemaker_session.sagemaker_client.describe_endpoint_config(name=endpoint_name)
assert "Could not find endpoint" in str(exception.value)
def test_multi_data_model_deploy_trained_model_from_framework_estimator(
container_image, sagemaker_session, cpu_instance_type):
timestamp = sagemaker_timestamp()
endpoint_name = "test-multimodel-endpoint-{}".format(timestamp)
model_name = "test-multimodel-{}".format(timestamp)
mxnet_version = "1.4.1"
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
mxnet_model_1 = __mxnet_training_job(sagemaker_session,
container_image, mxnet_version,
cpu_instance_type, 0.1)
model_data_prefix = os.path.join("s3://",
sagemaker_session.default_bucket(),
"multimodel-{}/".format(timestamp))
multi_data_model = MultiDataModel(
name=model_name,
model_data_prefix=model_data_prefix,
model=mxnet_model_1,
sagemaker_session=sagemaker_session,
)
# Add model before deploy
multi_data_model.add_model(mxnet_model_1.model_data,
PRETRAINED_MODEL_PATH_1)
# Deploy model to an endpoint
multi_data_model.deploy(1,
cpu_instance_type,
endpoint_name=endpoint_name)
# Train another model
mxnet_model_2 = __mxnet_training_job(sagemaker_session,
container_image, mxnet_version,
cpu_instance_type, 0.01)
# Deploy newly trained model
multi_data_model.add_model(mxnet_model_2.model_data,
PRETRAINED_MODEL_PATH_2)
endpoint_models = []
for model_path in multi_data_model.list_models():
endpoint_models.append(model_path)
assert PRETRAINED_MODEL_PATH_1 in endpoint_models
assert PRETRAINED_MODEL_PATH_2 in endpoint_models
# Define a predictor to set `serializer` parameter with npy_serializer
# instead of `json_serializer` in the default predictor returned by `MXNetPredictor`
# Since we are using a placeholder container image the prediction results are not accurate.
predictor = RealTimePredictor(
endpoint=endpoint_name,
sagemaker_session=sagemaker_session,
serializer=npy_serializer,
deserializer=string_deserializer,
)
data = numpy.zeros(shape=(1, 1, 28, 28))
# Prediction result for the first model
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_1)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_1)
# Prediction result for the second model
result = predictor.predict(data, target_model=PRETRAINED_MODEL_PATH_2)
assert result == "Invoked model: {}".format(PRETRAINED_MODEL_PATH_2)
# Cleanup
sagemaker_session.sagemaker_client.delete_endpoint_config(
EndpointConfigName=endpoint_name)
multi_data_model.delete_model()
with pytest.raises(Exception) as exception:
sagemaker_session.sagemaker_client.describe_model(ModelName=model_name)
assert "Could not find model" in str(exception.value)
sagemaker_session.sagemaker_client.describe_endpoint_config(
name=endpoint_name)
assert "Could not find endpoint" in str(exception.value)