def test_deploy_replace_in_asynchronous_mode_returns_before_endpoint_creation_completes(
pretrained_model, sagemaker_client):
endpoint_update_latency = 10
get_sagemaker_backend(
sagemaker_client.meta.region_name).set_endpoint_update_latency(
endpoint_update_latency)
app_name = "test-app"
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE,
synchronous=True)
update_start_time = time.time()
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_REPLACE,
synchronous=False,
archive=True)
update_end_time = time.time()
assert (update_end_time - update_start_time) < endpoint_update_latency
endpoint_description = sagemaker_client.describe_endpoint(
EndpointName=app_name)
assert endpoint_description["EndpointStatus"] == Endpoint.STATUS_UPDATING
def test_deploy_in_create_mode_throws_exception_after_endpoint_creation_fails(
pretrained_model, sagemaker_client):
endpoint_creation_latency = 10
sagemaker_backend = get_sagemaker_backend(
sagemaker_client.meta.region_name)
sagemaker_backend.set_endpoint_update_latency(endpoint_creation_latency)
boto_caller = botocore.client.BaseClient._make_api_call
def fail_endpoint_creations(self, operation_name, operation_kwargs):
"""
Processes all boto3 client operations according to the following rules:
- If the operation is an endpoint creation, create the endpoint and set its status to
``Endpoint.STATUS_FAILED``.
- Else, execute the client operation as normal
"""
result = boto_caller(self, operation_name, operation_kwargs)
if operation_name == "CreateEndpoint":
endpoint_name = operation_kwargs["EndpointName"]
sagemaker_backend.set_endpoint_latest_operation(
endpoint_name=endpoint_name,
operation=EndpointOperation.create_unsuccessful(
latency_seconds=endpoint_creation_latency))
return result
with mock.patch("botocore.client.BaseClient._make_api_call", new=fail_endpoint_creations),\
pytest.raises(MlflowException) as exc:
mfs.deploy(app_name="test-app",
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE)
assert "deployment operation failed" in exc.value.message
assert exc.value.error_code == ErrorCode.Name(INTERNAL_ERROR)
def test_deploy_creates_sagemaker_and_s3_resources_with_expected_names_from_local(
pretrained_model, sagemaker_client):
app_name = "test-app"
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE)
region_name = sagemaker_client.meta.region_name
s3_client = boto3.client("s3", region_name=region_name)
default_bucket = mfs._get_default_s3_bucket(region_name)
endpoint_description = sagemaker_client.describe_endpoint(
EndpointName=app_name)
endpoint_production_variants = endpoint_description["ProductionVariants"]
assert len(endpoint_production_variants) == 1
model_name = endpoint_production_variants[0]["VariantName"]
assert model_name in [
model["ModelName"]
for model in sagemaker_client.list_models()["Models"]
]
object_names = [
entry["Key"]
for entry in s3_client.list_objects(Bucket=default_bucket)["Contents"]
]
assert any([model_name in object_name for object_name in object_names])
assert any([
app_name in config["EndpointConfigName"] for config in
sagemaker_client.list_endpoint_configs()["EndpointConfigs"]
])
assert app_name in [
endpoint["EndpointName"]
for endpoint in sagemaker_client.list_endpoints()["Endpoints"]
]
def test_deployment_with_unsupported_flavor_raises_exception(pretrained_model):
unsupported_flavor = "this is not a valid flavor"
with pytest.raises(MlflowException) as exc:
mfs.deploy(app_name="bad_flavor",
model_uri=pretrained_model.model_uri,
flavor=unsupported_flavor)
assert exc.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_deployment_with_missing_flavor_raises_exception(pretrained_model):
missing_flavor = "mleap"
with pytest.raises(MlflowException) as exc:
mfs.deploy(app_name="missing-flavor",
model_uri=pretrained_model.model_uri,
flavor=missing_flavor)
assert exc.value.error_code == ErrorCode.Name(RESOURCE_DOES_NOT_EXIST)
def test_attempting_to_deploy_in_asynchronous_mode_without_archiving_throws_exception(
pretrained_model):
with pytest.raises(MlflowException) as exc:
mfs.deploy(app_name="test-app",
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE,
archive=False,
synchronous=False)
assert "Resources must be archived" in exc.value.message
assert exc.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_deploying_application_with_preexisting_name_in_create_mode_throws_exception(
pretrained_model):
app_name = "test-app"
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE)
with pytest.raises(MlflowException) as exc:
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE)
assert "an application with the same name already exists" in exc.value.message
assert exc.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_deployment_of_model_with_no_supported_flavors_raises_exception(
pretrained_model):
logged_model_path = _download_artifact_from_uri(pretrained_model.model_uri)
model_config_path = os.path.join(logged_model_path, "MLmodel")
model_config = Model.load(model_config_path)
del model_config.flavors[kiwi.pyfunc.FLAVOR_NAME]
model_config.save(path=model_config_path)
with pytest.raises(MlflowException) as exc:
mfs.deploy(app_name="missing-flavor",
model_uri=logged_model_path,
flavor=None)
assert exc.value.error_code == ErrorCode.Name(RESOURCE_DOES_NOT_EXIST)
def test_deploy_in_replace_mode_waits_for_endpoint_update_completion_before_deleting_resources(
pretrained_model, sagemaker_client):
endpoint_update_latency = 10
sagemaker_backend = get_sagemaker_backend(
sagemaker_client.meta.region_name)
sagemaker_backend.set_endpoint_update_latency(endpoint_update_latency)
app_name = "test-app"
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE)
endpoint_config_name_before_replacement = sagemaker_client.describe_endpoint(
EndpointName=app_name)["EndpointConfigName"]
boto_caller = botocore.client.BaseClient._make_api_call
update_start_time = time.time()
def validate_deletes(self, operation_name, operation_kwargs):
"""
Processes all boto3 client operations according to the following rules:
- If the operation deletes an S3 or SageMaker resource, ensure that the deletion was
initiated after the completion of the endpoint update
- Else, execute the client operation as normal
"""
result = boto_caller(self, operation_name, operation_kwargs)
if "Delete" in operation_name:
# Confirm that a successful endpoint update occurred prior to the invocation of this
# delete operation
endpoint_info = sagemaker_client.describe_endpoint(
EndpointName=app_name)
assert endpoint_info[
"EndpointStatus"] == Endpoint.STATUS_IN_SERVICE
assert endpoint_info[
"EndpointConfigName"] != endpoint_config_name_before_replacement
assert time.time() - update_start_time >= endpoint_update_latency
return result
with mock.patch("botocore.client.BaseClient._make_api_call",
new=validate_deletes):
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_REPLACE,
archive=False)
def test_deploy_in_synchronous_mode_waits_for_endpoint_creation_to_complete_before_returning(
pretrained_model, sagemaker_client):
endpoint_creation_latency = 10
get_sagemaker_backend(
sagemaker_client.meta.region_name).set_endpoint_update_latency(
endpoint_creation_latency)
app_name = "test-app"
deployment_start_time = time.time()
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE,
synchronous=True)
deployment_end_time = time.time()
assert (deployment_end_time -
deployment_start_time) >= endpoint_creation_latency
endpoint_description = sagemaker_client.describe_endpoint(
EndpointName=app_name)
assert endpoint_description["EndpointStatus"] == Endpoint.STATUS_IN_SERVICE
def test_deploy_creates_sagemaker_and_s3_resources_with_expected_names_from_s3(
pretrained_model, sagemaker_client):
local_model_path = _download_artifact_from_uri(pretrained_model.model_uri)
artifact_path = "model"
region_name = sagemaker_client.meta.region_name
default_bucket = mfs._get_default_s3_bucket(region_name)
s3_artifact_repo = S3ArtifactRepository('s3://{}'.format(default_bucket))
s3_artifact_repo.log_artifacts(local_model_path,
artifact_path=artifact_path)
model_s3_uri = 's3://{bucket_name}/{artifact_path}'.format(
bucket_name=default_bucket, artifact_path=pretrained_model.model_path)
app_name = "test-app"
mfs.deploy(app_name=app_name,
model_uri=model_s3_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE)
endpoint_description = sagemaker_client.describe_endpoint(
EndpointName=app_name)
endpoint_production_variants = endpoint_description["ProductionVariants"]
assert len(endpoint_production_variants) == 1
model_name = endpoint_production_variants[0]["VariantName"]
assert model_name in [
model["ModelName"]
for model in sagemaker_client.list_models()["Models"]
]
s3_client = boto3.client("s3", region_name=region_name)
object_names = [
entry["Key"]
for entry in s3_client.list_objects(Bucket=default_bucket)["Contents"]
]
assert any([model_name in object_name for object_name in object_names])
assert any([
app_name in config["EndpointConfigName"] for config in
sagemaker_client.list_endpoint_configs()["EndpointConfigs"]
])
assert app_name in [
endpoint["EndpointName"]
for endpoint in sagemaker_client.list_endpoints()["Endpoints"]
]
def test_deploy_in_add_mode_adds_new_model_to_existing_endpoint(
pretrained_model, sagemaker_client):
app_name = "test-app"
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE)
models_added = 1
for _ in range(11):
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_ADD,
archive=True,
synchronous=False)
models_added += 1
endpoint_response = sagemaker_client.describe_endpoint(
EndpointName=app_name)
endpoint_config_name = endpoint_response["EndpointConfigName"]
endpoint_config_response = sagemaker_client.describe_endpoint_config(
EndpointConfigName=endpoint_config_name)
production_variants = endpoint_config_response["ProductionVariants"]
assert len(production_variants) == models_added
def test_deploy_in_replace_model_removes_preexisting_models_from_endpoint(
pretrained_model, sagemaker_client):
app_name = "test-app"
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_ADD)
for _ in range(11):
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_ADD,
archive=True,
synchronous=False)
endpoint_response_before_replacement = sagemaker_client.describe_endpoint(
EndpointName=app_name)
endpoint_config_name_before_replacement =\
endpoint_response_before_replacement["EndpointConfigName"]
endpoint_config_response_before_replacement = sagemaker_client.describe_endpoint_config(
EndpointConfigName=endpoint_config_name_before_replacement)
production_variants_before_replacement =\
endpoint_config_response_before_replacement["ProductionVariants"]
deployed_models_before_replacement = [
variant["ModelName"]
for variant in production_variants_before_replacement
]
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_REPLACE,
archive=True,
synchronous=False)
endpoint_response_after_replacement = sagemaker_client.describe_endpoint(
EndpointName=app_name)
endpoint_config_name_after_replacement =\
endpoint_response_after_replacement["EndpointConfigName"]
endpoint_config_response_after_replacement = sagemaker_client.describe_endpoint_config(
EndpointConfigName=endpoint_config_name_after_replacement)
production_variants_after_replacement =\
endpoint_config_response_after_replacement["ProductionVariants"]
deployed_models_after_replacement = [
variant["ModelName"]
for variant in production_variants_after_replacement
]
assert len(deployed_models_after_replacement) == 1
assert all([
model_name not in deployed_models_after_replacement
for model_name in deployed_models_before_replacement
])
def test_deploy_in_replace_mode_with_archiving_does_not_delete_resources(
pretrained_model, sagemaker_client):
region_name = sagemaker_client.meta.region_name
sagemaker_backend = get_sagemaker_backend(region_name)
sagemaker_backend.set_endpoint_update_latency(5)
app_name = "test-app"
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE)
s3_client = boto3.client("s3", region_name=region_name)
default_bucket = mfs._get_default_s3_bucket(region_name)
object_names_before_replacement = [
entry["Key"]
for entry in s3_client.list_objects(Bucket=default_bucket)["Contents"]
]
endpoint_configs_before_replacement = [
config["EndpointConfigName"] for config in
sagemaker_client.list_endpoint_configs()["EndpointConfigs"]
]
models_before_replacement = [
model["ModelName"]
for model in sagemaker_client.list_models()["Models"]
]
model_uri = "runs:/{run_id}/{artifact_path}".format(
run_id=pretrained_model.run_id,
artifact_path=pretrained_model.model_path)
sk_model = kiwi.sklearn.load_model(model_uri=model_uri)
new_artifact_path = "model"
with kiwi.start_run():
kiwi.sklearn.log_model(sk_model=sk_model,
artifact_path=new_artifact_path)
new_model_uri = "runs:/{run_id}/{artifact_path}".format(
run_id=kiwi.active_run().info.run_id,
artifact_path=new_artifact_path)
mfs.deploy(app_name=app_name,
model_uri=new_model_uri,
mode=mfs.DEPLOYMENT_MODE_REPLACE,
archive=True,
synchronous=True)
object_names_after_replacement = [
entry["Key"]
for entry in s3_client.list_objects(Bucket=default_bucket)["Contents"]
]
endpoint_configs_after_replacement = [
config["EndpointConfigName"] for config in
sagemaker_client.list_endpoint_configs()["EndpointConfigs"]
]
models_after_replacement = [
model["ModelName"]
for model in sagemaker_client.list_models()["Models"]
]
assert all([
object_name in object_names_after_replacement
for object_name in object_names_before_replacement
])
assert all([
endpoint_config in endpoint_configs_after_replacement
for endpoint_config in endpoint_configs_before_replacement
])
assert all([
model in models_after_replacement
for model in models_before_replacement
])
