def test_deploy_in_create_mode_throws_exception_after_endpoint_creation_fails(
pretrained_model, sagemaker_client):
endpoint_creation_latency = 10
sagemaker_backend = get_sagemaker_backend(
sagemaker_client.meta.region_name)
sagemaker_backend.set_endpoint_update_latency(endpoint_creation_latency)
boto_caller = botocore.client.BaseClient._make_api_call
def fail_endpoint_creations(self, operation_name, operation_kwargs):
"""
Processes all boto3 client operations according to the following rules:
- If the operation is an endpoint creation, create the endpoint and set its status to
``Endpoint.STATUS_FAILED``.
- Else, execute the client operation as normal
"""
result = boto_caller(self, operation_name, operation_kwargs)
if operation_name == "CreateEndpoint":
endpoint_name = operation_kwargs["EndpointName"]
sagemaker_backend.set_endpoint_latest_operation(
endpoint_name=endpoint_name,
operation=EndpointOperation.create_unsuccessful(
latency_seconds=endpoint_creation_latency))
return result
with mock.patch("botocore.client.BaseClient._make_api_call", new=fail_endpoint_creations),\
pytest.raises(MlflowException) as exc:
mfs.deploy(app_name="test-app",
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE)
assert "deployment operation failed" in exc.value.message
assert exc.value.error_code == ErrorCode.Name(INTERNAL_ERROR)
def test_log_batch_validates_entity_names_and_values():
bad_kwargs = {
"metrics": [
[Metric(key="../bad/metric/name", value=0.3, timestamp=3, step=0)],
[
Metric(key="ok-name",
value="non-numerical-value",
timestamp=3,
step=0)
],
[
Metric(key="ok-name",
value=0.3,
timestamp="non-numerical-timestamp",
step=0)
],
],
"params": [[Param(key="../bad/param/name", value="my-val")]],
"tags": [[Param(key="../bad/tag/name", value="my-val")]],
}
with start_run() as active_run:
for kwarg, bad_values in bad_kwargs.items():
for bad_kwarg_value in bad_values:
final_kwargs = {
"run_id": active_run.info.run_id,
"metrics": [],
"params": [],
"tags": [],
}
final_kwargs[kwarg] = bad_kwarg_value
with pytest.raises(MlflowException) as e:
tracking.MlflowClient().log_batch(**final_kwargs)
assert e.value.error_code == ErrorCode.Name(
INVALID_PARAMETER_VALUE)
def __init__(self, json):
error_code = json.get('error_code', ErrorCode.Name(INTERNAL_ERROR))
message = "%s: %s" % (error_code, json['message'] if 'message' in json
else "Response: " + str(json))
super(RestException,
self).__init__(message, error_code=ErrorCode.Value(error_code))
self.json = json
def _validate_with_rest_endpoint(scoring_proc, host_port, df, x, sk_model):
with RestEndpoint(proc=scoring_proc, port=host_port) as endpoint:
for content_type in [
CONTENT_TYPE_JSON_SPLIT_ORIENTED, CONTENT_TYPE_CSV,
CONTENT_TYPE_JSON
]:
scoring_response = endpoint.invoke(df, content_type)
assert scoring_response.status_code == 200, "Failed to serve prediction, got " \
"response %s" % scoring_response.text
np.testing.assert_array_equal(
np.array(json.loads(scoring_response.text)),
sk_model.predict(x))
# Try examples of bad input, verify we get a non-200 status code
for content_type in [
CONTENT_TYPE_JSON_SPLIT_ORIENTED, CONTENT_TYPE_CSV,
CONTENT_TYPE_JSON
]:
scoring_response = endpoint.invoke(data="",
content_type=content_type)
assert scoring_response.status_code == 500, \
"Expected server failure with error code 500, got response with status code %s " \
"and body %s" % (scoring_response.status_code, scoring_response.text)
scoring_response_dict = json.loads(scoring_response.content)
assert "error_code" in scoring_response_dict
assert scoring_response_dict["error_code"] == ErrorCode.Name(
MALFORMED_REQUEST)
assert "message" in scoring_response_dict
assert "stack_trace" in scoring_response_dict
def test_validate_tag_name():
for good_name in GOOD_METRIC_OR_PARAM_NAMES:
_validate_tag_name(good_name)
for bad_name in BAD_METRIC_OR_PARAM_NAMES:
with pytest.raises(MlflowException, match="Invalid tag name") as e:
_validate_tag_name(bad_name)
assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_deployment_with_missing_flavor_raises_exception(pretrained_model):
missing_flavor = "mleap"
with pytest.raises(MlflowException) as exc:
mfs.deploy(app_name="missing-flavor",
model_uri=pretrained_model.model_uri,
flavor=missing_flavor)
assert exc.value.error_code == ErrorCode.Name(RESOURCE_DOES_NOT_EXIST)
def test_deployment_with_unsupported_flavor_raises_exception(pretrained_model):
unsupported_flavor = "this is not a valid flavor"
with pytest.raises(MlflowException) as exc:
mfs.deploy(app_name="bad_flavor",
model_uri=pretrained_model.model_uri,
flavor=unsupported_flavor)
assert exc.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def __init__(self, message, error_code=INTERNAL_ERROR, **kwargs):
"""
:param message: The message describing the error that occured. This will be included in the
exception's serialized JSON representation.
:param error_code: An appropriate error code for the error that occured; it will be included
in the exception's serialized JSON representation. This should be one of
the codes listed in the `mlflow.protos.databricks_pb2` proto.
:param kwargs: Additional key-value pairs to include in the serialized JSON representation
of the MlflowException.
"""
try:
self.error_code = ErrorCode.Name(error_code)
except (ValueError, TypeError):
self.error_code = ErrorCode.Name(INTERNAL_ERROR)
self.message = message
self.json_kwargs = kwargs
super(MlflowException, self).__init__(message)
def test_log_metric_validation():
with start_run() as active_run:
run_id = active_run.info.run_id
with pytest.raises(MlflowException) as e:
kiwi.log_metric("name_1", "apple")
assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
finished_run = tracking.MlflowClient().get_run(run_id)
assert len(finished_run.data.metrics) == 0
def test_log_batch_api_req(mock_get_request_json):
mock_get_request_json.return_value = "a" * (MAX_BATCH_LOG_REQUEST_SIZE + 1)
response = _log_batch()
assert response.status_code == 400
json_response = json.loads(response.get_data())
assert json_response["error_code"] == ErrorCode.Name(
INVALID_PARAMETER_VALUE)
assert ("Batched logging API requests must be at most %s bytes" %
MAX_BATCH_LOG_REQUEST_SIZE in json_response["message"])
def test_catch_mlflow_exception():
@catch_mlflow_exception
def test_handler():
raise MlflowException('test error', error_code=INTERNAL_ERROR)
# pylint: disable=assignment-from-no-return
response = test_handler()
json_response = json.loads(response.get_data())
assert response.status_code == 500
assert json_response['error_code'] == ErrorCode.Name(INTERNAL_ERROR)
assert json_response['message'] == 'test error'
def test_validate_run_id():
for good_id in [
"a" * 32, "f0" * 16, "abcdef0123456789" * 2, "a" * 33, "a" * 31,
"a" * 256, "A" * 32, "g" * 32, "a_" * 32,
"abcdefghijklmnopqrstuvqxyz"
]:
_validate_run_id(good_id)
for bad_id in ["a/bc" * 8, "", "a" * 400, "*" * 5]:
with pytest.raises(MlflowException, match="Invalid run ID") as e:
_validate_run_id(bad_id)
assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_attempting_to_deploy_in_asynchronous_mode_without_archiving_throws_exception(
pretrained_model):
with pytest.raises(MlflowException) as exc:
mfs.deploy(app_name="test-app",
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE,
archive=False,
synchronous=False)
assert "Resources must be archived" in exc.value.message
assert exc.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_run_databricks_throws_exception_when_spec_uses_existing_cluster():
with mock.patch.dict(os.environ, {
'DATABRICKS_HOST': 'test-host',
'DATABRICKS_TOKEN': 'foo'
}):
existing_cluster_spec = {
"existing_cluster_id": "1000-123456-clust1",
}
with pytest.raises(MlflowException) as exc:
run_databricks_project(cluster_spec=existing_cluster_spec)
assert "execution against existing clusters is not currently supported" in exc.value.message
assert exc.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_scoring_server_responds_to_invalid_json_input_with_stacktrace_and_error_code(
sklearn_model, model_path):
kiwi.sklearn.save_model(sk_model=sklearn_model.model, path=model_path)
incorrect_json_content = json.dumps({"not": "a serialized dataframe"})
response = pyfunc_serve_and_score_model(
model_uri=os.path.abspath(model_path),
data=incorrect_json_content,
content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED)
response_json = json.loads(response.content)
assert "error_code" in response_json
assert response_json["error_code"] == ErrorCode.Name(MALFORMED_REQUEST)
assert "message" in response_json
assert "stack_trace" in response_json
def test_scoring_server_responds_to_incompatible_inference_dataframe_with_stacktrace_and_error_code(
sklearn_model, model_path):
kiwi.sklearn.save_model(sk_model=sklearn_model.model, path=model_path)
incompatible_df = pd.DataFrame(np.array(range(10)))
response = pyfunc_serve_and_score_model(
model_uri=os.path.abspath(model_path),
data=incompatible_df,
content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED)
response_json = json.loads(response.content)
assert "error_code" in response_json
assert response_json["error_code"] == ErrorCode.Name(BAD_REQUEST)
assert "message" in response_json
assert "stack_trace" in response_json
def test_scoring_server_responds_to_malformed_json_input_with_stacktrace_and_error_code(
sklearn_model, model_path):
kiwi.sklearn.save_model(sk_model=sklearn_model.model, path=model_path)
malformed_json_content = "this is,,,, not valid json"
response = pyfunc_serve_and_score_model(
model_uri=os.path.abspath(model_path),
data=malformed_json_content,
content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED)
response_json = json.loads(response.content)
assert "error_code" in response_json
assert response_json["error_code"] == ErrorCode.Name(MALFORMED_REQUEST)
assert "message" in response_json
assert "stack_trace" in response_json
def test_deploying_application_with_preexisting_name_in_create_mode_throws_exception(
pretrained_model):
app_name = "test-app"
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE)
with pytest.raises(MlflowException) as exc:
mfs.deploy(app_name=app_name,
model_uri=pretrained_model.model_uri,
mode=mfs.DEPLOYMENT_MODE_CREATE)
assert "an application with the same name already exists" in exc.value.message
assert exc.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_deployment_of_model_with_no_supported_flavors_raises_exception(
pretrained_model):
logged_model_path = _download_artifact_from_uri(pretrained_model.model_uri)
model_config_path = os.path.join(logged_model_path, "MLmodel")
model_config = Model.load(model_config_path)
del model_config.flavors[kiwi.pyfunc.FLAVOR_NAME]
model_config.save(path=model_config_path)
with pytest.raises(MlflowException) as exc:
mfs.deploy(app_name="missing-flavor",
model_uri=logged_model_path,
flavor=None)
assert exc.value.error_code == ErrorCode.Name(RESOURCE_DOES_NOT_EXIST)
def test_scoring_server_responds_to_invalid_csv_input_with_stacktrace_and_error_code(
sklearn_model, model_path):
kiwi.sklearn.save_model(sk_model=sklearn_model.model, path=model_path)
# Any empty string is not valid pandas CSV
incorrect_csv_content = ""
response = pyfunc_serve_and_score_model(
model_uri=os.path.abspath(model_path),
data=incorrect_csv_content,
content_type=pyfunc_scoring_server.CONTENT_TYPE_CSV)
response_json = json.loads(response.content)
assert "error_code" in response_json
assert response_json["error_code"] == ErrorCode.Name(MALFORMED_REQUEST)
assert "message" in response_json
assert "stack_trace" in response_json
def test_register_model_raises_exception_with_unsupported_registry_store():
"""
This test case ensures that the `register_model` operation fails with an informative error
message when the registry store URI refers to a store that does not support Model Registry
features (e.g., FileStore).
"""
with TempDir() as tmp:
old_registry_uri = get_registry_uri()
try:
set_registry_uri(tmp.path())
with pytest.raises(MlflowException) as exc:
register_model(model_uri="runs:/1234/some_model",
name="testmodel")
assert exc.value.error_code == ErrorCode.Name(FEATURE_DISABLED)
finally:
set_registry_uri(old_registry_uri)
def test_scoring_server_responds_to_invalid_pandas_input_format_with_stacktrace_and_error_code(
sklearn_model, model_path):
kiwi.sklearn.save_model(sk_model=sklearn_model.model, path=model_path)
# The pyfunc scoring server expects a serialized Pandas Dataframe in `split` or `records`
# format; passing a serialized Dataframe in `table` format should yield a readable error
pandas_table_content = pd.DataFrame(
sklearn_model.inference_data).to_json(orient="table")
response = pyfunc_serve_and_score_model(
model_uri=os.path.abspath(model_path),
data=pandas_table_content,
content_type=pyfunc_scoring_server.CONTENT_TYPE_JSON_SPLIT_ORIENTED)
response_json = json.loads(response.content)
assert "error_code" in response_json
assert response_json["error_code"] == ErrorCode.Name(MALFORMED_REQUEST)
assert "message" in response_json
assert "stack_trace" in response_json
def test_databricks_rest_store_get_experiment_by_name(self):
creds = MlflowHostCreds('https://hello')
store = DatabricksRestStore(lambda: creds)
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
# Verify that Databricks REST client won't fall back to ListExperiments for 500-level
# errors that are not ENDPOINT_NOT_FOUND
def rate_limit_response_fn(*args, **kwargs):
# pylint: disable=unused-argument
raise MlflowException("Some internal error!", INTERNAL_ERROR)
mock_http.side_effect = rate_limit_response_fn
with pytest.raises(MlflowException) as exc_info:
store.get_experiment_by_name("abc")
assert exc_info.value.error_code == ErrorCode.Name(INTERNAL_ERROR)
assert exc_info.value.message == "Some internal error!"
expected_message0 = GetExperimentByName(experiment_name="abc")
self._verify_requests(mock_http, creds,
"experiments/get-by-name", "GET",
message_to_json(expected_message0))
assert mock_http.call_count == 1
def test_client_registry_operations_raise_exception_with_unsupported_registry_store(
):
"""
This test case ensures that Model Registry operations invoked on the `MlflowClient`
fail with an informative error message when the registry store URI refers to a
store that does not support Model Registry features (e.g., FileStore).
"""
with TempDir() as tmp:
client = MlflowClient(registry_uri=tmp.path())
expected_failure_functions = [
client._get_registry_client,
lambda: client.create_registered_model("test"),
lambda: client.get_registered_model("test"),
lambda: client.create_model_version("test", "source", "run_id"),
lambda: client.get_model_version("test", 1),
]
for func in expected_failure_functions:
with pytest.raises(MlflowException) as exc:
func()
assert exc.value.error_code == ErrorCode.Name(FEATURE_DISABLED)
def register_model(model_uri, name):
"""
Create a new model version in model registry for the model files specified by ``model_uri``.
Note that this method assumes the model registry backend URI is the same as that of the
tracking backend.
:param model_uri: URI referring to the MLmodel directory. Use a ``runs:/`` URI if you want to
record the run ID with the model in model registry. ``models:/`` URIs are
currently not supported.
:param name: Name of the registered model under which to create a new model version. If a
registered model with the given name does not exist, it will be created
automatically.
:return: Single :py:class:`mlflow.entities.model_registry.ModelVersion` object created by
backend.
"""
client = MlflowClient()
try:
create_model_response = client.create_registered_model(name)
eprint("Successfully registered model '%s'." %
create_model_response.name)
except MlflowException as e:
if e.error_code == ErrorCode.Name(RESOURCE_ALREADY_EXISTS):
eprint(
"Registered model '%s' already exists. Creating a new version of this model..."
% name)
else:
raise e
if RunsArtifactRepository.is_runs_uri(model_uri):
source = RunsArtifactRepository.get_underlying_uri(model_uri)
(run_id, _) = RunsArtifactRepository.parse_runs_uri(model_uri)
create_version_response = client.create_model_version(
name, source, run_id)
else:
create_version_response = client.create_model_version(name,
source=model_uri,
run_id=None)
eprint("Created version '{version}' of model '{model_name}'.".format(
version=create_version_response.version,
model_name=create_version_response.name))
return create_version_response
def test_rest_exception_error_code_and_no_message():
exc = RestException({"error_code": ErrorCode.Name(RESOURCE_DOES_NOT_EXIST),
"messages": "something important."})
assert "something important." in str(exc)
assert "RESOURCE_DOES_NOT_EXIST" in str(exc)
json.loads(exc.serialize_as_json())
def test_get_experiment_by_name(self, store_class):
creds = MlflowHostCreds('https://hello')
store = store_class(lambda: creds)
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
response = mock.MagicMock
response.status_code = 200
experiment = Experiment(
experiment_id="123", name="abc", artifact_location="/abc",
lifecycle_stage=LifecycleStage.ACTIVE)
response.text = json.dumps({
"experiment": json.loads(message_to_json(experiment.to_proto()))})
mock_http.return_value = response
result = store.get_experiment_by_name("abc")
expected_message0 = GetExperimentByName(experiment_name="abc")
self._verify_requests(mock_http, creds,
"experiments/get-by-name", "GET",
message_to_json(expected_message0))
assert result.experiment_id == experiment.experiment_id
assert result.name == experiment.name
assert result.artifact_location == experiment.artifact_location
assert result.lifecycle_stage == experiment.lifecycle_stage
# Test GetExperimentByName against nonexistent experiment
mock_http.reset_mock()
nonexistent_exp_response = mock.MagicMock
nonexistent_exp_response.status_code = 404
nonexistent_exp_response.text =\
MlflowException("Exp doesn't exist!", RESOURCE_DOES_NOT_EXIST).serialize_as_json()
mock_http.return_value = nonexistent_exp_response
assert store.get_experiment_by_name("nonexistent-experiment") is None
expected_message1 = GetExperimentByName(experiment_name="nonexistent-experiment")
self._verify_requests(mock_http, creds,
"experiments/get-by-name", "GET",
message_to_json(expected_message1))
assert mock_http.call_count == 1
# Test REST client behavior against a mocked old server, which has handler for
# ListExperiments but not GetExperimentByName
mock_http.reset_mock()
list_exp_response = mock.MagicMock
list_exp_response.text = json.dumps({
"experiments": [json.loads(message_to_json(experiment.to_proto()))]})
list_exp_response.status_code = 200
def response_fn(*args, **kwargs):
# pylint: disable=unused-argument
if kwargs.get('endpoint') == "/api/2.0/mlflow/experiments/get-by-name":
raise MlflowException("GetExperimentByName is not implemented",
ENDPOINT_NOT_FOUND)
else:
return list_exp_response
mock_http.side_effect = response_fn
result = store.get_experiment_by_name("abc")
expected_message2 = ListExperiments(view_type=ViewType.ALL)
self._verify_requests(mock_http, creds,
"experiments/get-by-name", "GET",
message_to_json(expected_message0))
self._verify_requests(mock_http, creds,
"experiments/list", "GET",
message_to_json(expected_message2))
assert result.experiment_id == experiment.experiment_id
assert result.name == experiment.name
assert result.artifact_location == experiment.artifact_location
assert result.lifecycle_stage == experiment.lifecycle_stage
# Verify that REST client won't fall back to ListExperiments for 429 errors (hitting
# rate limits)
mock_http.reset_mock()
def rate_limit_response_fn(*args, **kwargs):
# pylint: disable=unused-argument
raise MlflowException("Hit rate limit on GetExperimentByName",
REQUEST_LIMIT_EXCEEDED)
mock_http.side_effect = rate_limit_response_fn
with pytest.raises(MlflowException) as exc_info:
store.get_experiment_by_name("imspamming")
assert exc_info.value.error_code == ErrorCode.Name(REQUEST_LIMIT_EXCEEDED)
assert mock_http.call_count == 1
import json
from kiwi.protos.databricks_pb2 import INTERNAL_ERROR, TEMPORARILY_UNAVAILABLE, \
ENDPOINT_NOT_FOUND, PERMISSION_DENIED, REQUEST_LIMIT_EXCEEDED, BAD_REQUEST, \
INVALID_PARAMETER_VALUE, RESOURCE_DOES_NOT_EXIST, INVALID_STATE, RESOURCE_ALREADY_EXISTS, \
ErrorCode
ERROR_CODE_TO_HTTP_STATUS = {
ErrorCode.Name(INTERNAL_ERROR): 500,
ErrorCode.Name(INVALID_STATE): 500,
ErrorCode.Name(TEMPORARILY_UNAVAILABLE): 503,
ErrorCode.Name(REQUEST_LIMIT_EXCEEDED): 429,
ErrorCode.Name(ENDPOINT_NOT_FOUND): 404,
ErrorCode.Name(RESOURCE_DOES_NOT_EXIST): 404,
ErrorCode.Name(PERMISSION_DENIED): 403,
ErrorCode.Name(BAD_REQUEST): 400,
ErrorCode.Name(RESOURCE_ALREADY_EXISTS): 400,
ErrorCode.Name(INVALID_PARAMETER_VALUE): 400
}
class MlflowException(Exception):
"""
Generic exception thrown to surface failure information about external-facing operations.
The error message associated with this exception may be exposed to clients in HTTP responses
for debugging purposes. If the error text is sensitive, raise a generic `Exception` object
instead.
"""
def __init__(self, message, error_code=INTERNAL_ERROR, **kwargs):
"""
:param message: The message describing the error that occured. This will be included in the