def test_start_and_end_run():
# Use the start_run() and end_run() APIs without a `with` block, verify they work.
with start_run() as active_run:
mlflow.log_metric("name_1", 25)
finished_run = tracking.MlflowClient().get_run(active_run.info.run_id)
# Validate metrics
assert len(finished_run.data.metrics) == 1
assert finished_run.data.metrics["name_1"] == 25
def test_start_run_context_manager():
with start_run() as first_run:
first_uuid = first_run.info.run_id
# Check that start_run() causes the run information to be persisted in the store
persisted_run = tracking.MlflowClient().get_run(first_uuid)
assert persisted_run is not None
assert persisted_run.info == first_run.info
finished_run = tracking.MlflowClient().get_run(first_uuid)
assert finished_run.info.status == RunStatus.to_string(RunStatus.FINISHED)
# Launch a separate run that fails, verify the run status is FAILED and the run UUID is
# different
with pytest.raises(Exception):
with start_run() as second_run:
second_run_id = second_run.info.run_id
raise Exception("Failing run!")
assert second_run_id != first_uuid
finished_run2 = tracking.MlflowClient().get_run(second_run_id)
assert finished_run2.info.status == RunStatus.to_string(RunStatus.FAILED)
def test_create_experiment_with_duplicate_name(tracking_uri_mock):
name = "popular_name"
exp_id = mlflow.create_experiment(name)
with pytest.raises(MlflowException):
mlflow.create_experiment(name)
tracking.MlflowClient().delete_experiment(exp_id)
with pytest.raises(MlflowException):
mlflow.create_experiment(name)
def test_log_param(tracking_uri_mock):
with start_run() as active_run:
run_id = active_run.info.run_id
mlflow.log_param("name_1", "a")
mlflow.log_param("name_2", "b")
mlflow.log_param("name_1", "c")
mlflow.log_param("nested/nested/name", 5)
finished_run = tracking.MlflowClient().get_run(run_id)
# Validate params
assert finished_run.data.params == {"name_1": "c", "name_2": "b", "nested/nested/name": "5"}
def test_log_batch_duplicate_entries_raises():
with start_run() as active_run:
run_id = active_run.info.run_id
with pytest.raises(
MlflowException, match=r"Duplicate parameter keys have been submitted."
) as e:
tracking.MlflowClient().log_batch(
run_id=run_id, params=[Param("a", "1"), Param("a", "2")]
)
assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_set_experiment_with_deleted_experiment_name(tracking_uri_mock):
name = "dead_exp"
mlflow.set_experiment(name)
with start_run() as run:
exp_id = run.info.experiment_id
tracking.MlflowClient().delete_experiment(exp_id)
with pytest.raises(MlflowException):
mlflow.set_experiment(name)
def dump(run_id, artifact_max_level):
print("Options:")
print(" run_id:",run_id)
print(" artifact_max_level:",artifact_max_level)
client = tracking.MlflowClient()
run = client.get_run(run_id)
dump_run(run)
dump_artifacts(client, run_id,"",INDENT_INC, artifact_max_level)
def test_start_deleted_run():
run_id = None
with mlflow.start_run() as active_run:
run_id = active_run.info.run_uuid
tracking.MlflowClient().delete_run(run_id)
with pytest.raises(MlflowException,
matches='because it is in the deleted state.'):
with mlflow.start_run(run_uuid=run_id):
pass
assert mlflow.active_run() is None
def _print_description_and_log_tags(self):
eprint("=== Launched MLflow run as Databricks job run with ID %s. Getting run status "
"page URL... ===" % self._databricks_run_id)
run_info = self._job_runner.jobs_runs_get(self._databricks_run_id)
jobs_page_url = run_info["run_page_url"]
eprint("=== Check the run's status at %s ===" % jobs_page_url)
host_creds = databricks_utils.get_databricks_host_creds(self._job_runner.databricks_profile)
tracking.MlflowClient().set_tag(self._mlflow_run_id,
MLFLOW_DATABRICKS_RUN_URL, jobs_page_url)
tracking.MlflowClient().set_tag(self._mlflow_run_id,
MLFLOW_DATABRICKS_SHELL_JOB_RUN_ID, self._databricks_run_id)
tracking.MlflowClient().set_tag(self._mlflow_run_id,
MLFLOW_DATABRICKS_WEBAPP_URL, host_creds.host)
job_id = run_info.get('job_id')
# In some releases of Databricks we do not return the job ID. We start including it in DB
# releases 2.80 and above.
if job_id is not None:
tracking.MlflowClient().set_tag(self._mlflow_run_id,
MLFLOW_DATABRICKS_SHELL_JOB_ID, job_id)
def test_log_batch(tracking_uri_mock, tmpdir):
expected_metrics = {"metric-key0": 1.0, "metric-key1": 4.0}
expected_params = {"param-key0": "param-val0", "param-key1": "param-val1"}
exact_expected_tags = {"tag-key0": "tag-val0", "tag-key1": "tag-val1"}
approx_expected_tags = set([MLFLOW_SOURCE_NAME, MLFLOW_SOURCE_TYPE])
t = int(time.time())
sorted_expected_metrics = sorted(expected_metrics.items(),
key=lambda kv: kv[0])
metrics = [
Metric(key=key, value=value, timestamp=t, step=i)
for i, (key, value) in enumerate(sorted_expected_metrics)
]
params = [
Param(key=key, value=value) for key, value in expected_params.items()
]
tags = [
RunTag(key=key, value=value)
for key, value in exact_expected_tags.items()
]
with start_run() as active_run:
run_uuid = active_run.info.run_uuid
mlflow.tracking.MlflowClient().log_batch(run_id=run_uuid,
metrics=metrics,
params=params,
tags=tags)
finished_run = tracking.MlflowClient().get_run(run_uuid)
# Validate metrics
assert len(finished_run.data.metrics) == 2
for key, value in finished_run.data.metrics.items():
assert expected_metrics[key] == value
# TODO: use client get_metric_history API here instead once it exists
fs = FileStore(os.path.join(tmpdir.strpath, "mlruns"))
metric_history0 = fs.get_metric_history(run_uuid, "metric-key0")
assert set([(m.value, m.timestamp, m.step)
for m in metric_history0]) == set([
(1.0, t, 0),
])
metric_history1 = fs.get_metric_history(run_uuid, "metric-key1")
assert set([(m.value, m.timestamp, m.step)
for m in metric_history1]) == set([
(4.0, t, 1),
])
# Validate tags (for automatically-set tags)
assert len(finished_run.data.tags
) == len(exact_expected_tags) + len(approx_expected_tags)
for tag_key, tag_value in finished_run.data.tags.items():
if tag_key in approx_expected_tags:
pass
else:
assert exact_expected_tags[tag_key] == tag_value
# Validate params
assert finished_run.data.params == expected_params
def _build_docker_image(work_dir, project, active_run):
"""
Build a docker image containing the project in `work_dir`, using the base image and tagging the
built image with the project name specified by `project`.
"""
if not project.name:
raise ExecutionException(
"Project name in MLProject must be specified when using docker "
"for image tagging.")
tag_name = "mlflow-{name}-{version}".format(
name=(project.name if project.name else "docker-project"),
version=_get_git_commit(work_dir)[:7],
)
dockerfile = ("FROM {imagename}\n"
"LABEL Name={tag_name}\n"
"COPY {build_context_path}/* /mlflow/projects/code/\n"
"WORKDIR /mlflow/projects/code/\n").format(
imagename=project.docker_env.get('image'),
tag_name=tag_name,
build_context_path=_PROJECT_TAR_ARCHIVE_NAME)
build_ctx_path = _create_docker_build_ctx(work_dir, dockerfile)
with open(build_ctx_path, 'rb') as docker_build_ctx:
_logger.info("=== Building docker image %s ===", tag_name)
client = docker.from_env()
image = client.images.build(tag=tag_name,
forcerm=True,
dockerfile=posixpath.join(
_PROJECT_TAR_ARCHIVE_NAME,
_GENERATED_DOCKERFILE_NAME),
fileobj=docker_build_ctx,
custom_context=True,
encoding="gzip")
try:
os.remove(build_ctx_path)
except Exception: # pylint: disable=broad-except
_logger.info("Temporary docker context file %s was not deleted.",
build_ctx_path)
tracking.MlflowClient().set_tag(active_run.info.run_uuid,
MLFLOW_DOCKER_IMAGE_NAME, tag_name)
tracking.MlflowClient().set_tag(active_run.info.run_uuid,
MLFLOW_DOCKER_IMAGE_ID, image[0].id)
return tag_name
def test_log_batch_validates_entity_names_and_values():
with start_run() as active_run:
run_id = active_run.info.run_id
metrics = [
Metric(key="../bad/metric/name", value=0.3, timestamp=3, step=0)
]
with pytest.raises(MlflowException, match="Invalid metric name") as e:
tracking.MlflowClient().log_batch(run_id, metrics=metrics)
assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
metrics = [
Metric(key="ok-name",
value="non-numerical-value",
timestamp=3,
step=0)
]
with pytest.raises(MlflowException, match="Got invalid value") as e:
tracking.MlflowClient().log_batch(run_id, metrics=metrics)
assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
metrics = [
Metric(key="ok-name",
value=0.3,
timestamp="non-numerical-timestamp",
step=0)
]
with pytest.raises(MlflowException,
match="Got invalid timestamp") as e:
tracking.MlflowClient().log_batch(run_id, metrics=metrics)
assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
params = [Param(key="../bad/param/name", value="my-val")]
with pytest.raises(MlflowException,
match="Invalid parameter name") as e:
tracking.MlflowClient().log_batch(run_id, params=params)
assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
tags = [Param(key="../bad/tag/name", value="my-val")]
with pytest.raises(MlflowException, match="Invalid tag name") as e:
tracking.MlflowClient().log_batch(run_id, tags=tags)
assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_log_metric_validation():
try:
tracking.set_tracking_uri(tempfile.mkdtemp())
active_run = start_run()
run_uuid = active_run.info.run_uuid
with active_run:
mlflow.log_metric("name_1", "apple")
finished_run = tracking.MlflowClient().get_run(run_uuid)
assert len(finished_run.data.metrics) == 0
finally:
tracking.set_tracking_uri(None)
def test_log_params(tracking_uri_mock):
expected_params = {"name_1": "c", "name_2": "b", "nested/nested/name": "5"}
active_run = start_run()
run_uuid = active_run.info.run_uuid
with active_run:
mlflow.log_params(expected_params)
finished_run = tracking.MlflowClient().get_run(run_uuid)
# Validate params
assert len(finished_run.data.params) == 3
for param in finished_run.data.params:
assert expected_params[param.key] == param.value
def test_start_and_end_run(tracking_uri_mock):
# Use the start_run() and end_run() APIs without a `with` block, verify they work.
active_run = start_run()
mlflow.log_metric("name_1", 25)
end_run()
finished_run = tracking.MlflowClient().get_run(active_run.info.run_uuid)
# Validate metrics
assert len(finished_run.data.metrics) == 1
expected_pairs = {"name_1": 25}
for metric in finished_run.data.metrics:
assert expected_pairs[metric.key] == metric.value
def _resolve_runs_uri(uri):
from mlflow.utils import mlflow_tags
client = tracking.MlflowClient()
toks = uri.split("/")
if len(toks) < 2:
raise Exception("Bad runs URI")
run_id = toks[1]
run = client.get_run(run_id)
uri = _get_tag(run, mlflow_tags.MLFLOW_SOURCE_NAME)
version = _get_tag(run, mlflow_tags.MLFLOW_GIT_COMMIT)
return uri
def test_log_params(tracking_uri_mock):
expected_params = {"name_1": "c", "name_2": "b", "nested/nested/name": "5"}
with start_run() as active_run:
run_uuid = active_run.info.run_uuid
mlflow.log_params(expected_params)
finished_run = tracking.MlflowClient().get_run(run_uuid)
# Validate params
assert finished_run.data.params == {
"name_1": "c",
"name_2": "b",
"nested/nested/name": "5"
}
def test_list_experiments_paginated_errors():
client = tracking.MlflowClient()
# test that providing a completely invalid page token throws
with pytest.raises(MlflowException, match="Invalid page token") as exception_context:
client.list_experiments(page_token="evilhax", max_results=20)
assert exception_context.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
# test that providing too large of a max_results throws
with pytest.raises(
MlflowException, match="Invalid value for request parameter max_results"
) as exception_context:
client.list_experiments(page_token=None, max_results=int(1e15))
assert exception_context.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_log_metrics(tracking_uri_mock):
active_run = start_run()
run_uuid = active_run.info.run_uuid
expected_metrics = {"name_1": 30, "name_2": -3, "nested/nested/name": 40}
with active_run:
mlflow.log_metrics(expected_metrics)
finished_run = tracking.MlflowClient().get_run(run_uuid)
# Validate metric key/values match what we expect, and that all metrics have the same timestamp
common_timestamp = finished_run.data.metrics[0].timestamp
assert len(finished_run.data.metrics) == len(expected_metrics)
for metric in finished_run.data.metrics:
assert expected_metrics[metric.key] == metric.value
assert metric.timestamp == common_timestamp
def test_log_params_duplicate_keys_raises():
params = {"a": "1", "b": "2"}
with start_run() as active_run:
run_id = active_run.info.run_id
mlflow.log_params(params)
with pytest.raises(
expected_exception=MlflowException,
match=r"Changing param values is not allowed. Param with key=",
) as e:
mlflow.log_param("a", "3")
assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
finished_run = tracking.MlflowClient().get_run(run_id)
assert finished_run.data.params == params
def test_log_batch(tracking_uri_mock, tmpdir):
expected_metrics = {"metric-key0": 1.0, "metric-key1": 4.0}
expected_params = {"param-key0": "param-val0", "param-key1": "param-val1"}
exact_expected_tags = {"tag-key0": "tag-val0", "tag-key1": "tag-val1"}
approx_expected_tags = set([MLFLOW_USER, MLFLOW_SOURCE_NAME, MLFLOW_SOURCE_TYPE])
t = int(time.time())
sorted_expected_metrics = sorted(expected_metrics.items(), key=lambda kv: kv[0])
metrics = [Metric(key=key, value=value, timestamp=t, step=i)
for i, (key, value) in enumerate(sorted_expected_metrics)]
params = [Param(key=key, value=value) for key, value in expected_params.items()]
tags = [RunTag(key=key, value=value) for key, value in exact_expected_tags.items()]
with start_run() as active_run:
run_id = active_run.info.run_id
mlflow.tracking.MlflowClient().log_batch(run_id=run_id, metrics=metrics, params=params,
tags=tags)
client = tracking.MlflowClient()
finished_run = client.get_run(run_id)
# Validate metrics
assert len(finished_run.data.metrics) == 2
for key, value in finished_run.data.metrics.items():
assert expected_metrics[key] == value
metric_history0 = client.get_metric_history(run_id, "metric-key0")
assert set([(m.value, m.timestamp, m.step) for m in metric_history0]) == set([
(1.0, t, 0),
])
metric_history1 = client.get_metric_history(run_id, "metric-key1")
assert set([(m.value, m.timestamp, m.step) for m in metric_history1]) == set([
(4.0, t, 1),
])
# Validate tags (for automatically-set tags)
assert len(finished_run.data.tags) == len(exact_expected_tags) + len(approx_expected_tags)
for tag_key, tag_value in finished_run.data.tags.items():
if tag_key in approx_expected_tags:
pass
else:
assert exact_expected_tags[tag_key] == tag_value
# Validate params
assert finished_run.data.params == expected_params
# test that log_batch works with fewer params
new_tags = {"1": "2", "3": "4", "5": "6"}
tags = [RunTag(key=key, value=value) for key, value in new_tags.items()]
client.log_batch(run_id=run_id, tags=tags)
finished_run_2 = client.get_run(run_id)
# Validate tags (for automatically-set tags)
assert len(finished_run_2.data.tags) == len(finished_run.data.tags) + 3
for tag_key, tag_value in finished_run_2.data.tags.items():
if tag_key in new_tags:
assert new_tags[tag_key] == tag_value
def test_set_tags():
exact_expected_tags = {"name_1": "c", "name_2": "b", "nested/nested/name": 5}
approx_expected_tags = set([MLFLOW_USER, MLFLOW_SOURCE_NAME, MLFLOW_SOURCE_TYPE])
with start_run() as active_run:
run_id = active_run.info.run_id
mlflow.set_tags(exact_expected_tags)
finished_run = tracking.MlflowClient().get_run(run_id)
# Validate tags
assert len(finished_run.data.tags) == len(exact_expected_tags) + len(approx_expected_tags)
for tag_key, tag_val in finished_run.data.tags.items():
if tag_key in approx_expected_tags:
pass
else:
assert str(exact_expected_tags[tag_key]) == tag_val
def test_create_experiment_with_duplicate_name():
name = "popular_name"
exp_id = mlflow.create_experiment(name)
with pytest.raises(
MlflowException,
match=re.escape(f"Experiment(name={name}) already exists")):
mlflow.create_experiment(name)
tracking.MlflowClient().delete_experiment(exp_id)
with pytest.raises(
MlflowException,
match=re.escape(f"Experiment(name={name}) already exists")):
mlflow.create_experiment(name)
def test_log_metric(tracking_uri_mock):
active_run = start_run()
run_uuid = active_run.info.run_uuid
with active_run:
mlflow.log_metric("name_1", 25)
mlflow.log_metric("name_2", -3)
mlflow.log_metric("name_1", 30)
mlflow.log_metric("nested/nested/name", 40)
finished_run = tracking.MlflowClient().get_run(run_uuid)
# Validate metrics
assert len(finished_run.data.metrics) == 3
expected_pairs = {"name_1": 30, "name_2": -3, "nested/nested/name": 40}
for metric in finished_run.data.metrics:
assert expected_pairs[metric.key] == metric.value
def test_start_run_context_manager():
try:
tracking.set_tracking_uri(tempfile.mkdtemp())
first_run = start_run()
first_uuid = first_run.info.run_uuid
with first_run:
# Check that start_run() causes the run information to be persisted in the store
persisted_run = tracking.MlflowClient().get_run(first_uuid)
assert persisted_run is not None
assert persisted_run.info == first_run.info
finished_run = tracking.MlflowClient().get_run(first_uuid)
assert finished_run.info.status == RunStatus.FINISHED
# Launch a separate run that fails, verify the run status is FAILED and the run UUID is
# different
second_run = start_run()
assert second_run.info.run_uuid != first_uuid
with pytest.raises(Exception):
with second_run:
raise Exception("Failing run!")
finished_run2 = tracking.MlflowClient().get_run(second_run.info.run_uuid)
assert finished_run2.info.status == RunStatus.FAILED
finally:
tracking.set_tracking_uri(None)
def test_log_param(tracking_uri_mock):
print(tracking.get_tracking_uri())
active_run = start_run()
run_uuid = active_run.info.run_uuid
with active_run:
mlflow.log_param("name_1", "a")
mlflow.log_param("name_2", "b")
mlflow.log_param("name_1", "c")
mlflow.log_param("nested/nested/name", 5)
finished_run = tracking.MlflowClient().get_run(run_uuid)
# Validate params
assert len(finished_run.data.params) == 3
expected_pairs = {"name_1": "c", "name_2": "b", "nested/nested/name": "5"}
for param in finished_run.data.params:
assert expected_pairs[param.key] == param.value
def test_set_experiment_with_deleted_experiment():
name = "dead_exp"
mlflow.set_experiment(name)
with start_run() as run:
exp_id = run.info.experiment_id
tracking.MlflowClient().delete_experiment(exp_id)
with pytest.raises(MlflowException, match="Cannot set a deleted experiment") as exc:
mlflow.set_experiment(name)
assert exc.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
with pytest.raises(MlflowException, match="Cannot set a deleted experiment") as exc:
mlflow.set_experiment(experiment_id=exp_id)
assert exc.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_log_metric():
with start_run() as active_run, mock.patch("time.time") as time_mock:
time_mock.side_effect = [123 for _ in range(100)]
run_id = active_run.info.run_id
mlflow.log_metric("name_1", 25)
mlflow.log_metric("name_2", -3)
mlflow.log_metric("name_1", 30, 5)
mlflow.log_metric("name_1", 40, -2)
mlflow.log_metric("nested/nested/name", 40)
finished_run = tracking.MlflowClient().get_run(run_id)
# Validate metrics
assert len(finished_run.data.metrics) == 3
expected_pairs = {"name_1": 30, "name_2": -3, "nested/nested/name": 40}
for key, value in finished_run.data.metrics.items():
assert expected_pairs[key] == value
client = tracking.MlflowClient()
metric_history_name1 = client.get_metric_history(run_id, "name_1")
assert set([(m.value, m.timestamp, m.step) for m in metric_history_name1]) == set(
[(25, 123 * 1000, 0), (30, 123 * 1000, 5), (40, 123 * 1000, -2)]
)
metric_history_name2 = client.get_metric_history(run_id, "name_2")
assert set([(m.value, m.timestamp, m.step) for m in metric_history_name2]) == set(
[(-3, 123 * 1000, 0)]
)
def test_log_metrics_uses_millisecond_timestamp_resolution_fluent():
with start_run() as active_run, mock.patch("time.time") as time_mock:
time_mock.side_effect = lambda: 123
mlflow.log_metrics({"name_1": 25, "name_2": -3})
mlflow.log_metrics({"name_1": 30})
mlflow.log_metrics({"name_1": 40})
run_id = active_run.info.run_id
client = tracking.MlflowClient()
metric_history_name1 = client.get_metric_history(run_id, "name_1")
assert set([(m.value, m.timestamp) for m in metric_history_name1]) == set(
[(25, 123 * 1000), (30, 123 * 1000), (40, 123 * 1000)]
)
metric_history_name2 = client.get_metric_history(run_id, "name_2")
assert set([(m.value, m.timestamp) for m in metric_history_name2]) == set([(-3, 123 * 1000)])
def test_log_metrics(tracking_uri_mock, step_kwarg):
expected_metrics = {"name_1": 30, "name_2": -3, "nested/nested/name": 40}
with start_run() as active_run:
run_uuid = active_run.info.run_uuid
mlflow.log_metrics(expected_metrics, step=step_kwarg)
finished_run = tracking.MlflowClient().get_run(run_uuid)
# Validate metric key/values match what we expect, and that all metrics have the same timestamp
assert len(finished_run.data.metrics) == len(expected_metrics)
for key, value in finished_run.data.metrics.items():
assert expected_metrics[key] == value
common_timestamp = finished_run.data._metric_objs[0].timestamp
expected_step = step_kwarg if step_kwarg is not None else 0
for metric_obj in finished_run.data._metric_objs:
assert metric_obj.timestamp == common_timestamp
assert metric_obj.step == expected_step
