def test_set_tags(self):
fs = FileStore(self.test_root)
run_uuid = self.exp_data[0]["runs"][0]
fs.set_tag(run_uuid, RunTag("tag0", "value0"))
fs.set_tag(run_uuid, RunTag("tag1", "value1"))
tags = [(t.key, t.value) for t in fs.get_run(run_uuid).data.tags]
assert set(tags) == {
("tag0", "value0"),
("tag1", "value1"),
}
# Can overwrite tags.
fs.set_tag(run_uuid, RunTag("tag0", "value2"))
tags = [(t.key, t.value) for t in fs.get_run(run_uuid).data.tags]
assert set(tags) == {
("tag0", "value2"),
("tag1", "value1"),
}
# Can set multiline tags.
fs.set_tag(run_uuid, RunTag("multiline_tag", "value2\nvalue2\nvalue2"))
tags = [(t.key, t.value) for t in fs.get_run(run_uuid).data.tags]
assert set(tags) == {
("tag0", "value2"),
("tag1", "value1"),
("multiline_tag", "value2\nvalue2\nvalue2"),
}
def test_malformed_run(self):
fs = FileStore(self.test_root)
exp_0 = fs.get_experiment(Experiment.DEFAULT_EXPERIMENT_ID)
all_runs = fs.search_runs([exp_0.experiment_id], [],
run_view_type=ViewType.ALL)
all_run_ids = self.exp_data[exp_0.experiment_id]["runs"]
assert len(all_runs) == len(all_run_ids)
# delete metadata file.
bad_run_id = self.exp_data[exp_0.experiment_id]['runs'][0]
path = os.path.join(self.test_root, str(exp_0.experiment_id),
str(bad_run_id), "meta.yaml")
os.remove(path)
with pytest.raises(MissingConfigException) as e:
fs.get_run(bad_run_id)
assert e.message.contains("does not exist")
valid_runs = fs.search_runs([exp_0.experiment_id], [],
run_view_type=ViewType.ALL)
assert len(valid_runs) == len(all_runs) - 1
for rid in all_run_ids:
if rid != bad_run_id:
fs.get_run(rid)
def test_delete_tags(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]['runs'][0]
fs.set_tag(run_id, RunTag("tag0", "value0"))
fs.set_tag(run_id, RunTag("tag1", "value1"))
tags = fs.get_run(run_id).data.tags
assert tags["tag0"] == "value0"
assert tags["tag1"] == "value1"
fs.delete_tag(run_id, "tag0")
new_tags = fs.get_run(run_id).data.tags
assert "tag0" not in new_tags.keys()
# test that you cannot delete tags that don't exist.
with pytest.raises(MlflowException):
fs.delete_tag(run_id, "fakeTag")
# test that you cannot delete tags for nonexistent runs
with pytest.raises(MlflowException):
fs.delete_tag("random_id", "tag0")
fs = FileStore(self.test_root)
fs.delete_run(run_id)
# test that you cannot delete tags for deleted runs.
assert fs.get_run(
run_id).info.lifecycle_stage == LifecycleStage.DELETED
with pytest.raises(MlflowException):
fs.delete_tag(run_id, "tag0")
def test_bad_experiment_id_recorded_for_run(self):
fs = FileStore(self.test_root)
exp_0 = fs.get_experiment(Experiment.DEFAULT_EXPERIMENT_ID)
all_runs = fs.search_runs([exp_0.experiment_id], [], run_view_type=ViewType.ALL)
all_run_ids = self.exp_data[exp_0.experiment_id]["runs"]
assert len(all_runs) == len(all_run_ids)
# change experiment pointer in run
bad_run_id = str(self.exp_data[exp_0.experiment_id]['runs'][0])
path = os.path.join(self.test_root, str(exp_0.experiment_id), bad_run_id)
experiment_data = read_yaml(path, "meta.yaml")
experiment_data["experiment_id"] = 1
write_yaml(path, "meta.yaml", experiment_data, True)
with pytest.raises(MlflowException) as e:
fs.get_run(bad_run_id)
assert e.message.contains("not found")
valid_runs = fs.search_runs([exp_0.experiment_id], [], run_view_type=ViewType.ALL)
assert len(valid_runs) == len(all_runs) - 1
for rid in all_run_ids:
if rid != bad_run_id:
fs.get_run(rid)
def test_delete_restore_run(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]['runs'][0]
# Should not throw.
assert fs.get_run(run_id).info.lifecycle_stage == 'active'
fs.delete_run(run_id)
assert fs.get_run(run_id).info.lifecycle_stage == 'deleted'
fs.restore_run(run_id)
assert fs.get_run(run_id).info.lifecycle_stage == 'active'
def test_log_metric_allows_multiple_values_at_same_step_and_run_data_uses_max_step_value(self):
fs = FileStore(self.test_root)
run_id = self._create_run(fs).info.run_id
metric_name = "test-metric-1"
# Check that we get the max of (step, timestamp, value) in that order
tuples_to_log = [
(0, 100, 1000),
(3, 40, 100), # larger step wins even though it has smaller value
(3, 50, 10), # larger timestamp wins even though it has smaller value
(3, 50, 20), # tiebreak by max value
(3, 50, 20), # duplicate metrics with same (step, timestamp, value) are ok
# verify that we can log steps out of order / negative steps
(-3, 900, 900),
(-1, 800, 800),
]
for step, timestamp, value in reversed(tuples_to_log):
fs.log_metric(run_id, Metric(metric_name, value, timestamp, step))
metric_history = fs.get_metric_history(run_id, metric_name)
logged_tuples = [(m.step, m.timestamp, m.value) for m in metric_history]
assert set(logged_tuples) == set(tuples_to_log)
run_data = fs.get_run(run_id).data
run_metrics = run_data.metrics
assert len(run_metrics) == 1
assert run_metrics[metric_name] == 20
metric_obj = run_data._metric_objs[0]
assert metric_obj.key == metric_name
assert metric_obj.step == 3
assert metric_obj.timestamp == 50
assert metric_obj.value == 20
def test_log_metric_allows_multiple_values_at_same_ts_and_run_data_uses_max_ts_value(
self):
fs = FileStore(self.test_root)
run_uuid = self._create_run(fs).info.run_uuid
metric_name = "test-metric-1"
timestamp_values_mapping = {
1000: [float(i) for i in range(-20, 20)],
2000: [float(i) for i in range(-10, 10)],
}
logged_values = []
for timestamp, value_range in timestamp_values_mapping.items():
for value in reversed(value_range):
fs.log_metric(run_uuid, Metric(metric_name, value, timestamp))
logged_values.append(value)
six.assertCountEqual(self, [
metric.value
for metric in fs.get_metric_history(run_uuid, metric_name)
], logged_values)
run_metrics = fs.get_run(run_uuid).data.metrics
assert len(run_metrics) == 1
logged_metric_val = run_metrics[metric_name]
max_timestamp = max(timestamp_values_mapping)
assert logged_metric_val == max(
timestamp_values_mapping[max_timestamp])
def test_log_batch(self):
fs = FileStore(self.test_root)
run = fs.create_run(experiment_id=Experiment.DEFAULT_EXPERIMENT_ID,
user_id='user',
run_name=None,
source_type='source_type',
source_name='source_name',
entry_point_name='entry_point_name',
start_time=0,
source_version=None,
tags=[],
parent_run_id=None)
run_uuid = run.info.run_uuid
metric_entities = [
Metric("m1", 0.87, 12345),
Metric("m2", 0.49, 12345)
]
param_entities = [Param("p1", "p1val"), Param("p2", "p2val")]
tag_entities = [RunTag("t1", "t1val"), RunTag("t2", "t2val")]
fs.log_batch(run_id=run_uuid,
metrics=metric_entities,
params=param_entities,
tags=tag_entities)
run = fs.get_run(run_uuid)
tags = [(t.key, t.value) for t in run.data.tags]
metrics = [(m.key, m.value, m.timestamp) for m in run.data.metrics]
params = [(p.key, p.value) for p in run.data.params]
assert set(tags) == set([("t1", "t1val"), ("t2", "t2val")])
assert set(metrics) == set([("m1", 0.87, 12345), ("m2", 0.49, 12345)])
assert set(params) == set([("p1", "p1val"), ("p2", "p2val")])
def test_unicode_tag(self):
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
value = u"𝐼 𝓈𝑜𝓁𝑒𝓂𝓃𝓁𝓎 𝓈𝓌𝑒𝒶𝓇 𝓉𝒽𝒶𝓉 𝐼 𝒶𝓂 𝓊𝓅 𝓉𝑜 𝓃𝑜 𝑔𝑜𝑜𝒹"
fs.set_tag(run_id, RunTag("message", value))
tags = fs.get_run(run_id).data.tags
assert tags["message"] == value
def test_weird_tag_names(self):
WEIRD_TAG_NAME = "this is/a weird/but valid tag"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.set_tag(run_id, RunTag(WEIRD_TAG_NAME, "Muhahaha!"))
run = fs.get_run(run_id)
assert run.data.tags[WEIRD_TAG_NAME] == "Muhahaha!"
def test_log_empty_str(self):
PARAM_NAME = "new param"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_param(run_id, Param(PARAM_NAME, ""))
run = fs.get_run(run_id)
assert run.data.params[PARAM_NAME] == ""
def test_weird_param_names(self):
WEIRD_PARAM_NAME = "this is/a weird/but valid param"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_param(run_id, Param(WEIRD_PARAM_NAME, "Value"))
run = fs.get_run(run_id)
assert run.data.params[WEIRD_PARAM_NAME] == "Value"
def test_run(tmpdir, tracking_uri_mock, use_start_run): # pylint: disable=unused-argument
submitted_run = mlflow.projects.run(
TEST_PROJECT_DIR,
entry_point="test_tracking",
parameters={"use_start_run": use_start_run},
use_conda=False,
experiment_id=0)
assert submitted_run.run_id is not None
# Blocking runs should be finished when they return
validate_exit_status(submitted_run.get_status(), RunStatus.FINISHED)
# Test that we can call wait() on a synchronous run & that the run has the correct
# status after calling wait().
submitted_run.wait()
validate_exit_status(submitted_run.get_status(), RunStatus.FINISHED)
# Validate run contents in the FileStore
run_uuid = submitted_run.run_id
store = FileStore(tmpdir.strpath)
run_infos = store.list_run_infos(experiment_id=0,
run_view_type=ViewType.ACTIVE_ONLY)
assert len(run_infos) == 1
store_run_uuid = run_infos[0].run_uuid
assert run_uuid == store_run_uuid
run = store.get_run(run_uuid)
expected_params = {"use_start_run": use_start_run}
assert run.info.status == RunStatus.FINISHED
assert len(run.data.params) == len(expected_params)
for param in run.data.params:
assert param.value == expected_params[param.key]
expected_metrics = {"some_key": 3}
assert len(run.data.metrics) == len(expected_metrics)
for metric in run.data.metrics:
assert metric.value == expected_metrics[metric.key]
def test_weird_tag_names(self):
WEIRD_TAG_NAME = "this is/a weird/but valid tag"
fs = FileStore(self.test_root)
run_uuid = self.exp_data[0]["runs"][0]
fs.set_tag(run_uuid, RunTag(WEIRD_TAG_NAME, "Muhahaha!"))
tag = fs.get_run(run_uuid).data.tags[0]
assert tag.key == WEIRD_TAG_NAME
assert tag.value == "Muhahaha!"
def test_unicode_tag(self):
fs = FileStore(self.test_root)
run_uuid = self.exp_data[0]["runs"][0]
value = u"𝐼 𝓈𝑜𝓁𝑒𝓂𝓃𝓁𝓎 𝓈𝓌𝑒𝒶𝓇 𝓉𝒽𝒶𝓉 𝐼 𝒶𝓂 𝓊𝓅 𝓉𝑜 𝓃𝑜 𝑔𝑜𝑜𝒹"
fs.set_tag(run_uuid, RunTag("message", value))
tag = fs.get_run(run_uuid).data.tags[0]
assert tag.key == "message"
assert tag.value == value
def test_get_deleted_run(self):
"""
Getting metrics/tags/params/run info should be allowed on deleted runs.
"""
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]['runs'][0]
fs.delete_run(run_id)
assert fs.get_run(run_id)
def test_create_run_with_parent_id(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run = fs.create_run(exp_id, 'user', 'name', 'source_type',
'source_name', 'entry_point_name', 0, None, [],
'test_parent_run_id')
assert fs.get_run(
run.info.run_uuid
).data.tags[MLFLOW_PARENT_RUN_ID] == 'test_parent_run_id'
def test_set_tags(self):
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.set_tag(run_id, RunTag("tag0", "value0"))
fs.set_tag(run_id, RunTag("tag1", "value1"))
tags = fs.get_run(run_id).data.tags
assert tags["tag0"] == "value0"
assert tags["tag1"] == "value1"
# Can overwrite tags.
fs.set_tag(run_id, RunTag("tag0", "value2"))
tags = fs.get_run(run_id).data.tags
assert tags["tag0"] == "value2"
assert tags["tag1"] == "value1"
# Can set multiline tags.
fs.set_tag(run_id, RunTag("multiline_tag", "value2\nvalue2\nvalue2"))
tags = fs.get_run(run_id).data.tags
assert tags["multiline_tag"] == "value2\nvalue2\nvalue2"
def test_get_run(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
runs = self.exp_data[exp_id]["runs"]
for run_uuid in runs:
run = fs.get_run(run_uuid)
run_info = self.run_data[run_uuid]
run_info.pop("metrics")
run_info.pop("params")
self.assertEqual(run_info, dict(run.info))
def test_log_empty_str(self):
PARAM_NAME = "new param"
fs = FileStore(self.test_root)
run_uuid = self.exp_data[0]["runs"][0]
fs.log_param(run_uuid, Param(PARAM_NAME, ""))
run = fs.get_run(run_uuid)
my_params = [p for p in run.data.params if p.key == PARAM_NAME]
assert len(my_params) == 1
param = my_params[0]
assert param.key == PARAM_NAME
assert param.value == ""
def test_weird_param_names(self):
WEIRD_PARAM_NAME = "this is/a weird/but valid param"
fs = FileStore(self.test_root)
run_uuid = self.exp_data[0]["runs"][0]
fs.log_param(run_uuid, Param(WEIRD_PARAM_NAME, "Value"))
run = fs.get_run(run_uuid)
my_params = [p for p in run.data.params if p.key == WEIRD_PARAM_NAME]
assert len(my_params) == 1
param = my_params[0]
assert param.key == WEIRD_PARAM_NAME
assert param.value == "Value"
def test_get_run(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
runs = self.exp_data[exp_id]["runs"]
for run_uuid in runs:
run = fs.get_run(run_uuid)
run_info = self.run_data[run_uuid]
run_info.pop("metrics")
run_info.pop("params")
run_info.pop("tags")
run_info['lifecycle_stage'] = RunInfo.ACTIVE_LIFECYCLE
self.assertEqual(run_info, dict(run.info))
def test_weird_metric_names(self):
WEIRD_METRIC_NAME = "this is/a weird/but valid metric"
fs = FileStore(self.test_root)
run_uuid = self.exp_data[0]["runs"][0]
fs.log_metric(run_uuid, Metric(WEIRD_METRIC_NAME, 10, 1234))
run = fs.get_run(run_uuid)
my_metrics = [m for m in run.data.metrics if m.key == WEIRD_METRIC_NAME]
assert len(my_metrics) == 1
metric = my_metrics[0]
assert metric.key == WEIRD_METRIC_NAME
assert metric.value == 10
assert metric.timestamp == 1234
def test_weird_metric_names(self):
WEIRD_METRIC_NAME = "this is/a weird/but valid metric"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_metric(run_id, Metric(WEIRD_METRIC_NAME, 10, 1234, 0))
run = fs.get_run(run_id)
assert run.data.metrics[WEIRD_METRIC_NAME] == 10
history = fs.get_metric_history(run_id, WEIRD_METRIC_NAME)
assert len(history) == 1
metric = history[0]
assert metric.key == WEIRD_METRIC_NAME
assert metric.value == 10
assert metric.timestamp == 1234
def test_run_local_git_repo(
tmpdir,
local_git_repo,
local_git_repo_uri,
tracking_uri_mock, # pylint: disable=unused-argument
use_start_run,
version):
if version is not None:
uri = local_git_repo_uri + "#" + TEST_PROJECT_NAME
else:
uri = os.path.join("%s/" % local_git_repo, TEST_PROJECT_NAME)
if version == "git-commit":
version = _get_version_local_git_repo(local_git_repo)
submitted_run = mlflow.projects.run(
uri,
entry_point="test_tracking",
version=version,
parameters={"use_start_run": use_start_run},
use_conda=False,
experiment_id=0)
# Blocking runs should be finished when they return
validate_exit_status(submitted_run.get_status(), RunStatus.FINISHED)
# Test that we can call wait() on a synchronous run & that the run has the correct
# status after calling wait().
submitted_run.wait()
validate_exit_status(submitted_run.get_status(), RunStatus.FINISHED)
# Validate run contents in the FileStore
run_uuid = submitted_run.run_id
store = FileStore(tmpdir.strpath)
run_infos = store.list_run_infos(experiment_id=0,
run_view_type=ViewType.ACTIVE_ONLY)
assert "file:" in run_infos[0].source_name
assert len(run_infos) == 1
store_run_uuid = run_infos[0].run_uuid
assert run_uuid == store_run_uuid
run = store.get_run(run_uuid)
expected_params = {"use_start_run": use_start_run}
assert run.info.status == RunStatus.FINISHED
assert len(run.data.params) == len(expected_params)
for param in run.data.params:
assert param.value == expected_params[param.key]
expected_metrics = {"some_key": 3}
assert len(run.data.metrics) == len(expected_metrics)
for metric in run.data.metrics:
assert metric.value == expected_metrics[metric.key]
# Validate the branch name tag is logged
if version == "master":
expected_tags = {"mlflow.gitBranchName": "master"}
for tag in run.data.tags:
assert tag.value == expected_tags[tag.key]
def test_log_parameters():
""" Test that we log provided parameters when running a project. """
with TempDir() as tmp, mock.patch("mlflow.tracking.get_tracking_uri") as get_tracking_uri_mock:
tmp_dir = tmp.path()
get_tracking_uri_mock.return_value = tmp_dir
mlflow.projects.run(
TEST_PROJECT_DIR, entry_point="greeter", parameters={"name": "friend"},
use_conda=False, experiment_id=0)
store = FileStore(tmp_dir)
run_uuid = store.list_run_infos(experiment_id=0)[0].run_uuid
run = store.get_run(run_uuid)
expected_params = {"name": "friend"}
assert len(run.data.params) == len(expected_params)
for param in run.data.params:
assert param.value == expected_params[param.key]
def test_set_deleted_run(self):
"""
Setting metrics/tags/params/updating run info should not be allowed on deleted runs.
"""
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]['runs'][0]
fs.delete_run(run_id)
assert fs.get_run(run_id).info.lifecycle_stage == LifecycleStage.DELETED
with pytest.raises(MlflowException):
fs.set_tag(run_id, RunTag('a', 'b'))
with pytest.raises(MlflowException):
fs.log_metric(run_id, Metric('a', 0.0, timestamp=0))
with pytest.raises(MlflowException):
fs.log_param(run_id, Param('a', 'b'))
def test_run():
for use_start_run in map(str, [0, 1]):
with TempDir() as tmp, mock.patch("mlflow.tracking.get_tracking_uri")\
as get_tracking_uri_mock:
tmp_dir = tmp.path()
get_tracking_uri_mock.return_value = tmp_dir
submitted_run = mlflow.projects.run(
TEST_PROJECT_DIR,
entry_point="test_tracking",
parameters={"use_start_run": use_start_run},
use_conda=False,
experiment_id=0)
# Blocking runs should be finished when they return
validate_exit_status(submitted_run.get_status(),
RunStatus.FINISHED)
# Test that we can call wait() on a synchronous run & that the run has the correct
# status after calling wait().
submitted_run.wait()
validate_exit_status(submitted_run.get_status(),
RunStatus.FINISHED)
# Validate run contents in the FileStore
run_uuid = submitted_run.run_id
store = FileStore(tmp_dir)
run_infos = store.list_run_infos(experiment_id=0)
assert len(run_infos) == 1
store_run_uuid = run_infos[0].run_uuid
assert run_uuid == store_run_uuid
run = store.get_run(run_uuid)
expected_params = {"use_start_run": use_start_run}
assert run.info.status == RunStatus.FINISHED
assert len(run.data.params) == len(expected_params)
for param in run.data.params:
assert param.value == expected_params[param.key]
expected_metrics = {"some_key": 3}
for metric in run.data.metrics:
assert metric.value == expected_metrics[metric.key]
