def test_log_batch_accepts_empty_payload(self):
fs = FileStore(self.test_root)
run = self._create_run(fs)
fs.log_batch(run.info.run_id, metrics=[], params=[], tags=[])
self._verify_logged(fs,
run.info.run_id,
metrics=[],
params=[],
tags=[])
def test_log_batch_same_metric_repeated_multiple_reqs(self):
fs = FileStore(self.test_root)
run = self._create_run(fs)
metric0 = Metric(key="metric-key", value=1, timestamp=2, step=0)
metric1 = Metric(key="metric-key", value=2, timestamp=3, step=0)
fs.log_batch(run.info.run_id, params=[], metrics=[metric0], tags=[])
self._verify_logged(fs, run.info.run_id, params=[], metrics=[metric0], tags=[])
fs.log_batch(run.info.run_id, params=[], metrics=[metric1], tags=[])
self._verify_logged(fs, run.info.run_id, params=[], metrics=[metric0, metric1], tags=[])
def test_get_deleted_run(self):
"""
Getting metrics/tags/params/run info should be allowed on deleted runs.
"""
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]['runs'][0]
fs.delete_run(run_id)
assert fs.get_run(run_id)
def test_search_with_deterministic_max_results(self):
fs = FileStore(self.test_root)
exp = fs.create_experiment("test_search_with_deterministic_max_results")
# Create 10 runs with the same start_time.
# Sort based on run_id
runs = sorted([fs.create_run(exp, "user", 1000, []).info.run_id for r in range(10)])
for n in [0, 1, 2, 4, 8, 10, 20]:
assert runs[: min(10, n)] == self._search(fs, exp, max_results=n)
def test_log_batch_allows_tag_overwrite_single_req(self):
fs = FileStore(self.test_root)
run = self._create_run(fs)
tags = [RunTag("t-key", "val"), RunTag("t-key", "newval")]
fs.log_batch(run.info.run_id, metrics=[], params=[], tags=tags)
self._verify_logged(fs,
run.info.run_id,
metrics=[],
params=[],
tags=[tags[-1]])
def test_get_experiment(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
self._verify_experiment(fs, exp_id)
# test that fake experiments dont exist.
# look for random experiment ids between 8000, 15000 since created ones are (100, 2000)
for exp_id in set(random_int(8000, 15000) for x in range(20)):
with self.assertRaises(Exception):
fs.get_experiment(exp_id)
def test_delete_restore_run(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]['runs'][0]
# Should not throw.
assert fs.get_run(run_id).info.lifecycle_stage == 'active'
fs.delete_run(run_id)
assert fs.get_run(run_id).info.lifecycle_stage == 'deleted'
fs.restore_run(run_id)
assert fs.get_run(run_id).info.lifecycle_stage == 'active'
def test_log_batch_params_idempotency(self):
fs = FileStore(self.test_root)
run = self._create_run(fs)
params = [Param("p-key", "p-val")]
fs.log_batch(run.info.run_id, metrics=[], params=params, tags=[])
fs.log_batch(run.info.run_id, metrics=[], params=params, tags=[])
self._verify_logged(fs,
run.info.run_id,
metrics=[],
params=params,
tags=[])
def test_log_batch(self):
fs = FileStore(self.test_root)
run = fs.create_run(
experiment_id=FileStore.DEFAULT_EXPERIMENT_ID, user_id='user', start_time=0, tags=[])
run_id = run.info.run_id
metric_entities = [Metric("m1", 0.87, 12345, 0), Metric("m2", 0.49, 12345, 0)]
param_entities = [Param("p1", "p1val"), Param("p2", "p2val")]
tag_entities = [RunTag("t1", "t1val"), RunTag("t2", "t2val")]
fs.log_batch(
run_id=run_id, metrics=metric_entities, params=param_entities, tags=tag_entities)
self._verify_logged(fs, run_id, metric_entities, param_entities, tag_entities)
def test_search_tags(self):
fs = FileStore(self.test_root)
experiment_id = self.experiments[0]
r1 = fs.create_run(experiment_id, "user", 0, []).info.run_id
r2 = fs.create_run(experiment_id, "user", 0, []).info.run_id
fs.set_tag(r1, RunTag("generic_tag", "p_val"))
fs.set_tag(r2, RunTag("generic_tag", "p_val"))
fs.set_tag(r1, RunTag("generic_2", "some value"))
fs.set_tag(r2, RunTag("generic_2", "another value"))
fs.set_tag(r1, RunTag("p_a", "abc"))
fs.set_tag(r2, RunTag("p_b", "ABC"))
# test search returns both runs
self.assertCountEqual(
[r1, r2], self._search(fs, experiment_id, filter_str="tags.generic_tag = 'p_val'")
)
# test search returns appropriate run (same key different values per run)
self.assertCountEqual(
[r1], self._search(fs, experiment_id, filter_str="tags.generic_2 = 'some value'")
)
self.assertCountEqual(
[r2], self._search(fs, experiment_id, filter_str="tags.generic_2='another value'")
)
self.assertCountEqual(
[], self._search(fs, experiment_id, filter_str="tags.generic_tag = 'wrong_val'")
)
self.assertCountEqual(
[], self._search(fs, experiment_id, filter_str="tags.generic_tag != 'p_val'")
)
self.assertCountEqual(
[r1, r2], self._search(fs, experiment_id, filter_str="tags.generic_tag != 'wrong_val'"),
)
self.assertCountEqual(
[r1, r2], self._search(fs, experiment_id, filter_str="tags.generic_2 != 'wrong_val'"),
)
self.assertCountEqual([r1], self._search(fs, experiment_id, filter_str="tags.p_a = 'abc'"))
self.assertCountEqual([r2], self._search(fs, experiment_id, filter_str="tags.p_b = 'ABC'"))
self.assertCountEqual(
[r2], self._search(fs, experiment_id, filter_str="tags.generic_2 LIKE '%other%'")
)
self.assertCountEqual(
[], self._search(fs, experiment_id, filter_str="tags.generic_2 LIKE 'other%'")
)
self.assertCountEqual(
[], self._search(fs, experiment_id, filter_str="tags.generic_2 LIKE '%other'")
)
self.assertCountEqual(
[r2], self._search(fs, experiment_id, filter_str="tags.generic_2 ILIKE '%OTHER%'")
)
def test_get_all_metrics(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
runs = self.exp_data[exp_id]["runs"]
for run_id in runs:
run_info = self.run_data[run_id]
metrics = fs.get_all_metrics(run_id)
metrics_dict = run_info.pop("metrics")
for metric in metrics:
expected_timestamp, expected_value = max(metrics_dict[metric.key])
self.assertEqual(metric.timestamp, expected_timestamp)
self.assertEqual(metric.value, expected_value)
def test_create_run_appends_to_artifact_uri_path_correctly(self):
cases = [
("path/to/local/folder", "path/to/local/folder/{e}/{r}/artifacts"),
("/path/to/local/folder",
"/path/to/local/folder/{e}/{r}/artifacts"),
("#path/to/local/folder?",
"#path/to/local/folder?/{e}/{r}/artifacts"),
("file:path/to/local/folder",
"file:path/to/local/folder/{e}/{r}/artifacts"),
("file:///path/to/local/folder",
"file:///path/to/local/folder/{e}/{r}/artifacts"),
("file:path/to/local/folder?param=value",
"file:path/to/local/folder/{e}/{r}/artifacts?param=value"),
("file:///path/to/local/folder",
"file:///path/to/local/folder/{e}/{r}/artifacts"),
(
"file:///path/to/local/folder?param=value#fragment",
"file:///path/to/local/folder/{e}/{r}/artifacts?param=value#fragment",
),
("s3://bucket/path/to/root",
"s3://bucket/path/to/root/{e}/{r}/artifacts"),
(
"s3://bucket/path/to/root?creds=mycreds",
"s3://bucket/path/to/root/{e}/{r}/artifacts?creds=mycreds",
),
(
"dbscheme+driver://root@host/dbname?creds=mycreds#myfragment",
"dbscheme+driver://root@host/dbname/{e}/{r}/artifacts?creds=mycreds#myfragment",
),
(
"dbscheme+driver://root:[email protected]?creds=mycreds#myfragment",
"dbscheme+driver://root:[email protected]/{e}/{r}/artifacts"
"?creds=mycreds#myfragment",
),
(
"dbscheme+driver://root:[email protected]/mydb?creds=mycreds#myfragment",
"dbscheme+driver://root:[email protected]/mydb/{e}/{r}/artifacts"
"?creds=mycreds#myfragment",
),
]
for artifact_root_uri, expected_artifact_uri_format in cases:
with TempDir() as tmp:
fs = FileStore(tmp.path(), artifact_root_uri)
exp_id = fs.create_experiment("exp")
run = fs.create_run(experiment_id=exp_id,
user_id='user',
start_time=0,
tags=[])
self.assertEqual(
run.info.artifact_uri,
expected_artifact_uri_format.format(e=exp_id,
r=run.info.run_id))
def test_log_param_enforces_value_immutability(self):
param_name = "new param"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_param(run_id, Param(param_name, "value1"))
# Duplicate calls to `log_param` with the same key and value should succeed
fs.log_param(run_id, Param(param_name, "value1"))
with pytest.raises(MlflowException) as exc:
fs.log_param(run_id, Param(param_name, "value2"))
assert exc.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
run = fs.get_run(run_id)
assert run.data.params[param_name] == "value1"
def test_list_run_infos(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
run_infos = fs.list_run_infos(exp_id, run_view_type=ViewType.ALL)
for run_info in run_infos:
run_id = run_info.run_id
dict_run_info = self.run_data[run_id]
dict_run_info.pop("metrics")
dict_run_info.pop("params")
dict_run_info.pop("tags")
dict_run_info["lifecycle_stage"] = LifecycleStage.ACTIVE
dict_run_info["status"] = RunStatus.to_string(dict_run_info["status"])
self.assertEqual(dict_run_info, dict(run_info))
def test_search_runs(self):
# replace with test with code is implemented
fs = FileStore(self.test_root)
# Expect 2 runs for each experiment
assert len(
self._search(fs,
self.experiments[0],
run_view_type=ViewType.ACTIVE_ONLY)) == 2
assert len(self._search(fs, self.experiments[0])) == 2
assert len(
self._search(fs,
self.experiments[0],
run_view_type=ViewType.DELETED_ONLY)) == 0
def test_hard_delete_run(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]["runs"][0]
fs._hard_delete_run(run_id)
with self.assertRaises(MlflowException):
fs.get_run(run_id)
with self.assertRaises(MlflowException):
fs.get_all_tags(run_id)
with self.assertRaises(MlflowException):
fs.get_all_metrics(run_id)
with self.assertRaises(MlflowException):
fs.get_all_params(run_id)
def test_weird_metric_names(self):
WEIRD_METRIC_NAME = "this is/a weird/but valid metric"
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.log_metric(run_id, Metric(WEIRD_METRIC_NAME, 10, 1234, 0))
run = fs.get_run(run_id)
assert run.data.metrics[WEIRD_METRIC_NAME] == 10
history = fs.get_metric_history(run_id, WEIRD_METRIC_NAME)
assert len(history) == 1
metric = history[0]
assert metric.key == WEIRD_METRIC_NAME
assert metric.value == 10
assert metric.timestamp == 1234
def test_search_runs_pagination(self):
fs = FileStore(self.test_root)
exp = fs.create_experiment("test_search_runs_pagination")
# test returned token behavior
runs = sorted([fs.create_run(exp, "user", 1000, []).info.run_id for r in range(10)])
result = fs.search_runs([exp], None, ViewType.ALL, max_results=4)
assert [r.info.run_id for r in result] == runs[0:4]
assert result.token is not None
result = fs.search_runs([exp], None, ViewType.ALL, max_results=4, page_token=result.token)
assert [r.info.run_id for r in result] == runs[4:8]
assert result.token is not None
result = fs.search_runs([exp], None, ViewType.ALL, max_results=4, page_token=result.token)
assert [r.info.run_id for r in result] == runs[8:]
assert result.token is None
def test_get_experiment_by_name(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
name = self.exp_data[exp_id]["name"]
exp = fs.get_experiment_by_name(name)
self.assertEqual(exp.experiment_id, exp_id)
self.assertEqual(exp.name, self.exp_data[exp_id]["name"])
self.assertEqual(exp.artifact_location, self.exp_data[exp_id]["artifact_location"])
# test that fake experiments dont exist.
# look up experiments with names of length 15 since created ones are of length 10
for exp_names in set(random_str(15) for x in range(20)):
exp = fs.get_experiment_by_name(exp_names)
self.assertIsNone(exp)
def test_get_metric_history(self):
fs = FileStore(self.test_root)
for exp_id in self.experiments:
runs = self.exp_data[exp_id]["runs"]
for run_id in runs:
run_info = self.run_data[run_id]
metrics = run_info.pop("metrics")
for metric_name, values in metrics.items():
metric_history = fs.get_metric_history(run_id, metric_name)
sorted_values = sorted(values, reverse=True)
for metric in metric_history:
timestamp, metric_value = sorted_values.pop()
self.assertEqual(metric.timestamp, timestamp)
self.assertEqual(metric.key, metric_name)
self.assertEqual(metric.value, metric_value)
def test_malformed_experiment(self):
fs = FileStore(self.test_root)
exp_0 = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert exp_0.experiment_id == FileStore.DEFAULT_EXPERIMENT_ID
experiments = len(fs.list_experiments(ViewType.ALL))
# delete metadata file.
path = os.path.join(self.test_root, str(exp_0.experiment_id), "meta.yaml")
os.remove(path)
with pytest.raises(MissingConfigException) as e:
fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert e.message.contains("does not exist")
assert len(fs.list_experiments(ViewType.ALL)) == experiments - 1
def test_delete_tags(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]["runs"][0]
fs.set_tag(run_id, RunTag("tag0", "value0"))
fs.set_tag(run_id, RunTag("tag1", "value1"))
tags = fs.get_run(run_id).data.tags
assert tags["tag0"] == "value0"
assert tags["tag1"] == "value1"
fs.delete_tag(run_id, "tag0")
new_tags = fs.get_run(run_id).data.tags
assert "tag0" not in new_tags.keys()
# test that you cannot delete tags that don't exist.
with pytest.raises(MlflowException):
fs.delete_tag(run_id, "fakeTag")
# test that you cannot delete tags for nonexistent runs
with pytest.raises(MlflowException):
fs.delete_tag("random_id", "tag0")
fs = FileStore(self.test_root)
fs.delete_run(run_id)
# test that you cannot delete tags for deleted runs.
assert fs.get_run(run_id).info.lifecycle_stage == LifecycleStage.DELETED
with pytest.raises(MlflowException):
fs.delete_tag(run_id, "tag0")
def test_list_experiments_paginated(self):
fs = FileStore(self.test_root)
for _ in range(10):
fs.create_experiment(random_str(12))
exps1 = fs.list_experiments(max_results=4, page_token=None)
self.assertEqual(len(exps1), 4)
self.assertIsNotNone(exps1.token)
exps2 = fs.list_experiments(max_results=4, page_token=None)
self.assertEqual(len(exps2), 4)
self.assertIsNotNone(exps2.token)
self.assertNotEqual(exps1, exps2)
exps3 = fs.list_experiments(max_results=500, page_token=exps2.token)
self.assertLessEqual(len(exps3), 500)
if len(exps3) < 500:
self.assertIsNone(exps3.token)
def test_set_deleted_run(self):
"""
Setting metrics/tags/params/updating run info should not be allowed on deleted runs.
"""
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run_id = self.exp_data[exp_id]["runs"][0]
fs.delete_run(run_id)
assert fs.get_run(run_id).info.lifecycle_stage == LifecycleStage.DELETED
with pytest.raises(MlflowException):
fs.set_tag(run_id, RunTag("a", "b"))
with pytest.raises(MlflowException):
fs.log_metric(run_id, Metric("a", 0.0, timestamp=0, step=0))
with pytest.raises(MlflowException):
fs.log_param(run_id, Param("a", "b"))
def test_log_batch_tags_idempotency(self):
fs = FileStore(self.test_root)
run = self._create_run(fs)
fs.log_batch(run.info.run_id,
metrics=[],
params=[],
tags=[RunTag("t-key", "t-val")])
fs.log_batch(run.info.run_id,
metrics=[],
params=[],
tags=[RunTag("t-key", "t-val")])
self._verify_logged(fs,
run.info.run_id,
metrics=[],
params=[],
tags=[RunTag("t-key", "t-val")])
def test_search_with_max_results(self):
fs = FileStore(self.test_root)
exp = fs.create_experiment("search_with_max_results")
runs = [fs.create_run(exp, "user", r, []).info.run_id for r in range(10)]
runs.reverse()
print(runs)
print(self._search(fs, exp))
assert runs[:10] == self._search(fs, exp)
for n in [0, 1, 2, 4, 8, 10, 20, 50, 100, 500, 1000, 1200, 2000]:
assert runs[: min(1200, n)] == self._search(fs, exp, max_results=n)
with self.assertRaises(MlflowException) as e:
self._search(fs, exp, None, max_results=int(1e10))
self.assertIn("Invalid value for request parameter max_results. It ", e.exception.message)
def test_set_tags(self):
fs = FileStore(self.test_root)
run_id = self.exp_data[FileStore.DEFAULT_EXPERIMENT_ID]["runs"][0]
fs.set_tag(run_id, RunTag("tag0", "value0"))
fs.set_tag(run_id, RunTag("tag1", "value1"))
tags = fs.get_run(run_id).data.tags
assert tags["tag0"] == "value0"
assert tags["tag1"] == "value1"
# Can overwrite tags.
fs.set_tag(run_id, RunTag("tag0", "value2"))
tags = fs.get_run(run_id).data.tags
assert tags["tag0"] == "value2"
assert tags["tag1"] == "value1"
# Can set multiline tags.
fs.set_tag(run_id, RunTag("multiline_tag", "value2\nvalue2\nvalue2"))
tags = fs.get_run(run_id).data.tags
assert tags["multiline_tag"] == "value2\nvalue2\nvalue2"
def test_create_experiment_appends_to_artifact_uri_path_correctly(self):
cases = [
("path/to/local/folder", "path/to/local/folder/{e}"),
("/path/to/local/folder", "/path/to/local/folder/{e}"),
("#path/to/local/folder?", "#path/to/local/folder?/{e}"),
("file:path/to/local/folder", "file:path/to/local/folder/{e}"),
("file:///path/to/local/folder",
"file:///path/to/local/folder/{e}"),
("file:path/to/local/folder?param=value",
"file:path/to/local/folder/{e}?param=value"),
("file:///path/to/local/folder",
"file:///path/to/local/folder/{e}"),
(
"file:///path/to/local/folder?param=value#fragment",
"file:///path/to/local/folder/{e}?param=value#fragment",
),
("s3://bucket/path/to/root", "s3://bucket/path/to/root/{e}"),
(
"s3://bucket/path/to/root?creds=mycreds",
"s3://bucket/path/to/root/{e}?creds=mycreds",
),
(
"dbscheme+driver://root@host/dbname?creds=mycreds#myfragment",
"dbscheme+driver://root@host/dbname/{e}?creds=mycreds#myfragment",
),
(
"dbscheme+driver://root:[email protected]?creds=mycreds#myfragment",
"dbscheme+driver://root:[email protected]/{e}?creds=mycreds#myfragment",
),
(
"dbscheme+driver://root:[email protected]/mydb?creds=mycreds#myfragment",
"dbscheme+driver://root:[email protected]/mydb/{e}?creds=mycreds#myfragment",
),
]
for artifact_root_uri, expected_artifact_uri_format in cases:
with TempDir() as tmp:
fs = FileStore(tmp.path(), artifact_root_uri)
exp_id = fs.create_experiment("exp")
exp = fs.get_experiment(exp_id)
self.assertEqual(exp.artifact_location,
expected_artifact_uri_format.format(e=exp_id))
def test_set_experiment_tags(self):
fs = FileStore(self.test_root)
fs.set_experiment_tag(FileStore.DEFAULT_EXPERIMENT_ID,
ExperimentTag("tag0", "value0"))
fs.set_experiment_tag(FileStore.DEFAULT_EXPERIMENT_ID,
ExperimentTag("tag1", "value1"))
experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert len(experiment.tags) == 2
assert experiment.tags["tag0"] == "value0"
assert experiment.tags["tag1"] == "value1"
# test that updating a tag works
fs.set_experiment_tag(FileStore.DEFAULT_EXPERIMENT_ID,
ExperimentTag("tag0", "value00000"))
experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert experiment.tags["tag0"] == "value00000"
assert experiment.tags["tag1"] == "value1"
# test that setting a tag on 1 experiment does not impact another experiment.
exp_id = None
for exp in self.experiments:
if exp != FileStore.DEFAULT_EXPERIMENT_ID:
exp_id = exp
break
experiment = fs.get_experiment(exp_id)
assert len(experiment.tags) == 0
# setting a tag on different experiments maintains different values across experiments
fs.set_experiment_tag(exp_id, ExperimentTag("tag1", "value11111"))
experiment = fs.get_experiment(exp_id)
assert len(experiment.tags) == 1
assert experiment.tags["tag1"] == "value11111"
experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert experiment.tags["tag0"] == "value00000"
assert experiment.tags["tag1"] == "value1"
# test can set multi-line tags
fs.set_experiment_tag(
exp_id, ExperimentTag("multiline_tag", "value2\nvalue2\nvalue2"))
experiment = fs.get_experiment(exp_id)
assert experiment.tags["multiline_tag"] == "value2\nvalue2\nvalue2"
# test cannot set tags on deleted experiments
fs.delete_experiment(exp_id)
with pytest.raises(MlflowException):
fs.set_experiment_tag(exp_id, ExperimentTag("should", "notset"))
def test_mismatching_experiment_id(self):
fs = FileStore(self.test_root)
exp_0 = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert exp_0.experiment_id == FileStore.DEFAULT_EXPERIMENT_ID
experiments = len(fs.list_experiments(ViewType.ALL))
# mv experiment folder
target = "1"
path_orig = os.path.join(self.test_root, str(exp_0.experiment_id))
path_new = os.path.join(self.test_root, str(target))
os.rename(path_orig, path_new)
with pytest.raises(MlflowException) as e:
fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert e.message.contains("Could not find experiment with ID")
with pytest.raises(MlflowException) as e:
fs.get_experiment(target)
assert e.message.contains("does not exist")
assert len(fs.list_experiments(ViewType.ALL)) == experiments - 1
