def test_create_run_in_deleted_experiment(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
# delete it
fs.delete_experiment(exp_id)
with pytest.raises(Exception):
fs.create_run(exp_id, 'user', 0, [])
def test_search_tags(self):
fs = FileStore(self.test_root)
experiment_id = self.experiments[0]
r1 = fs.create_run(experiment_id, 'user', 0, []).info.run_id
r2 = fs.create_run(experiment_id, 'user', 0, []).info.run_id
fs.set_tag(r1, RunTag('generic_tag', 'p_val'))
fs.set_tag(r2, RunTag('generic_tag', 'p_val'))
fs.set_tag(r1, RunTag('generic_2', 'some value'))
fs.set_tag(r2, RunTag('generic_2', 'another value'))
fs.set_tag(r1, RunTag('p_a', 'abc'))
fs.set_tag(r2, RunTag('p_b', 'ABC'))
# test search returns both runs
six.assertCountEqual(
self, [r1, r2],
self._search(fs,
experiment_id,
filter_str="tags.generic_tag = 'p_val'"))
# test search returns appropriate run (same key different values per run)
six.assertCountEqual(
self, [r1],
self._search(fs,
experiment_id,
filter_str="tags.generic_2 = 'some value'"))
six.assertCountEqual(
self, [r2],
self._search(fs,
experiment_id,
filter_str="tags.generic_2='another value'"))
six.assertCountEqual(
self, [],
self._search(fs,
experiment_id,
filter_str="tags.generic_tag = 'wrong_val'"))
six.assertCountEqual(
self, [],
self._search(fs,
experiment_id,
filter_str="tags.generic_tag != 'p_val'"))
six.assertCountEqual(
self, [r1, r2],
self._search(fs,
experiment_id,
filter_str="tags.generic_tag != 'wrong_val'"))
six.assertCountEqual(
self, [r1, r2],
self._search(fs,
experiment_id,
filter_str="tags.generic_2 != 'wrong_val'"))
six.assertCountEqual(
self, [r1],
self._search(fs, experiment_id, filter_str="tags.p_a = 'abc'"))
six.assertCountEqual(
self, [r2],
self._search(fs, experiment_id, filter_str="tags.p_b = 'ABC'"))
def test_search_with_deterministic_max_results(self):
fs = FileStore(self.test_root)
exp = fs.create_experiment("test_search_with_deterministic_max_results")
# Create 10 runs with the same start_time.
# Sort based on run_id
runs = sorted([fs.create_run(exp, "user", 1000, []).info.run_id for r in range(10)])
for n in [0, 1, 2, 4, 8, 10, 20]:
assert runs[: min(10, n)] == self._search(fs, exp, max_results=n)
def test_log_batch(self):
fs = FileStore(self.test_root)
run = fs.create_run(
experiment_id=FileStore.DEFAULT_EXPERIMENT_ID, user_id='user', start_time=0, tags=[])
run_id = run.info.run_id
metric_entities = [Metric("m1", 0.87, 12345, 0), Metric("m2", 0.49, 12345, 0)]
param_entities = [Param("p1", "p1val"), Param("p2", "p2val")]
tag_entities = [RunTag("t1", "t1val"), RunTag("t2", "t2val")]
fs.log_batch(
run_id=run_id, metrics=metric_entities, params=param_entities, tags=tag_entities)
self._verify_logged(fs, run_id, metric_entities, param_entities, tag_entities)
def test_search_tags(self):
fs = FileStore(self.test_root)
experiment_id = self.experiments[0]
r1 = fs.create_run(experiment_id, "user", 0, []).info.run_id
r2 = fs.create_run(experiment_id, "user", 0, []).info.run_id
fs.set_tag(r1, RunTag("generic_tag", "p_val"))
fs.set_tag(r2, RunTag("generic_tag", "p_val"))
fs.set_tag(r1, RunTag("generic_2", "some value"))
fs.set_tag(r2, RunTag("generic_2", "another value"))
fs.set_tag(r1, RunTag("p_a", "abc"))
fs.set_tag(r2, RunTag("p_b", "ABC"))
# test search returns both runs
self.assertCountEqual(
[r1, r2], self._search(fs, experiment_id, filter_str="tags.generic_tag = 'p_val'")
)
# test search returns appropriate run (same key different values per run)
self.assertCountEqual(
[r1], self._search(fs, experiment_id, filter_str="tags.generic_2 = 'some value'")
)
self.assertCountEqual(
[r2], self._search(fs, experiment_id, filter_str="tags.generic_2='another value'")
)
self.assertCountEqual(
[], self._search(fs, experiment_id, filter_str="tags.generic_tag = 'wrong_val'")
)
self.assertCountEqual(
[], self._search(fs, experiment_id, filter_str="tags.generic_tag != 'p_val'")
)
self.assertCountEqual(
[r1, r2], self._search(fs, experiment_id, filter_str="tags.generic_tag != 'wrong_val'"),
)
self.assertCountEqual(
[r1, r2], self._search(fs, experiment_id, filter_str="tags.generic_2 != 'wrong_val'"),
)
self.assertCountEqual([r1], self._search(fs, experiment_id, filter_str="tags.p_a = 'abc'"))
self.assertCountEqual([r2], self._search(fs, experiment_id, filter_str="tags.p_b = 'ABC'"))
self.assertCountEqual(
[r2], self._search(fs, experiment_id, filter_str="tags.generic_2 LIKE '%other%'")
)
self.assertCountEqual(
[], self._search(fs, experiment_id, filter_str="tags.generic_2 LIKE 'other%'")
)
self.assertCountEqual(
[], self._search(fs, experiment_id, filter_str="tags.generic_2 LIKE '%other'")
)
self.assertCountEqual(
[r2], self._search(fs, experiment_id, filter_str="tags.generic_2 ILIKE '%OTHER%'")
)
def test_create_run_returns_expected_run_data(self):
fs = FileStore(self.test_root)
no_tags_run = fs.create_run(
experiment_id=FileStore.DEFAULT_EXPERIMENT_ID, user_id="user", start_time=0, tags=[]
)
assert isinstance(no_tags_run.data, RunData)
assert len(no_tags_run.data.tags) == 0
tags_dict = {
"my_first_tag": "first",
"my-second-tag": "2nd",
}
tags_entities = [RunTag(key, value) for key, value in tags_dict.items()]
tags_run = fs.create_run(
experiment_id=FileStore.DEFAULT_EXPERIMENT_ID,
user_id="user",
start_time=0,
tags=tags_entities,
)
assert isinstance(tags_run.data, RunData)
assert tags_run.data.tags == tags_dict
def test_create_run_appends_to_artifact_uri_path_correctly(self):
cases = [
("path/to/local/folder", "path/to/local/folder/{e}/{r}/artifacts"),
("/path/to/local/folder",
"/path/to/local/folder/{e}/{r}/artifacts"),
("#path/to/local/folder?",
"#path/to/local/folder?/{e}/{r}/artifacts"),
("file:path/to/local/folder",
"file:path/to/local/folder/{e}/{r}/artifacts"),
("file:///path/to/local/folder",
"file:///path/to/local/folder/{e}/{r}/artifacts"),
("file:path/to/local/folder?param=value",
"file:path/to/local/folder/{e}/{r}/artifacts?param=value"),
("file:///path/to/local/folder",
"file:///path/to/local/folder/{e}/{r}/artifacts"),
(
"file:///path/to/local/folder?param=value#fragment",
"file:///path/to/local/folder/{e}/{r}/artifacts?param=value#fragment",
),
("s3://bucket/path/to/root",
"s3://bucket/path/to/root/{e}/{r}/artifacts"),
(
"s3://bucket/path/to/root?creds=mycreds",
"s3://bucket/path/to/root/{e}/{r}/artifacts?creds=mycreds",
),
(
"dbscheme+driver://root@host/dbname?creds=mycreds#myfragment",
"dbscheme+driver://root@host/dbname/{e}/{r}/artifacts?creds=mycreds#myfragment",
),
(
"dbscheme+driver://root:[email protected]?creds=mycreds#myfragment",
"dbscheme+driver://root:[email protected]/{e}/{r}/artifacts"
"?creds=mycreds#myfragment",
),
(
"dbscheme+driver://root:[email protected]/mydb?creds=mycreds#myfragment",
"dbscheme+driver://root:[email protected]/mydb/{e}/{r}/artifacts"
"?creds=mycreds#myfragment",
),
]
for artifact_root_uri, expected_artifact_uri_format in cases:
with TempDir() as tmp:
fs = FileStore(tmp.path(), artifact_root_uri)
exp_id = fs.create_experiment("exp")
run = fs.create_run(experiment_id=exp_id,
user_id='user',
start_time=0,
tags=[])
self.assertEqual(
run.info.artifact_uri,
expected_artifact_uri_format.format(e=exp_id,
r=run.info.run_id))
def test_search_runs_pagination(self):
fs = FileStore(self.test_root)
exp = fs.create_experiment("test_search_runs_pagination")
# test returned token behavior
runs = sorted([fs.create_run(exp, "user", 1000, []).info.run_id for r in range(10)])
result = fs.search_runs([exp], None, ViewType.ALL, max_results=4)
assert [r.info.run_id for r in result] == runs[0:4]
assert result.token is not None
result = fs.search_runs([exp], None, ViewType.ALL, max_results=4, page_token=result.token)
assert [r.info.run_id for r in result] == runs[4:8]
assert result.token is not None
result = fs.search_runs([exp], None, ViewType.ALL, max_results=4, page_token=result.token)
assert [r.info.run_id for r in result] == runs[8:]
assert result.token is None
def test_search_with_max_results(self):
fs = FileStore(self.test_root)
exp = fs.create_experiment("search_with_max_results")
runs = [fs.create_run(exp, "user", r, []).info.run_id for r in range(10)]
runs.reverse()
print(runs)
print(self._search(fs, exp))
assert runs[:10] == self._search(fs, exp)
for n in [0, 1, 2, 4, 8, 10, 20, 50, 100, 500, 1000, 1200, 2000]:
assert runs[: min(1200, n)] == self._search(fs, exp, max_results=n)
with self.assertRaises(MlflowException) as e:
self._search(fs, exp, None, max_results=int(1e10))
self.assertIn("Invalid value for request parameter max_results. It ", e.exception.message)
