def test_create_run_in_deleted_experiment(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
# delete it
fs.delete_experiment(exp_id)
with pytest.raises(Exception):
fs.create_run(exp_id, 'user', 0, [])
def test_create_run_in_deleted_experiment(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
# delete it
fs.delete_experiment(exp_id)
with pytest.raises(Exception):
fs.create_run(exp_id, 'user', 'name', 'source_type', 'source_name', 'entry_point_name',
0, None, [], None)
def test_create_run_with_parent_id(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run = fs.create_run(exp_id, 'user', 'name', 'source_type', 'source_name',
'entry_point_name', 0, None, [], 'test_parent_run_id')
assert any([t.key == MLFLOW_PARENT_RUN_ID and t.value == 'test_parent_run_id'
for t in fs.get_all_tags(run.info.run_uuid)])
def test_log_batch(self):
fs = FileStore(self.test_root)
run = fs.create_run(experiment_id=Experiment.DEFAULT_EXPERIMENT_ID,
user_id='user',
run_name=None,
source_type='source_type',
source_name='source_name',
entry_point_name='entry_point_name',
start_time=0,
source_version=None,
tags=[],
parent_run_id=None)
run_uuid = run.info.run_uuid
metric_entities = [
Metric("m1", 0.87, 12345),
Metric("m2", 0.49, 12345)
]
param_entities = [Param("p1", "p1val"), Param("p2", "p2val")]
tag_entities = [RunTag("t1", "t1val"), RunTag("t2", "t2val")]
fs.log_batch(run_id=run_uuid,
metrics=metric_entities,
params=param_entities,
tags=tag_entities)
self._verify_logged(fs, run_uuid, metric_entities, param_entities,
tag_entities)
def test_log_batch(self):
fs = FileStore(self.test_root)
run = fs.create_run(experiment_id=Experiment.DEFAULT_EXPERIMENT_ID,
user_id='user',
run_name=None,
source_type='source_type',
source_name='source_name',
entry_point_name='entry_point_name',
start_time=0,
source_version=None,
tags=[],
parent_run_id=None)
run_uuid = run.info.run_uuid
metric_entities = [
Metric("m1", 0.87, 12345),
Metric("m2", 0.49, 12345)
]
param_entities = [Param("p1", "p1val"), Param("p2", "p2val")]
tag_entities = [RunTag("t1", "t1val"), RunTag("t2", "t2val")]
fs.log_batch(run_id=run_uuid,
metrics=metric_entities,
params=param_entities,
tags=tag_entities)
run = fs.get_run(run_uuid)
tags = [(t.key, t.value) for t in run.data.tags]
metrics = [(m.key, m.value, m.timestamp) for m in run.data.metrics]
params = [(p.key, p.value) for p in run.data.params]
assert set(tags) == set([("t1", "t1val"), ("t2", "t2val")])
assert set(metrics) == set([("m1", 0.87, 12345), ("m2", 0.49, 12345)])
assert set(params) == set([("p1", "p1val"), ("p2", "p2val")])
def test_search_tags(self):
fs = FileStore(self.test_root)
experiment_id = self.experiments[0]
r1 = fs.create_run(experiment_id, 'user', 0, []).info.run_id
r2 = fs.create_run(experiment_id, 'user', 0, []).info.run_id
fs.set_tag(r1, RunTag('generic_tag', 'p_val'))
fs.set_tag(r2, RunTag('generic_tag', 'p_val'))
fs.set_tag(r1, RunTag('generic_2', 'some value'))
fs.set_tag(r2, RunTag('generic_2', 'another value'))
fs.set_tag(r1, RunTag('p_a', 'abc'))
fs.set_tag(r2, RunTag('p_b', 'ABC'))
# test search returns both runs
six.assertCountEqual(
self, [r1, r2],
self._search(fs,
experiment_id,
filter_str="tags.generic_tag = 'p_val'"))
# test search returns appropriate run (same key different values per run)
six.assertCountEqual(
self, [r1],
self._search(fs,
experiment_id,
filter_str="tags.generic_2 = 'some value'"))
six.assertCountEqual(
self, [r2],
self._search(fs,
experiment_id,
filter_str="tags.generic_2='another value'"))
six.assertCountEqual(
self, [],
self._search(fs,
experiment_id,
filter_str="tags.generic_tag = 'wrong_val'"))
six.assertCountEqual(
self, [],
self._search(fs,
experiment_id,
filter_str="tags.generic_tag != 'p_val'"))
six.assertCountEqual(
self, [r1, r2],
self._search(fs,
experiment_id,
filter_str="tags.generic_tag != 'wrong_val'"))
six.assertCountEqual(
self, [r1, r2],
self._search(fs,
experiment_id,
filter_str="tags.generic_2 != 'wrong_val'"))
six.assertCountEqual(
self, [r1],
self._search(fs, experiment_id, filter_str="tags.p_a = 'abc'"))
six.assertCountEqual(
self, [r2],
self._search(fs, experiment_id, filter_str="tags.p_b = 'ABC'"))
def test_create_run_with_parent_id(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
run = fs.create_run(exp_id, 'user', 'name', 'source_type',
'source_name', 'entry_point_name', 0, None, [],
'test_parent_run_id')
assert fs.get_run(
run.info.run_uuid
).data.tags[MLFLOW_PARENT_RUN_ID] == 'test_parent_run_id'
def test_search_with_deterministic_max_results(self):
fs = FileStore(self.test_root)
exp = fs.create_experiment("test_search_with_deterministic_max_results")
# Create 10 runs with the same start_time.
# Sort based on run_id
runs = sorted([fs.create_run(exp, 'user', 1000, []).info.run_id
for r in range(10)])
for n in [0, 1, 2, 4, 8, 10, 20]:
assert(runs[:min(10, n)] == self._search(fs, exp, max_results=n))
def test_log_batch(self):
fs = FileStore(self.test_root)
run = fs.create_run(
experiment_id=FileStore.DEFAULT_EXPERIMENT_ID, user_id='user', start_time=0, tags=[])
run_id = run.info.run_id
metric_entities = [Metric("m1", 0.87, 12345, 0), Metric("m2", 0.49, 12345, 0)]
param_entities = [Param("p1", "p1val"), Param("p2", "p2val")]
tag_entities = [RunTag("t1", "t1val"), RunTag("t2", "t2val")]
fs.log_batch(
run_id=run_id, metrics=metric_entities, params=param_entities, tags=tag_entities)
self._verify_logged(fs, run_id, metric_entities, param_entities, tag_entities)
def test_create_run_returns_expected_run_data(self):
fs = FileStore(self.test_root)
no_tags_run = fs.create_run(
experiment_id=FileStore.DEFAULT_EXPERIMENT_ID, user_id='user', start_time=0, tags=[])
assert isinstance(no_tags_run.data, RunData)
assert len(no_tags_run.data.tags) == 0
tags_dict = {
"my_first_tag": "first",
"my-second-tag": "2nd",
}
tags_entities = [
RunTag(key, value) for key, value in tags_dict.items()
]
tags_run = fs.create_run(
experiment_id=FileStore.DEFAULT_EXPERIMENT_ID,
user_id='user',
start_time=0,
tags=tags_entities)
assert isinstance(tags_run.data, RunData)
assert tags_run.data.tags == tags_dict
def test_search_runs_pagination(self):
fs = FileStore(self.test_root)
exp = fs.create_experiment("test_search_runs_pagination")
# test returned token behavior
runs = sorted([fs.create_run(exp, 'user', 1000, []).info.run_id
for r in range(10)])
result = fs.search_runs([exp], None, ViewType.ALL, max_results=4)
assert [r.info.run_id for r in result] == runs[0:4]
assert result.token is not None
result = fs.search_runs([exp], None, ViewType.ALL, max_results=4,
page_token=result.token)
assert [r.info.run_id for r in result] == runs[4:8]
assert result.token is not None
result = fs.search_runs([exp], None, ViewType.ALL, max_results=4,
page_token=result.token)
assert [r.info.run_id for r in result] == runs[8:]
assert result.token is None
def test_search_with_max_results(self):
fs = FileStore(self.test_root)
exp = fs.create_experiment("search_with_max_results")
runs = [fs.create_run(exp, 'user', r, []).info.run_id
for r in range(10)]
runs.reverse()
print(runs)
print(self._search(fs, exp))
assert(runs[:10] == self._search(fs, exp))
for n in [0, 1, 2, 4, 8, 10, 20, 50, 100, 500, 1000, 1200, 2000]:
assert(runs[:min(1200, n)] == self._search(fs, exp, max_results=n))
with self.assertRaises(MlflowException) as e:
self._search(fs, exp, None, max_results=int(1e10))
self.assertIn("Invalid value for request parameter max_results. It ", e.exception.message)
