def test_bad_experiment_id_recorded_for_run(self):
fs = FileStore(self.test_root)
exp_0 = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
all_runs = self._search(fs, exp_0.experiment_id)
all_run_ids = self.exp_data[exp_0.experiment_id]["runs"]
assert len(all_runs) == len(all_run_ids)
# change experiment pointer in run
bad_run_id = str(self.exp_data[exp_0.experiment_id]['runs'][0])
path = os.path.join(self.test_root, str(exp_0.experiment_id),
bad_run_id)
experiment_data = read_yaml(path, "meta.yaml")
experiment_data["experiment_id"] = 1
write_yaml(path, "meta.yaml", experiment_data, True)
with pytest.raises(MlflowException) as e:
fs.get_run(bad_run_id)
assert e.message.contains("not found")
valid_runs = self._search(fs, exp_0.experiment_id)
assert len(valid_runs) == len(all_runs) - 1
for rid in all_run_ids:
if rid != bad_run_id:
fs.get_run(rid)
def test_malformed_experiment(self):
fs = FileStore(self.test_root)
exp_0 = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert exp_0.experiment_id == FileStore.DEFAULT_EXPERIMENT_ID
experiments = len(fs.list_experiments(ViewType.ALL))
# delete metadata file.
path = os.path.join(self.test_root, str(exp_0.experiment_id),
"meta.yaml")
os.remove(path)
with pytest.raises(MissingConfigException) as e:
fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert e.message.contains("does not exist")
assert len(fs.list_experiments(ViewType.ALL)) == experiments - 1
def test_create_experiment(self):
fs = FileStore(self.test_root)
# Error cases
with self.assertRaises(Exception):
fs.create_experiment(None)
with self.assertRaises(Exception):
fs.create_experiment("")
exp_id_ints = (int(exp_id) for exp_id in self.experiments)
next_id = str(max(exp_id_ints) + 1)
name = random_str(25) # since existing experiments are 10 chars long
created_id = fs.create_experiment(name)
# test that newly created experiment matches expected id
self.assertEqual(created_id, next_id)
# get the new experiment (by id) and verify (by name)
exp1 = fs.get_experiment(created_id)
self.assertEqual(exp1.name, name)
self.assertEqual(
exp1.artifact_location,
path_to_local_file_uri(posixpath.join(self.test_root, created_id)))
# get the new experiment (by name) and verify (by id)
exp2 = fs.get_experiment_by_name(name)
self.assertEqual(exp2.experiment_id, created_id)
def test_mismatching_experiment_id(self):
fs = FileStore(self.test_root)
exp_0 = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert exp_0.experiment_id == FileStore.DEFAULT_EXPERIMENT_ID
experiments = len(fs.list_experiments(ViewType.ALL))
# mv experiment folder
target = "1"
path_orig = os.path.join(self.test_root, str(exp_0.experiment_id))
path_new = os.path.join(self.test_root, str(target))
os.rename(path_orig, path_new)
with pytest.raises(MlflowException) as e:
fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert e.message.contains("Could not find experiment with ID")
with pytest.raises(MlflowException) as e:
fs.get_experiment(target)
assert e.message.contains("does not exist")
assert len(fs.list_experiments(ViewType.ALL)) == experiments - 1
def test_rename_experiment(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
# Error cases
with self.assertRaises(Exception):
fs.rename_experiment(exp_id, None)
with self.assertRaises(Exception):
# test that names of existing experiments are checked before renaming
other_exp_id = None
for exp in self.experiments:
if exp != exp_id:
other_exp_id = exp
break
fs.rename_experiment(exp_id, fs.get_experiment(other_exp_id).name)
exp_name = self.exp_data[exp_id]["name"]
new_name = exp_name + "!!!"
self.assertNotEqual(exp_name, new_name)
self.assertEqual(fs.get_experiment(exp_id).name, exp_name)
fs.rename_experiment(exp_id, new_name)
self.assertEqual(fs.get_experiment(exp_id).name, new_name)
# Ensure that we cannot rename deleted experiments.
fs.delete_experiment(exp_id)
with pytest.raises(Exception) as e:
fs.rename_experiment(exp_id, exp_name)
assert "non-active lifecycle" in str(e.value)
self.assertEqual(fs.get_experiment(exp_id).name, new_name)
# Restore the experiment, and confirm that we acn now rename it.
fs.restore_experiment(exp_id)
self.assertEqual(fs.get_experiment(exp_id).name, new_name)
fs.rename_experiment(exp_id, exp_name)
self.assertEqual(fs.get_experiment(exp_id).name, exp_name)
def test_rename_experiment(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
exp_name = self.exp_data[exp_id]["name"]
new_name = exp_name + "!!!"
self.assertNotEqual(exp_name, new_name)
self.assertEqual(fs.get_experiment(exp_id).name, exp_name)
fs.rename_experiment(exp_id, new_name)
self.assertEqual(fs.get_experiment(exp_id).name, new_name)
# Ensure that we cannot rename deleted experiments.
fs.delete_experiment(exp_id)
with pytest.raises(Exception) as e:
fs.rename_experiment(exp_id, exp_name)
assert 'non-active lifecycle' in str(e.value)
self.assertEqual(fs.get_experiment(exp_id).name, new_name)
# Restore the experiment, and confirm that we acn now rename it.
fs.restore_experiment(exp_id)
self.assertEqual(fs.get_experiment(exp_id).name, new_name)
fs.rename_experiment(exp_id, exp_name)
self.assertEqual(fs.get_experiment(exp_id).name, exp_name)
def test_delete_restore_experiment(self):
fs = FileStore(self.test_root)
exp_id = self.experiments[random_int(0, len(self.experiments) - 1)]
exp_name = self.exp_data[exp_id]["name"]
# delete it
fs.delete_experiment(exp_id)
self.assertTrue(exp_id not in self._extract_ids(fs.list_experiments(ViewType.ACTIVE_ONLY)))
self.assertTrue(exp_id in self._extract_ids(fs.list_experiments(ViewType.DELETED_ONLY)))
self.assertTrue(exp_id in self._extract_ids(fs.list_experiments(ViewType.ALL)))
self.assertEqual(fs.get_experiment(exp_id).lifecycle_stage, LifecycleStage.DELETED)
# restore it
fs.restore_experiment(exp_id)
restored_1 = fs.get_experiment(exp_id)
self.assertEqual(restored_1.experiment_id, exp_id)
self.assertEqual(restored_1.name, exp_name)
restored_2 = fs.get_experiment_by_name(exp_name)
self.assertEqual(restored_2.experiment_id, exp_id)
self.assertEqual(restored_2.name, exp_name)
self.assertTrue(exp_id in self._extract_ids(fs.list_experiments(ViewType.ACTIVE_ONLY)))
self.assertTrue(exp_id not in self._extract_ids(fs.list_experiments(ViewType.DELETED_ONLY)))
self.assertTrue(exp_id in self._extract_ids(fs.list_experiments(ViewType.ALL)))
self.assertEqual(fs.get_experiment(exp_id).lifecycle_stage, LifecycleStage.ACTIVE)
def test_create_experiment_appends_to_artifact_uri_path_correctly(self):
cases = [
("path/to/local/folder", "path/to/local/folder/{e}"),
("/path/to/local/folder", "/path/to/local/folder/{e}"),
("#path/to/local/folder?", "#path/to/local/folder?/{e}"),
("file:path/to/local/folder", "file:path/to/local/folder/{e}"),
("file:///path/to/local/folder",
"file:///path/to/local/folder/{e}"),
("file:path/to/local/folder?param=value",
"file:path/to/local/folder/{e}?param=value"),
("file:///path/to/local/folder",
"file:///path/to/local/folder/{e}"),
(
"file:///path/to/local/folder?param=value#fragment",
"file:///path/to/local/folder/{e}?param=value#fragment",
),
("s3://bucket/path/to/root", "s3://bucket/path/to/root/{e}"),
(
"s3://bucket/path/to/root?creds=mycreds",
"s3://bucket/path/to/root/{e}?creds=mycreds",
),
(
"dbscheme+driver://root@host/dbname?creds=mycreds#myfragment",
"dbscheme+driver://root@host/dbname/{e}?creds=mycreds#myfragment",
),
(
"dbscheme+driver://root:[email protected]?creds=mycreds#myfragment",
"dbscheme+driver://root:[email protected]/{e}?creds=mycreds#myfragment",
),
(
"dbscheme+driver://root:[email protected]/mydb?creds=mycreds#myfragment",
"dbscheme+driver://root:[email protected]/mydb/{e}?creds=mycreds#myfragment",
),
]
for artifact_root_uri, expected_artifact_uri_format in cases:
with TempDir() as tmp:
fs = FileStore(tmp.path(), artifact_root_uri)
exp_id = fs.create_experiment("exp")
exp = fs.get_experiment(exp_id)
self.assertEqual(exp.artifact_location,
expected_artifact_uri_format.format(e=exp_id))
def test_malformed_run(self):
fs = FileStore(self.test_root)
exp_0 = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
all_runs = self._search(fs, exp_0.experiment_id)
all_run_ids = self.exp_data[exp_0.experiment_id]["runs"]
assert len(all_runs) == len(all_run_ids)
# delete metadata file.
bad_run_id = self.exp_data[exp_0.experiment_id]["runs"][0]
path = os.path.join(self.test_root, str(exp_0.experiment_id), str(bad_run_id), "meta.yaml")
os.remove(path)
with pytest.raises(MissingConfigException) as e:
fs.get_run(bad_run_id)
assert e.message.contains("does not exist")
valid_runs = self._search(fs, exp_0.experiment_id)
assert len(valid_runs) == len(all_runs) - 1
for rid in all_run_ids:
if rid != bad_run_id:
fs.get_run(rid)
def test_set_experiment_tags(self):
fs = FileStore(self.test_root)
fs.set_experiment_tag(FileStore.DEFAULT_EXPERIMENT_ID,
ExperimentTag("tag0", "value0"))
fs.set_experiment_tag(FileStore.DEFAULT_EXPERIMENT_ID,
ExperimentTag("tag1", "value1"))
experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert len(experiment.tags) == 2
assert experiment.tags["tag0"] == "value0"
assert experiment.tags["tag1"] == "value1"
# test that updating a tag works
fs.set_experiment_tag(FileStore.DEFAULT_EXPERIMENT_ID,
ExperimentTag("tag0", "value00000"))
experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert experiment.tags["tag0"] == "value00000"
assert experiment.tags["tag1"] == "value1"
# test that setting a tag on 1 experiment does not impact another experiment.
exp_id = None
for exp in self.experiments:
if exp != FileStore.DEFAULT_EXPERIMENT_ID:
exp_id = exp
break
experiment = fs.get_experiment(exp_id)
assert len(experiment.tags) == 0
# setting a tag on different experiments maintains different values across experiments
fs.set_experiment_tag(exp_id, ExperimentTag("tag1", "value11111"))
experiment = fs.get_experiment(exp_id)
assert len(experiment.tags) == 1
assert experiment.tags["tag1"] == "value11111"
experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert experiment.tags["tag0"] == "value00000"
assert experiment.tags["tag1"] == "value1"
# test can set multi-line tags
fs.set_experiment_tag(
exp_id, ExperimentTag("multiline_tag", "value2\nvalue2\nvalue2"))
experiment = fs.get_experiment(exp_id)
assert experiment.tags["multiline_tag"] == "value2\nvalue2\nvalue2"
# test cannot set tags on deleted experiments
fs.delete_experiment(exp_id)
with pytest.raises(MlflowException):
fs.set_experiment_tag(exp_id, ExperimentTag("should", "notset"))
def test_default_experiment_initialization(self):
fs = FileStore(self.test_root)
fs.delete_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
fs = FileStore(self.test_root)
experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID)
assert experiment.lifecycle_stage == LifecycleStage.DELETED
