def test_bad_experiment_id_recorded_for_run(self): fs = FileStore(self.test_root) exp_0 = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) all_runs = self._search(fs, exp_0.experiment_id) all_run_ids = self.exp_data[exp_0.experiment_id]["runs"] assert len(all_runs) == len(all_run_ids) # change experiment pointer in run bad_run_id = str(self.exp_data[exp_0.experiment_id]['runs'][0]) path = os.path.join(self.test_root, str(exp_0.experiment_id), bad_run_id) experiment_data = read_yaml(path, "meta.yaml") experiment_data["experiment_id"] = 1 write_yaml(path, "meta.yaml", experiment_data, True) with pytest.raises(MlflowException) as e: fs.get_run(bad_run_id) assert e.message.contains("not found") valid_runs = self._search(fs, exp_0.experiment_id) assert len(valid_runs) == len(all_runs) - 1 for rid in all_run_ids: if rid != bad_run_id: fs.get_run(rid)
def test_malformed_experiment(self): fs = FileStore(self.test_root) exp_0 = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) assert exp_0.experiment_id == FileStore.DEFAULT_EXPERIMENT_ID experiments = len(fs.list_experiments(ViewType.ALL)) # delete metadata file. path = os.path.join(self.test_root, str(exp_0.experiment_id), "meta.yaml") os.remove(path) with pytest.raises(MissingConfigException) as e: fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) assert e.message.contains("does not exist") assert len(fs.list_experiments(ViewType.ALL)) == experiments - 1
def test_create_experiment(self): fs = FileStore(self.test_root) # Error cases with self.assertRaises(Exception): fs.create_experiment(None) with self.assertRaises(Exception): fs.create_experiment("") exp_id_ints = (int(exp_id) for exp_id in self.experiments) next_id = str(max(exp_id_ints) + 1) name = random_str(25) # since existing experiments are 10 chars long created_id = fs.create_experiment(name) # test that newly created experiment matches expected id self.assertEqual(created_id, next_id) # get the new experiment (by id) and verify (by name) exp1 = fs.get_experiment(created_id) self.assertEqual(exp1.name, name) self.assertEqual( exp1.artifact_location, path_to_local_file_uri(posixpath.join(self.test_root, created_id))) # get the new experiment (by name) and verify (by id) exp2 = fs.get_experiment_by_name(name) self.assertEqual(exp2.experiment_id, created_id)
def test_mismatching_experiment_id(self): fs = FileStore(self.test_root) exp_0 = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) assert exp_0.experiment_id == FileStore.DEFAULT_EXPERIMENT_ID experiments = len(fs.list_experiments(ViewType.ALL)) # mv experiment folder target = "1" path_orig = os.path.join(self.test_root, str(exp_0.experiment_id)) path_new = os.path.join(self.test_root, str(target)) os.rename(path_orig, path_new) with pytest.raises(MlflowException) as e: fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) assert e.message.contains("Could not find experiment with ID") with pytest.raises(MlflowException) as e: fs.get_experiment(target) assert e.message.contains("does not exist") assert len(fs.list_experiments(ViewType.ALL)) == experiments - 1
def test_rename_experiment(self): fs = FileStore(self.test_root) exp_id = self.experiments[random_int(0, len(self.experiments) - 1)] # Error cases with self.assertRaises(Exception): fs.rename_experiment(exp_id, None) with self.assertRaises(Exception): # test that names of existing experiments are checked before renaming other_exp_id = None for exp in self.experiments: if exp != exp_id: other_exp_id = exp break fs.rename_experiment(exp_id, fs.get_experiment(other_exp_id).name) exp_name = self.exp_data[exp_id]["name"] new_name = exp_name + "!!!" self.assertNotEqual(exp_name, new_name) self.assertEqual(fs.get_experiment(exp_id).name, exp_name) fs.rename_experiment(exp_id, new_name) self.assertEqual(fs.get_experiment(exp_id).name, new_name) # Ensure that we cannot rename deleted experiments. fs.delete_experiment(exp_id) with pytest.raises(Exception) as e: fs.rename_experiment(exp_id, exp_name) assert "non-active lifecycle" in str(e.value) self.assertEqual(fs.get_experiment(exp_id).name, new_name) # Restore the experiment, and confirm that we acn now rename it. fs.restore_experiment(exp_id) self.assertEqual(fs.get_experiment(exp_id).name, new_name) fs.rename_experiment(exp_id, exp_name) self.assertEqual(fs.get_experiment(exp_id).name, exp_name)
def test_rename_experiment(self): fs = FileStore(self.test_root) exp_id = self.experiments[random_int(0, len(self.experiments) - 1)] exp_name = self.exp_data[exp_id]["name"] new_name = exp_name + "!!!" self.assertNotEqual(exp_name, new_name) self.assertEqual(fs.get_experiment(exp_id).name, exp_name) fs.rename_experiment(exp_id, new_name) self.assertEqual(fs.get_experiment(exp_id).name, new_name) # Ensure that we cannot rename deleted experiments. fs.delete_experiment(exp_id) with pytest.raises(Exception) as e: fs.rename_experiment(exp_id, exp_name) assert 'non-active lifecycle' in str(e.value) self.assertEqual(fs.get_experiment(exp_id).name, new_name) # Restore the experiment, and confirm that we acn now rename it. fs.restore_experiment(exp_id) self.assertEqual(fs.get_experiment(exp_id).name, new_name) fs.rename_experiment(exp_id, exp_name) self.assertEqual(fs.get_experiment(exp_id).name, exp_name)
def test_delete_restore_experiment(self): fs = FileStore(self.test_root) exp_id = self.experiments[random_int(0, len(self.experiments) - 1)] exp_name = self.exp_data[exp_id]["name"] # delete it fs.delete_experiment(exp_id) self.assertTrue(exp_id not in self._extract_ids(fs.list_experiments(ViewType.ACTIVE_ONLY))) self.assertTrue(exp_id in self._extract_ids(fs.list_experiments(ViewType.DELETED_ONLY))) self.assertTrue(exp_id in self._extract_ids(fs.list_experiments(ViewType.ALL))) self.assertEqual(fs.get_experiment(exp_id).lifecycle_stage, LifecycleStage.DELETED) # restore it fs.restore_experiment(exp_id) restored_1 = fs.get_experiment(exp_id) self.assertEqual(restored_1.experiment_id, exp_id) self.assertEqual(restored_1.name, exp_name) restored_2 = fs.get_experiment_by_name(exp_name) self.assertEqual(restored_2.experiment_id, exp_id) self.assertEqual(restored_2.name, exp_name) self.assertTrue(exp_id in self._extract_ids(fs.list_experiments(ViewType.ACTIVE_ONLY))) self.assertTrue(exp_id not in self._extract_ids(fs.list_experiments(ViewType.DELETED_ONLY))) self.assertTrue(exp_id in self._extract_ids(fs.list_experiments(ViewType.ALL))) self.assertEqual(fs.get_experiment(exp_id).lifecycle_stage, LifecycleStage.ACTIVE)
def test_create_experiment_appends_to_artifact_uri_path_correctly(self): cases = [ ("path/to/local/folder", "path/to/local/folder/{e}"), ("/path/to/local/folder", "/path/to/local/folder/{e}"), ("#path/to/local/folder?", "#path/to/local/folder?/{e}"), ("file:path/to/local/folder", "file:path/to/local/folder/{e}"), ("file:///path/to/local/folder", "file:///path/to/local/folder/{e}"), ("file:path/to/local/folder?param=value", "file:path/to/local/folder/{e}?param=value"), ("file:///path/to/local/folder", "file:///path/to/local/folder/{e}"), ( "file:///path/to/local/folder?param=value#fragment", "file:///path/to/local/folder/{e}?param=value#fragment", ), ("s3://bucket/path/to/root", "s3://bucket/path/to/root/{e}"), ( "s3://bucket/path/to/root?creds=mycreds", "s3://bucket/path/to/root/{e}?creds=mycreds", ), ( "dbscheme+driver://root@host/dbname?creds=mycreds#myfragment", "dbscheme+driver://root@host/dbname/{e}?creds=mycreds#myfragment", ), ( "dbscheme+driver://root:[email protected]?creds=mycreds#myfragment", "dbscheme+driver://root:[email protected]/{e}?creds=mycreds#myfragment", ), ( "dbscheme+driver://root:[email protected]/mydb?creds=mycreds#myfragment", "dbscheme+driver://root:[email protected]/mydb/{e}?creds=mycreds#myfragment", ), ] for artifact_root_uri, expected_artifact_uri_format in cases: with TempDir() as tmp: fs = FileStore(tmp.path(), artifact_root_uri) exp_id = fs.create_experiment("exp") exp = fs.get_experiment(exp_id) self.assertEqual(exp.artifact_location, expected_artifact_uri_format.format(e=exp_id))
def test_malformed_run(self): fs = FileStore(self.test_root) exp_0 = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) all_runs = self._search(fs, exp_0.experiment_id) all_run_ids = self.exp_data[exp_0.experiment_id]["runs"] assert len(all_runs) == len(all_run_ids) # delete metadata file. bad_run_id = self.exp_data[exp_0.experiment_id]["runs"][0] path = os.path.join(self.test_root, str(exp_0.experiment_id), str(bad_run_id), "meta.yaml") os.remove(path) with pytest.raises(MissingConfigException) as e: fs.get_run(bad_run_id) assert e.message.contains("does not exist") valid_runs = self._search(fs, exp_0.experiment_id) assert len(valid_runs) == len(all_runs) - 1 for rid in all_run_ids: if rid != bad_run_id: fs.get_run(rid)
def test_set_experiment_tags(self): fs = FileStore(self.test_root) fs.set_experiment_tag(FileStore.DEFAULT_EXPERIMENT_ID, ExperimentTag("tag0", "value0")) fs.set_experiment_tag(FileStore.DEFAULT_EXPERIMENT_ID, ExperimentTag("tag1", "value1")) experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) assert len(experiment.tags) == 2 assert experiment.tags["tag0"] == "value0" assert experiment.tags["tag1"] == "value1" # test that updating a tag works fs.set_experiment_tag(FileStore.DEFAULT_EXPERIMENT_ID, ExperimentTag("tag0", "value00000")) experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) assert experiment.tags["tag0"] == "value00000" assert experiment.tags["tag1"] == "value1" # test that setting a tag on 1 experiment does not impact another experiment. exp_id = None for exp in self.experiments: if exp != FileStore.DEFAULT_EXPERIMENT_ID: exp_id = exp break experiment = fs.get_experiment(exp_id) assert len(experiment.tags) == 0 # setting a tag on different experiments maintains different values across experiments fs.set_experiment_tag(exp_id, ExperimentTag("tag1", "value11111")) experiment = fs.get_experiment(exp_id) assert len(experiment.tags) == 1 assert experiment.tags["tag1"] == "value11111" experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) assert experiment.tags["tag0"] == "value00000" assert experiment.tags["tag1"] == "value1" # test can set multi-line tags fs.set_experiment_tag( exp_id, ExperimentTag("multiline_tag", "value2\nvalue2\nvalue2")) experiment = fs.get_experiment(exp_id) assert experiment.tags["multiline_tag"] == "value2\nvalue2\nvalue2" # test cannot set tags on deleted experiments fs.delete_experiment(exp_id) with pytest.raises(MlflowException): fs.set_experiment_tag(exp_id, ExperimentTag("should", "notset"))
def test_default_experiment_initialization(self): fs = FileStore(self.test_root) fs.delete_experiment(FileStore.DEFAULT_EXPERIMENT_ID) fs = FileStore(self.test_root) experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) assert experiment.lifecycle_stage == LifecycleStage.DELETED