def test_delete_restore_experiment(self): fs = FileStore(self.test_root) exp_id = self.experiments[random_int(0, len(self.experiments) - 1)] exp_name = self.exp_data[exp_id]["name"] # delete it fs.delete_experiment(exp_id) self.assertTrue(exp_id not in self._extract_ids(fs.list_experiments(ViewType.ACTIVE_ONLY))) self.assertTrue(exp_id in self._extract_ids(fs.list_experiments(ViewType.DELETED_ONLY))) self.assertTrue(exp_id in self._extract_ids(fs.list_experiments(ViewType.ALL))) self.assertEqual(fs.get_experiment(exp_id).lifecycle_stage, Experiment.DELETED_LIFECYCLE) # restore it fs.restore_experiment(exp_id) restored_1 = fs.get_experiment(exp_id) self.assertEqual(restored_1.experiment_id, exp_id) self.assertEqual(restored_1.name, exp_name) restored_2 = fs.get_experiment_by_name(exp_name) self.assertEqual(restored_2.experiment_id, exp_id) self.assertEqual(restored_2.name, exp_name) self.assertTrue(exp_id in self._extract_ids(fs.list_experiments(ViewType.ACTIVE_ONLY))) self.assertTrue(exp_id not in self._extract_ids(fs.list_experiments(ViewType.DELETED_ONLY))) self.assertTrue(exp_id in self._extract_ids(fs.list_experiments(ViewType.ALL))) self.assertEqual(fs.get_experiment(exp_id).lifecycle_stage, Experiment.ACTIVE_LIFECYCLE)
def test_get_experiment(self): fs = FileStore(self.test_root) for exp_id in self.experiments: self._verify_experiment(fs, exp_id) # test that fake experiments dont exist. # look for random experiment ids between 8000, 15000 since created ones are (100, 2000) for exp_id in set(random_int(8000, 15000) for x in range(20)): with self.assertRaises(Exception): fs.get_experiment(exp_id)
def test_get_experiment(self): fs = FileStore(self.test_root) for exp_id in self.experiments: exp = fs.get_experiment(exp_id) self.assertEqual(exp.experiment_id, exp_id) self.assertEqual(exp.name, self.exp_data[exp_id]["name"]) self.assertEqual(exp.artifact_location, self.exp_data[exp_id]["artifact_location"]) # test that fake experiments dont exist. # look for random experiment ids between 8000, 15000 since created ones are (100, 2000) for exp_id in set(random_int(8000, 15000) for x in range(20)): with self.assertRaises(Exception): fs.get_experiment(exp_id)
def test_malformed_experiment(self): fs = FileStore(self.test_root) exp_0 = fs.get_experiment(Experiment.DEFAULT_EXPERIMENT_ID) assert exp_0.experiment_id == Experiment.DEFAULT_EXPERIMENT_ID experiments = len(fs.list_experiments(ViewType.ALL)) # delete metadata file. path = os.path.join(self.test_root, str(exp_0.experiment_id), "meta.yaml") os.remove(path) with pytest.raises(MissingConfigException) as e: fs.get_experiment(Experiment.DEFAULT_EXPERIMENT_ID) assert e.message.contains("does not exist") assert len(fs.list_experiments(ViewType.ALL)) == experiments - 1
def test_bad_experiment_id_recorded_for_run(self): fs = FileStore(self.test_root) exp_0 = fs.get_experiment(Experiment.DEFAULT_EXPERIMENT_ID) all_runs = fs.search_runs([exp_0.experiment_id], [], run_view_type=ViewType.ALL) all_run_ids = self.exp_data[exp_0.experiment_id]["runs"] assert len(all_runs) == len(all_run_ids) # change experiment pointer in run bad_run_id = str(self.exp_data[exp_0.experiment_id]['runs'][0]) path = os.path.join(self.test_root, str(exp_0.experiment_id), bad_run_id) experiment_data = read_yaml(path, "meta.yaml") experiment_data["experiment_id"] = 1 write_yaml(path, "meta.yaml", experiment_data, True) with pytest.raises(MlflowException) as e: fs.get_run(bad_run_id) assert e.message.contains("not found") valid_runs = fs.search_runs([exp_0.experiment_id], [], run_view_type=ViewType.ALL) assert len(valid_runs) == len(all_runs) - 1 for rid in all_run_ids: if rid != bad_run_id: fs.get_run(rid)
def test_create_experiment(self): fs = FileStore(self.test_root) # Error cases with self.assertRaises(Exception): fs.create_experiment(None) with self.assertRaises(Exception): fs.create_experiment("") exp_id_ints = (int(exp_id) for exp_id in self.experiments) next_id = str(max(exp_id_ints) + 1) name = random_str(25) # since existing experiments are 10 chars long created_id = fs.create_experiment(name) # test that newly created experiment matches expected id self.assertEqual(created_id, next_id) # get the new experiment (by id) and verify (by name) exp1 = fs.get_experiment(created_id) self.assertEqual(exp1.name, name) self.assertEqual( exp1.artifact_location, path_to_local_file_uri(posixpath.join(self.test_root, created_id))) # get the new experiment (by name) and verify (by id) exp2 = fs.get_experiment_by_name(name) self.assertEqual(exp2.experiment_id, created_id)
def test_malformed_run(self): fs = FileStore(self.test_root) exp_0 = fs.get_experiment(Experiment.DEFAULT_EXPERIMENT_ID) all_runs = fs.search_runs([exp_0.experiment_id], [], run_view_type=ViewType.ALL) all_run_ids = self.exp_data[exp_0.experiment_id]["runs"] assert len(all_runs) == len(all_run_ids) # delete metadata file. bad_run_id = self.exp_data[exp_0.experiment_id]['runs'][0] path = os.path.join(self.test_root, str(exp_0.experiment_id), str(bad_run_id), "meta.yaml") os.remove(path) with pytest.raises(MissingConfigException) as e: fs.get_run(bad_run_id) assert e.message.contains("does not exist") valid_runs = fs.search_runs([exp_0.experiment_id], [], run_view_type=ViewType.ALL) assert len(valid_runs) == len(all_runs) - 1 for rid in all_run_ids: if rid != bad_run_id: fs.get_run(rid)
def test_mismatching_experiment_id(self): fs = FileStore(self.test_root) exp_0 = fs.get_experiment(Experiment.DEFAULT_EXPERIMENT_ID) assert exp_0.experiment_id == Experiment.DEFAULT_EXPERIMENT_ID experiments = len(fs.list_experiments(ViewType.ALL)) # mv experiment folder target = 1 path_orig = os.path.join(self.test_root, str(exp_0.experiment_id)) path_new = os.path.join(self.test_root, str(target)) os.rename(path_orig, path_new) with pytest.raises(MlflowException) as e: fs.get_experiment(Experiment.DEFAULT_EXPERIMENT_ID) assert e.message.contains("Could not find experiment with ID") with pytest.raises(MlflowException) as e: fs.get_experiment(target) assert e.message.contains("does not exist") assert len(fs.list_experiments(ViewType.ALL)) == experiments - 1
def test_rename_experiment(self): fs = FileStore(self.test_root) exp_id = self.experiments[random_int(0, len(self.experiments) - 1)] exp_name = self.exp_data[exp_id]["name"] new_name = exp_name + "!!!" self.assertNotEqual(exp_name, new_name) self.assertEqual(fs.get_experiment(exp_id).name, exp_name) fs.rename_experiment(exp_id, new_name) self.assertEqual(fs.get_experiment(exp_id).name, new_name) # Ensure that we cannot rename deleted experiments. fs.delete_experiment(exp_id) with pytest.raises(Exception) as e: fs.rename_experiment(exp_id, exp_name) assert 'non-active lifecycle' in str(e.value) self.assertEqual(fs.get_experiment(exp_id).name, new_name) # Restore the experiment, and confirm that we acn now rename it. fs.restore_experiment(exp_id) self.assertEqual(fs.get_experiment(exp_id).name, new_name) fs.rename_experiment(exp_id, exp_name) self.assertEqual(fs.get_experiment(exp_id).name, exp_name)
def test_create_experiment(self): fs = FileStore(self.test_root) next_id = max(self.experiments) + 1 name = random_str(25) # since existing experiments are 10 chars long created_id = fs.create_experiment(name) # test that newly created experiment matches expected id self.assertEqual(created_id, next_id) # get the new experiment (by id) and verify (by name) exp1 = fs.get_experiment(created_id) self.assertEqual(exp1.name, name) # get the new experiment (by name) and verify (by id) exp2 = fs.get_experiment_by_name(name) self.assertEqual(exp2.experiment_id, created_id)
def test_set_experiment_tags(self): fs = FileStore(self.test_root) fs.set_experiment_tag(FileStore.DEFAULT_EXPERIMENT_ID, ExperimentTag("tag0", "value0")) fs.set_experiment_tag(FileStore.DEFAULT_EXPERIMENT_ID, ExperimentTag("tag1", "value1")) experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) assert len(experiment.tags) == 2 assert experiment.tags["tag0"] == "value0" assert experiment.tags["tag1"] == "value1" # test that updating a tag works fs.set_experiment_tag(FileStore.DEFAULT_EXPERIMENT_ID, ExperimentTag("tag0", "value00000")) experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) assert experiment.tags["tag0"] == "value00000" assert experiment.tags["tag1"] == "value1" # test that setting a tag on 1 experiment does not impact another experiment. exp_id = None for exp in self.experiments: if exp != FileStore.DEFAULT_EXPERIMENT_ID: exp_id = exp break experiment = fs.get_experiment(exp_id) assert len(experiment.tags) == 0 # setting a tag on different experiments maintains different values across experiments fs.set_experiment_tag(exp_id, ExperimentTag("tag1", "value11111")) experiment = fs.get_experiment(exp_id) assert len(experiment.tags) == 1 assert experiment.tags["tag1"] == "value11111" experiment = fs.get_experiment(FileStore.DEFAULT_EXPERIMENT_ID) assert experiment.tags["tag0"] == "value00000" assert experiment.tags["tag1"] == "value1" # test can set multi-line tags fs.set_experiment_tag(exp_id, ExperimentTag("multiline_tag", "value2\nvalue2\nvalue2")) experiment = fs.get_experiment(exp_id) assert experiment.tags["multiline_tag"] == "value2\nvalue2\nvalue2" # test cannot set tags on deleted experiments fs.delete_experiment(exp_id) with pytest.raises(MlflowException): fs.set_experiment_tag(exp_id, ExperimentTag("should", "notset"))
def test_default_experiment_initialization(self): fs = FileStore(self.test_root) fs.delete_experiment(Experiment.DEFAULT_EXPERIMENT_ID) fs = FileStore(self.test_root) assert fs.get_experiment(0).lifecycle_stage == LifecycleStage.DELETED