def test_create_experiment(self): fs = FileStore(self.test_root) # Error cases with self.assertRaises(Exception): fs.create_experiment(None) with self.assertRaises(Exception): fs.create_experiment("") exp_id_ints = (int(exp_id) for exp_id in self.experiments) next_id = str(max(exp_id_ints) + 1) name = random_str(25) # since existing experiments are 10 chars long created_id = fs.create_experiment(name) # test that newly created experiment matches expected id self.assertEqual(created_id, next_id) # get the new experiment (by id) and verify (by name) exp1 = fs.get_experiment(created_id) self.assertEqual(exp1.name, name) self.assertEqual( exp1.artifact_location, path_to_local_file_uri(posixpath.join(self.test_root, created_id))) # get the new experiment (by name) and verify (by id) exp2 = fs.get_experiment_by_name(name) self.assertEqual(exp2.experiment_id, created_id)
def test_create_first_experiment(self): fs = FileStore(self.test_root) fs.list_experiments = mock.Mock(return_value=[]) fs._create_experiment_with_id = mock.Mock() fs.create_experiment(random_str(1)) fs._create_experiment_with_id.assert_called_once() experiment_id = fs._create_experiment_with_id.call_args[0][1] self.assertEqual(experiment_id, 0)
def test_search_with_deterministic_max_results(self): fs = FileStore(self.test_root) exp = fs.create_experiment("test_search_with_deterministic_max_results") # Create 10 runs with the same start_time. # Sort based on run_id runs = sorted([fs.create_run(exp, 'user', 1000, []).info.run_id for r in range(10)]) for n in [0, 1, 2, 4, 8, 10, 20]: assert(runs[:min(10, n)] == self._search(fs, exp, max_results=n))
def test_create_experiment(self): fs = FileStore(self.test_root) # Error cases with self.assertRaises(Exception): fs.create_experiment(None) with self.assertRaises(Exception): fs.create_experiment("") next_id = max(self.experiments) + 1 name = random_str(25) # since existing experiments are 10 chars long created_id = fs.create_experiment(name) # test that newly created experiment matches expected id self.assertEqual(created_id, next_id) # get the new experiment (by id) and verify (by name) exp1 = fs.get_experiment(created_id) self.assertEqual(exp1.name, name) # get the new experiment (by name) and verify (by id) exp2 = fs.get_experiment_by_name(name) self.assertEqual(exp2.experiment_id, created_id)
def test_search_runs_pagination(self): fs = FileStore(self.test_root) exp = fs.create_experiment("test_search_runs_pagination") # test returned token behavior runs = sorted([fs.create_run(exp, 'user', 1000, []).info.run_id for r in range(10)]) result = fs.search_runs([exp], None, ViewType.ALL, max_results=4) assert [r.info.run_id for r in result] == runs[0:4] assert result.token is not None result = fs.search_runs([exp], None, ViewType.ALL, max_results=4, page_token=result.token) assert [r.info.run_id for r in result] == runs[4:8] assert result.token is not None result = fs.search_runs([exp], None, ViewType.ALL, max_results=4, page_token=result.token) assert [r.info.run_id for r in result] == runs[8:] assert result.token is None
def test_search_with_max_results(self): fs = FileStore(self.test_root) exp = fs.create_experiment("search_with_max_results") runs = [fs.create_run(exp, 'user', r, []).info.run_id for r in range(10)] runs.reverse() print(runs) print(self._search(fs, exp)) assert(runs[:10] == self._search(fs, exp)) for n in [0, 1, 2, 4, 8, 10, 20, 50, 100, 500, 1000, 1200, 2000]: assert(runs[:min(1200, n)] == self._search(fs, exp, max_results=n)) with self.assertRaises(MlflowException) as e: self._search(fs, exp, None, max_results=int(1e10)) self.assertIn("Invalid value for request parameter max_results. It ", e.exception.message)
def test_create_duplicate_experiments(self): fs = FileStore(self.test_root) for exp_id in self.experiments: name = self.exp_data[exp_id]["name"] with self.assertRaises(Exception): fs.create_experiment(name)