class TestExperimentAssistant(object): """ Tests the experiment assistant. """ EAss = None param_defs = None def setup(self): """ Tests whether the initialization works correctly. Tests: - optimizer correct - minimization correct - param_defs correct """ optimizer = "RandomSearch" name = "test_init_experiment" self.param_defs = { "x": MinMaxNumericParamDef(0, 1), "name": NominalParamDef(["A", "B", "C"]) } minimization = True optimizer_params = { "multiprocessing": "none" } self.EAss = ExperimentAssistant(optimizer, optimizer_arguments=optimizer_params) self.EAss.init_experiment(name, param_defs=self.param_defs, minimization=minimization) assert_equal(self.EAss._optimizer.__class__.__name__, optimizer) assert_equal(self.EAss._optimizer_arguments, optimizer_params) assert_equal(self.EAss._experiment.minimization_problem, minimization) def test_init_experiment(self): optimizer = "RandomSearch" name = "test_init_experiment" self.param_defs = { "x": MinMaxNumericParamDef(0, 1), "name": NominalParamDef(["A", "B", "C"]) } minimization = True optimizer_params = { "multiprocessing": "none" } self.EAss = ExperimentAssistant(optimizer, optimizer_arguments=optimizer_params) self.EAss.init_experiment(name, param_defs=self.param_defs, minimization=minimization) with assert_raises(ValueError): self.EAss.init_experiment(name, param_defs=self.param_defs, minimization=minimization) with assert_raises(ValueError): self.EAss.set_experiment("this value does not matter.") def teardown(self): self.EAss.set_exit() def test_get_next_candidate(self): """ Tests the get next candidate function. Tests: - The candidate's parameters are acceptable """ cand = None counter = 0 while cand is None and counter < 20: cand = self.EAss.get_next_candidate() time.sleep(0.1) counter += 1 if counter == 20: raise Exception("Received no result in the first 2 seconds.") assert_is_none(cand.result) params = cand.params assert_less_equal(params["x"], 1) assert_greater_equal(params["x"], 0) assert_in(params["name"], self.param_defs["name"].values) self.EAss.update(cand, "pausing") time.sleep(1) new_cand = None while new_cand is None and counter < 20: new_cand = self.EAss.get_next_candidate() time.sleep(0.1) counter += 1 if counter == 20: raise Exception("Received no result in the first 2 seconds.") assert_equal(new_cand, cand) def test_update(self): """ Tests whether update works. - candidate exists in the list - result is equal - the status message incorrect error works - the candidate instance check works """ cand = self.EAss.get_next_candidate() cand.result = 1 self.EAss.update(cand) assert_items_equal(self.EAss._experiment.candidates_finished, [cand]) assert_equal(self.EAss._experiment.candidates_finished[0].result, 1) self.EAss.update(cand, "pausing") self.EAss.update(cand, "working") with assert_raises(ValueError): self.EAss.update(cand, status="No status.") with assert_raises(ValueError): self.EAss.update(False) def test_get_best_candidate(self): """ Tests whether get_best_candidate works. - Whether the best of the two candidates is the one it should be. """ cand_one = self.EAss.get_next_candidate() cand_one.result = 1 self.EAss.update(cand_one) cand_two = self.EAss.get_next_candidate() cand_two.result = 0 self.EAss.update(cand_two) assert_equal(cand_two, self.EAss.get_best_candidate()) def test_all_plots_working(self): """ Tests whether all of the plot functions work. Does not test for correctness. """ cand = self.EAss.get_next_candidate() cand.result = 1 self.EAss.update(cand) cand = self.EAss.get_next_candidate() cand.result = 0 cand = self.EAss.get_next_candidate() cand.result = 2 self.EAss.plot_result_per_step() def test_get_candidates_dict(self): candidates_dict = self.EAss.get_candidates() assert_true(isinstance(candidates_dict, dict)) for l in ["finished", "pending", "working"]: assert_in(l, candidates_dict) assert_true(isinstance(candidates_dict[l], list))
def clone_experiments_by_id(self, exp_id, optimizer, optimizer_arguments, new_exp_name): """ Take an existing experiment managed by this lab assistant, fully clone it and store it under a new name to use it with a new optimizer. This functionality can be used to initialize several experiments of several optimizers with the same points. For the given exp_name all underlying experiment instances are cloned and renamed. Then a new experiment assistant is instantiated given the cloned and renamed experiment using the given optimizer. The new experiment assistants are stored and managed inside this lab assistant. The old experiment is not touched and continues to be part of this lab assistant. The parameter definitions and other experiment specific configuration is copied over from the old to the new experiment. Parameters ---------- exp_id : string The id of the experiment to be cloned. new_exp_name: string, optional The name the cloned experiment will have after creation. If None, the old name is reused. optimizer : Optimizer instance or string This is an optimizer implementing the corresponding functions: It gets an experiment instance, and returns one or multiple candidates which should be evaluated next. Alternatively, it can be a string corresponding to the optimizer, as defined by apsis.utilities.optimizer_utils. optimizer_arguments : dict, optional These are arguments for the optimizer. Refer to their documentation as to which are available. """ while True: new_exp_id = uuid.uuid4().hex if new_exp_id not in self._exp_assistants.keys(): break self._exp_assistants[new_exp_id] = [] #every experiment has self.cv many assistants for i in range(len(self._exp_assistants[exp_id])): old_exp_assistant = self._exp_assistants[exp_id][i] old_exp = old_exp_assistant._experiment #clone and rename experiment new_exp = old_exp.clone() new_name_cved = new_exp_name + "_" + str(i) new_exp.name = new_name_cved #recreate exp assistant new_exp_assistant = ExperimentAssistant(optimizer, optimizer_arguments=optimizer_arguments, write_directory_base=self._lab_run_directory, csv_write_frequency=1) new_exp_assistant.set_experiment(new_exp) self._exp_assistants[new_exp_id].append(new_exp_assistant) self.candidates_pending[new_exp_id] = copy.deepcopy(self.candidates_pending[exp_id]) self._logger.info("Experiment " + str(exp_id) + " cloned to " + str(new_exp_id) + " and successfully initialized.") return new_exp_id