def init_experiment(self, name, optimizer, param_defs, exp_id=None, notes=None, optimizer_arguments=None, minimization=True): """ Initializes an experiment. Parameters ---------- name : string name of the experiment. optimizer : string String representation of the optimizer. param_defs : dict of parameter definitions Dictionary of parameter definition classes. optimizer_arguments : dict, optional A dictionary defining the operation of the optimizer. See the respective documentation of the optimizers. Default is None, which are default values. exp_id : string or None, optional The id of the experiment, which will be used to reference it. Should be a proper uuid, and especially has to be unique. If it is not, an error may be returned. notes : jsonable object or None, optional Any note that you'd like to put in the experiment. Could be used to provide some details on the experiment, on the start time or the user starting it. minimization : bool, optional Whether the problem is one of minimization. Defaults to True. Returns ------- exp_id : string String representing the id of the experiment or "failed" if failed. Raises ------ ValueError : Iff there already is an experiment with the exp_id for this lab assistant. Does not occur if no exp_id is given. """ if exp_id in self._exp_assistants.keys(): raise ValueError("Already an experiment with id %s registered." %exp_id) if exp_id is None: while True: exp_id = uuid.uuid4().hex if exp_id not in self._exp_assistants.keys(): break exp_ass = ExperimentAssistant(optimizer, optimizer_arguments=optimizer_arguments, write_directory_base=self._lab_run_directory, csv_write_frequency=1) exp_ass.init_experiment(name, param_defs, exp_id, notes, minimization) self._exp_assistants[exp_id] = exp_ass self._logger.info("Experiment initialized successfully.") return exp_id
def init_experiment(self, name, optimizer, param_defs, exp_id=None, notes=None, optimizer_arguments=None, minimization=True): """ Initializes a new experiment. This actually initializes self.cv many experiments. Internally, the experiments are called name_i, where name is the experiment_name and i is the number of the experiment. Parameters ---------- name : string The name of the experiment. This has to be unique. optimizer : Optimizer instance or string This is an optimizer implementing the corresponding functions: It gets an experiment instance, and returns one or multiple candidates which should be evaluated next. Alternatively, it can be a string corresponding to the optimizer, as defined by apsis.utilities.optimizer_utils. param_defs : dict of ParamDef. This is the parameter space defining the experiment. optimizer_arguments : dict, optional These are arguments for the optimizer. Refer to their documentation as to which are available. minimization : bool, optional Whether the problem is one of minimization or maximization. """ self._logger.info("Initializing new experiment \"%s\". " " Parameter definitions: %s. Minimization is %s" %(name, param_defs, minimization)) if exp_id in self._exp_assistants: raise ValueError("Already an experiment with id %s registered." %name) if exp_id is None: while True: exp_id = uuid.uuid4().hex if exp_id not in self._exp_assistants.keys(): break self._exp_assistants[exp_id] = [] self.candidates_pending[exp_id] = [] for i in range(self.cv): exp_ass = ExperimentAssistant(optimizer, optimizer_arguments=optimizer_arguments, write_directory_base=self._lab_run_directory, csv_write_frequency=1) exp_ass.init_experiment(name + "_" + str(i), param_defs, exp_id, notes, minimization) self._exp_assistants[exp_id].append(exp_ass) self.candidates_pending[exp_id].append([]) self._logger.info("Experiment initialized successfully.") return exp_id
class TestExperimentAssistant(object): """ Tests the experiment assistant. """ EAss = None param_defs = None def setup(self): """ Tests whether the initialization works correctly. Tests: - optimizer correct - minimization correct - param_defs correct """ optimizer = "RandomSearch" name = "test_init_experiment" self.param_defs = { "x": MinMaxNumericParamDef(0, 1), "name": NominalParamDef(["A", "B", "C"]) } minimization = True optimizer_params = { "multiprocessing": "none" } self.EAss = ExperimentAssistant(optimizer, optimizer_arguments=optimizer_params) self.EAss.init_experiment(name, param_defs=self.param_defs, minimization=minimization) assert_equal(self.EAss._optimizer.__class__.__name__, optimizer) assert_equal(self.EAss._optimizer_arguments, optimizer_params) assert_equal(self.EAss._experiment.minimization_problem, minimization) def test_init_experiment(self): optimizer = "RandomSearch" name = "test_init_experiment" self.param_defs = { "x": MinMaxNumericParamDef(0, 1), "name": NominalParamDef(["A", "B", "C"]) } minimization = True optimizer_params = { "multiprocessing": "none" } self.EAss = ExperimentAssistant(optimizer, optimizer_arguments=optimizer_params) self.EAss.init_experiment(name, param_defs=self.param_defs, minimization=minimization) with assert_raises(ValueError): self.EAss.init_experiment(name, param_defs=self.param_defs, minimization=minimization) with assert_raises(ValueError): self.EAss.set_experiment("this value does not matter.") def teardown(self): self.EAss.set_exit() def test_get_next_candidate(self): """ Tests the get next candidate function. Tests: - The candidate's parameters are acceptable """ cand = None counter = 0 while cand is None and counter < 20: cand = self.EAss.get_next_candidate() time.sleep(0.1) counter += 1 if counter == 20: raise Exception("Received no result in the first 2 seconds.") assert_is_none(cand.result) params = cand.params assert_less_equal(params["x"], 1) assert_greater_equal(params["x"], 0) assert_in(params["name"], self.param_defs["name"].values) self.EAss.update(cand, "pausing") time.sleep(1) new_cand = None while new_cand is None and counter < 20: new_cand = self.EAss.get_next_candidate() time.sleep(0.1) counter += 1 if counter == 20: raise Exception("Received no result in the first 2 seconds.") assert_equal(new_cand, cand) def test_update(self): """ Tests whether update works. - candidate exists in the list - result is equal - the status message incorrect error works - the candidate instance check works """ cand = self.EAss.get_next_candidate() cand.result = 1 self.EAss.update(cand) assert_items_equal(self.EAss._experiment.candidates_finished, [cand]) assert_equal(self.EAss._experiment.candidates_finished[0].result, 1) self.EAss.update(cand, "pausing") self.EAss.update(cand, "working") with assert_raises(ValueError): self.EAss.update(cand, status="No status.") with assert_raises(ValueError): self.EAss.update(False) def test_get_best_candidate(self): """ Tests whether get_best_candidate works. - Whether the best of the two candidates is the one it should be. """ cand_one = self.EAss.get_next_candidate() cand_one.result = 1 self.EAss.update(cand_one) cand_two = self.EAss.get_next_candidate() cand_two.result = 0 self.EAss.update(cand_two) assert_equal(cand_two, self.EAss.get_best_candidate()) def test_all_plots_working(self): """ Tests whether all of the plot functions work. Does not test for correctness. """ cand = self.EAss.get_next_candidate() cand.result = 1 self.EAss.update(cand) cand = self.EAss.get_next_candidate() cand.result = 0 cand = self.EAss.get_next_candidate() cand.result = 2 self.EAss.plot_result_per_step() def test_get_candidates_dict(self): candidates_dict = self.EAss.get_candidates() assert_true(isinstance(candidates_dict, dict)) for l in ["finished", "pending", "working"]: assert_in(l, candidates_dict) assert_true(isinstance(candidates_dict[l], list))