def get_factorial_experiment( has_optimization_config: bool = True, with_batch: bool = False, with_status_quo: bool = False, ) -> Experiment: exp = Experiment( name="factorial_test_experiment", search_space=get_factorial_search_space(), optimization_config=OptimizationConfig( objective=Objective(metric=get_factorial_metric()) ) if has_optimization_config else None, runner=SyntheticRunner(), is_test=True, tracking_metrics=[get_factorial_metric("secondary_metric")], ) if with_status_quo: exp.status_quo = Arm( parameters={ "factor1": "level11", "factor2": "level21", "factor3": "level31", } ) if with_batch: factorial_generator = get_factorial(search_space=exp.search_space) factorial_run = factorial_generator.gen(n=-1) exp.new_batch_trial(optimize_for_power=with_status_quo).add_generator_run( factorial_run ) return exp
def get_branin_experiment( has_optimization_config: bool = True, with_batch: bool = False, with_status_quo: bool = False, with_fidelity_parameter: bool = False, with_choice_parameter: bool = False, search_space: Optional[SearchSpace] = None, ) -> Experiment: search_space = search_space or get_branin_search_space( with_fidelity_parameter=with_fidelity_parameter, with_choice_parameter=with_choice_parameter, ) exp = Experiment( name="branin_test_experiment", search_space=search_space, optimization_config=get_branin_optimization_config() if has_optimization_config else None, runner=SyntheticRunner(), is_test=True, ) if with_status_quo: exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0}) if with_batch: sobol_generator = get_sobol(search_space=exp.search_space) sobol_run = sobol_generator.gen(n=15) exp.new_batch_trial(optimize_for_power=with_status_quo).add_generator_run( sobol_run ) return exp
def get_branin_experiment_with_multi_objective( has_optimization_config: bool = True, has_objective_thresholds: bool = False, with_batch: bool = False, with_status_quo: bool = False, with_fidelity_parameter: bool = False, num_objectives: int = 2, ) -> Experiment: exp = Experiment( name="branin_test_experiment", search_space=get_branin_search_space( with_fidelity_parameter=with_fidelity_parameter), optimization_config=get_branin_multi_objective_optimization_config( has_objective_thresholds=has_objective_thresholds, num_objectives=num_objectives, ) if has_optimization_config else None, runner=SyntheticRunner(), is_test=True, ) if with_status_quo: # Experiment chooses the name "status_quo" by default exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0}) if with_batch: sobol_generator = get_sobol(search_space=exp.search_space, seed=TEST_SOBOL_SEED) sobol_run = sobol_generator.gen(n=5) exp.new_batch_trial( optimize_for_power=with_status_quo).add_generator_run(sobol_run) return exp
def get_branin_experiment( has_optimization_config: bool = True, with_batch: bool = False, with_trial: bool = False, with_status_quo: bool = False, with_fidelity_parameter: bool = False, with_choice_parameter: bool = False, with_str_choice_param: bool = False, search_space: Optional[SearchSpace] = None, minimize: bool = False, named: bool = True, with_completed_trial: bool = False, ) -> Experiment: search_space = search_space or get_branin_search_space( with_fidelity_parameter=with_fidelity_parameter, with_choice_parameter=with_choice_parameter, with_str_choice_param=with_str_choice_param, ) exp = Experiment( name="branin_test_experiment" if named else None, search_space=search_space, optimization_config=get_branin_optimization_config( minimize=minimize) if has_optimization_config else None, runner=SyntheticRunner(), is_test=True, ) if with_status_quo: exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0}) if with_batch: sobol_generator = get_sobol(search_space=exp.search_space) sobol_run = sobol_generator.gen(n=15) exp.new_batch_trial( optimize_for_power=with_status_quo).add_generator_run(sobol_run) if with_trial or with_completed_trial: sobol_generator = get_sobol(search_space=exp.search_space) sobol_run = sobol_generator.gen(n=1) trial = exp.new_trial(generator_run=sobol_run) if with_completed_trial: trial.mark_running(no_runner_required=True) exp.attach_data( get_branin_data(trials=[trial])) # Add data for one trial trial.mark_completed() return exp
def get_branin_with_multi_task(with_multi_objective: bool = False): exp = Experiment( name="branin_test_experiment", search_space=get_branin_search_space(), optimization_config=get_branin_multi_objective_optimization_config( has_objective_thresholds=True, ) if with_multi_objective else get_branin_optimization_config(), runner=SyntheticRunner(), is_test=True, ) exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0}, name="status_quo") sobol_generator = get_sobol(search_space=exp.search_space, seed=TEST_SOBOL_SEED) sobol_run = sobol_generator.gen(n=5) exp.new_batch_trial(optimize_for_power=True).add_generator_run(sobol_run) not_none(exp.trials.get(0)).run() exp.new_batch_trial(optimize_for_power=True).add_generator_run(sobol_run) not_none(exp.trials.get(1)).run() return exp
def get_branin_experiment_with_timestamp_map_metric( with_status_quo: bool = False, rate: Optional[float] = None, ) -> Experiment: exp = Experiment( name="branin_with_timestamp_map_metric", search_space=get_branin_search_space(), optimization_config=OptimizationConfig(objective=Objective( metric=BraninTimestampMapMetric( name="branin_map", param_names=["x1", "x2"], rate=rate), minimize=True, )), tracking_metrics=[ BraninMetric(name="branin", param_names=["x1", "x2"]) ], runner=SyntheticRunner(), default_data_type=DataType.MAP_DATA, ) if with_status_quo: exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0}) return exp
def get_branin_experiment( has_optimization_config: bool = True, with_batch: bool = False, with_status_quo: bool = False, ) -> Experiment: exp = Experiment( name="branin_test_experiment", search_space=get_branin_search_space(), optimization_config=get_branin_optimization_config() if has_optimization_config else None, runner=SyntheticRunner(), is_test=True, ) if with_status_quo: exp.status_quo = Arm(parameters={"x1": 0, "x2": 0}) if with_batch: sobol_generator = get_sobol(search_space=exp.search_space) sobol_run = sobol_generator.gen(n=15) exp.new_batch_trial().add_generator_run(sobol_run) return exp
def testStatusQuoSetter(self): sq_parameters = self.experiment.status_quo.parameters self.experiment.status_quo = None self.assertIsNone(self.experiment.status_quo) # Verify normal update sq_parameters["w"] = 3.5 self.experiment.status_quo = Arm(sq_parameters) self.assertEqual(self.experiment.status_quo.parameters["w"], 3.5) self.assertEqual(self.experiment.status_quo.name, "status_quo") self.assertTrue("status_quo" in self.experiment.arms_by_name) # Verify all None values self.experiment.status_quo = Arm( {n: None for n in sq_parameters.keys()}) self.assertIsNone(self.experiment.status_quo.parameters["w"]) # Try extra param sq_parameters["a"] = 4 with self.assertRaises(ValueError): self.experiment.status_quo = Arm(sq_parameters) # Try wrong type sq_parameters.pop("a") sq_parameters["w"] = "hello" with self.assertRaises(ValueError): self.experiment.status_quo = Arm(sq_parameters) # Verify arms_by_signature, arms_by_name only contains status_quo self.assertEqual(len(self.experiment.arms_by_signature), 1) self.assertEqual(len(self.experiment.arms_by_name), 1) # Change status quo, verify still just 1 arm sq_parameters["w"] = 3.6 self.experiment.status_quo = Arm(sq_parameters) self.assertEqual(len(self.experiment.arms_by_signature), 1) self.assertEqual(len(self.experiment.arms_by_name), 1) # Make a batch, add status quo to it, then change exp status quo, verify 2 arms batch = self.experiment.new_batch_trial() batch.set_status_quo_with_weight(self.experiment.status_quo, 1) sq_parameters["w"] = 3.7 self.experiment.status_quo = Arm(sq_parameters) self.assertEqual(len(self.experiment.arms_by_signature), 2) self.assertEqual(len(self.experiment.arms_by_name), 2) self.assertEqual(self.experiment.status_quo.name, "status_quo_e0") self.assertTrue("status_quo_e0" in self.experiment.arms_by_name) # Try missing param sq_parameters.pop("w") with self.assertRaises(ValueError): self.experiment.status_quo = Arm(sq_parameters) # Actually name the status quo. exp = Experiment( name="test3", search_space=get_branin_search_space(), tracking_metrics=[ BraninMetric(name="b", param_names=["x1", "x2"]) ], runner=SyntheticRunner(), ) batch = exp.new_batch_trial() arms = get_branin_arms(n=1, seed=0) batch.add_arms_and_weights(arms=arms) self.assertIsNone(exp.status_quo) exp.status_quo = arms[0] self.assertEqual(exp.status_quo.name, "0_0") # Try setting sq to existing arm with different name with self.assertRaises(ValueError): exp.status_quo = Arm(arms[0].parameters, name="new_name")