def testUnimplementedEvaluationFunction(self) -> None: experiment = SimpleExperiment( name="test_branin", search_space=get_branin_search_space(), objective_name="sum", ) with self.assertRaises(Exception): experiment.evaluation_function(parameterization={}) experiment.evaluation_function = sum_evaluation_function
def testOptionalObjectiveName(self) -> None: experiment = SimpleExperiment( name="test_branin", search_space=get_branin_search_space(), evaluation_function=sum_evaluation_function_v2, ) for i in range(len(self.arms)): experiment.new_trial(generator_run=GeneratorRun(arms=[self.arms[i]])) self.assertFalse(experiment.eval().df.empty)
def testEvaluationFunctionV4Numpy(self) -> None: experiment = SimpleExperiment( name="test_branin", search_space=get_branin_search_space(), objective_name="sum", evaluation_function=sum_evaluation_function_v4_numpy, ) for i in range(len(self.arms)): experiment.new_trial(generator_run=GeneratorRun(arms=[self.arms[i]])) self.assertFalse(experiment.eval().df.empty)
def setUp(self) -> None: self.experiment = SimpleExperiment( name="test_branin", search_space=get_branin_search_space(), evaluation_function=sum_evaluation_function, objective_name="sum", ) self.arms = [ Arm(parameters={"x1": 0.75, "x2": 1}), Arm(parameters={"x1": 2, "x2": 7}), Arm(parameters={"x1": 10, "x2": 8}), Arm(parameters={"x1": -2, "x2": 10}), ]
def testExperimentWithoutName(self) -> Experiment: exp = Experiment( search_space=get_branin_search_space(), tracking_metrics=[ BraninMetric(name="b", param_names=["x1", "x2"]) ], runner=SyntheticRunner(), ) self.assertEqual("Experiment(None)", str(exp)) batch = exp.new_batch_trial() batch.add_arms_and_weights(arms=get_branin_arms(n=5, seed=0)) batch.run() self.assertEqual(batch.run_metadata, {"name": "0"})
def _setupBraninExperiment(self, n: int) -> Experiment: exp = Experiment( name="test3", search_space=get_branin_search_space(), tracking_metrics=[ BraninMetric(name="b", param_names=["x1", "x2"]) ], runner=SyntheticRunner(), ) batch = exp.new_batch_trial() batch.add_arms_and_weights(arms=get_branin_arms(n=n, seed=0)) batch.run() (exp.new_batch_trial().add_arms_and_weights( arms=get_branin_arms(n=3 * n, seed=1)).run()) return exp
def testRelativeConstraint(self): branin_rel = BenchmarkProblem( name="constrained_branin", fbest=0.397887, optimization_config=OptimizationConfig( objective=Objective( metric=BraninMetric(name="branin_objective", param_names=["x1", "x2"], noise_sd=5.0), minimize=True, ), outcome_constraints=[ OutcomeConstraint( metric=L2NormMetric( name="branin_constraint", param_names=["x1", "x2"], noise_sd=5.0, ), op=ComparisonOp.LEQ, bound=5.0, relative=True, ) ], ), search_space=get_branin_search_space(), ) suite = BOBenchmarkingSuite() suite.run( num_runs=1, total_iterations=5, bo_strategies=[ GenerationStrategy( [GenerationStep(model=Models.SOBOL, num_arms=5)]) ], bo_problems=[branin_rel], ) with self.assertRaises(ValueError): suite.generate_report()
def testLowerBound(self): branin_lb = BenchmarkProblem( name="constrained_branin", fbest=0.397887, optimization_config=OptimizationConfig( objective=Objective( metric=BraninMetric(name="branin_objective", param_names=["x1", "x2"], noise_sd=5.0), minimize=True, ), outcome_constraints=[ OutcomeConstraint( metric=L2NormMetric( name="branin_constraint", param_names=["x1", "x2"], noise_sd=5.0, ), op=ComparisonOp.GEQ, bound=5.0, relative=False, ) ], ), search_space=get_branin_search_space(), ) suite = BOBenchmarkingSuite() suite.run( num_runs=1, batch_size=2, total_iterations=4, bo_strategies=[ GenerationStrategy( [GenerationStep(model=Models.SOBOL, num_arms=5)]) ], bo_problems=[branin_lb], ) suite.generate_report(include_individual=True)
def testStatusQuoSetter(self): sq_parameters = self.experiment.status_quo.parameters self.experiment.status_quo = None self.assertIsNone(self.experiment.status_quo) # Verify normal update sq_parameters["w"] = 3.5 self.experiment.status_quo = Arm(sq_parameters) self.assertEqual(self.experiment.status_quo.parameters["w"], 3.5) self.assertEqual(self.experiment.status_quo.name, "status_quo") # Verify all None values self.experiment.status_quo = Arm( {n: None for n in sq_parameters.keys()}) self.assertIsNone(self.experiment.status_quo.parameters["w"]) # Try extra param sq_parameters["a"] = 4 with self.assertRaises(ValueError): self.experiment.status_quo = Arm(sq_parameters) # Try wrong type sq_parameters.pop("a") sq_parameters["w"] = "hello" with self.assertRaises(ValueError): self.experiment.status_quo = Arm(sq_parameters) # Verify arms_by_signature only contains status_quo self.assertEqual(len(self.experiment.arms_by_signature), 1) # Change status quo, verify still just 1 arm sq_parameters["w"] = 3.6 self.experiment.status_quo = Arm(sq_parameters) self.assertEqual(len(self.experiment.arms_by_signature), 1) # Make a batch, then change exp status quo, verify 2 arms self.experiment.new_batch_trial() sq_parameters["w"] = 3.7 self.experiment.status_quo = Arm(sq_parameters) self.assertEqual(len(self.experiment.arms_by_signature), 2) self.assertEqual(self.experiment.status_quo.name, "status_quo_e0") # Try missing param sq_parameters.pop("w") with self.assertRaises(ValueError): self.experiment.status_quo = Arm(sq_parameters) # Actually name the status quo. exp = Experiment( name="test3", search_space=get_branin_search_space(), tracking_metrics=[ BraninMetric(name="b", param_names=["x1", "x2"]) ], runner=SyntheticRunner(), ) batch = exp.new_batch_trial() arms = get_branin_arms(n=1, seed=0) batch.add_arms_and_weights(arms=arms) self.assertIsNone(exp.status_quo) exp.status_quo = arms[0] self.assertEqual(exp.status_quo.name, "0_0") # Try setting sq to existing arm with different name with self.assertRaises(ValueError): exp.status_quo = Arm(arms[0].parameters, name="new_name")
name: str fbest: float optimization_config: OptimizationConfig search_space: SearchSpace # Branin problems branin = BenchmarkProblem( name=branin_function.name, fbest=branin_function.fmin, optimization_config=OptimizationConfig(objective=Objective( metric=BraninMetric( name="branin_objective", param_names=["x1", "x2"], noise_sd=5.0), minimize=True, )), search_space=get_branin_search_space(), ) branin_max = BenchmarkProblem( name=branin_function.name, fbest=branin_function.fmax, optimization_config=OptimizationConfig(objective=Objective( metric=NegativeBraninMetric( name="neg_branin", param_names=["x1", "x2"], noise_sd=5.0), minimize=False, )), search_space=get_branin_search_space(), ) # Hartmann 6 problems