def testIsFactorial(self): self.assertFalse(self.batch.is_factorial) # Insufficient factors small_experiment = Experiment( name="small_test", search_space=SearchSpace( [FixedParameter("a", ParameterType.INT, 4)]), ) small_trial = small_experiment.new_batch_trial().add_arm(Arm({"a": 4})) self.assertFalse(small_trial.is_factorial) new_batch_trial = self.experiment.new_batch_trial() new_batch_trial.add_arms_and_weights(arms=[ Arm(parameters={ "w": 0.75, "x": 1, "y": "foo", "z": True }), Arm(parameters={ "w": 0.75, "x": 2, "y": "foo", "z": True }), Arm(parameters={ "w": 0.77, "x": 1, "y": "foo", "z": True }), ]) self.assertFalse(new_batch_trial.is_factorial) new_batch_trial = self.experiment.new_batch_trial() new_batch_trial.add_arms_and_weights(arms=[ Arm(parameters={ "w": 0.77, "x": 1, "y": "foo", "z": True }), Arm(parameters={ "w": 0.77, "x": 2, "y": "foo", "z": True }), Arm(parameters={ "w": 0.75, "x": 1, "y": "foo", "z": True }), Arm(parameters={ "w": 0.75, "x": 2, "y": "foo", "z": True }), ]) self.assertTrue(new_batch_trial.is_factorial)
def testStatusQuoSetter(self): sq_parameters = self.experiment.status_quo.parameters self.experiment.status_quo = None self.assertIsNone(self.experiment.status_quo) # Verify normal update sq_parameters["w"] = 3.5 self.experiment.status_quo = Arm(sq_parameters) self.assertEqual(self.experiment.status_quo.parameters["w"], 3.5) self.assertEqual(self.experiment.status_quo.name, "status_quo") self.assertTrue("status_quo" in self.experiment.arms_by_name) # Verify all None values self.experiment.status_quo = Arm( {n: None for n in sq_parameters.keys()}) self.assertIsNone(self.experiment.status_quo.parameters["w"]) # Try extra param sq_parameters["a"] = 4 with self.assertRaises(ValueError): self.experiment.status_quo = Arm(sq_parameters) # Try wrong type sq_parameters.pop("a") sq_parameters["w"] = "hello" with self.assertRaises(ValueError): self.experiment.status_quo = Arm(sq_parameters) # Verify arms_by_signature, arms_by_name only contains status_quo self.assertEqual(len(self.experiment.arms_by_signature), 1) self.assertEqual(len(self.experiment.arms_by_name), 1) # Change status quo, verify still just 1 arm sq_parameters["w"] = 3.6 self.experiment.status_quo = Arm(sq_parameters) self.assertEqual(len(self.experiment.arms_by_signature), 1) self.assertEqual(len(self.experiment.arms_by_name), 1) # Make a batch, add status quo to it, then change exp status quo, verify 2 arms batch = self.experiment.new_batch_trial() batch.set_status_quo_with_weight(self.experiment.status_quo, 1) sq_parameters["w"] = 3.7 self.experiment.status_quo = Arm(sq_parameters) self.assertEqual(len(self.experiment.arms_by_signature), 2) self.assertEqual(len(self.experiment.arms_by_name), 2) self.assertEqual(self.experiment.status_quo.name, "status_quo_e0") self.assertTrue("status_quo_e0" in self.experiment.arms_by_name) # Try missing param sq_parameters.pop("w") with self.assertRaises(ValueError): self.experiment.status_quo = Arm(sq_parameters) # Actually name the status quo. exp = Experiment( name="test3", search_space=get_branin_search_space(), tracking_metrics=[ BraninMetric(name="b", param_names=["x1", "x2"]) ], runner=SyntheticRunner(), ) batch = exp.new_batch_trial() arms = get_branin_arms(n=1, seed=0) batch.add_arms_and_weights(arms=arms) self.assertIsNone(exp.status_quo) exp.status_quo = arms[0] self.assertEqual(exp.status_quo.name, "0_0") # Try setting sq to existing arm with different name with self.assertRaises(ValueError): exp.status_quo = Arm(arms[0].parameters, name="new_name")
def test_REMBOStrategy(self, mock_fit_gpytorch_model, mock_optimize_acqf): # Construct a high-D test experiment with multiple metrics hartmann_search_space = SearchSpace( parameters=[ RangeParameter( name=f"x{i}", parameter_type=ParameterType.FLOAT, lower=0.0, upper=1.0, ) for i in range(20) ] ) exp = Experiment( name="test", search_space=hartmann_search_space, optimization_config=OptimizationConfig( objective=Objective( metric=Hartmann6Metric( name="hartmann6", param_names=[f"x{i}" for i in range(6)] ), minimize=True, ), outcome_constraints=[ OutcomeConstraint( metric=L2NormMetric( name="l2norm", param_names=[f"x{i}" for i in range(6)], noise_sd=0.2, ), op=ComparisonOp.LEQ, bound=1.25, relative=False, ) ], ), runner=SyntheticRunner(), ) # Instantiate the strategy gs = REMBOStrategy(D=20, d=6, k=4, init_per_proj=4) # Check that arms and data are correctly segmented by projection exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2)).run() self.assertEqual(len(gs.arms_by_proj[0]), 2) self.assertEqual(len(gs.arms_by_proj[1]), 0) exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2)).run() self.assertEqual(len(gs.arms_by_proj[0]), 2) self.assertEqual(len(gs.arms_by_proj[1]), 2) # Iterate until the first projection fits a GP for _ in range(4): exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2)).run() mock_fit_gpytorch_model.assert_not_called() self.assertEqual(len(gs.arms_by_proj[0]), 4) self.assertEqual(len(gs.arms_by_proj[1]), 4) self.assertEqual(len(gs.arms_by_proj[2]), 2) self.assertEqual(len(gs.arms_by_proj[3]), 2) # Keep iterating until GP is used for gen for i in range(4): # First two trials will go towards 3rd and 4th proj. getting enough if i < 1: # data for GP. self.assertLess(len(gs.arms_by_proj[2]), 4) if i < 2: self.assertLess(len(gs.arms_by_proj[3]), 4) exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2)).run() if i < 2: mock_fit_gpytorch_model.assert_not_called() else: # After all proj. have > 4 arms' worth of data, GP can be fit. self.assertFalse(any(len(x) < 4 for x in gs.arms_by_proj.values())) mock_fit_gpytorch_model.assert_called() self.assertTrue(len(gs.model_transitions) > 0) gs2 = gs.clone_reset() self.assertEqual(gs2.D, 20) self.assertEqual(gs2.d, 6)