def testIsFactorial(self):
self.assertFalse(self.batch.is_factorial)
# Insufficient factors
small_experiment = Experiment(
name="small_test",
search_space=SearchSpace(
[FixedParameter("a", ParameterType.INT, 4)]),
)
small_trial = small_experiment.new_batch_trial().add_arm(Arm({"a": 4}))
self.assertFalse(small_trial.is_factorial)
new_batch_trial = self.experiment.new_batch_trial()
new_batch_trial.add_arms_and_weights(arms=[
Arm(parameters={
"w": 0.75,
"x": 1,
"y": "foo",
"z": True
}),
Arm(parameters={
"w": 0.75,
"x": 2,
"y": "foo",
"z": True
}),
Arm(parameters={
"w": 0.77,
"x": 1,
"y": "foo",
"z": True
}),
])
self.assertFalse(new_batch_trial.is_factorial)
new_batch_trial = self.experiment.new_batch_trial()
new_batch_trial.add_arms_and_weights(arms=[
Arm(parameters={
"w": 0.77,
"x": 1,
"y": "foo",
"z": True
}),
Arm(parameters={
"w": 0.77,
"x": 2,
"y": "foo",
"z": True
}),
Arm(parameters={
"w": 0.75,
"x": 1,
"y": "foo",
"z": True
}),
Arm(parameters={
"w": 0.75,
"x": 2,
"y": "foo",
"z": True
}),
])
self.assertTrue(new_batch_trial.is_factorial)
def testStatusQuoSetter(self):
sq_parameters = self.experiment.status_quo.parameters
self.experiment.status_quo = None
self.assertIsNone(self.experiment.status_quo)
# Verify normal update
sq_parameters["w"] = 3.5
self.experiment.status_quo = Arm(sq_parameters)
self.assertEqual(self.experiment.status_quo.parameters["w"], 3.5)
self.assertEqual(self.experiment.status_quo.name, "status_quo")
self.assertTrue("status_quo" in self.experiment.arms_by_name)
# Verify all None values
self.experiment.status_quo = Arm(
{n: None
for n in sq_parameters.keys()})
self.assertIsNone(self.experiment.status_quo.parameters["w"])
# Try extra param
sq_parameters["a"] = 4
with self.assertRaises(ValueError):
self.experiment.status_quo = Arm(sq_parameters)
# Try wrong type
sq_parameters.pop("a")
sq_parameters["w"] = "hello"
with self.assertRaises(ValueError):
self.experiment.status_quo = Arm(sq_parameters)
# Verify arms_by_signature, arms_by_name only contains status_quo
self.assertEqual(len(self.experiment.arms_by_signature), 1)
self.assertEqual(len(self.experiment.arms_by_name), 1)
# Change status quo, verify still just 1 arm
sq_parameters["w"] = 3.6
self.experiment.status_quo = Arm(sq_parameters)
self.assertEqual(len(self.experiment.arms_by_signature), 1)
self.assertEqual(len(self.experiment.arms_by_name), 1)
# Make a batch, add status quo to it, then change exp status quo, verify 2 arms
batch = self.experiment.new_batch_trial()
batch.set_status_quo_with_weight(self.experiment.status_quo, 1)
sq_parameters["w"] = 3.7
self.experiment.status_quo = Arm(sq_parameters)
self.assertEqual(len(self.experiment.arms_by_signature), 2)
self.assertEqual(len(self.experiment.arms_by_name), 2)
self.assertEqual(self.experiment.status_quo.name, "status_quo_e0")
self.assertTrue("status_quo_e0" in self.experiment.arms_by_name)
# Try missing param
sq_parameters.pop("w")
with self.assertRaises(ValueError):
self.experiment.status_quo = Arm(sq_parameters)
# Actually name the status quo.
exp = Experiment(
name="test3",
search_space=get_branin_search_space(),
tracking_metrics=[
BraninMetric(name="b", param_names=["x1", "x2"])
],
runner=SyntheticRunner(),
)
batch = exp.new_batch_trial()
arms = get_branin_arms(n=1, seed=0)
batch.add_arms_and_weights(arms=arms)
self.assertIsNone(exp.status_quo)
exp.status_quo = arms[0]
self.assertEqual(exp.status_quo.name, "0_0")
# Try setting sq to existing arm with different name
with self.assertRaises(ValueError):
exp.status_quo = Arm(arms[0].parameters, name="new_name")
def test_REMBOStrategy(self, mock_fit_gpytorch_model, mock_optimize_acqf):
# Construct a high-D test experiment with multiple metrics
hartmann_search_space = SearchSpace(
parameters=[
RangeParameter(
name=f"x{i}",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0,
)
for i in range(20)
]
)
exp = Experiment(
name="test",
search_space=hartmann_search_space,
optimization_config=OptimizationConfig(
objective=Objective(
metric=Hartmann6Metric(
name="hartmann6", param_names=[f"x{i}" for i in range(6)]
),
minimize=True,
),
outcome_constraints=[
OutcomeConstraint(
metric=L2NormMetric(
name="l2norm",
param_names=[f"x{i}" for i in range(6)],
noise_sd=0.2,
),
op=ComparisonOp.LEQ,
bound=1.25,
relative=False,
)
],
),
runner=SyntheticRunner(),
)
# Instantiate the strategy
gs = REMBOStrategy(D=20, d=6, k=4, init_per_proj=4)
# Check that arms and data are correctly segmented by projection
exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2)).run()
self.assertEqual(len(gs.arms_by_proj[0]), 2)
self.assertEqual(len(gs.arms_by_proj[1]), 0)
exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2)).run()
self.assertEqual(len(gs.arms_by_proj[0]), 2)
self.assertEqual(len(gs.arms_by_proj[1]), 2)
# Iterate until the first projection fits a GP
for _ in range(4):
exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2)).run()
mock_fit_gpytorch_model.assert_not_called()
self.assertEqual(len(gs.arms_by_proj[0]), 4)
self.assertEqual(len(gs.arms_by_proj[1]), 4)
self.assertEqual(len(gs.arms_by_proj[2]), 2)
self.assertEqual(len(gs.arms_by_proj[3]), 2)
# Keep iterating until GP is used for gen
for i in range(4):
# First two trials will go towards 3rd and 4th proj. getting enough
if i < 1: # data for GP.
self.assertLess(len(gs.arms_by_proj[2]), 4)
if i < 2:
self.assertLess(len(gs.arms_by_proj[3]), 4)
exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2)).run()
if i < 2:
mock_fit_gpytorch_model.assert_not_called()
else:
# After all proj. have > 4 arms' worth of data, GP can be fit.
self.assertFalse(any(len(x) < 4 for x in gs.arms_by_proj.values()))
mock_fit_gpytorch_model.assert_called()
self.assertTrue(len(gs.model_transitions) > 0)
gs2 = gs.clone_reset()
self.assertEqual(gs2.D, 20)
self.assertEqual(gs2.d, 6)
