def test_factorial_thompson_strategy(self, mock_update, mock_gen,
mock_discrete):
exp = get_branin_experiment()
factorial_thompson_generation_strategy = GenerationStrategy(steps=[
GenerationStep(model=Models.FACTORIAL, num_arms=1),
GenerationStep(model=Models.THOMPSON, num_arms=-1),
])
self.assertEqual(factorial_thompson_generation_strategy.name,
"factorial+thompson")
self.assertEqual(
factorial_thompson_generation_strategy.generator_changes, [1])
for i in range(2):
data = get_data() if i > 0 else None
factorial_thompson_generation_strategy.gen(experiment=exp,
new_data=data)
exp.new_batch_trial().add_arm(Arm(parameters={"x1": i, "x2": i}))
if i < 1:
mock_discrete.assert_called()
args, kwargs = mock_discrete.call_args
self.assertIsInstance(kwargs.get("model"),
FullFactorialGenerator)
exp.new_batch_trial()
else:
mock_discrete.assert_called()
args, kwargs = mock_discrete.call_args
self.assertIsInstance(
kwargs.get("model"),
(ThompsonSampler, EmpiricalBayesThompsonSampler),
)
def test_equality(self):
gs1 = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-1),
])
gs2 = gs1.clone_reset()
self.assertEqual(gs1, gs2)
gs1._data = get_data()
self.assertNotEqual(gs1, gs2)
gs2._data = get_data()
self.assertEqual(gs1, gs2)
gs1 = gs1.clone_reset()
gs2 = gs2.clone_reset()
gs1.gen(experiment=get_branin_experiment())
self.assertNotEqual(gs1, gs2)
gs2.gen(experiment=get_branin_experiment())
# Each generation strategy generated a different arm, so they are not
# equal even though they have the same setup and generated the same
# number of arms.
self.assertNotEqual(gs1, gs2)
def testEmptyMetrics(self):
empty_experiment = Experiment(name="test_experiment",
search_space=get_search_space())
self.assertEqual(empty_experiment.num_trials, 0)
with self.assertRaises(ValueError):
empty_experiment.fetch_data()
batch = empty_experiment.new_batch_trial()
self.assertEqual(empty_experiment.num_trials, 1)
with self.assertRaises(ValueError):
batch.fetch_data()
empty_experiment.add_tracking_metric(Metric(name="some_metric"))
empty_experiment.attach_data(get_data())
self.assertFalse(empty_experiment.fetch_data().df.empty)
def get_modelbridge(mock_gen_arms,
mock_observations_from_data,
status_quo_name: Optional[str] = None) -> ModelBridge:
exp = get_experiment()
modelbridge = ModelBridge(
search_space=get_search_space(),
model=FullFactorialGenerator(),
experiment=exp,
data=get_data(),
status_quo_name=status_quo_name,
)
modelbridge._predict = mock.MagicMock(
"ax.modelbridge.base.ModelBridge._predict",
autospec=True,
return_value=[get_observation().data],
)
return modelbridge