def test_sobol_GPEI_strategy(self, mock_GPEI_gen, mock_GPEI_update,
mock_GPEI_init):
exp = get_branin_experiment()
sobol_GPEI_generation_strategy = GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=2),
],
)
self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI")
self.assertEqual(sobol_GPEI_generation_strategy.generator_changes, [5])
exp.new_trial(
generator_run=sobol_GPEI_generation_strategy.gen(exp)).run()
for i in range(1, 8):
if i == 7:
# Check completeness error message.
with self.assertRaisesRegex(ValueError, "Generation strategy"):
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1))
else:
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1))
exp.new_trial(generator_run=g).run()
if i > 4:
self.assertIsInstance(sobol_GPEI_generation_strategy.model,
TorchModelBridge)
else:
self.assertEqual(g._model_key, "Sobol")
self.assertEqual(
g._model_kwargs,
{
"seed": None,
"deduplicate": False,
"init_position": 0,
"scramble": True,
},
)
self.assertEqual(
g._bridge_kwargs,
{
"optimization_config": None,
"status_quo_features": None,
"status_quo_name": None,
"transform_configs": None,
"transforms": Cont_X_trans,
},
)
# Check for "seen data" error message.
with self.assertRaisesRegex(ValueError, "Data for arm"):
sobol_GPEI_generation_strategy.gen(exp, exp.fetch_data())
def test_restore_from_generator_run(self):
gs = GenerationStrategy(
steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-1),
]
)
with self.assertRaises(ValueError):
gs._restore_model_from_generator_run()
gs.gen(experiment=get_branin_experiment())
model = gs.model
gs._restore_model_from_generator_run()
# Model should be reset.
self.assertIsNot(model, gs.model)
def testUpdateGenerationStrategy(self):
generation_strategy = get_generation_strategy()
save_generation_strategy(generation_strategy=generation_strategy)
# Add data, save, reload
generation_strategy._data = Data(
df=pd.DataFrame.from_records([{
"metric_name": "foo",
"mean": 1,
"arm_name": "bar"
}]))
save_generation_strategy(generation_strategy=generation_strategy)
loaded_generation_strategy = load_generation_strategy_by_id(
gs_id=generation_strategy._db_id)
self.assertEqual(generation_strategy, loaded_generation_strategy)
experiment = get_branin_experiment()
generation_strategy = get_generation_strategy()
save_experiment(experiment)
# add generator run, save, reload
experiment.new_trial(generator_run=generation_strategy.gen(experiment))
save_generation_strategy(generation_strategy=generation_strategy)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, loaded_generation_strategy)
# add another generator run, save, reload
experiment.new_trial(generator_run=generation_strategy.gen(
experiment, new_data=get_branin_data()))
save_generation_strategy(generation_strategy=generation_strategy)
save_experiment(experiment)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, loaded_generation_strategy)
# make sure that we can update the experiment too
experiment.description = "foobar"
save_experiment(experiment)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, loaded_generation_strategy)
self.assertEqual(generation_strategy._experiment.description,
experiment.description)
self.assertEqual(
generation_strategy._experiment.description,
loaded_generation_strategy._experiment.description,
)
def test_MTGP(self):
"""Tests MTGP instantiation."""
# Test Multi-type MTGP
exp = get_multi_type_experiment(add_trials=True)
mtgp = get_MTGP(experiment=exp, data=exp.fetch_data())
self.assertIsInstance(mtgp, TorchModelBridge)
# Test Single-type MTGP
exp = get_branin_experiment()
# Check that factory generates a valid sobol modelbridge.
sobol = get_sobol(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(n=1)
exp.new_batch_trial().add_generator_run(sobol_run).run()
mtgp = get_MTGP(experiment=exp, data=exp.fetch_data())
self.assertIsInstance(mtgp, TorchModelBridge)
def testEncodeDecodeGenerationStrategy(self):
# Cannot load generation strategy before it has been saved
with self.assertRaises(ValueError):
load_generation_strategy_by_id(gs_id=0)
# Check that we can encode and decode the generation strategy *before*
# it has generated some trials and been updated with some data.
generation_strategy = get_generation_strategy()
# Check that we can save a generation strategy without an experiment
# attached.
save_generation_strategy(generation_strategy=generation_strategy)
# Also try restoring this generation strategy by its ID in the DB.
new_generation_strategy = load_generation_strategy_by_id(
gs_id=generation_strategy._db_id)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsNone(generation_strategy._experiment)
self.assertEqual(len(generation_strategy._generated), 0)
self.assertEqual(len(generation_strategy._observed), 0)
# Cannot load generation strategy before it has been saved
experiment = get_branin_experiment()
save_experiment(experiment)
with self.assertRaises(ValueError):
load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
# Check that we can encode and decode the generation strategy *after*
# it has generated some trials and been updated with some data.
generation_strategy = new_generation_strategy
experiment.new_trial(generator_run=generation_strategy.gen(experiment))
experiment.new_trial(
generation_strategy.gen(experiment, new_data=get_branin_data()))
self.assertGreater(len(generation_strategy._generated), 0)
self.assertGreater(len(generation_strategy._observed), 0)
save_generation_strategy(generation_strategy=generation_strategy)
save_experiment(experiment)
# Try restoring the generation strategy using the experiment its
# attached to.
new_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
self.assertIsInstance(new_generation_strategy.model, ModelBridge)
self.assertEqual(len(new_generation_strategy._generator_runs), 2)
self.assertEqual(new_generation_strategy._experiment._name,
experiment._name)
def test_sobol_GPEI(self):
"""Tests sobol + GPEI instantiation."""
exp = get_branin_experiment()
# Check that factory generates a valid sobol modelbridge.
sobol = get_sobol(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(n=1)
exp.new_batch_trial().add_generator_run(sobol_run).run()
# Check that factory generates a valid GP+EI modelbridge.
exp.optimization_config = get_branin_optimization_config()
gpei = get_GPEI(experiment=exp, data=exp.fetch_data())
self.assertIsInstance(gpei, TorchModelBridge)
gpei = get_GPEI(experiment=exp,
data=exp.fetch_data(),
search_space=exp.search_space)
self.assertIsInstance(gpei, TorchModelBridge)
def test_enum_sobol_GPEI(self):
"""Tests Sobol instantiation through the Models enum."""
exp = get_branin_experiment()
# Check that factory generates a valid sobol modelbridge.
sobol = Models.SOBOL(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(n=1)
self.assertEqual(sobol_run._model_key, "Sobol")
exp.new_batch_trial().add_generator_run(sobol_run).run()
# Check that factory generates a valid GP+EI modelbridge.
exp.optimization_config = get_branin_optimization_config()
gpei = Models.GPEI(experiment=exp, data=exp.fetch_data())
self.assertIsInstance(gpei, TorchModelBridge)
gpei = Models.GPEI(experiment=exp,
data=exp.fetch_data(),
search_space=exp.search_space)
self.assertIsInstance(gpei, TorchModelBridge)
def test_with_factory_function(self):
"""Checks that generation strategy works with custom factory functions.
No information about the model should be saved on generator run."""
def get_sobol(search_space: SearchSpace) -> RandomModelBridge:
return RandomModelBridge(
search_space=search_space,
model=SobolGenerator(),
transforms=Cont_X_trans,
)
exp = get_branin_experiment()
sobol_generation_strategy = GenerationStrategy(
steps=[GenerationStep(model=get_sobol, num_arms=5)])
g = sobol_generation_strategy.gen(exp)
self.assertIsInstance(sobol_generation_strategy.model,
RandomModelBridge)
self.assertIsNone(g._model_key)
self.assertIsNone(g._model_kwargs)
self.assertIsNone(g._bridge_kwargs)
def test_do_not_enforce_min_observations(self):
# We should be able to move on to the next model if there is not
# enough data observed if `enforce_num_arms` setting is False, in which
# case the previous model should be used until there is enough data.
exp = get_branin_experiment()
gs = GenerationStrategy(steps=[
GenerationStep(
model=Models.SOBOL,
num_arms=5,
min_arms_observed=5,
enforce_num_arms=False,
),
GenerationStep(model=Models.GPEI, num_arms=1),
])
for _ in range(5):
gs.gen(exp)
sobol = gs._model
gs.gen(exp)
# Make sure the same model is used to generate the 6th point.
self.assertIs(gs._model, sobol)
def test_transform_callback_unitx(self, _):
exp = get_branin_experiment()
parameters = [
RangeParameter(name="x1",
parameter_type=ParameterType.FLOAT,
lower=0,
upper=10),
RangeParameter(name="x2",
parameter_type=ParameterType.FLOAT,
lower=0,
upper=100),
]
gpei = TorchModelBridge(
experiment=exp,
data=exp.fetch_data(),
search_space=SearchSpace(parameters=parameters),
model=BotorchModel(),
transforms=[UnitX],
)
transformed = gpei._transform_callback([0.75, 0.35])
self.assertTrue(np.allclose(transformed, [0.75, 0.35]))
def testDecodeGenerationStrategy(self):
generation_strategy = get_generation_strategy()
experiment = get_branin_experiment()
gs_json = object_to_json(generation_strategy)
new_generation_strategy = generation_strategy_from_json(
experiment, gs_json)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertGreater(len(new_generation_strategy._steps), 0)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
# Model has not yet been initialized on this GS since it hasn't generated
# anything yet.
self.assertIsNone(new_generation_strategy.model)
# Check that we can encode and decode the generation strategy after
# it has generated some trials.
generation_strategy = new_generation_strategy
experiment.new_trial(generator_run=generation_strategy.gen(experiment))
gs_json = object_to_json(generation_strategy)
new_generation_strategy = generation_strategy_from_json(
experiment, gs_json)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
# Since this GS has now generated one generator run, model should have
# been initialized and restored when decoding from JSON.
self.assertIsInstance(new_generation_strategy.model, ModelBridge)
# Check that we can encode and decode the generation strategy after
# it has generated some trials and been updated with some data.
generation_strategy = new_generation_strategy
experiment.new_trial(
generation_strategy.gen(experiment, new_data=get_branin_data()))
self.assertGreater(len(generation_strategy._generated), 0)
self.assertGreater(len(generation_strategy._observed), 0)
gs_json = object_to_json(generation_strategy)
new_generation_strategy = generation_strategy_from_json(
experiment, gs_json)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
self.assertIsInstance(new_generation_strategy.model, ModelBridge)
def test_transform_callback_int(self, _):
exp = get_branin_experiment()
parameters = [
RangeParameter(name="x1",
parameter_type=ParameterType.INT,
lower=1,
upper=10),
RangeParameter(name="x2",
parameter_type=ParameterType.INT,
lower=5,
upper=15),
]
gpei = TorchModelBridge(
experiment=exp,
data=exp.fetch_data(),
search_space=SearchSpace(parameters=parameters),
model=BotorchModel(),
transforms=[IntToFloat],
torch_dtype=torch.double,
)
transformed = gpei._transform_callback([5.4, 7.6])
self.assertTrue(np.allclose(transformed, [5, 8]))
def test_sobol_GPEI_strategy_batches(
self, mock_GPEI_gen, mock_GPEI_update, mock_GPEI_init
):
exp = get_branin_experiment()
sobol_GPEI_generation_strategy = GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=8),
],
)
self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI")
self.assertEqual(sobol_GPEI_generation_strategy.generator_changes, [5])
exp.new_batch_trial(
generator_run=sobol_GPEI_generation_strategy.gen(exp, n=2)
).run()
for i in range(1, 8):
if i == 2:
with self.assertRaisesRegex(ValueError, "Cannot generate 2 new"):
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1), n=2
)
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1)
)
elif i == 7:
# Check completeness error message.
with self.assertRaisesRegex(ValueError, "Generation strategy"):
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1), n=2
)
else:
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1), n=2
)
exp.new_batch_trial(generator_run=g).run()
with self.assertRaises(ValueError):
sobol_GPEI_generation_strategy.gen(exp, exp.fetch_data())
self.assertIsInstance(sobol_GPEI_generation_strategy.model, TorchModelBridge)
def test_MTGP(self):
"""Tests MTGP instantiation."""
# Test Multi-type MTGP
exp = get_multi_type_experiment(add_trials=True)
mtgp = get_MTGP(experiment=exp, data=exp.fetch_data())
self.assertIsInstance(mtgp, TorchModelBridge)
# Test Single-type MTGP
exp = get_branin_experiment()
# Check that factory generates a valid sobol modelbridge.
sobol = get_sobol(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(n=1)
exp.new_batch_trial().add_generator_run(sobol_run).run()
mtgp = get_MTGP(experiment=exp,
is_multi_type=False,
data=exp.fetch_data())
self.assertIsInstance(mtgp, TorchModelBridge)
# Test wrong call of Multi-type MTGP. The type of the input experiment
# should be MultiTypeExperiment.
with self.assertRaises(ValueError):
get_MTGP(experiment=exp, is_multi_type=True, data=exp.fetch_data())
def test_sobol_GPEI_strategy_keep_generating(self, mock_GPEI_gen,
mock_GPEI_update,
mock_GPEI_init):
exp = get_branin_experiment()
sobol_GPEI_generation_strategy = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-1),
])
self.assertEqual(sobol_GPEI_generation_strategy.name, "sobol+GPEI")
self.assertEqual(sobol_GPEI_generation_strategy.generator_changes, [5])
exp.new_trial(
generator_run=sobol_GPEI_generation_strategy.gen(exp)).run()
for i in range(1, 15):
# Passing in all experiment data should cause an error as only
# new data should be passed into `gen`.
if i > 1:
with self.assertRaisesRegex(ValueError, "Data for arm"):
g = sobol_GPEI_generation_strategy.gen(
exp, exp.fetch_data())
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1))
exp.new_trial(generator_run=g).run()
if i > 4:
mock_GPEI_init.assert_called()
def test_uniform(self):
exp = get_branin_experiment()
uniform = get_uniform(exp.search_space)
self.assertIsInstance(uniform, RandomModelBridge)
uniform_run = uniform.gen(n=5)
self.assertEqual(len(uniform_run.arms), 5)
