def testUpdateGenerationStrategy(self):
generation_strategy = get_generation_strategy()
save_generation_strategy(generation_strategy=generation_strategy)
experiment = get_branin_experiment()
generation_strategy = get_generation_strategy()
save_experiment(experiment)
# add generator run, save, reload
experiment.new_trial(generator_run=generation_strategy.gen(experiment))
save_generation_strategy(generation_strategy=generation_strategy)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name
)
# `_seen_trial_indices_by_status` attribute of a GS is not saved in DB,
# so it will be None in the restored version of the GS.
# Hackily removing it from the original GS to check equality.
generation_strategy._seen_trial_indices_by_status = None
self.assertEqual(generation_strategy, loaded_generation_strategy)
# add another generator run, save, reload
experiment.new_trial(
generator_run=generation_strategy.gen(experiment, data=get_branin_data())
)
save_generation_strategy(generation_strategy=generation_strategy)
save_experiment(experiment)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name
)
# `_seen_trial_indices_by_status` attribute of a GS is not saved in DB,
# so it will be None in the restored version of the GS.
# Hackily removing it from the original GS to check equality.
generation_strategy._seen_trial_indices_by_status = None
self.assertEqual(generation_strategy, loaded_generation_strategy)
# make sure that we can update the experiment too
experiment.description = "foobar"
save_experiment(experiment)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name
)
self.assertEqual(generation_strategy, loaded_generation_strategy)
self.assertEqual(
generation_strategy._experiment.description, experiment.description
)
self.assertEqual(
generation_strategy._experiment.description,
loaded_generation_strategy._experiment.description,
)
def testUpdateGenerationStrategy(self):
generation_strategy = get_generation_strategy()
save_generation_strategy(generation_strategy=generation_strategy)
# Add data, save, reload
generation_strategy._data = Data(
df=pd.DataFrame.from_records([{
"metric_name": "foo",
"mean": 1,
"arm_name": "bar"
}]))
save_generation_strategy(generation_strategy=generation_strategy)
loaded_generation_strategy = load_generation_strategy_by_id(
gs_id=generation_strategy._db_id)
self.assertEqual(generation_strategy, loaded_generation_strategy)
experiment = get_branin_experiment()
generation_strategy = get_generation_strategy()
save_experiment(experiment)
# add generator run, save, reload
experiment.new_trial(generator_run=generation_strategy.gen(experiment))
save_generation_strategy(generation_strategy=generation_strategy)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, loaded_generation_strategy)
# add another generator run, save, reload
experiment.new_trial(generator_run=generation_strategy.gen(
experiment, new_data=get_branin_data()))
save_generation_strategy(generation_strategy=generation_strategy)
save_experiment(experiment)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, loaded_generation_strategy)
# make sure that we can update the experiment too
experiment.description = "foobar"
save_experiment(experiment)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, loaded_generation_strategy)
self.assertEqual(generation_strategy._experiment.description,
experiment.description)
self.assertEqual(
generation_strategy._experiment.description,
loaded_generation_strategy._experiment.description,
)
def testDecodeGenerationStrategy(self):
generation_strategy = get_generation_strategy()
experiment = get_branin_experiment()
gs_json = object_to_json(generation_strategy)
new_generation_strategy = generation_strategy_from_json(gs_json)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertGreater(len(new_generation_strategy._steps), 0)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
# Model has not yet been initialized on this GS since it hasn't generated
# anything yet.
self.assertIsNone(new_generation_strategy.model)
# Check that we can encode and decode the generation strategy after
# it has generated some generator runs.
generation_strategy = new_generation_strategy
gr = generation_strategy.gen(experiment)
gs_json = object_to_json(generation_strategy)
new_generation_strategy = generation_strategy_from_json(gs_json)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
# Since this GS has now generated one generator run, model should have
# been initialized and restored when decoding from JSON.
self.assertIsInstance(new_generation_strategy.model, ModelBridge)
# Check that we can encode and decode the generation strategy after
# it has generated some trials and been updated with some data.
generation_strategy = new_generation_strategy
experiment.new_trial(gr) # Add previously generated GR as trial.
# Make generation strategy aware of the trial's data via `gen`.
generation_strategy.gen(experiment, data=get_branin_data())
gs_json = object_to_json(generation_strategy)
new_generation_strategy = generation_strategy_from_json(gs_json)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
self.assertIsInstance(new_generation_strategy.model, ModelBridge)
def get_random_generation_strategy(self) -> GenerationStrategy:
"""Get an GenerationStrategy instance with random name."""
generation_strategy = get_generation_strategy()
gs_name = "".join(random.choice(string.ascii_letters) for i in range(8))
generation_strategy._name = gs_name
return generation_strategy
def testUpdateGenerationStrategy(self):
generation_strategy = get_generation_strategy()
save_generation_strategy(generation_strategy=generation_strategy)
experiment = get_branin_experiment()
generation_strategy = get_generation_strategy()
save_experiment(experiment)
# add generator run, save, reload
experiment.new_trial(generator_run=generation_strategy.gen(experiment))
save_generation_strategy(generation_strategy=generation_strategy)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, loaded_generation_strategy)
# add another generator run, save, reload
experiment.new_trial(generator_run=generation_strategy.gen(
experiment, data=get_branin_data()))
save_generation_strategy(generation_strategy=generation_strategy)
save_experiment(experiment)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
# During restoration of generation strategy's model from its last generator
# run, we set `_seen_trial_indices_by_status` to that of the experiment,
# from which we are grabbing the data to restore the model with. When the
# experiment was updated more recently than the last `gen` from generation
# strategy, the generation strategy prior to save might not have 'seen'
# some recently added trials, so we update the mappings to match and check
# that the generation strategies are equal otherwise.
generation_strategy._seen_trial_indices_by_status[
TrialStatus.CANDIDATE].add(1)
self.assertEqual(generation_strategy, loaded_generation_strategy)
# make sure that we can update the experiment too
experiment.description = "foobar"
save_experiment(experiment)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, loaded_generation_strategy)
self.assertEqual(generation_strategy._experiment.description,
experiment.description)
self.assertEqual(
generation_strategy._experiment.description,
loaded_generation_strategy._experiment.description,
)
def test_get_standard_plots(self):
exp = get_branin_experiment()
self.assertEqual(
len(
get_standard_plots(
experiment=exp,
generation_strategy=get_generation_strategy())),
0,
)
def test_save_load_experiment_and_generation_strategy(self):
exp, gs = load_experiment_and_generation_strategy(
self.exp.name, self.db_settings)
self.assertIsNone(gs)
gs = get_generation_strategy()
gs._experiment = self.exp
save_experiment_and_generation_strategy(self.exp, gs, self.db_settings)
exp, gs = load_experiment_and_generation_strategy(
self.exp.name, self.db_settings)
self.assertIsNotNone(gs)
def setUp(self):
self.generation_strategy = get_generation_strategy(
with_experiment=True)
self.experiment = self.generation_strategy.experiment
init_test_engine_and_session_factory(force_init=True)
self.with_db_settings = WithDBSettingsBase(db_settings=DBSettings(
url="sqlite://"))
_save_experiment(self.experiment,
encoder=self.with_db_settings.db_settings.encoder)
_save_generation_strategy(
generation_strategy=self.generation_strategy,
encoder=self.with_db_settings.db_settings.encoder,
)
def testUpdateGenerationStrategy(self):
generation_strategy = get_generation_strategy()
save_generation_strategy(generation_strategy=generation_strategy)
experiment = get_branin_experiment()
generation_strategy = get_generation_strategy()
save_experiment(experiment)
# add generator run, save, reload
experiment.new_trial(generator_run=generation_strategy.gen(experiment))
save_generation_strategy(generation_strategy=generation_strategy)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, loaded_generation_strategy)
# add another generator run, save, reload
experiment.new_trial(generator_run=generation_strategy.gen(
experiment, data=get_branin_data()))
save_generation_strategy(generation_strategy=generation_strategy)
save_experiment(experiment)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, loaded_generation_strategy)
# make sure that we can update the experiment too
experiment.description = "foobar"
save_experiment(experiment)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, loaded_generation_strategy)
self.assertEqual(generation_strategy._experiment.description,
experiment.description)
self.assertEqual(
generation_strategy._experiment.description,
loaded_generation_strategy._experiment.description,
)
def testEncodeDecodeGenerationStrategy(self):
# Cannot load generation strategy before it has been saved
with self.assertRaises(ValueError):
load_generation_strategy_by_id(gs_id=0)
# Check that we can encode and decode the generation strategy *before*
# it has generated some trials and been updated with some data.
generation_strategy = get_generation_strategy()
# Check that we can save a generation strategy without an experiment
# attached.
save_generation_strategy(generation_strategy=generation_strategy)
# Also try restoring this generation strategy by its ID in the DB.
new_generation_strategy = load_generation_strategy_by_id(
gs_id=generation_strategy._db_id
)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsNone(generation_strategy._experiment)
# Cannot load generation strategy before it has been saved
experiment = get_branin_experiment()
save_experiment(experiment)
with self.assertRaises(ValueError):
load_generation_strategy_by_experiment_name(experiment_name=experiment.name)
# Check that we can encode and decode the generation strategy *after*
# it has generated some trials and been updated with some data.
generation_strategy = new_generation_strategy
experiment.new_trial(generation_strategy.gen(experiment=experiment))
experiment.new_trial(
generation_strategy.gen(experiment, data=get_branin_data())
)
save_generation_strategy(generation_strategy=generation_strategy)
save_experiment(experiment)
# Try restoring the generation strategy using the experiment its
# attached to.
new_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name
)
# `_seen_trial_indices_by_status` attribute of a GS is not saved in DB,
# so it will be None in the restored version of the GS.
# Hackily removing it from the original GS to check equality.
generation_strategy._seen_trial_indices_by_status = None
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
self.assertIsInstance(new_generation_strategy.model, ModelBridge)
self.assertEqual(len(new_generation_strategy._generator_runs), 2)
self.assertEqual(new_generation_strategy._experiment._name, experiment._name)
def testEncodeDecodeGenerationStrategy(self):
# Cannot load generation strategy before it has been saved
with self.assertRaises(ValueError):
load_generation_strategy_by_id(gs_id=0)
# Check that we can encode and decode the generation strategy *before*
# it has generated some trials and been updated with some data.
generation_strategy = get_generation_strategy()
# Check that we can save a generation strategy without an experiment
# attached.
save_generation_strategy(generation_strategy=generation_strategy)
# Also try restoring this generation strategy by its ID in the DB.
new_generation_strategy = load_generation_strategy_by_id(
gs_id=generation_strategy._db_id)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsNone(generation_strategy._experiment)
self.assertEqual(len(generation_strategy._generated), 0)
self.assertEqual(len(generation_strategy._observed), 0)
# Cannot load generation strategy before it has been saved
experiment = get_branin_experiment()
save_experiment(experiment)
with self.assertRaises(ValueError):
load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
# Check that we can encode and decode the generation strategy *after*
# it has generated some trials and been updated with some data.
generation_strategy = new_generation_strategy
experiment.new_trial(generator_run=generation_strategy.gen(experiment))
experiment.new_trial(
generation_strategy.gen(experiment, new_data=get_branin_data()))
self.assertGreater(len(generation_strategy._generated), 0)
self.assertGreater(len(generation_strategy._observed), 0)
save_generation_strategy(generation_strategy=generation_strategy)
save_experiment(experiment)
# Try restoring the generation strategy using the experiment its
# attached to.
new_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
self.assertIsInstance(new_generation_strategy.model, ModelBridge)
self.assertEqual(len(new_generation_strategy._generator_runs), 2)
self.assertEqual(new_generation_strategy._experiment._name,
experiment._name)
def testDecodeGenerationStrategy(self):
generation_strategy = get_generation_strategy()
experiment = get_branin_experiment()
gs_json = object_to_json(generation_strategy)
new_generation_strategy = generation_strategy_from_json(gs_json)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertGreater(len(new_generation_strategy._steps), 0)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
# Model has not yet been initialized on this GS since it hasn't generated
# anything yet.
self.assertIsNone(new_generation_strategy.model)
# Check that we can encode and decode the generation strategy after
# it has generated some trials.
generation_strategy = new_generation_strategy
experiment.new_trial(generator_run=generation_strategy.gen(experiment))
gs_json = object_to_json(generation_strategy)
new_generation_strategy = generation_strategy_from_json(gs_json)
# `_seen_trial_indices_by_status` attribute of a GS is not saved in DB,
# so it will be None in the restored version of the GS.
# Hackily removing it from the original GS to check equality.
generation_strategy._seen_trial_indices_by_status = None
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
# Since this GS has now generated one generator run, model should have
# been initialized and restored when decoding from JSON.
self.assertIsInstance(new_generation_strategy.model, ModelBridge)
# Check that we can encode and decode the generation strategy after
# it has generated some trials and been updated with some data.
generation_strategy = new_generation_strategy
experiment.new_trial(
generation_strategy.gen(experiment, data=get_branin_data()))
gs_json = object_to_json(generation_strategy)
new_generation_strategy = generation_strategy_from_json(gs_json)
# `_seen_trial_indices_by_status` attribute of a GS is not saved in DB,
# so it will be None in the restored version of the GS.
# Hackily removing it from the original GS to check equality.
generation_strategy._seen_trial_indices_by_status = None
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
self.assertIsInstance(new_generation_strategy.model, ModelBridge)
def test_get_standard_plots(self):
exp = get_branin_experiment()
self.assertEqual(
len(
get_standard_plots(
experiment=exp, model=get_generation_strategy().model
)
),
0,
)
exp = get_branin_experiment(with_batch=True, minimize=True)
exp.trials[0].run()
gs = choose_generation_strategy(search_space=exp.search_space)
gs._model = Models.BOTORCH(
experiment=exp,
data=exp.fetch_data(),
)
plots = get_standard_plots(experiment=exp, model=gs.model)
self.assertEqual(len(plots), 3)
self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots))
def test_get_standard_plots(self):
exp = get_branin_experiment()
self.assertEqual(
len(
get_standard_plots(experiment=exp,
model=get_generation_strategy().model)),
0,
)
exp = get_branin_experiment(with_batch=True, minimize=True)
exp.trials[0].run()
plots = get_standard_plots(
experiment=exp,
model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()),
)
self.assertEqual(len(plots), 6)
self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots))
exp = get_branin_experiment_with_multi_objective(with_batch=True)
exp.optimization_config.objective.objectives[0].minimize = False
exp.optimization_config.objective.objectives[1].minimize = True
exp.optimization_config._objective_thresholds = [
ObjectiveThreshold(metric=exp.metrics["branin_a"],
op=ComparisonOp.GEQ,
bound=-100.0),
ObjectiveThreshold(metric=exp.metrics["branin_b"],
op=ComparisonOp.LEQ,
bound=100.0),
]
exp.trials[0].run()
plots = get_standard_plots(experiment=exp,
model=Models.MOO(experiment=exp,
data=exp.fetch_data()))
self.assertEqual(len(plots), 7)
# All plots are successfully created when objective thresholds are absent
exp.optimization_config._objective_thresholds = []
plots = get_standard_plots(experiment=exp,
model=Models.MOO(experiment=exp,
data=exp.fetch_data()))
self.assertEqual(len(plots), 7)
exp = get_branin_experiment_with_timestamp_map_metric(
with_status_quo=True)
exp.new_trial().add_arm(exp.status_quo)
exp.trials[0].run()
exp.new_trial(generator_run=Models.SOBOL(
search_space=exp.search_space).gen(n=1))
exp.trials[1].run()
plots = get_standard_plots(
experiment=exp,
model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()),
true_objective_metric_name="branin",
)
self.assertEqual(len(plots), 9)
self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots))
self.assertIn(
"Objective branin_map vs. True Objective Metric branin",
[p.layout.title.text for p in plots],
)
with self.assertRaisesRegex(
ValueError, "Please add a valid true_objective_metric_name"):
plots = get_standard_plots(
experiment=exp,
model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()),
true_objective_metric_name="not_present",
)
def testDecodeGenerationStrategy(self):
generation_strategy = get_generation_strategy()
experiment = get_branin_experiment()
gs_json = object_to_json(
generation_strategy,
encoder_registry=DEPRECATED_ENCODER_REGISTRY,
class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY,
)
new_generation_strategy = generation_strategy_from_json(
gs_json,
decoder_registry=DEPRECATED_DECODER_REGISTRY,
class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY,
)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertGreater(len(new_generation_strategy._steps), 0)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
# Model has not yet been initialized on this GS since it hasn't generated
# anything yet.
self.assertIsNone(new_generation_strategy.model)
# Check that we can encode and decode the generation strategy after
# it has generated some generator runs. Since we now need to `gen`,
# we remove the fake callable kwarg we added, since model does not
# expect it.
generation_strategy = get_generation_strategy(with_callable_model_kwarg=False)
gr = generation_strategy.gen(experiment)
gs_json = object_to_json(
generation_strategy,
encoder_registry=DEPRECATED_ENCODER_REGISTRY,
class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY,
)
new_generation_strategy = generation_strategy_from_json(
gs_json,
decoder_registry=DEPRECATED_DECODER_REGISTRY,
class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY,
)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
# Since this GS has now generated one generator run, model should have
# been initialized and restored when decoding from JSON.
self.assertIsInstance(new_generation_strategy.model, ModelBridge)
# Check that we can encode and decode the generation strategy after
# it has generated some trials and been updated with some data.
generation_strategy = new_generation_strategy
experiment.new_trial(gr) # Add previously generated GR as trial.
# Make generation strategy aware of the trial's data via `gen`.
generation_strategy.gen(experiment, data=get_branin_data())
gs_json = object_to_json(
generation_strategy,
encoder_registry=DEPRECATED_ENCODER_REGISTRY,
class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY,
)
new_generation_strategy = generation_strategy_from_json(
gs_json,
decoder_registry=DEPRECATED_DECODER_REGISTRY,
class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY,
)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
self.assertIsInstance(new_generation_strategy.model, ModelBridge)