def test_enum_sobol_GPEI(self):
"""Tests Soboland GPEI instantiation through the Models enum."""
exp = get_branin_experiment()
# Check that factory generates a valid sobol modelbridge.
sobol = Models.SOBOL(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(n=1)
self.assertEqual(sobol_run._model_key, "Sobol")
exp.new_batch_trial().add_generator_run(sobol_run).run()
# Check that factory generates a valid GP+EI modelbridge.
exp.optimization_config = get_branin_optimization_config()
gpei = Models.GPEI(experiment=exp, data=exp.fetch_data())
self.assertIsInstance(gpei, TorchModelBridge)
self.assertEqual(gpei._model_key, "GPEI")
botorch_defaults = "ax.models.torch.botorch_defaults"
# Check that the callable kwargs and the torch kwargs were recorded.
self.assertEqual(
gpei._model_kwargs,
{
"acqf_constructor": {
"is_callable_as_path": True,
"value": f"{botorch_defaults}.get_NEI",
},
"acqf_optimizer": {
"is_callable_as_path": True,
"value": f"{botorch_defaults}.scipy_optimizer",
},
"model_constructor": {
"is_callable_as_path": True,
"value": f"{botorch_defaults}.get_and_fit_model",
},
"model_predictor": {
"is_callable_as_path": True,
"value": f"{botorch_defaults}.predict_from_model",
},
"refit_on_cv": False,
"refit_on_update": True,
"warm_start_refitting": True,
},
)
self.assertEqual(
gpei._bridge_kwargs,
{
"transform_configs": None,
"torch_dtype": torch_float64,
"torch_device": torch_device(type="cpu"),
"status_quo_name": None,
"status_quo_features": None,
"optimization_config": None,
"transforms": Cont_X_trans + Y_trans,
},
)
gpei = Models.GPEI(experiment=exp,
data=exp.fetch_data(),
search_space=exp.search_space)
self.assertIsInstance(gpei, TorchModelBridge)
def plot_progress(self, ax_client):
model = Models.GPEI(experiment=ax_client.experiment,
data=ax_client.experiment.fetch_data())
html_elements = [
plot_config_to_html(ax_client.get_optimization_trace())
]
model_params = get_range_parameters(model)
try:
if len(model_params) > 1:
html_elements.append(
plot_config_to_html(
interact_contour(
model=model,
metric_name=self.YR.args.eval_primary_metric)))
else:
html_elements.append(
plot_config_to_html(
interact_slice(
model=model,
param_name=model_params[0].name,
metric_name=self.YR.args.eval_primary_metric)))
except TypeError:
pass
with open(
os.path.join(self.bayes_opt_root_experiment_folder,
"optimization_plots.html"), 'w') as f:
f.write(render_report_elements(self.experiment_name,
html_elements))
def test_get_model_from_generator_run(self):
"""Tests that it is possible to restore a model from a generator run it
produced, if `Models` registry was used.
"""
exp = get_branin_experiment()
initial_sobol = Models.SOBOL(experiment=exp, seed=239)
gr = initial_sobol.gen(n=1)
# Restore the model as it was before generation.
sobol = get_model_from_generator_run(generator_run=gr,
experiment=exp,
data=exp.fetch_data())
self.assertEqual(sobol.model.init_position, 0)
self.assertEqual(sobol.model.seed, 239)
# Restore the model as it was after generation (to resume generation).
sobol_after_gen = get_model_from_generator_run(generator_run=gr,
experiment=exp,
data=exp.fetch_data(),
after_gen=True)
self.assertEqual(sobol_after_gen.model.init_position, 1)
self.assertEqual(sobol_after_gen.model.seed, 239)
self.assertEqual(
initial_sobol.gen(n=1).arms,
sobol_after_gen.gen(n=1).arms)
exp.new_trial(generator_run=gr)
# Check restoration of GPEI, to ensure proper restoration of callable kwargs
gpei = Models.GPEI(experiment=exp, data=get_branin_data())
# Punch GPEI model + bridge kwargs into the Sobol generator run, to avoid
# a slow call to `gpei.gen`.
gr._model_key = "GPEI"
gr._model_kwargs = gpei._model_kwargs
gr._bridge_kwargs = gpei._bridge_kwargs
gpei_restored = get_model_from_generator_run(gr,
experiment=exp,
data=get_branin_data())
for key in gpei.__dict__:
self.assertIn(key, gpei_restored.__dict__)
original, restored = gpei.__dict__[key], gpei_restored.__dict__[
key]
# Fit times are set in instantiation so not same and model compared below
if key in ["fit_time", "fit_time_since_gen", "model"]:
continue # Fit times are set in instantiation so won't be same
if isinstance(original, OrderedDict) and isinstance(
restored, OrderedDict):
original, restored = list(original.keys()), list(
restored.keys())
if isinstance(original, Model) and isinstance(restored, Model):
continue # Model equality is tough to compare.
self.assertEqual(original, restored)
for key in gpei.model.__dict__:
self.assertIn(key, gpei_restored.model.__dict__)
original, restored = (
gpei.model.__dict__[key],
gpei_restored.model.__dict__[key],
)
# Botorch model equality is tough to compare and training data
# is unnecessary to compare, because data passed to model was the same
if key in ["model", "warm_start_refitting", "Xs", "Ys"]:
continue
self.assertEqual(original, restored)
def test_enum_sobol_GPEI(self):
"""Tests Sobol instantiation through the Models enum."""
exp = get_branin_experiment()
# Check that factory generates a valid sobol modelbridge.
sobol = Models.SOBOL(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(n=1)
self.assertEqual(sobol_run._model_key, "Sobol")
exp.new_batch_trial().add_generator_run(sobol_run).run()
# Check that factory generates a valid GP+EI modelbridge.
exp.optimization_config = get_branin_optimization_config()
gpei = Models.GPEI(experiment=exp, data=exp.fetch_data())
self.assertIsInstance(gpei, TorchModelBridge)
gpei = Models.GPEI(experiment=exp,
data=exp.fetch_data(),
search_space=exp.search_space)
self.assertIsInstance(gpei, TorchModelBridge)
def plot_progress(self, ax_client):
model = Models.GPEI(experiment=ax_client.experiment, data=ax_client.experiment.fetch_data())
html_elements = []
html_elements.append(plot_config_to_html(ax_client.get_optimization_trace()))
try:
html_elements.append(plot_config_to_html(interact_contour(model=model, metric_name=self.YR.args.eval_primary_metric)))
except:
pass
with open(os.path.join(self.bayes_opt_root_experiment_folder, "optimization_plots.html"), 'w') as f:
f.write(render_report_elements(self.experiment_name, html_elements))
parameter_type=ParameterType.FLOAT),
RangeParameter(name="leafes", lower=2, upper=1000,
parameter_type=ParameterType.INT)]
)
experiment = SimpleExperiment(
name = f"weather_lbgm_{dt.datetime.today().strftime('%d-%m-%Y')}",
search_space = search_space,
evaluation_function = mdl.obj_fun,
)
sobol = Models.SOBOL(experiment.search_space)
for i in range(n_random_trials):
experiment.new_trial(generator_run=sobol.gen(1))
best_arm = None
for i in range(n_searches):
gpei = Models.GPEI(experiment=experiment, data=experiment.eval())
generator_run = gpei.gen(1)
best_arm, _ = generator_run.best_arm_predictions
experiment.new_trial(generator_run=generator_run)
best_para_ax = best_arm.parameters
n_oos = 0
params['num_boost_round'] = 200
mdl = Model(data_mat, lags, n_oos, n_val, prediction_range, target_vars_inds, params)
mdl.fit(best_para_ax)
X_pred = mdl.standing_forecast()
