def test_get_model_from_generator_run(self):
"""Tests that it is possible to restore a model from a generator run it
produced, if `Models` registry was used.
"""
exp = get_branin_experiment()
initial_sobol = Models.SOBOL(experiment=exp, seed=239)
gr = initial_sobol.gen(n=1)
# Restore the model as it was before generation.
sobol = get_model_from_generator_run(generator_run=gr,
experiment=exp,
data=exp.fetch_data())
self.assertEqual(sobol.model.init_position, 0)
self.assertEqual(sobol.model.seed, 239)
# Restore the model as it was after generation (to resume generation).
sobol_after_gen = get_model_from_generator_run(generator_run=gr,
experiment=exp,
data=exp.fetch_data(),
after_gen=True)
self.assertEqual(sobol_after_gen.model.init_position, 1)
self.assertEqual(sobol_after_gen.model.seed, 239)
self.assertEqual(
initial_sobol.gen(n=1).arms,
sobol_after_gen.gen(n=1).arms)
exp.new_trial(generator_run=gr)
# Check restoration of GPEI, to ensure proper restoration of callable kwargs
gpei = Models.GPEI(experiment=exp, data=get_branin_data())
# Punch GPEI model + bridge kwargs into the Sobol generator run, to avoid
# a slow call to `gpei.gen`.
gr._model_key = "GPEI"
gr._model_kwargs = gpei._model_kwargs
gr._bridge_kwargs = gpei._bridge_kwargs
gpei_restored = get_model_from_generator_run(gr,
experiment=exp,
data=get_branin_data())
for key in gpei.__dict__:
self.assertIn(key, gpei_restored.__dict__)
original, restored = gpei.__dict__[key], gpei_restored.__dict__[
key]
# Fit times are set in instantiation so not same and model compared below
if key in ["fit_time", "fit_time_since_gen", "model"]:
continue # Fit times are set in instantiation so won't be same
if isinstance(original, OrderedDict) and isinstance(
restored, OrderedDict):
original, restored = list(original.keys()), list(
restored.keys())
if isinstance(original, Model) and isinstance(restored, Model):
continue # Model equality is tough to compare.
self.assertEqual(original, restored)
for key in gpei.model.__dict__:
self.assertIn(key, gpei_restored.model.__dict__)
original, restored = (
gpei.model.__dict__[key],
gpei_restored.model.__dict__[key],
)
# Botorch model equality is tough to compare and training data
# is unnecessary to compare, because data passed to model was the same
if key in ["model", "warm_start_refitting", "Xs", "Ys"]:
continue
self.assertEqual(original, restored)
def _restore_model_from_generator_run(
self,
models_enum: Optional[Type[ModelRegistryBase]] = None) -> None:
"""Reinstantiates the most recent model on this generation strategy
from the last generator run it produced.
NOTE: Uses model and model bridge kwargs stored on the generator run, as well
as the model state attributes stored on the generator run.
"""
generator_run = self.last_generator_run
if generator_run is None:
raise ValueError(
"No generator run was stored on generation strategy.")
if self._experiment is None: # pragma: no cover
raise ValueError(
"No experiment was set on this generation strategy.")
data = self.experiment.lookup_data()
self._model = get_model_from_generator_run(
generator_run=generator_run,
experiment=self.experiment,
# pyre-fixme [6]: Incompat param: Expect `Data` got `AbstractDataFrameData`
data=data,
models_enum=models_enum,
)
self._save_seen_trial_indices()
def _restore_model_from_generator_run(
self, models_enum: Optional[Type[Models]] = None
) -> None:
generator_run = self.last_generator_run
if generator_run is None:
raise ValueError("No generator run was stored on generation strategy.")
if self._experiment is None: # pragma: no cover
raise ValueError("No experiment was set on this generation strategy.")
self._model = get_model_from_generator_run(
generator_run=generator_run,
experiment=self.experiment,
data=self.experiment.fetch_data(),
models_enum=models_enum,
)
def test_get_model_from_generator_run(self):
"""Tests that it is possible to restore a model from a generator run it
produced, if `Models` registry was used.
"""
exp = get_branin_experiment()
initial_sobol = Models.SOBOL(experiment=exp, seed=239)
gr = initial_sobol.gen(n=1)
# Restore the model as it was before generation.
sobol = get_model_from_generator_run(generator_run=gr,
experiment=exp,
data=exp.fetch_data())
self.assertEqual(sobol.model.init_position, 0)
self.assertEqual(sobol.model.seed, 239)
# Restore the model as it was after generation (to resume generation).
sobol_after_gen = get_model_from_generator_run(generator_run=gr,
experiment=exp,
data=exp.fetch_data(),
after_gen=True)
self.assertEqual(sobol_after_gen.model.init_position, 1)
self.assertEqual(sobol_after_gen.model.seed, 239)
self.assertEqual(
initial_sobol.gen(n=1).arms,
sobol_after_gen.gen(n=1).arms)
def _restore_model_from_generator_run(
self, models_enum: Optional[Type[Models]] = None
) -> None:
"""Reinstantiates the most recent model on this generation strategy
from the last generator run it produced.
NOTE: Uses model and model bridge kwargs stored on the generator run, as well
as the model state attributes stored on the generator run.
"""
generator_run = self.last_generator_run
if generator_run is None:
raise ValueError("No generator run was stored on generation strategy.")
if self._experiment is None: # pragma: no cover
raise ValueError("No experiment was set on this generation strategy.")
self._model = get_model_from_generator_run(
generator_run=generator_run,
experiment=self.experiment,
data=self.experiment.fetch_data(),
models_enum=models_enum,
)
def get_best_from_model_predictions_with_trial_index(
experiment: Experiment,
) -> Optional[Tuple[int, TParameterization, Optional[TModelPredictArm]]]:
"""Given an experiment, returns the best predicted parameterization and corresponding
prediction based on the most recent Trial with predictions. If no trials have
predictions returns None.
Only some models return predictions. For instance GPEI does while Sobol does not.
TModelPredictArm is of the form:
({metric_name: mean}, {metric_name_1: {metric_name_2: cov_1_2}})
Args:
experiment: Experiment, on which to identify best raw objective arm.
Returns:
Tuple of parameterization and model predictions for it.
"""
# pyre-ignore [16]
if isinstance(experiment.optimization_config.objective, MultiObjective):
logger.warning(
"get_best_from_model_predictions is deprecated for multi-objective "
"optimization configs. This method will return an arbitrary point on "
"the pareto frontier.")
for idx, trial in sorted(experiment.trials.items(),
key=lambda x: x[0],
reverse=True):
gr = None
if isinstance(trial, Trial):
gr = trial.generator_run
elif isinstance(trial, BatchTrial):
if len(trial.generator_run_structs) > 0:
# In theory batch_trial can have >1 gr, grab the first
gr = trial.generator_run_structs[0].generator_run
if gr is not None and gr.best_arm_predictions is not None: # pragma: no cover
data = experiment.lookup_data()
if not isinstance(data, Data):
return _gr_to_prediction_with_trial_index(idx, gr)
model = get_model_from_generator_run(generator_run=gr,
experiment=experiment,
data=data)
# If model is not ArrayModelBridge, just use the best arm frmo the
# last good generator run
if not isinstance(model, ArrayModelBridge):
return _gr_to_prediction_with_trial_index(idx, gr)
# Check to see if the model is worth using
cv_results = cross_validate(model=model)
diagnostics = compute_diagnostics(result=cv_results)
assess_model_fit_results = assess_model_fit(
diagnostics=diagnostics)
objective_name = experiment.optimization_config.objective.metric.name
# If model fit is bad use raw results
if (objective_name in
assess_model_fit_results.bad_fit_metrics_to_fisher_score):
logger.warning(
"Model fit is poor; falling back on raw data for best point."
)
if not _is_all_noiseless(df=data.df,
metric_name=objective_name):
logger.warning(
"Model fit is poor and data on objective metric " +
f"{objective_name} is noisy; interpret best points " +
"results carefully.")
return _get_best_poor_model_fit(experiment=experiment)
res = model.model_best_point()
if res is None:
return _gr_to_prediction_with_trial_index(idx, gr)
best_arm, best_arm_predictions = res
return idx, not_none(best_arm).parameters, best_arm_predictions
return None
