def test_get_model_from_generator_run(self): """Tests that it is possible to restore a model from a generator run it produced, if `Models` registry was used. """ exp = get_branin_experiment() initial_sobol = Models.SOBOL(experiment=exp, seed=239) gr = initial_sobol.gen(n=1) # Restore the model as it was before generation. sobol = get_model_from_generator_run(generator_run=gr, experiment=exp, data=exp.fetch_data()) self.assertEqual(sobol.model.init_position, 0) self.assertEqual(sobol.model.seed, 239) # Restore the model as it was after generation (to resume generation). sobol_after_gen = get_model_from_generator_run(generator_run=gr, experiment=exp, data=exp.fetch_data(), after_gen=True) self.assertEqual(sobol_after_gen.model.init_position, 1) self.assertEqual(sobol_after_gen.model.seed, 239) self.assertEqual( initial_sobol.gen(n=1).arms, sobol_after_gen.gen(n=1).arms) exp.new_trial(generator_run=gr) # Check restoration of GPEI, to ensure proper restoration of callable kwargs gpei = Models.GPEI(experiment=exp, data=get_branin_data()) # Punch GPEI model + bridge kwargs into the Sobol generator run, to avoid # a slow call to `gpei.gen`. gr._model_key = "GPEI" gr._model_kwargs = gpei._model_kwargs gr._bridge_kwargs = gpei._bridge_kwargs gpei_restored = get_model_from_generator_run(gr, experiment=exp, data=get_branin_data()) for key in gpei.__dict__: self.assertIn(key, gpei_restored.__dict__) original, restored = gpei.__dict__[key], gpei_restored.__dict__[ key] # Fit times are set in instantiation so not same and model compared below if key in ["fit_time", "fit_time_since_gen", "model"]: continue # Fit times are set in instantiation so won't be same if isinstance(original, OrderedDict) and isinstance( restored, OrderedDict): original, restored = list(original.keys()), list( restored.keys()) if isinstance(original, Model) and isinstance(restored, Model): continue # Model equality is tough to compare. self.assertEqual(original, restored) for key in gpei.model.__dict__: self.assertIn(key, gpei_restored.model.__dict__) original, restored = ( gpei.model.__dict__[key], gpei_restored.model.__dict__[key], ) # Botorch model equality is tough to compare and training data # is unnecessary to compare, because data passed to model was the same if key in ["model", "warm_start_refitting", "Xs", "Ys"]: continue self.assertEqual(original, restored)
def _restore_model_from_generator_run( self, models_enum: Optional[Type[ModelRegistryBase]] = None) -> None: """Reinstantiates the most recent model on this generation strategy from the last generator run it produced. NOTE: Uses model and model bridge kwargs stored on the generator run, as well as the model state attributes stored on the generator run. """ generator_run = self.last_generator_run if generator_run is None: raise ValueError( "No generator run was stored on generation strategy.") if self._experiment is None: # pragma: no cover raise ValueError( "No experiment was set on this generation strategy.") data = self.experiment.lookup_data() self._model = get_model_from_generator_run( generator_run=generator_run, experiment=self.experiment, # pyre-fixme [6]: Incompat param: Expect `Data` got `AbstractDataFrameData` data=data, models_enum=models_enum, ) self._save_seen_trial_indices()
def _restore_model_from_generator_run( self, models_enum: Optional[Type[Models]] = None ) -> None: generator_run = self.last_generator_run if generator_run is None: raise ValueError("No generator run was stored on generation strategy.") if self._experiment is None: # pragma: no cover raise ValueError("No experiment was set on this generation strategy.") self._model = get_model_from_generator_run( generator_run=generator_run, experiment=self.experiment, data=self.experiment.fetch_data(), models_enum=models_enum, )
def test_get_model_from_generator_run(self): """Tests that it is possible to restore a model from a generator run it produced, if `Models` registry was used. """ exp = get_branin_experiment() initial_sobol = Models.SOBOL(experiment=exp, seed=239) gr = initial_sobol.gen(n=1) # Restore the model as it was before generation. sobol = get_model_from_generator_run(generator_run=gr, experiment=exp, data=exp.fetch_data()) self.assertEqual(sobol.model.init_position, 0) self.assertEqual(sobol.model.seed, 239) # Restore the model as it was after generation (to resume generation). sobol_after_gen = get_model_from_generator_run(generator_run=gr, experiment=exp, data=exp.fetch_data(), after_gen=True) self.assertEqual(sobol_after_gen.model.init_position, 1) self.assertEqual(sobol_after_gen.model.seed, 239) self.assertEqual( initial_sobol.gen(n=1).arms, sobol_after_gen.gen(n=1).arms)
def _restore_model_from_generator_run( self, models_enum: Optional[Type[Models]] = None ) -> None: """Reinstantiates the most recent model on this generation strategy from the last generator run it produced. NOTE: Uses model and model bridge kwargs stored on the generator run, as well as the model state attributes stored on the generator run. """ generator_run = self.last_generator_run if generator_run is None: raise ValueError("No generator run was stored on generation strategy.") if self._experiment is None: # pragma: no cover raise ValueError("No experiment was set on this generation strategy.") self._model = get_model_from_generator_run( generator_run=generator_run, experiment=self.experiment, data=self.experiment.fetch_data(), models_enum=models_enum, )
def get_best_from_model_predictions_with_trial_index( experiment: Experiment, ) -> Optional[Tuple[int, TParameterization, Optional[TModelPredictArm]]]: """Given an experiment, returns the best predicted parameterization and corresponding prediction based on the most recent Trial with predictions. If no trials have predictions returns None. Only some models return predictions. For instance GPEI does while Sobol does not. TModelPredictArm is of the form: ({metric_name: mean}, {metric_name_1: {metric_name_2: cov_1_2}}) Args: experiment: Experiment, on which to identify best raw objective arm. Returns: Tuple of parameterization and model predictions for it. """ # pyre-ignore [16] if isinstance(experiment.optimization_config.objective, MultiObjective): logger.warning( "get_best_from_model_predictions is deprecated for multi-objective " "optimization configs. This method will return an arbitrary point on " "the pareto frontier.") for idx, trial in sorted(experiment.trials.items(), key=lambda x: x[0], reverse=True): gr = None if isinstance(trial, Trial): gr = trial.generator_run elif isinstance(trial, BatchTrial): if len(trial.generator_run_structs) > 0: # In theory batch_trial can have >1 gr, grab the first gr = trial.generator_run_structs[0].generator_run if gr is not None and gr.best_arm_predictions is not None: # pragma: no cover data = experiment.lookup_data() if not isinstance(data, Data): return _gr_to_prediction_with_trial_index(idx, gr) model = get_model_from_generator_run(generator_run=gr, experiment=experiment, data=data) # If model is not ArrayModelBridge, just use the best arm frmo the # last good generator run if not isinstance(model, ArrayModelBridge): return _gr_to_prediction_with_trial_index(idx, gr) # Check to see if the model is worth using cv_results = cross_validate(model=model) diagnostics = compute_diagnostics(result=cv_results) assess_model_fit_results = assess_model_fit( diagnostics=diagnostics) objective_name = experiment.optimization_config.objective.metric.name # If model fit is bad use raw results if (objective_name in assess_model_fit_results.bad_fit_metrics_to_fisher_score): logger.warning( "Model fit is poor; falling back on raw data for best point." ) if not _is_all_noiseless(df=data.df, metric_name=objective_name): logger.warning( "Model fit is poor and data on objective metric " + f"{objective_name} is noisy; interpret best points " + "results carefully.") return _get_best_poor_model_fit(experiment=experiment) res = model.model_best_point() if res is None: return _gr_to_prediction_with_trial_index(idx, gr) best_arm, best_arm_predictions = res return idx, not_none(best_arm).parameters, best_arm_predictions return None