def test_cross_validate_raises_not_implemented_error_for_non_cv_model_with_data( self, ): exp = get_branin_experiment(with_batch=True) exp.trials[0].run().complete() sobol = Models.SOBOL( experiment=exp, search_space=exp.search_space, data=exp.fetch_data() ) with self.assertRaises(NotImplementedError): cross_validate(model=sobol)
def test_cross_validation(self): exp = get_branin_experiment(with_batch=True) exp.trials[0].run() model = Models.BOTORCH( # Model bridge kwargs experiment=exp, data=exp.fetch_data(), ) cv = cross_validate(model) # Assert that each type of plot can be constructed successfully plot = interact_cross_validation_plotly(cv) self.assertIsInstance(plot, go.Figure) plot = interact_cross_validation(cv) self.assertIsInstance(plot, AxPlotConfig)
def cross_validate( self, ) -> Tuple[Optional[List[CVResult]], Optional[CVDiagnostics]]: """ Call cross_validate, compute_diagnostics and cache the results If the model cannot be cross validated, warn and return None """ if self._cv_results is not None and self._diagnostics is not None: return self._cv_results, self._diagnostics self._assert_fitted() try: self._cv_results = cross_validate( model=self.fitted_model, **(self.model_cv_kwargs or {}), ) except NotImplementedError: warnings.warn(f"{self.model_enum.value} cannot be cross validated") return None, None self._diagnostics = compute_diagnostics(self._cv_results) return self._cv_results, self._diagnostics
def testCrossValidate(self): # Prepare input and output data ma = mock.MagicMock() ma.get_training_data = mock.MagicMock( "ax.modelbridge.base.ModelBridge.get_training_data", autospec=True, return_value=self.training_data, ) ma.cross_validate = mock.MagicMock( "ax.modelbridge.base.ModelBridge.cross_validate", autospec=True, return_value=self.observation_data, ) # Do cross validation with self.assertRaises(ValueError): cross_validate(model=ma, folds=4) with self.assertRaises(ValueError): cross_validate(model=ma, folds=0) # First 2-fold result = cross_validate(model=ma, folds=2) self.assertEqual(len(result), 4) # Check that ModelBridge.cross_validate was called correctly. z = ma.cross_validate.mock_calls self.assertEqual(len(z), 2) train = [[ obs.features.parameters["x"] for obs in r[2]["cv_training_data"] ] for r in z] test = [[obsf.parameters["x"] for obsf in r[2]["cv_test_points"]] for r in z] # Test no overlap between train and test sets, and all points used for i in range(2): self.assertEqual(len(set(train[i]).intersection(test[i])), 0) self.assertEqual(len(train[i]) + len(test[i]), 4) # Test all points used as test points all_test = np.hstack(test) self.assertTrue( np.array_equal(sorted(all_test), np.array([2.0, 2.0, 3.0, 4.0]))) # Test LOO result = cross_validate(model=ma, folds=-1) self.assertEqual(len(result), 4) z = ma.cross_validate.mock_calls[2:] self.assertEqual(len(z), 3) train = [[ obs.features.parameters["x"] for obs in r[2]["cv_training_data"] ] for r in z] test = [[obsf.parameters["x"] for obsf in r[2]["cv_test_points"]] for r in z] # Test no overlap between train and test sets, and all points used for i in range(3): self.assertEqual(len(set(train[i]).intersection(test[i])), 0) self.assertEqual(len(train[i]) + len(test[i]), 4) # Test all points used as test points all_test = np.hstack(test) self.assertTrue( np.array_equal(sorted(all_test), np.array([2.0, 2.0, 3.0, 4.0]))) # Test selector def test_selector(obs): return obs.features.parameters["x"] != 4.0 result = cross_validate(model=ma, folds=-1, test_selector=test_selector) self.assertEqual(len(result), 3) z = ma.cross_validate.mock_calls[5:] self.assertEqual(len(z), 2) all_test = np.hstack( [[obsf.parameters["x"] for obsf in r[2]["cv_test_points"]] for r in z]) self.assertTrue( np.array_equal(sorted(all_test), np.array([2.0, 2.0, 3.0])))
def get_best_from_model_predictions_with_trial_index( experiment: Experiment, ) -> Optional[Tuple[int, TParameterization, Optional[TModelPredictArm]]]: """Given an experiment, returns the best predicted parameterization and corresponding prediction based on the most recent Trial with predictions. If no trials have predictions returns None. Only some models return predictions. For instance GPEI does while Sobol does not. TModelPredictArm is of the form: ({metric_name: mean}, {metric_name_1: {metric_name_2: cov_1_2}}) Args: experiment: Experiment, on which to identify best raw objective arm. Returns: Tuple of parameterization and model predictions for it. """ # pyre-ignore [16] if isinstance(experiment.optimization_config.objective, MultiObjective): logger.warning( "get_best_from_model_predictions is deprecated for multi-objective " "optimization configs. This method will return an arbitrary point on " "the pareto frontier.") for idx, trial in sorted(experiment.trials.items(), key=lambda x: x[0], reverse=True): gr = None if isinstance(trial, Trial): gr = trial.generator_run elif isinstance(trial, BatchTrial): if len(trial.generator_run_structs) > 0: # In theory batch_trial can have >1 gr, grab the first gr = trial.generator_run_structs[0].generator_run if gr is not None and gr.best_arm_predictions is not None: # pragma: no cover data = experiment.lookup_data() if not isinstance(data, Data): return _gr_to_prediction_with_trial_index(idx, gr) model = get_model_from_generator_run(generator_run=gr, experiment=experiment, data=data) # If model is not ArrayModelBridge, just use the best arm frmo the # last good generator run if not isinstance(model, ArrayModelBridge): return _gr_to_prediction_with_trial_index(idx, gr) # Check to see if the model is worth using cv_results = cross_validate(model=model) diagnostics = compute_diagnostics(result=cv_results) assess_model_fit_results = assess_model_fit( diagnostics=diagnostics) objective_name = experiment.optimization_config.objective.metric.name # If model fit is bad use raw results if (objective_name in assess_model_fit_results.bad_fit_metrics_to_fisher_score): logger.warning( "Model fit is poor; falling back on raw data for best point." ) if not _is_all_noiseless(df=data.df, metric_name=objective_name): logger.warning( "Model fit is poor and data on objective metric " + f"{objective_name} is noisy; interpret best points " + "results carefully.") return _get_best_poor_model_fit(experiment=experiment) res = model.model_best_point() if res is None: return _gr_to_prediction_with_trial_index(idx, gr) best_arm, best_arm_predictions = res return idx, not_none(best_arm).parameters, best_arm_predictions return None
def _get_cross_validation_plots(model: ModelBridge) -> List[go.Figure]: cv = cross_validate(model=model) return [ interact_cross_validation_plotly(cv_results=cv, caption=CROSS_VALIDATION_CAPTION) ]
def test_cross_validate_gives_a_useful_error_for_model_with_no_data(self): exp = get_branin_experiment() sobol = Models.SOBOL(experiment=exp, search_space=exp.search_space) with self.assertRaisesRegex(ValueError, "no training data"): cross_validate(model=sobol)
def _get_cross_validation_plot(model: ModelBridge) -> go.Figure: cv = cross_validate(model) return interact_cross_validation_plotly(cv)
def _get_cross_validation_plots(model: ModelBridge) -> List[go.Figure]: cv = cross_validate(model=model) return [interact_cross_validation_plotly(cv_results=cv)]
contour_plot = interact_contour(model=gpei, metric_name=metric) plot(contour_plot.data, filename=os.path.join(output_dir, 'contour_plot', '{}.html'.format(metric))) # Tradeoff Plot tradeoff_plot = plot_objective_vs_constraints(gpei, 'f1', rel=False) plot(tradeoff_plot.data, filename=os.path.join(output_dir, 'tradeoff_plot.html')) # Slice Plot # show the metric outcome as a function of one parameter while fixing the others os.makedirs(os.path.join(output_dir, 'slice_plot')) for param in ["lr", "decay", "warmups", "eps"]: slice_plot = plot_slice(gpei, param, "f1") plot(slice_plot.data, filename=os.path.join(output_dir, 'slice_plot', '{}.html'.format(param))) # Tile Plot # the effect of each arm tile_plot = interact_fitted(gpei, rel=False) plot(tile_plot.data, filename=os.path.join(output_dir, 'tile_plot.html')) # Cross Validation plot # splits the model's train data into train/test folds and makes out-of-sample predictions on the test folds. cv_results = cross_validate(gpei) cv_plot = interact_cross_validation(cv_results) plot(cv_plot.data, filename=os.path.join(output_dir, 'cv_plot.html'))