def test_cross_validate_raises_not_implemented_error_for_non_cv_model_with_data(
self,
):
exp = get_branin_experiment(with_batch=True)
exp.trials[0].run().complete()
sobol = Models.SOBOL(
experiment=exp, search_space=exp.search_space, data=exp.fetch_data()
)
with self.assertRaises(NotImplementedError):
cross_validate(model=sobol)
def test_cross_validation(self):
exp = get_branin_experiment(with_batch=True)
exp.trials[0].run()
model = Models.BOTORCH(
# Model bridge kwargs
experiment=exp,
data=exp.fetch_data(),
)
cv = cross_validate(model)
# Assert that each type of plot can be constructed successfully
plot = interact_cross_validation_plotly(cv)
self.assertIsInstance(plot, go.Figure)
plot = interact_cross_validation(cv)
self.assertIsInstance(plot, AxPlotConfig)
def cross_validate(
self,
) -> Tuple[Optional[List[CVResult]], Optional[CVDiagnostics]]:
"""
Call cross_validate, compute_diagnostics and cache the results
If the model cannot be cross validated, warn and return None
"""
if self._cv_results is not None and self._diagnostics is not None:
return self._cv_results, self._diagnostics
self._assert_fitted()
try:
self._cv_results = cross_validate(
model=self.fitted_model,
**(self.model_cv_kwargs or {}),
)
except NotImplementedError:
warnings.warn(f"{self.model_enum.value} cannot be cross validated")
return None, None
self._diagnostics = compute_diagnostics(self._cv_results)
return self._cv_results, self._diagnostics
def testCrossValidate(self):
# Prepare input and output data
ma = mock.MagicMock()
ma.get_training_data = mock.MagicMock(
"ax.modelbridge.base.ModelBridge.get_training_data",
autospec=True,
return_value=self.training_data,
)
ma.cross_validate = mock.MagicMock(
"ax.modelbridge.base.ModelBridge.cross_validate",
autospec=True,
return_value=self.observation_data,
)
# Do cross validation
with self.assertRaises(ValueError):
cross_validate(model=ma, folds=4)
with self.assertRaises(ValueError):
cross_validate(model=ma, folds=0)
# First 2-fold
result = cross_validate(model=ma, folds=2)
self.assertEqual(len(result), 4)
# Check that ModelBridge.cross_validate was called correctly.
z = ma.cross_validate.mock_calls
self.assertEqual(len(z), 2)
train = [[
obs.features.parameters["x"] for obs in r[2]["cv_training_data"]
] for r in z]
test = [[obsf.parameters["x"] for obsf in r[2]["cv_test_points"]]
for r in z]
# Test no overlap between train and test sets, and all points used
for i in range(2):
self.assertEqual(len(set(train[i]).intersection(test[i])), 0)
self.assertEqual(len(train[i]) + len(test[i]), 4)
# Test all points used as test points
all_test = np.hstack(test)
self.assertTrue(
np.array_equal(sorted(all_test), np.array([2.0, 2.0, 3.0, 4.0])))
# Test LOO
result = cross_validate(model=ma, folds=-1)
self.assertEqual(len(result), 4)
z = ma.cross_validate.mock_calls[2:]
self.assertEqual(len(z), 3)
train = [[
obs.features.parameters["x"] for obs in r[2]["cv_training_data"]
] for r in z]
test = [[obsf.parameters["x"] for obsf in r[2]["cv_test_points"]]
for r in z]
# Test no overlap between train and test sets, and all points used
for i in range(3):
self.assertEqual(len(set(train[i]).intersection(test[i])), 0)
self.assertEqual(len(train[i]) + len(test[i]), 4)
# Test all points used as test points
all_test = np.hstack(test)
self.assertTrue(
np.array_equal(sorted(all_test), np.array([2.0, 2.0, 3.0, 4.0])))
# Test selector
def test_selector(obs):
return obs.features.parameters["x"] != 4.0
result = cross_validate(model=ma,
folds=-1,
test_selector=test_selector)
self.assertEqual(len(result), 3)
z = ma.cross_validate.mock_calls[5:]
self.assertEqual(len(z), 2)
all_test = np.hstack(
[[obsf.parameters["x"] for obsf in r[2]["cv_test_points"]]
for r in z])
self.assertTrue(
np.array_equal(sorted(all_test), np.array([2.0, 2.0, 3.0])))
def get_best_from_model_predictions_with_trial_index(
experiment: Experiment,
) -> Optional[Tuple[int, TParameterization, Optional[TModelPredictArm]]]:
"""Given an experiment, returns the best predicted parameterization and corresponding
prediction based on the most recent Trial with predictions. If no trials have
predictions returns None.
Only some models return predictions. For instance GPEI does while Sobol does not.
TModelPredictArm is of the form:
({metric_name: mean}, {metric_name_1: {metric_name_2: cov_1_2}})
Args:
experiment: Experiment, on which to identify best raw objective arm.
Returns:
Tuple of parameterization and model predictions for it.
"""
# pyre-ignore [16]
if isinstance(experiment.optimization_config.objective, MultiObjective):
logger.warning(
"get_best_from_model_predictions is deprecated for multi-objective "
"optimization configs. This method will return an arbitrary point on "
"the pareto frontier.")
for idx, trial in sorted(experiment.trials.items(),
key=lambda x: x[0],
reverse=True):
gr = None
if isinstance(trial, Trial):
gr = trial.generator_run
elif isinstance(trial, BatchTrial):
if len(trial.generator_run_structs) > 0:
# In theory batch_trial can have >1 gr, grab the first
gr = trial.generator_run_structs[0].generator_run
if gr is not None and gr.best_arm_predictions is not None: # pragma: no cover
data = experiment.lookup_data()
if not isinstance(data, Data):
return _gr_to_prediction_with_trial_index(idx, gr)
model = get_model_from_generator_run(generator_run=gr,
experiment=experiment,
data=data)
# If model is not ArrayModelBridge, just use the best arm frmo the
# last good generator run
if not isinstance(model, ArrayModelBridge):
return _gr_to_prediction_with_trial_index(idx, gr)
# Check to see if the model is worth using
cv_results = cross_validate(model=model)
diagnostics = compute_diagnostics(result=cv_results)
assess_model_fit_results = assess_model_fit(
diagnostics=diagnostics)
objective_name = experiment.optimization_config.objective.metric.name
# If model fit is bad use raw results
if (objective_name in
assess_model_fit_results.bad_fit_metrics_to_fisher_score):
logger.warning(
"Model fit is poor; falling back on raw data for best point."
)
if not _is_all_noiseless(df=data.df,
metric_name=objective_name):
logger.warning(
"Model fit is poor and data on objective metric " +
f"{objective_name} is noisy; interpret best points " +
"results carefully.")
return _get_best_poor_model_fit(experiment=experiment)
res = model.model_best_point()
if res is None:
return _gr_to_prediction_with_trial_index(idx, gr)
best_arm, best_arm_predictions = res
return idx, not_none(best_arm).parameters, best_arm_predictions
return None
def _get_cross_validation_plots(model: ModelBridge) -> List[go.Figure]:
cv = cross_validate(model=model)
return [
interact_cross_validation_plotly(cv_results=cv,
caption=CROSS_VALIDATION_CAPTION)
]
def test_cross_validate_gives_a_useful_error_for_model_with_no_data(self):
exp = get_branin_experiment()
sobol = Models.SOBOL(experiment=exp, search_space=exp.search_space)
with self.assertRaisesRegex(ValueError, "no training data"):
cross_validate(model=sobol)
def _get_cross_validation_plot(model: ModelBridge) -> go.Figure:
cv = cross_validate(model)
return interact_cross_validation_plotly(cv)
def _get_cross_validation_plots(model: ModelBridge) -> List[go.Figure]:
cv = cross_validate(model=model)
return [interact_cross_validation_plotly(cv_results=cv)]
contour_plot = interact_contour(model=gpei, metric_name=metric)
plot(contour_plot.data,
filename=os.path.join(output_dir, 'contour_plot',
'{}.html'.format(metric)))
# Tradeoff Plot
tradeoff_plot = plot_objective_vs_constraints(gpei, 'f1', rel=False)
plot(tradeoff_plot.data,
filename=os.path.join(output_dir, 'tradeoff_plot.html'))
# Slice Plot
# show the metric outcome as a function of one parameter while fixing the others
os.makedirs(os.path.join(output_dir, 'slice_plot'))
for param in ["lr", "decay", "warmups", "eps"]:
slice_plot = plot_slice(gpei, param, "f1")
plot(slice_plot.data,
filename=os.path.join(output_dir, 'slice_plot',
'{}.html'.format(param)))
# Tile Plot
# the effect of each arm
tile_plot = interact_fitted(gpei, rel=False)
plot(tile_plot.data,
filename=os.path.join(output_dir, 'tile_plot.html'))
# Cross Validation plot
# splits the model's train data into train/test folds and makes out-of-sample predictions on the test folds.
cv_results = cross_validate(gpei)
cv_plot = interact_cross_validation(cv_results)
plot(cv_plot.data, filename=os.path.join(output_dir, 'cv_plot.html'))
