def testAssessModelFit(self):
# Construct diagnostics
result = []
for i, obs in enumerate(self.training_data):
result.append(
CVResult(observed=obs, predicted=self.observation_data[i]))
diag = compute_diagnostics(result=result)
for v in diag.values():
self.assertEqual(set(v.keys()), {"a", "b"})
# Check for correct computation, relative to manually computed result
self.assertAlmostEqual(diag["Fisher exact test p"]["a"], 0.10)
self.assertAlmostEqual(diag["Fisher exact test p"]["b"], 0.166666666)
good_fit_metrics, bad_fit_metrics = assess_model_fit(
diag, significance_level=0.05)
self.assertTrue("a" in bad_fit_metrics)
self.assertTrue("b" in bad_fit_metrics)
good_fit_metrics, bad_fit_metrics = assess_model_fit(
diag, significance_level=0.15)
self.assertTrue("a" in good_fit_metrics)
self.assertTrue("b" in bad_fit_metrics)
good_fit_metrics, bad_fit_metrics = assess_model_fit(
diag, significance_level=0.2)
self.assertTrue("a" in good_fit_metrics)
self.assertTrue("b" in good_fit_metrics)
def testComputeDiagnostics(self):
# Construct CVResults
result = []
for i, obs in enumerate(self.training_data):
result.append(
CVResult(observed=obs, predicted=self.observation_data[i]))
# Compute diagnostics
diag = compute_diagnostics(result=result)
for v in diag.values():
self.assertEqual(set(v.keys()), {"a", "b"})
# Check for correct computation, relative to manually computed result
self.assertAlmostEqual(diag["MAPE"]["a"], 0.4984126984126984)
self.assertAlmostEqual(diag["Total raw effect"]["a"], 3.5)
self.assertAlmostEqual(diag["Total raw effect"]["b"], 1.5)
def testHasGoodOptConfigModelFit(self):
# Construct diagnostics
result = []
for i, obs in enumerate(self.training_data):
result.append(
CVResult(observed=obs, predicted=self.observation_data[i]))
diag = compute_diagnostics(result=result)
assess_model_fit_result = assess_model_fit(
diagnostics=diag,
significance_level=0.05,
)
# Test single objective
optimization_config = OptimizationConfig(objective=Objective(
metric=Metric("a")))
has_good_fit = has_good_opt_config_model_fit(
optimization_config=optimization_config,
assess_model_fit_result=assess_model_fit_result,
)
self.assertFalse(has_good_fit)
# Test multi objective
optimization_config = MultiObjectiveOptimizationConfig(
objective=MultiObjective(
metrics=[Metric("a"), Metric("b")]))
has_good_fit = has_good_opt_config_model_fit(
optimization_config=optimization_config,
assess_model_fit_result=assess_model_fit_result,
)
self.assertFalse(has_good_fit)
# Test constraints
optimization_config = OptimizationConfig(
objective=Objective(metric=Metric("a")),
outcome_constraints=[
OutcomeConstraint(metric=Metric("b"),
op=ComparisonOp.GEQ,
bound=0.1)
],
)
has_good_fit = has_good_opt_config_model_fit(
optimization_config=optimization_config,
assess_model_fit_result=assess_model_fit_result,
)
self.assertFalse(has_good_fit)
def cross_validate(
self,
) -> Tuple[Optional[List[CVResult]], Optional[CVDiagnostics]]:
"""
Call cross_validate, compute_diagnostics and cache the results
If the model cannot be cross validated, warn and return None
"""
if self._cv_results is not None and self._diagnostics is not None:
return self._cv_results, self._diagnostics
self._assert_fitted()
try:
self._cv_results = cross_validate(
model=self.fitted_model,
**(self.model_cv_kwargs or {}),
)
except NotImplementedError:
warnings.warn(f"{self.model_enum.value} cannot be cross validated")
return None, None
self._diagnostics = compute_diagnostics(self._cv_results)
return self._cv_results, self._diagnostics
def get_best_from_model_predictions_with_trial_index(
experiment: Experiment,
) -> Optional[Tuple[int, TParameterization, Optional[TModelPredictArm]]]:
"""Given an experiment, returns the best predicted parameterization and corresponding
prediction based on the most recent Trial with predictions. If no trials have
predictions returns None.
Only some models return predictions. For instance GPEI does while Sobol does not.
TModelPredictArm is of the form:
({metric_name: mean}, {metric_name_1: {metric_name_2: cov_1_2}})
Args:
experiment: Experiment, on which to identify best raw objective arm.
Returns:
Tuple of parameterization and model predictions for it.
"""
# pyre-ignore [16]
if isinstance(experiment.optimization_config.objective, MultiObjective):
logger.warning(
"get_best_from_model_predictions is deprecated for multi-objective "
"optimization configs. This method will return an arbitrary point on "
"the pareto frontier.")
for idx, trial in sorted(experiment.trials.items(),
key=lambda x: x[0],
reverse=True):
gr = None
if isinstance(trial, Trial):
gr = trial.generator_run
elif isinstance(trial, BatchTrial):
if len(trial.generator_run_structs) > 0:
# In theory batch_trial can have >1 gr, grab the first
gr = trial.generator_run_structs[0].generator_run
if gr is not None and gr.best_arm_predictions is not None: # pragma: no cover
data = experiment.lookup_data()
if not isinstance(data, Data):
return _gr_to_prediction_with_trial_index(idx, gr)
model = get_model_from_generator_run(generator_run=gr,
experiment=experiment,
data=data)
# If model is not ArrayModelBridge, just use the best arm frmo the
# last good generator run
if not isinstance(model, ArrayModelBridge):
return _gr_to_prediction_with_trial_index(idx, gr)
# Check to see if the model is worth using
cv_results = cross_validate(model=model)
diagnostics = compute_diagnostics(result=cv_results)
assess_model_fit_results = assess_model_fit(
diagnostics=diagnostics)
objective_name = experiment.optimization_config.objective.metric.name
# If model fit is bad use raw results
if (objective_name in
assess_model_fit_results.bad_fit_metrics_to_fisher_score):
logger.warning(
"Model fit is poor; falling back on raw data for best point."
)
if not _is_all_noiseless(df=data.df,
metric_name=objective_name):
logger.warning(
"Model fit is poor and data on objective metric " +
f"{objective_name} is noisy; interpret best points " +
"results carefully.")
return _get_best_poor_model_fit(experiment=experiment)
res = model.model_best_point()
if res is None:
return _gr_to_prediction_with_trial_index(idx, gr)
best_arm, best_arm_predictions = res
return idx, not_none(best_arm).parameters, best_arm_predictions
return None
