def test_best_raw_objective_point(self): exp = get_branin_experiment() with self.assertRaisesRegex(ValueError, "Cannot identify best "): get_best_raw_objective_point(exp) self.assertEqual(get_best_parameters(exp), None) exp.new_trial( generator_run=GeneratorRun(arms=[Arm(parameters={"x1": 5.0, "x2": 5.0})]) ).run() opt_conf = exp.optimization_config.clone() opt_conf.objective.metric._name = "not_branin" with self.assertRaisesRegex(ValueError, "No data has been logged"): get_best_raw_objective_point(exp, opt_conf)
def test_best_raw_objective_point_scalarized(self): exp = get_branin_experiment() exp.optimization_config = OptimizationConfig( ScalarizedObjective(metrics=[get_branin_metric()], minimize=False) ) with self.assertRaisesRegex(ValueError, "Cannot identify best "): get_best_raw_objective_point(exp) self.assertEqual(get_best_parameters(exp, Models), None) exp.new_trial( generator_run=GeneratorRun(arms=[Arm(parameters={"x1": 5.0, "x2": 5.0})]) ).run() self.assertEqual(get_best_raw_objective_point(exp)[0], {"x1": 5.0, "x2": 5.0})
def get_best_parameters( self, ) -> Optional[Tuple[TParameterization, Optional[TModelPredictArm]]]: return best_point_utils.get_best_parameters(self.experiment)
def test_best_from_model_prediction(self): exp = get_branin_experiment() for _ in range(3): sobol = Models.SOBOL(search_space=exp.search_space) generator_run = sobol.gen(n=1) trial = exp.new_trial(generator_run=generator_run) trial.run() trial.mark_completed() exp.attach_data(exp.fetch_data()) gpei = Models.BOTORCH(experiment=exp, data=exp.lookup_data()) generator_run = gpei.gen(n=1) trial = exp.new_trial(generator_run=generator_run) trial.run() trial.mark_completed() with patch.object( ArrayModelBridge, "model_best_point", return_value=( ( Arm( name="0_0", parameters={"x1": -4.842811906710267, "x2": 11.887089014053345}, ), ( {"branin": 34.76260622783635}, {"branin": {"branin": 0.00028306433439807734}}, ), ) ), ) as mock_model_best_point, self.assertLogs( logger="ax.service.utils.best_point", level="WARN" ) as lg: # Test bad model fit causes function to resort back to raw data with patch( "ax.service.utils.best_point.assess_model_fit", return_value=AssessModelFitResult( good_fit_metrics_to_fisher_score={}, bad_fit_metrics_to_fisher_score={ "branin": 0, }, ), ): self.assertIsNotNone(get_best_parameters(exp, Models)) self.assertTrue( any("Model fit is poor" in warning for warning in lg.output), msg=lg.output, ) mock_model_best_point.assert_not_called() # Test model best point is used when fit is good with patch( "ax.service.utils.best_point.assess_model_fit", return_value=AssessModelFitResult( good_fit_metrics_to_fisher_score={ "branin": 0, }, bad_fit_metrics_to_fisher_score={}, ), ): self.assertIsNotNone(get_best_parameters(exp, Models)) mock_model_best_point.assert_called() # Assert the non-mocked method works correctly as well self.assertIsNotNone(get_best_parameters(exp, Models))