def test_default_generation_strategy(self) -> None: """Test that Sobol+GPEI is used if no GenerationStrategy is provided.""" ax_client = AxClient() ax_client.create_experiment( parameters=[ # pyre-fixme[6]: expected union that should include {"name": "x1", "type": "range", "bounds": [-5.0, 10.0]}, {"name": "x2", "type": "range", "bounds": [0.0, 15.0]}, ], objective_name="branin", minimize=True, ) self.assertEqual( [s.model for s in not_none(ax_client.generation_strategy)._steps], [Models.SOBOL, Models.GPEI], ) with self.assertRaisesRegex(ValueError, ".* no trials."): ax_client.get_optimization_trace(objective_optimum=branin.fmin) for i in range(6): parameterization, trial_index = ax_client.get_next_trial() x1, x2 = parameterization.get("x1"), parameterization.get("x2") ax_client.complete_trial( trial_index, raw_data={ "branin": ( checked_cast( float, branin(checked_cast(float, x1), checked_cast(float, x2)), ), 0.0, ) }, sample_size=i, ) if i < 5: with self.assertRaisesRegex(ValueError, "Could not obtain contour"): ax_client.get_contour_plot(param_x="x1", param_y="x2") ax_client.get_optimization_trace(objective_optimum=branin.fmin) ax_client.get_contour_plot() self.assertIn("x1", ax_client.get_trials_data_frame()) self.assertIn("x2", ax_client.get_trials_data_frame()) self.assertIn("branin", ax_client.get_trials_data_frame()) self.assertEqual(len(ax_client.get_trials_data_frame()), 6) # Test that Sobol is chosen when all parameters are choice. ax_client = AxClient() ax_client.create_experiment( parameters=[ # pyre-fixme[6]: expected union that should include {"name": "x1", "type": "choice", "values": [1, 2, 3]}, {"name": "x2", "type": "choice", "values": [1, 2, 3]}, ] ) self.assertEqual( [s.model for s in not_none(ax_client.generation_strategy)._steps], [Models.SOBOL], ) self.assertEqual(ax_client.get_recommended_max_parallelism(), [(-1, -1)]) self.assertTrue(ax_client.get_trials_data_frame().empty)
def test_default_generation_strategy_continuous(self, _a, _b, _c, _d) -> None: """Test that Sobol+GPEI is used if no GenerationStrategy is provided.""" ax_client = AxClient() ax_client.create_experiment( parameters=[ # pyre-fixme[6]: expected union that should include { "name": "x", "type": "range", "bounds": [-5.0, 10.0] }, { "name": "y", "type": "range", "bounds": [0.0, 15.0] }, ], objective_name="a", minimize=True, ) self.assertEqual( [s.model for s in not_none(ax_client.generation_strategy)._steps], [Models.SOBOL, Models.GPEI], ) with self.assertRaisesRegex(ValueError, ".* no trials"): ax_client.get_optimization_trace(objective_optimum=branin.fmin) for i in range(6): parameterization, trial_index = ax_client.get_next_trial() x, y = parameterization.get("x"), parameterization.get("y") ax_client.complete_trial( trial_index, raw_data={ "a": ( checked_cast( float, branin(checked_cast(float, x), checked_cast(float, y)), ), 0.0, ) }, sample_size=i, ) self.assertEqual(ax_client.generation_strategy.model._model_key, "GPEI") ax_client.get_optimization_trace(objective_optimum=branin.fmin) ax_client.get_contour_plot() ax_client.get_feature_importances() trials_df = ax_client.get_trials_data_frame() self.assertIn("x", trials_df) self.assertIn("y", trials_df) self.assertIn("a", trials_df) self.assertEqual(len(trials_df), 6)
def test_plotting_validation(self): ax_client = AxClient() ax_client.create_experiment(parameters=[{ "name": "x3", "type": "fixed", "value": 2, "value_type": "int" }]) with self.assertRaisesRegex(ValueError, ".* there are no trials"): ax_client.get_contour_plot() with self.assertRaisesRegex(ValueError, ".* there are no trials"): ax_client.get_feature_importances() ax_client.get_next_trial() with self.assertRaisesRegex(ValueError, ".* less than 2 parameters"): ax_client.get_contour_plot() ax_client = AxClient() ax_client.create_experiment(parameters=[ { "name": "x", "type": "range", "bounds": [-5.0, 10.0] }, { "name": "y", "type": "range", "bounds": [0.0, 15.0] }, ]) ax_client.get_next_trial() with self.assertRaisesRegex(ValueError, "If `param_x` is provided"): ax_client.get_contour_plot(param_x="y") with self.assertRaisesRegex(ValueError, "If `param_x` is provided"): ax_client.get_contour_plot(param_y="y") with self.assertRaisesRegex(ValueError, 'Parameter "x3"'): ax_client.get_contour_plot(param_x="x3", param_y="x3") with self.assertRaisesRegex(ValueError, 'Parameter "x4"'): ax_client.get_contour_plot(param_x="x", param_y="x4") with self.assertRaisesRegex(ValueError, 'Metric "nonexistent"'): ax_client.get_contour_plot(param_x="x", param_y="y", metric_name="nonexistent") with self.assertRaisesRegex(UnsupportedPlotError, "Could not obtain contour"): ax_client.get_contour_plot(param_x="x", param_y="y", metric_name="objective") with self.assertRaisesRegex(ValueError, "Could not obtain feature"): ax_client.get_feature_importances()
# for i in range(25): # parameters, trial_index = ax_client.get_next_trial() # # Local evaluation here can be replaced with deployment to external system. # ax_client.complete_trial(trial_index=trial_index, raw_data=evaluate(parameters)) # # _, trial_index = ax_client.get_next_trial() # ax_client.log_trial_failure(trial_index=trial_index) # # ax_client.get_trials_data_frame().sort_values('trial_index') # best_parameters, values = ax_client.get_best_parameters() from ax.utils.notebook.plotting import render, init_notebook_plotting from ax.plot.contour import plot_contour plot = plot_contour( model=gpei, param_x=opt_list[0], param_y=opt_list[1], metric_name="base", ) render(plot) ax_client.generation_strategy.model = gpei init_notebook_plotting(offline=True) # render(ax_client.get_contour_plot()) render(ax_client.get_contour_plot(param_x=opt_list[0], param_y=opt_list[0])) #, metric_name=base)) # render(ax_client.get_optimization_trace(objective_optimum=hartmann6.fmin)) # Objective_optimum is optional. ax_client.save_to_json_file() # For custom filepath, pass `filepath` argument. restored_ax_client = AxClient.load_from_json_file( ) # For custom filepath, pass `filepath` argument.