def test_default_generation_strategy(self) -> None:
"""Test that Sobol+GPEI is used if no GenerationStrategy is provided."""
ax_client = AxClient()
ax_client.create_experiment(
parameters=[ # pyre-fixme[6]: expected union that should include
{"name": "x1", "type": "range", "bounds": [-5.0, 10.0]},
{"name": "x2", "type": "range", "bounds": [0.0, 15.0]},
],
objective_name="branin",
minimize=True,
)
self.assertEqual(
[s.model for s in not_none(ax_client.generation_strategy)._steps],
[Models.SOBOL, Models.GPEI],
)
with self.assertRaisesRegex(ValueError, ".* no trials."):
ax_client.get_optimization_trace(objective_optimum=branin.fmin)
for i in range(6):
parameterization, trial_index = ax_client.get_next_trial()
x1, x2 = parameterization.get("x1"), parameterization.get("x2")
ax_client.complete_trial(
trial_index,
raw_data={
"branin": (
checked_cast(
float,
branin(checked_cast(float, x1), checked_cast(float, x2)),
),
0.0,
)
},
sample_size=i,
)
if i < 5:
with self.assertRaisesRegex(ValueError, "Could not obtain contour"):
ax_client.get_contour_plot(param_x="x1", param_y="x2")
ax_client.get_optimization_trace(objective_optimum=branin.fmin)
ax_client.get_contour_plot()
self.assertIn("x1", ax_client.get_trials_data_frame())
self.assertIn("x2", ax_client.get_trials_data_frame())
self.assertIn("branin", ax_client.get_trials_data_frame())
self.assertEqual(len(ax_client.get_trials_data_frame()), 6)
# Test that Sobol is chosen when all parameters are choice.
ax_client = AxClient()
ax_client.create_experiment(
parameters=[ # pyre-fixme[6]: expected union that should include
{"name": "x1", "type": "choice", "values": [1, 2, 3]},
{"name": "x2", "type": "choice", "values": [1, 2, 3]},
]
)
self.assertEqual(
[s.model for s in not_none(ax_client.generation_strategy)._steps],
[Models.SOBOL],
)
self.assertEqual(ax_client.get_recommended_max_parallelism(), [(-1, -1)])
self.assertTrue(ax_client.get_trials_data_frame().empty)
def test_default_generation_strategy_continuous(self, _a, _b, _c,
_d) -> None:
"""Test that Sobol+GPEI is used if no GenerationStrategy is provided."""
ax_client = AxClient()
ax_client.create_experiment(
parameters=[ # pyre-fixme[6]: expected union that should include
{
"name": "x",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "y",
"type": "range",
"bounds": [0.0, 15.0]
},
],
objective_name="a",
minimize=True,
)
self.assertEqual(
[s.model for s in not_none(ax_client.generation_strategy)._steps],
[Models.SOBOL, Models.GPEI],
)
with self.assertRaisesRegex(ValueError, ".* no trials"):
ax_client.get_optimization_trace(objective_optimum=branin.fmin)
for i in range(6):
parameterization, trial_index = ax_client.get_next_trial()
x, y = parameterization.get("x"), parameterization.get("y")
ax_client.complete_trial(
trial_index,
raw_data={
"a": (
checked_cast(
float,
branin(checked_cast(float, x),
checked_cast(float, y)),
),
0.0,
)
},
sample_size=i,
)
self.assertEqual(ax_client.generation_strategy.model._model_key,
"GPEI")
ax_client.get_optimization_trace(objective_optimum=branin.fmin)
ax_client.get_contour_plot()
ax_client.get_feature_importances()
trials_df = ax_client.get_trials_data_frame()
self.assertIn("x", trials_df)
self.assertIn("y", trials_df)
self.assertIn("a", trials_df)
self.assertEqual(len(trials_df), 6)
def test_plotting_validation(self):
ax_client = AxClient()
ax_client.create_experiment(parameters=[{
"name": "x3",
"type": "fixed",
"value": 2,
"value_type": "int"
}])
with self.assertRaisesRegex(ValueError, ".* there are no trials"):
ax_client.get_contour_plot()
with self.assertRaisesRegex(ValueError, ".* there are no trials"):
ax_client.get_feature_importances()
ax_client.get_next_trial()
with self.assertRaisesRegex(ValueError, ".* less than 2 parameters"):
ax_client.get_contour_plot()
ax_client = AxClient()
ax_client.create_experiment(parameters=[
{
"name": "x",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "y",
"type": "range",
"bounds": [0.0, 15.0]
},
])
ax_client.get_next_trial()
with self.assertRaisesRegex(ValueError, "If `param_x` is provided"):
ax_client.get_contour_plot(param_x="y")
with self.assertRaisesRegex(ValueError, "If `param_x` is provided"):
ax_client.get_contour_plot(param_y="y")
with self.assertRaisesRegex(ValueError, 'Parameter "x3"'):
ax_client.get_contour_plot(param_x="x3", param_y="x3")
with self.assertRaisesRegex(ValueError, 'Parameter "x4"'):
ax_client.get_contour_plot(param_x="x", param_y="x4")
with self.assertRaisesRegex(ValueError, 'Metric "nonexistent"'):
ax_client.get_contour_plot(param_x="x",
param_y="y",
metric_name="nonexistent")
with self.assertRaisesRegex(UnsupportedPlotError,
"Could not obtain contour"):
ax_client.get_contour_plot(param_x="x",
param_y="y",
metric_name="objective")
with self.assertRaisesRegex(ValueError, "Could not obtain feature"):
ax_client.get_feature_importances()
# for i in range(25):
# parameters, trial_index = ax_client.get_next_trial()
# # Local evaluation here can be replaced with deployment to external system.
# ax_client.complete_trial(trial_index=trial_index, raw_data=evaluate(parameters))
# # _, trial_index = ax_client.get_next_trial()
# ax_client.log_trial_failure(trial_index=trial_index)
#
# ax_client.get_trials_data_frame().sort_values('trial_index')
# best_parameters, values = ax_client.get_best_parameters()
from ax.utils.notebook.plotting import render, init_notebook_plotting
from ax.plot.contour import plot_contour
plot = plot_contour(
model=gpei,
param_x=opt_list[0],
param_y=opt_list[1],
metric_name="base",
)
render(plot)
ax_client.generation_strategy.model = gpei
init_notebook_plotting(offline=True)
# render(ax_client.get_contour_plot())
render(ax_client.get_contour_plot(param_x=opt_list[0],
param_y=opt_list[0])) #, metric_name=base))
# render(ax_client.get_optimization_trace(objective_optimum=hartmann6.fmin)) # Objective_optimum is optional.
ax_client.save_to_json_file() # For custom filepath, pass `filepath` argument.
restored_ax_client = AxClient.load_from_json_file(
) # For custom filepath, pass `filepath` argument.
