def test_recommended_parallelism(self):
ax_client = AxClient()
with self.assertRaisesRegex(ValueError, "No generation strategy"):
ax_client.get_recommended_max_parallelism()
ax_client.create_experiment(
parameters=[
{"name": "x1", "type": "range", "bounds": [-5.0, 10.0]},
{"name": "x2", "type": "range", "bounds": [0.0, 15.0]},
],
minimize=True,
)
self.assertEqual(ax_client.get_recommended_max_parallelism(), [(5, 5), (-1, 3)])
self.assertEqual(
run_trials_using_recommended_parallelism(
ax_client, ax_client.get_recommended_max_parallelism(), 20
),
0,
)
# With incorrect parallelism setting, the 'need more data' error should
# still be raised.
ax_client = AxClient()
ax_client.create_experiment(
parameters=[
{"name": "x1", "type": "range", "bounds": [-5.0, 10.0]},
{"name": "x2", "type": "range", "bounds": [0.0, 15.0]},
],
minimize=True,
)
with self.assertRaisesRegex(ValueError, "All trials for current model "):
run_trials_using_recommended_parallelism(ax_client, [(6, 6), (-1, 3)], 20)
def test_deprecated_save_load_method_errors(self):
ax_client = AxClient()
with self.assertRaises(NotImplementedError):
ax_client.save()
with self.assertRaises(NotImplementedError):
ax_client.load()
with self.assertRaises(NotImplementedError):
ax_client.load_experiment("test_experiment")
with self.assertRaises(NotImplementedError):
ax_client.get_recommended_max_parallelism()
def test_default_generation_strategy_discrete(self) -> None:
"""Test that Sobol is used if no GenerationStrategy is provided and
the search space is discrete.
"""
# Test that Sobol is chosen when all parameters are choice.
ax_client = AxClient()
ax_client.create_experiment(
parameters=[ # pyre-fixme[6]: expected union that should include
{
"name": "x",
"type": "choice",
"values": [1, 2, 3]
},
{
"name": "y",
"type": "choice",
"values": [1, 2, 3]
},
])
self.assertEqual(
[s.model for s in not_none(ax_client.generation_strategy)._steps],
[Models.SOBOL],
)
self.assertEqual(ax_client.get_recommended_max_parallelism(),
[(-1, -1)])
self.assertTrue(ax_client.get_trials_data_frame().empty)
def test_default_generation_strategy(self) -> None:
"""Test that Sobol+GPEI is used if no GenerationStrategy is provided."""
ax_client = AxClient()
ax_client.create_experiment(
parameters=[ # pyre-fixme[6]: expected union that should include
{"name": "x1", "type": "range", "bounds": [-5.0, 10.0]},
{"name": "x2", "type": "range", "bounds": [0.0, 15.0]},
],
objective_name="branin",
minimize=True,
)
self.assertEqual(
[s.model for s in not_none(ax_client.generation_strategy)._steps],
[Models.SOBOL, Models.GPEI],
)
with self.assertRaisesRegex(ValueError, ".* no trials."):
ax_client.get_optimization_trace(objective_optimum=branin.fmin)
for i in range(6):
parameterization, trial_index = ax_client.get_next_trial()
x1, x2 = parameterization.get("x1"), parameterization.get("x2")
ax_client.complete_trial(
trial_index,
raw_data={
"branin": (
checked_cast(
float,
branin(checked_cast(float, x1), checked_cast(float, x2)),
),
0.0,
)
},
sample_size=i,
)
if i < 5:
with self.assertRaisesRegex(ValueError, "Could not obtain contour"):
ax_client.get_contour_plot(param_x="x1", param_y="x2")
ax_client.get_optimization_trace(objective_optimum=branin.fmin)
ax_client.get_contour_plot()
self.assertIn("x1", ax_client.get_trials_data_frame())
self.assertIn("x2", ax_client.get_trials_data_frame())
self.assertIn("branin", ax_client.get_trials_data_frame())
self.assertEqual(len(ax_client.get_trials_data_frame()), 6)
# Test that Sobol is chosen when all parameters are choice.
ax_client = AxClient()
ax_client.create_experiment(
parameters=[ # pyre-fixme[6]: expected union that should include
{"name": "x1", "type": "choice", "values": [1, 2, 3]},
{"name": "x2", "type": "choice", "values": [1, 2, 3]},
]
)
self.assertEqual(
[s.model for s in not_none(ax_client.generation_strategy)._steps],
[Models.SOBOL],
)
self.assertEqual(ax_client.get_recommended_max_parallelism(), [(-1, -1)])
self.assertTrue(ax_client.get_trials_data_frame().empty)
def test_default_generation_strategy(self) -> None:
"""Test that Sobol+GPEI is used if no GenerationStrategy is provided."""
ax = AxClient()
ax.create_experiment(
parameters=[
{
"name": "x1",
"type": "range",
"bounds": [-5.0, 10.0]
},
{
"name": "x2",
"type": "range",
"bounds": [0.0, 15.0]
},
],
objective_name="branin",
minimize=True,
)
self.assertEqual(
[s.model for s in ax.generation_strategy._steps],
[Models.SOBOL, Models.GPEI],
)
for _ in range(6):
parameterization, trial_index = ax.get_next_trial()
x1, x2 = parameterization.get("x1"), parameterization.get("x2")
ax.complete_trial(trial_index,
raw_data={"branin": (branin(x1, x2), 0.0)})
# Test that Sobol is chosen when all parameters are choice.
ax = AxClient()
ax.create_experiment(parameters=[
{
"name": "x1",
"type": "choice",
"values": [1, 2, 3]
},
{
"name": "x2",
"type": "choice",
"values": [1, 2, 3]
},
])
self.assertEqual([s.model for s in ax.generation_strategy._steps],
[Models.SOBOL])
self.assertEqual(ax.get_recommended_max_parallelism(), [(-1, -1)])
