def test_current_generator_run_limit_unlimited_second_step(self):
NUM_INIT_TRIALS = 5
SECOND_STEP_PARALLELISM = 3
NUM_ROUNDS = 4
exp = get_branin_experiment()
sobol_gs_with_parallelism_limits = GenerationStrategy(steps=[
GenerationStep(
model=Models.SOBOL,
num_trials=NUM_INIT_TRIALS,
min_trials_observed=3,
),
GenerationStep(
model=Models.SOBOL,
num_trials=-1,
max_parallelism=SECOND_STEP_PARALLELISM,
),
])
sobol_gs_with_parallelism_limits._experiment = exp
could_gen = self._run_GS_for_N_rounds(
gs=sobol_gs_with_parallelism_limits,
exp=exp,
num_rounds=NUM_ROUNDS)
# We expect trials from first generation step + trials from remaining rounds in
# batches limited by parallelism setting in the second step.
self.assertEqual(
len(exp.trials),
NUM_INIT_TRIALS + (NUM_ROUNDS - 1) * SECOND_STEP_PARALLELISM,
)
self.assertTrue(all(t.status.is_completed
for t in exp.trials.values()))
self.assertEqual(could_gen, [NUM_INIT_TRIALS] +
[SECOND_STEP_PARALLELISM] * (NUM_ROUNDS - 1))
def setUp(self):
self.branin_experiment = get_branin_experiment()
self.branin_experiment._properties[
Keys.IMMUTABLE_SEARCH_SPACE_AND_OPT_CONF] = True
self.branin_experiment_no_impl_metrics = Experiment(
search_space=get_branin_search_space(),
optimization_config=OptimizationConfig(objective=Objective(
metric=Metric(name="branin"))),
)
self.sobol_GPEI_GS = choose_generation_strategy(
search_space=get_branin_search_space())
self.two_sobol_steps_GS = GenerationStrategy( # Contrived GS to ensure
steps=[ # that `DataRequiredError` is property handled in scheduler.
GenerationStep( # This error is raised when not enough trials
model=Models.
SOBOL, # have been observed to proceed to next
num_trials=5, # geneneration step.
min_trials_observed=3,
max_parallelism=2,
),
GenerationStep(model=Models.SOBOL,
num_trials=-1,
max_parallelism=3),
])
# GS to force the scheduler to poll completed trials after each ran trial.
self.sobol_GS_no_parallelism = GenerationStrategy(steps=[
GenerationStep(
model=Models.SOBOL, num_trials=-1, max_parallelism=1)
])
def test_sobol_GPEI_strategy_keep_generating(self, mock_GPEI_gen,
mock_GPEI_update,
mock_GPEI_init):
exp = get_branin_experiment()
sobol_GPEI_generation_strategy = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-1),
])
self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI")
self.assertEqual(sobol_GPEI_generation_strategy.generator_changes, [5])
exp.new_trial(
generator_run=sobol_GPEI_generation_strategy.gen(exp)).run()
for i in range(1, 15):
# Passing in all experiment data should cause an error as only
# new data should be passed into `gen`.
if i > 1:
with self.assertRaisesRegex(ValueError, "Data for arm"):
g = sobol_GPEI_generation_strategy.gen(
exp, exp.fetch_data())
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1))
exp.new_trial(generator_run=g).run()
if i > 4:
self.assertIsInstance(sobol_GPEI_generation_strategy.model,
TorchModelBridge)
def generation_strategy_from_sqa(
self, gs_sqa: SQAGenerationStrategy) -> GenerationStrategy:
"""Convert SQALchemy generation strategy to Ax `GenerationStrategy`."""
steps = object_from_json(gs_sqa.steps)
gs = GenerationStrategy(name=gs_sqa.name, steps=steps)
gs._curr = gs._steps[gs_sqa.curr_index]
gs._generator_runs = [
self.generator_run_from_sqa(gr) for gr in gs_sqa.generator_runs
]
if len(gs._generator_runs) > 0:
# Generation strategy had an initialized model.
# pyre-ignore[16]: SQAGenerationStrategy does not have `experiment` attr.
gs._experiment = self.experiment_from_sqa(gs_sqa.experiment)
# If model in the current step was not directly from the `Models` enum,
# pass its type to `restore_model_from_generator_run`, which will then
# attempt to use this type to recreate the model.
if type(gs._curr.model) != Models:
models_enum = type(gs._curr.model)
assert issubclass(models_enum, Models)
# pyre-ignore[6]: `models_enum` typing hackiness
gs._restore_model_from_generator_run(models_enum=models_enum)
else:
gs._restore_model_from_generator_run()
gs._db_id = gs_sqa.id
return gs
def _run_GS_for_N_rounds(self, gs: GenerationStrategy, exp: Experiment,
num_rounds: int) -> List[int]:
could_gen = []
for _ in range(num_rounds):
(
num_trials_to_gen,
opt_complete,
) = gs.current_generator_run_limit()
self.assertFalse(opt_complete)
could_gen.append(num_trials_to_gen)
trials = []
for _ in range(num_trials_to_gen):
gr = gs.gen(
experiment=exp,
pending_observations=get_pending(experiment=exp),
)
trials.append(
exp.new_trial(gr).mark_running(no_runner_required=True))
for trial in trials:
exp.attach_data(get_branin_data(trial_indices=[trial.index]))
trial.mark_completed()
return could_gen
def setUp(self):
self.gr = GeneratorRun(arms=[Arm(parameters={"x1": 1, "x2": 2})])
# Mock out slow GPEI.
self.torch_model_bridge_patcher = patch(
f"{TorchModelBridge.__module__}.TorchModelBridge", spec=True)
self.mock_torch_model_bridge = self.torch_model_bridge_patcher.start()
self.mock_torch_model_bridge.return_value.gen.return_value = self.gr
# Mock out slow TS.
self.discrete_model_bridge_patcher = patch(
f"{DiscreteModelBridge.__module__}.DiscreteModelBridge", spec=True)
self.mock_discrete_model_bridge = self.discrete_model_bridge_patcher.start(
)
self.mock_discrete_model_bridge.return_value.gen.return_value = self.gr
# Mock in `Models` registry
self.registry_setup_dict_patcher = patch.dict(
f"{Models.__module__}.MODEL_KEY_TO_MODEL_SETUP",
{
"Factorial":
MODEL_KEY_TO_MODEL_SETUP["Factorial"]._replace(
bridge_class=self.mock_discrete_model_bridge),
"Thompson":
MODEL_KEY_TO_MODEL_SETUP["Thompson"]._replace(
bridge_class=self.mock_discrete_model_bridge),
"GPEI":
MODEL_KEY_TO_MODEL_SETUP["GPEI"]._replace(
bridge_class=self.mock_torch_model_bridge),
},
)
self.mock_in_registry = self.registry_setup_dict_patcher.start()
# model bridges are mocked, which makes kwargs' validation difficult,
# so for now we will skip it in the generation strategy tests.
# NOTE: Starting with Python3.8 this is not a problem as `autospec=True`
# ensures that the mocks have correct signatures, but in earlier
# versions kwarg validation on mocks does not really work.
self.step_model_kwargs = {"silently_filter_kwargs": True}
self.hss_experiment = get_hierarchical_search_space_experiment()
self.sobol_GPEI_GS = GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(
model=Models.SOBOL,
num_trials=5,
model_kwargs=self.step_model_kwargs,
),
GenerationStep(model=Models.GPEI,
num_trials=2,
model_kwargs=self.step_model_kwargs),
],
)
self.sobol_GS = GenerationStrategy(steps=[
GenerationStep(
Models.SOBOL,
num_trials=-1,
should_deduplicate=True,
)
])
def test_sobol_GPEI_strategy_batches(self):
mock_GPEI_gen = self.mock_torch_model_bridge.return_value.gen
mock_GPEI_gen.return_value = GeneratorRun(
arms=[
Arm(parameters={"x1": 1, "x2": 2}),
Arm(parameters={"x1": 3, "x2": 4}),
]
)
exp = get_branin_experiment()
sobol_GPEI_generation_strategy = GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(model=Models.SOBOL, num_trials=1),
GenerationStep(model=Models.GPEI, num_trials=6),
],
)
self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI")
self.assertEqual(sobol_GPEI_generation_strategy.model_transitions, [1])
gr = sobol_GPEI_generation_strategy.gen(exp, n=2)
exp.new_batch_trial(generator_run=gr).run()
for i in range(1, 8):
if i == 7:
# Check completeness error message.
with self.assertRaises(GenerationStrategyCompleted):
g = sobol_GPEI_generation_strategy.gen(exp, n=2)
else:
g = sobol_GPEI_generation_strategy.gen(exp, n=2)
exp.new_batch_trial(generator_run=g).run()
self.assertIsInstance(sobol_GPEI_generation_strategy.model, TorchModelBridge)
def choose_generation_strategy(
search_space: SearchSpace,
arms_per_trial: int = 1,
enforce_sequential_optimization: bool = True,
random_seed: Optional[int] = None,
winsorize_botorch_model: bool = False,
winsorization_limits: Optional[Tuple[Optional[float], Optional[float]]] = None,
no_bayesian_optimization: bool = False,
) -> GenerationStrategy:
"""Select an appropriate generation strategy based on the properties of
the search space.
Args:
search_space: SearchSpace, based on the properties of which to select the
generation strategy.
arms_per_trial: If a trial is batched, how many arms will be in each batch.
Defaults to 1, which corresponds to a regular, non-batched, `Trial`.
enforce_sequential_optimization: Whether to enforce that the generation
strategy needs to be updated with `min_arms_observed` observations for
a given generation step before proceeding to the next one.
random_seed: Fixed random seed for the Sobol generator.
winsorize_botorch_model: Whether to apply the winsorization transform
prior to applying other transforms for fitting the BoTorch model.
winsorization_limits: Bounds for winsorization, if winsorizing, expressed
as percentile. Usually only the upper winsorization trim is used when
minimizing, and only the lower when maximizing.
no_bayesian_optimization: If True, Bayesian optimization generation
strategy will not be suggested and quasi-random strategy will be used.
"""
# If there are more discrete choices than continuous parameters, Sobol
# will do better than GP+EI.
if not no_bayesian_optimization and _should_use_gp(search_space=search_space):
# Ensure that number of arms per model is divisible by batch size.
sobol_arms = max(5, len(search_space.parameters))
if arms_per_trial != 1: # pragma: no cover
# If using batches, ensure that initialization sample is divisible by
# the batch size.
sobol_arms = ceil(sobol_arms / arms_per_trial) * arms_per_trial
gs = GenerationStrategy(
steps=[
_make_sobol_step(
num_arms=sobol_arms,
enforce_num_arms=enforce_sequential_optimization,
seed=random_seed,
),
_make_botorch_step(
recommended_max_parallelism=3,
winsorize=winsorize_botorch_model,
winsorization_limits=winsorization_limits,
),
]
)
logger.info(
f"Using Bayesian Optimization generation strategy: {gs}. Iterations "
f"after {sobol_arms} will take longer to generate due to model-fitting."
)
return gs
logger.info(f"Using Sobol generation strategy.")
return GenerationStrategy(steps=[_make_sobol_step(seed=random_seed)])
def test_factorial_thompson_strategy(self, mock_update, mock_gen,
mock_discrete):
exp = get_branin_experiment()
factorial_thompson_generation_strategy = GenerationStrategy(steps=[
GenerationStep(model=Models.FACTORIAL, num_arms=1),
GenerationStep(model=Models.THOMPSON, num_arms=-1),
])
self.assertEqual(factorial_thompson_generation_strategy.name,
"factorial+thompson")
self.assertEqual(
factorial_thompson_generation_strategy.generator_changes, [1])
for i in range(2):
data = get_data() if i > 0 else None
factorial_thompson_generation_strategy.gen(experiment=exp,
new_data=data)
exp.new_batch_trial().add_arm(Arm(parameters={"x1": i, "x2": i}))
if i < 1:
mock_discrete.assert_called()
args, kwargs = mock_discrete.call_args
self.assertIsInstance(kwargs.get("model"),
FullFactorialGenerator)
exp.new_batch_trial()
else:
mock_discrete.assert_called()
args, kwargs = mock_discrete.call_args
self.assertIsInstance(
kwargs.get("model"),
(ThompsonSampler, EmpiricalBayesThompsonSampler),
)
def test_factorial_thompson_strategy(self, _):
exp = get_branin_experiment()
factorial_thompson_generation_strategy = GenerationStrategy(steps=[
GenerationStep(
model=Models.FACTORIAL,
num_trials=1,
model_kwargs=self.step_model_kwargs,
),
GenerationStep(
model=Models.THOMPSON,
num_trials=-1,
model_kwargs=self.step_model_kwargs,
),
])
self.assertEqual(factorial_thompson_generation_strategy.name,
"Factorial+Thompson")
self.assertEqual(
factorial_thompson_generation_strategy.model_transitions, [1])
mock_model_bridge = self.mock_discrete_model_bridge.return_value
# Initial factorial batch.
exp.new_batch_trial(
factorial_thompson_generation_strategy.gen(experiment=exp))
args, kwargs = mock_model_bridge._set_kwargs_to_save.call_args
self.assertEqual(kwargs.get("model_key"), "Factorial")
# Subsequent Thompson sampling batch.
exp.new_batch_trial(
factorial_thompson_generation_strategy.gen(experiment=exp))
args, kwargs = mock_model_bridge._set_kwargs_to_save.call_args
self.assertEqual(kwargs.get("model_key"), "Thompson")
def test_store_experiment(self):
exp = get_branin_experiment()
sobol_generation_strategy = GenerationStrategy(
steps=[GenerationStep(model=Models.SOBOL, num_arms=5)])
self.assertIsNone(sobol_generation_strategy._experiment)
sobol_generation_strategy.gen(exp)
self.assertIsNotNone(sobol_generation_strategy._experiment)
def generation_strategy_from_json(
generation_strategy_json: Dict[str, Any],
experiment: Optional[Experiment] = None) -> GenerationStrategy:
"""Load generation strategy from JSON."""
steps = object_from_json(generation_strategy_json.pop("steps"))
gs = GenerationStrategy(steps=steps,
name=generation_strategy_json.pop("name"))
gs._db_id = object_from_json(generation_strategy_json.pop("db_id"))
gs._experiment = experiment or object_from_json(
generation_strategy_json.pop("experiment"))
gs._curr = gs._steps[generation_strategy_json.pop("curr_index")]
gs._generator_runs = object_from_json(
generation_strategy_json.pop("generator_runs"))
if generation_strategy_json.pop(
"had_initialized_model"): # pragma: no cover
# If model in the current step was not directly from the `Models` enum,
# pass its type to `restore_model_from_generator_run`, which will then
# attempt to use this type to recreate the model.
if type(gs._curr.model) != Models:
models_enum = type(gs._curr.model)
assert issubclass(models_enum, ModelRegistryBase)
# pyre-ignore[6]: `models_enum` typing hackiness
gs._restore_model_from_generator_run(models_enum=models_enum)
return gs
gs._restore_model_from_generator_run()
return gs
def test_clone_reset(self):
ftgs = GenerationStrategy(steps=[
GenerationStep(model=Models.FACTORIAL, num_arms=1),
GenerationStep(model=Models.THOMPSON, num_arms=2),
])
ftgs._curr = ftgs._steps[1]
self.assertEqual(ftgs._curr.index, 1)
self.assertEqual(ftgs.clone_reset()._curr.index, 0)
def test_custom_callables_for_models(self):
exp = get_branin_experiment()
sobol_factory_generation_strategy = GenerationStrategy(
steps=[GenerationStep(model=get_sobol, num_trials=-1)]
)
self.assertFalse(sobol_factory_generation_strategy._uses_registered_models)
self.assertTrue(sobol_factory_generation_strategy.uses_non_registered_models)
gr = sobol_factory_generation_strategy.gen(experiment=exp, n=1)
self.assertEqual(len(gr.arms), 1)
def test_max_parallelism_reached(self):
exp = get_branin_experiment()
sobol_generation_strategy = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_trials=5, max_parallelism=1)
])
exp.new_trial(generator_run=sobol_generation_strategy.gen(
experiment=exp)).mark_running(no_runner_required=True)
with self.assertRaises(MaxParallelismReachedException):
sobol_generation_strategy.gen(experiment=exp)
def test_kwargs_passed(self):
gs = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL,
num_arms=1,
model_kwargs={"scramble": False})
])
exp = get_branin_experiment()
gs.gen(exp, exp.fetch_data())
self.assertFalse(gs._model.model.scramble)
def test_sobol_GPEI_strategy_batches(self, mock_GPEI_gen, mock_GPEI_update,
mock_GPEI_init):
exp = get_branin_experiment()
sobol_GPEI_generation_strategy = GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=8),
],
)
self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI")
self.assertEqual(sobol_GPEI_generation_strategy.generator_changes, [5])
exp.new_batch_trial(
generator_run=sobol_GPEI_generation_strategy.gen(exp, n=2)).run()
for i in range(1, 8):
if i == 2:
with self.assertRaisesRegex(ValueError,
"Cannot generate 2 new"):
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1), n=2)
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1))
elif i == 7:
# Check completeness error message.
with self.assertRaisesRegex(ValueError, "Generation strategy"):
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1), n=2)
else:
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1), n=2)
exp.new_batch_trial(generator_run=g).run()
if i > 4:
mock_GPEI_init.assert_called()
with self.assertRaises(ValueError):
sobol_GPEI_generation_strategy.gen(exp, exp.fetch_data())
def test_sobol_GPEI_strategy(self):
exp = get_branin_experiment()
sobol_GPEI = GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(
model=Models.SOBOL,
num_trials=5,
model_kwargs=self.step_model_kwargs,
),
GenerationStep(model=Models.GPEI,
num_trials=2,
model_kwargs=self.step_model_kwargs),
],
)
self.assertEqual(sobol_GPEI.name, "Sobol+GPEI")
self.assertEqual(sobol_GPEI.model_transitions, [5])
# exp.new_trial(generator_run=sobol_GPEI.gen(exp)).run()
for i in range(7):
g = sobol_GPEI.gen(exp)
exp.new_trial(generator_run=g).run()
self.assertEqual(len(sobol_GPEI._generator_runs), i + 1)
if i > 4:
self.mock_torch_model_bridge.assert_called()
else:
self.assertEqual(g._model_key, "Sobol")
self.assertEqual(
g._model_kwargs,
{
"seed": None,
"deduplicate": False,
"init_position": i,
"scramble": True,
"generated_points": None,
},
)
self.assertEqual(
g._bridge_kwargs,
{
"optimization_config": None,
"status_quo_features": None,
"status_quo_name": None,
"transform_configs": None,
"transforms": Cont_X_trans,
"fit_out_of_design": False,
"fit_abandoned": False,
},
)
self.assertEqual(g._model_state_after_gen,
{"init_position": i + 1})
# Check completeness error message when GS should be done.
with self.assertRaises(GenerationStrategyCompleted):
g = sobol_GPEI.gen(exp)
def test_min_observed(self):
# We should fail to transition the next model if there is not
# enough data observed.
exp = get_branin_experiment(get_branin_experiment())
gs = GenerationStrategy(steps=[
GenerationStep(
model=Models.SOBOL, num_trials=5, min_trials_observed=5),
GenerationStep(model=Models.GPEI, num_trials=1),
])
self.assertFalse(gs.uses_non_registered_models)
for _ in range(5):
exp.new_trial(gs.gen(exp))
with self.assertRaises(DataRequiredError):
gs.gen(exp)
def test_sobol_strategy(self):
exp = get_branin_experiment()
sobol_generation_strategy = GenerationStrategy(steps=[
GenerationStep(
model=Models.SOBOL,
num_trials=5,
max_parallelism=10,
use_update=False,
enforce_num_trials=False,
)
])
for i in range(1, 6):
sobol_generation_strategy.gen(exp, n=1)
self.assertEqual(len(sobol_generation_strategy._generator_runs), i)
def test_current_generator_run_limit(self):
NUM_INIT_TRIALS = 5
SECOND_STEP_PARALLELISM = 3
NUM_ROUNDS = 4
exp = get_branin_experiment()
sobol_gs_with_parallelism_limits = GenerationStrategy(steps=[
GenerationStep(
model=Models.SOBOL,
num_trials=NUM_INIT_TRIALS,
min_trials_observed=3,
),
GenerationStep(
model=Models.SOBOL,
num_trials=-1,
max_parallelism=SECOND_STEP_PARALLELISM,
),
])
sobol_gs_with_parallelism_limits._experiment = exp
could_gen = []
for _ in range(NUM_ROUNDS):
(
num_trials_to_gen,
opt_complete,
) = sobol_gs_with_parallelism_limits.current_generator_run_limit()
self.assertFalse(opt_complete)
could_gen.append(num_trials_to_gen)
trials = []
for _ in range(num_trials_to_gen):
gr = sobol_gs_with_parallelism_limits.gen(
experiment=exp,
pending_observations=get_pending(experiment=exp),
)
trials.append(
exp.new_trial(gr).mark_running(no_runner_required=True))
for trial in trials:
exp.attach_data(get_branin_data(trial_indices=[trial.index]))
trial.mark_completed()
# We expect trials from first generation step + trials from remaining rounds in
# batches limited by parallelism setting in the second step.
self.assertEqual(
len(exp.trials),
NUM_INIT_TRIALS + (NUM_ROUNDS - 1) * SECOND_STEP_PARALLELISM,
)
self.assertTrue(all(t.status.is_completed
for t in exp.trials.values()))
self.assertEqual(could_gen, [NUM_INIT_TRIALS] +
[SECOND_STEP_PARALLELISM] * (NUM_ROUNDS - 1))
def choose_generation_strategy(
search_space: SearchSpace,
arms_per_trial: int = 1,
enforce_sequential_optimization: bool = True,
random_seed: Optional[int] = None,
) -> GenerationStrategy:
"""Select an appropriate generation strategy based on the properties of
the search space."""
model_kwargs = {"seed": random_seed} if (random_seed is not None) else None
num_continuous_parameters, num_discrete_choices = 0, 0
for parameter in search_space.parameters.values():
if isinstance(parameter, ChoiceParameter):
num_discrete_choices += len(parameter.values)
if isinstance(parameter, RangeParameter):
num_continuous_parameters += 1
# If there are more discrete choices than continuous parameters, Sobol
# will do better than GP+EI.
if num_continuous_parameters >= num_discrete_choices:
# Ensure that number of arms per model is divisible by batch size.
sobol_arms = (
ceil(max(5, len(search_space.parameters)) / arms_per_trial) *
arms_per_trial)
logger.info(
"Using Bayesian Optimization generation strategy. Iterations after "
f"{sobol_arms} will take longer to generate due to model-fitting.")
return GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(
model=Models.SOBOL,
num_arms=sobol_arms,
min_arms_observed=ceil(sobol_arms / 2),
enforce_num_arms=enforce_sequential_optimization,
model_kwargs=model_kwargs,
),
GenerationStep(model=Models.GPEI,
num_arms=-1,
recommended_max_parallelism=3),
],
)
else:
logger.info(f"Using Sobol generation strategy.")
return GenerationStrategy(
name="Sobol",
steps=[
GenerationStep(model=Models.SOBOL,
num_arms=-1,
model_kwargs=model_kwargs)
],
)
def test_equality(self):
gs1 = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-1),
])
gs2 = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-1),
])
self.assertEqual(gs1, gs2)
# Clone_reset() doesn't clone exactly, so they won't be equal.
gs3 = gs1.clone_reset()
self.assertNotEqual(gs1, gs3)
def test_string_representation(self):
gs1 = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_trials=5),
GenerationStep(model=Models.GPEI, num_trials=-1),
])
self.assertEqual(
str(gs1),
("GenerationStrategy(name='Sobol+GPEI', steps=[Sobol for 5 trials,"
" GPEI for subsequent trials])"),
)
gs2 = GenerationStrategy(
steps=[GenerationStep(model=Models.SOBOL, num_trials=-1)])
self.assertEqual(
str(gs2),
("GenerationStrategy(name='Sobol', steps=[Sobol for all trials])"))
def test_annotate_exception(self, _):
strategy0 = GenerationStrategy(
name="Sobol",
steps=[GenerationStep(model=Models.SOBOL, num_trials=-1)])
loop = OptimizationLoop.with_evaluation_function(
parameters=[
{
"name": "x1",
"type": "range",
"bounds": [-5.0, 10.0],
"value_type": "float",
"log_scale": False,
},
{
"name": "x2",
"type": "range",
"bounds": [0.0, 10.0]
},
],
experiment_name="test",
objective_name="branin",
minimize=True,
evaluation_function=_branin_evaluation_function,
total_trials=6,
generation_strategy=strategy0,
)
with self.assertRaisesRegex(
expected_exception=RuntimeError,
expected_regex="Cholesky errors typically occur",
):
loop.run_trial()
def test_custom_gs(self) -> None:
"""Managed loop with custom generation strategy"""
strategy0 = GenerationStrategy(
name="Sobol",
steps=[GenerationStep(model=Models.SOBOL, num_trials=-1)])
loop = OptimizationLoop.with_evaluation_function(
parameters=[
{
"name": "x1",
"type": "range",
"bounds": [-5.0, 10.0],
"value_type": "float",
"log_scale": False,
},
{
"name": "x2",
"type": "range",
"bounds": [0.0, 10.0]
},
],
experiment_name="test",
objective_name="branin",
minimize=True,
evaluation_function=_branin_evaluation_function,
total_trials=6,
generation_strategy=strategy0,
)
bp, _ = loop.full_run().get_best_point()
self.assertIn("x1", bp)
self.assertIn("x2", bp)
def _save_generation_strategy(generation_strategy: GenerationStrategy,
encoder: Encoder) -> int:
# If the generation strategy has not yet generated anything, there will be no
# experiment set on it.
if generation_strategy._experiment is None:
experiment_id = None
else:
# Experiment was set on the generation strategy, so need to check whether
# if has been saved and create a relationship b/w GS and experiment if so.
experiment_id = _get_experiment_id(
experiment=generation_strategy._experiment, encoder=encoder)
gs_sqa = encoder.generation_strategy_to_sqa(
generation_strategy=generation_strategy, experiment_id=experiment_id)
with session_scope() as session:
if generation_strategy._db_id is None:
session.add(gs_sqa)
session.flush() # Ensures generation strategy id is set.
generation_strategy._db_id = gs_sqa.id
else:
gs_sqa_class = encoder.config.class_to_sqa_class[
GenerationStrategy]
existing_gs_sqa = session.query(gs_sqa_class).get(
generation_strategy._db_id)
existing_gs_sqa.update(gs_sqa)
# our update logic ignores foreign keys, i.e. fields ending in _id,
# because we want SQLAlchemy to handle those relationships for us
# however, generation_strategy.experiment_id is an exception, so we
# need to update that manually
existing_gs_sqa.experiment_id = gs_sqa.experiment_id
return generation_strategy._db_id
def make_basic_generation_strategy(
name: str,
acquisition: str,
num_initial_trials: int = 14,
surrogate_model_constructor: Callable = singletask_gp_model_constructor,
) -> GenerationStrategy:
if acquisition not in ACQF_MODEL_MAP:
acquisition = "Sobol"
logger.warning(f"{acquisition} is not a supported "
"acquisition function. Defaulting to Sobol.")
return GenerationStrategy(
name=name,
steps=[
GenerationStep(
model=Models.SOBOL,
num_trials=num_initial_trials,
min_trials_observed=num_initial_trials,
),
GenerationStep(
model=ACQF_MODEL_MAP[acquisition],
num_trials=-1,
model_kwargs={
"model_constructor": surrogate_model_constructor,
"transforms": Cont_X_trans + Y_trans,
},
),
],
)
def test_optimize_graceful_exit_on_exception(self) -> None:
"""Tests optimization as a single call, with exception during
candidate generation.
"""
best, vals, exp, model = optimize(
parameters=[ # pyre-fixme[6]
{"name": "x1", "type": "range", "bounds": [-10.0, 10.0]},
{"name": "x2", "type": "range", "bounds": [-10.0, 10.0]},
],
# Booth function.
evaluation_function=lambda p: (
(p["x1"] + 2 * p["x2"] - 7) ** 2 + (2 * p["x1"] + p["x2"] - 5) ** 2,
None,
),
minimize=True,
total_trials=6,
generation_strategy=GenerationStrategy(
name="Sobol", steps=[GenerationStep(model=Models.SOBOL, num_trials=3)]
),
)
self.assertEqual(len(exp.trials), 3) # Check that we stopped at 3 trials.
# All the regular return values should still be present.
self.assertIn("x1", best)
self.assertIn("x2", best)
self.assertIsNotNone(vals)
self.assertIn("objective", vals[0])
self.assertIn("objective", vals[1])
self.assertIn("objective", vals[1]["objective"])
def test_sobol_GPEI_strategy_keep_generating(self):
exp = get_branin_experiment()
sobol_GPEI_generation_strategy = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-1),
])
self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI")
self.assertEqual(sobol_GPEI_generation_strategy.model_transitions, [5])
exp.new_trial(
generator_run=sobol_GPEI_generation_strategy.gen(exp)).run()
for i in range(1, 15):
g = sobol_GPEI_generation_strategy.gen(exp, exp.fetch_data())
exp.new_trial(generator_run=g).run()
if i > 4:
self.assertIsInstance(sobol_GPEI_generation_strategy.model,
TorchModelBridge)
