def setUp(self):
self.gr = GeneratorRun(arms=[Arm(parameters={"x1": 1, "x2": 2})])
# Mock out slow GPEI.
self.torch_model_bridge_patcher = patch(
f"{TorchModelBridge.__module__}.TorchModelBridge", spec=True)
self.mock_torch_model_bridge = self.torch_model_bridge_patcher.start()
self.mock_torch_model_bridge.return_value.gen.return_value = self.gr
# Mock out slow TS.
self.discrete_model_bridge_patcher = patch(
f"{DiscreteModelBridge.__module__}.DiscreteModelBridge", spec=True)
self.mock_discrete_model_bridge = self.discrete_model_bridge_patcher.start(
)
self.mock_discrete_model_bridge.return_value.gen.return_value = self.gr
# Mock in `Models` registry
self.registry_setup_dict_patcher = patch.dict(
f"{Models.__module__}.MODEL_KEY_TO_MODEL_SETUP",
{
"Factorial":
MODEL_KEY_TO_MODEL_SETUP["Factorial"]._replace(
bridge_class=self.mock_discrete_model_bridge),
"Thompson":
MODEL_KEY_TO_MODEL_SETUP["Thompson"]._replace(
bridge_class=self.mock_discrete_model_bridge),
"GPEI":
MODEL_KEY_TO_MODEL_SETUP["GPEI"]._replace(
bridge_class=self.mock_torch_model_bridge),
},
)
self.mock_in_registry = self.registry_setup_dict_patcher.start()
# model bridges are mocked, which makes kwargs' validation difficult,
# so for now we will skip it in the generation strategy tests.
# NOTE: Starting with Python3.8 this is not a problem as `autospec=True`
# ensures that the mocks have correct signatures, but in earlier
# versions kwarg validation on mocks does not really work.
self.step_model_kwargs = {"silently_filter_kwargs": True}
self.hss_experiment = get_hierarchical_search_space_experiment()
self.sobol_GPEI_GS = GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(
model=Models.SOBOL,
num_trials=5,
model_kwargs=self.step_model_kwargs,
),
GenerationStep(model=Models.GPEI,
num_trials=2,
model_kwargs=self.step_model_kwargs),
],
)
self.sobol_GS = GenerationStrategy(steps=[
GenerationStep(
Models.SOBOL,
num_trials=-1,
should_deduplicate=True,
)
])
def test_validation(self):
# num_arms can be positive or -1.
with self.assertRaises(ValueError):
GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-10),
])
# only last num_arms can be -1.
with self.assertRaises(ValueError):
GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=-1),
GenerationStep(model=Models.GPEI, num_arms=10),
])
exp = Experiment(
name="test",
search_space=SearchSpace(parameters=[get_choice_parameter()]))
factorial_thompson_generation_strategy = GenerationStrategy(steps=[
GenerationStep(model=Models.FACTORIAL, num_arms=1),
GenerationStep(model=Models.THOMPSON, num_arms=2),
])
self.assertTrue(
factorial_thompson_generation_strategy._uses_registered_models)
self.assertFalse(
factorial_thompson_generation_strategy.uses_non_registered_models)
with self.assertRaises(ValueError):
factorial_thompson_generation_strategy.gen(exp)
def choose_generation_strategy(
search_space: SearchSpace,
arms_per_trial: int = 1,
enforce_sequential_optimization: bool = True,
random_seed: Optional[int] = None,
winsorize_botorch_model: bool = False,
winsorization_limits: Optional[Tuple[Optional[float], Optional[float]]] = None,
no_bayesian_optimization: bool = False,
) -> GenerationStrategy:
"""Select an appropriate generation strategy based on the properties of
the search space.
Args:
search_space: SearchSpace, based on the properties of which to select the
generation strategy.
arms_per_trial: If a trial is batched, how many arms will be in each batch.
Defaults to 1, which corresponds to a regular, non-batched, `Trial`.
enforce_sequential_optimization: Whether to enforce that the generation
strategy needs to be updated with `min_arms_observed` observations for
a given generation step before proceeding to the next one.
random_seed: Fixed random seed for the Sobol generator.
winsorize_botorch_model: Whether to apply the winsorization transform
prior to applying other transforms for fitting the BoTorch model.
winsorization_limits: Bounds for winsorization, if winsorizing, expressed
as percentile. Usually only the upper winsorization trim is used when
minimizing, and only the lower when maximizing.
no_bayesian_optimization: If True, Bayesian optimization generation
strategy will not be suggested and quasi-random strategy will be used.
"""
# If there are more discrete choices than continuous parameters, Sobol
# will do better than GP+EI.
if not no_bayesian_optimization and _should_use_gp(search_space=search_space):
# Ensure that number of arms per model is divisible by batch size.
sobol_arms = max(5, len(search_space.parameters))
if arms_per_trial != 1: # pragma: no cover
# If using batches, ensure that initialization sample is divisible by
# the batch size.
sobol_arms = ceil(sobol_arms / arms_per_trial) * arms_per_trial
gs = GenerationStrategy(
steps=[
_make_sobol_step(
num_arms=sobol_arms,
enforce_num_arms=enforce_sequential_optimization,
seed=random_seed,
),
_make_botorch_step(
recommended_max_parallelism=3,
winsorize=winsorize_botorch_model,
winsorization_limits=winsorization_limits,
),
]
)
logger.info(
f"Using Bayesian Optimization generation strategy: {gs}. Iterations "
f"after {sobol_arms} will take longer to generate due to model-fitting."
)
return gs
logger.info(f"Using Sobol generation strategy.")
return GenerationStrategy(steps=[_make_sobol_step(seed=random_seed)])
def test_restore_from_generator_run(self):
gs = GenerationStrategy(
steps=[GenerationStep(model=Models.SOBOL, num_trials=5)])
# No generator runs on GS, so can't restore from one.
with self.assertRaises(ValueError):
gs._restore_model_from_generator_run()
exp = get_branin_experiment(with_batch=True)
gs.gen(experiment=exp)
model = gs.model
# Create a copy of the generation strategy and check that when
# we restore from last generator run, the model will be set
# correctly and that `_seen_trial_indices_by_status` is filled.
new_gs = GenerationStrategy(
steps=[GenerationStep(model=Models.SOBOL, num_trials=5)])
new_gs._experiment = exp
new_gs._generator_runs = gs._generator_runs
self.assertIsNone(new_gs._seen_trial_indices_by_status)
new_gs._restore_model_from_generator_run()
self.assertEqual(gs._seen_trial_indices_by_status,
exp.trial_indices_by_status)
# Model should be reset, but it should be the same model with same data.
self.assertIsNot(model, new_gs.model)
self.assertEqual(model.__class__,
new_gs.model.__class__) # Model bridge.
self.assertEqual(model.model.__class__,
new_gs.model.model.__class__) # Model.
self.assertEqual(model._training_data, new_gs.model._training_data)
def setUp(self):
self.branin_experiment = get_branin_experiment()
self.branin_experiment._properties[
Keys.IMMUTABLE_SEARCH_SPACE_AND_OPT_CONF] = True
self.branin_experiment_no_impl_metrics = Experiment(
search_space=get_branin_search_space(),
optimization_config=OptimizationConfig(objective=Objective(
metric=Metric(name="branin"))),
)
self.sobol_GPEI_GS = choose_generation_strategy(
search_space=get_branin_search_space())
self.two_sobol_steps_GS = GenerationStrategy( # Contrived GS to ensure
steps=[ # that `DataRequiredError` is property handled in scheduler.
GenerationStep( # This error is raised when not enough trials
model=Models.
SOBOL, # have been observed to proceed to next
num_trials=5, # geneneration step.
min_trials_observed=3,
max_parallelism=2,
),
GenerationStep(model=Models.SOBOL,
num_trials=-1,
max_parallelism=3),
])
# GS to force the scheduler to poll completed trials after each ran trial.
self.sobol_GS_no_parallelism = GenerationStrategy(steps=[
GenerationStep(
model=Models.SOBOL, num_trials=-1, max_parallelism=1)
])
def test_equality(self):
gs1 = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-1),
])
gs2 = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-1),
])
self.assertEqual(gs1, gs2)
# Clone_reset() doesn't clone exactly, so they won't be equal.
gs3 = gs1.clone_reset()
self.assertNotEqual(gs1, gs3)
def choose_generation_strategy(
search_space: SearchSpace,
arms_per_trial: int = 1,
enforce_sequential_optimization: bool = True,
random_seed: Optional[int] = None,
) -> GenerationStrategy:
"""Select an appropriate generation strategy based on the properties of
the search space."""
model_kwargs = {"seed": random_seed} if (random_seed is not None) else None
num_continuous_parameters, num_discrete_choices = 0, 0
for parameter in search_space.parameters.values():
if isinstance(parameter, ChoiceParameter):
num_discrete_choices += len(parameter.values)
if isinstance(parameter, RangeParameter):
num_continuous_parameters += 1
# If there are more discrete choices than continuous parameters, Sobol
# will do better than GP+EI.
if num_continuous_parameters >= num_discrete_choices:
# Ensure that number of arms per model is divisible by batch size.
sobol_arms = (
ceil(max(5, len(search_space.parameters)) / arms_per_trial) *
arms_per_trial)
logger.info(
"Using Bayesian Optimization generation strategy. Iterations after "
f"{sobol_arms} will take longer to generate due to model-fitting.")
return GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(
model=Models.SOBOL,
num_arms=sobol_arms,
min_arms_observed=ceil(sobol_arms / 2),
enforce_num_arms=enforce_sequential_optimization,
model_kwargs=model_kwargs,
),
GenerationStep(model=Models.GPEI,
num_arms=-1,
recommended_max_parallelism=3),
],
)
else:
logger.info(f"Using Sobol generation strategy.")
return GenerationStrategy(
name="Sobol",
steps=[
GenerationStep(model=Models.SOBOL,
num_arms=-1,
model_kwargs=model_kwargs)
],
)
def test_string_representation(self):
gs1 = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_trials=5),
GenerationStep(model=Models.GPEI, num_trials=-1),
])
self.assertEqual(
str(gs1),
("GenerationStrategy(name='Sobol+GPEI', steps=[Sobol for 5 trials,"
" GPEI for subsequent trials])"),
)
gs2 = GenerationStrategy(
steps=[GenerationStep(model=Models.SOBOL, num_trials=-1)])
self.assertEqual(
str(gs2),
("GenerationStrategy(name='Sobol', steps=[Sobol for all trials])"))
def generation_strategy_from_sqa(
self, gs_sqa: SQAGenerationStrategy) -> GenerationStrategy:
"""Convert SQALchemy generation strategy to Ax `GenerationStrategy`."""
steps = object_from_json(gs_sqa.steps)
gs = GenerationStrategy(name=gs_sqa.name, steps=steps)
gs._curr = gs._steps[gs_sqa.curr_index]
gs._generator_runs = [
self.generator_run_from_sqa(gr) for gr in gs_sqa.generator_runs
]
if len(gs._generator_runs) > 0:
# Generation strategy had an initialized model.
# pyre-ignore[16]: SQAGenerationStrategy does not have `experiment` attr.
gs._experiment = self.experiment_from_sqa(gs_sqa.experiment)
# If model in the current step was not directly from the `Models` enum,
# pass its type to `restore_model_from_generator_run`, which will then
# attempt to use this type to recreate the model.
if type(gs._curr.model) != Models:
models_enum = type(gs._curr.model)
assert issubclass(models_enum, Models)
# pyre-ignore[6]: `models_enum` typing hackiness
gs._restore_model_from_generator_run(models_enum=models_enum)
else:
gs._restore_model_from_generator_run()
gs._db_id = gs_sqa.id
return gs
def run_hartmann6_benchmarks(D, rep, random_subspace=False):
if D == 100:
problem = hartmann6_100
elif D == 1000 and not random_subspace:
problem = hartmann6_1000
elif D == 1000 and random_subspace:
problem = hartmann6_random_subspace_1000
strategy0 = GenerationStrategy(
name="Sobol",
steps=[
GenerationStep(model=Models.SOBOL,
num_arms=-1,
model_kwargs={'seed': rep + 1})
],
)
strategy1 = ALEBOStrategy(D=D, d=12, init_size=10)
strategy2 = REMBOStrategy(D=D, d=6, init_per_proj=2)
strategy3 = HeSBOStrategy(D=D, d=6, init_per_proj=10, name=f"HeSBO, d=d")
strategy4 = HeSBOStrategy(D=D, d=12, init_per_proj=10, name=f"HeSBO, d=2d")
all_benchmarks = full_benchmark_run(
num_replications=1, # Running them 1 at a time for distributed
num_trials=200,
batch_size=1,
methods=[strategy0, strategy1, strategy2, strategy3, strategy4],
problems=[problem],
)
rs_str = 'random_subspace_' if random_subspace else ''
with open(
f'results/hartmann6_{rs_str}{D}_alebo_rembo_hesbo_sobol_rep_{rep}.json',
"w") as fout:
json.dump(object_to_json(all_benchmarks), fout)
def test_use_update(self, mock_fetch_trials_data, mock_update):
exp = get_branin_experiment()
sobol_gs_with_update = GenerationStrategy(
steps=[GenerationStep(model=Models.SOBOL, num_trials=-1, use_update=True)]
)
# Try without passing data (generation strategy fetches data from experiment).
trial = exp.new_trial(generator_run=sobol_gs_with_update.gen(experiment=exp))
mock_update.assert_not_called()
trial._status = TrialStatus.COMPLETED
for i in range(3):
trial = exp.new_trial(
generator_run=sobol_gs_with_update.gen(experiment=exp)
)
self.assertEqual(
mock_fetch_trials_data.call_args[1].get("trial_indices"), {i}
)
trial._status = TrialStatus.COMPLETED
# Try with passing data.
sobol_gs_with_update.gen(
experiment=exp, data=get_branin_data(trial_indices=range(4))
)
# Only the data for the last completed trial should be considered new and passed
# to `update`.
self.assertEqual(
set(mock_update.call_args[1].get("new_data").df["trial_index"].values), {3}
)
def test_sobol_GPEI_strategy_batches(self):
mock_GPEI_gen = self.mock_torch_model_bridge.return_value.gen
mock_GPEI_gen.return_value = GeneratorRun(
arms=[
Arm(parameters={"x1": 1, "x2": 2}),
Arm(parameters={"x1": 3, "x2": 4}),
]
)
exp = get_branin_experiment()
sobol_GPEI_generation_strategy = GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(model=Models.SOBOL, num_trials=1),
GenerationStep(model=Models.GPEI, num_trials=6),
],
)
self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI")
self.assertEqual(sobol_GPEI_generation_strategy.model_transitions, [1])
gr = sobol_GPEI_generation_strategy.gen(exp, n=2)
exp.new_batch_trial(generator_run=gr).run()
for i in range(1, 8):
if i == 7:
# Check completeness error message.
with self.assertRaises(GenerationStrategyCompleted):
g = sobol_GPEI_generation_strategy.gen(exp, n=2)
else:
g = sobol_GPEI_generation_strategy.gen(exp, n=2)
exp.new_batch_trial(generator_run=g).run()
self.assertIsInstance(sobol_GPEI_generation_strategy.model, TorchModelBridge)
def test_sobol_GPEI_strategy(self, mock_GPEI_gen, mock_GPEI_update,
mock_GPEI_init):
exp = get_branin_experiment()
sobol_GPEI_generation_strategy = GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=2),
],
)
self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI")
self.assertEqual(sobol_GPEI_generation_strategy.generator_changes, [5])
exp.new_trial(
generator_run=sobol_GPEI_generation_strategy.gen(exp)).run()
for i in range(1, 8):
if i == 7:
# Check completeness error message.
with self.assertRaisesRegex(ValueError, "Generation strategy"):
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1))
else:
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1))
exp.new_trial(generator_run=g).run()
if i > 4:
mock_GPEI_init.assert_called()
# Check for "seen data" error message.
with self.assertRaisesRegex(ValueError, "Data for arm"):
sobol_GPEI_generation_strategy.gen(exp, exp.fetch_data())
def test_current_generator_run_limit_unlimited_second_step(self):
NUM_INIT_TRIALS = 5
SECOND_STEP_PARALLELISM = 3
NUM_ROUNDS = 4
exp = get_branin_experiment()
sobol_gs_with_parallelism_limits = GenerationStrategy(steps=[
GenerationStep(
model=Models.SOBOL,
num_trials=NUM_INIT_TRIALS,
min_trials_observed=3,
),
GenerationStep(
model=Models.SOBOL,
num_trials=-1,
max_parallelism=SECOND_STEP_PARALLELISM,
),
])
sobol_gs_with_parallelism_limits._experiment = exp
could_gen = self._run_GS_for_N_rounds(
gs=sobol_gs_with_parallelism_limits,
exp=exp,
num_rounds=NUM_ROUNDS)
# We expect trials from first generation step + trials from remaining rounds in
# batches limited by parallelism setting in the second step.
self.assertEqual(
len(exp.trials),
NUM_INIT_TRIALS + (NUM_ROUNDS - 1) * SECOND_STEP_PARALLELISM,
)
self.assertTrue(all(t.status.is_completed
for t in exp.trials.values()))
self.assertEqual(could_gen, [NUM_INIT_TRIALS] +
[SECOND_STEP_PARALLELISM] * (NUM_ROUNDS - 1))
def test_basic(self):
"""Run through the benchmarking loop."""
results = full_benchmark_run(
problem_groups={
self.CATEGORY_NAME: [
SimpleBenchmarkProblem(branin, noise_sd=0.4),
BenchmarkProblem(
name="Branin",
search_space=get_branin_search_space(),
optimization_config=get_branin_optimization_config(),
),
BenchmarkProblem(
search_space=get_branin_search_space(),
optimization_config=get_optimization_config(),
),
]
},
method_groups={
self.CATEGORY_NAME: [
GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_trials=-1)
])
]
},
num_replications=3,
num_trials=5,
# Just to have it be more telling if something is broken
raise_all_exceptions=True,
batch_size=[[1], [3], [1]],
)
self.assertEqual(len(results["Branin"]["Sobol"]), 3)
def test_store_experiment(self):
exp = get_branin_experiment()
sobol_generation_strategy = GenerationStrategy(
steps=[GenerationStep(model=Models.SOBOL, num_arms=5)])
self.assertIsNone(sobol_generation_strategy._experiment)
sobol_generation_strategy.gen(exp)
self.assertIsNotNone(sobol_generation_strategy._experiment)
def run_branin_and_gramacy_100_benchmarks(rep):
strategy0 = GenerationStrategy(
name="Sobol",
steps=[
GenerationStep(model=Models.SOBOL,
num_arms=-1,
model_kwargs={'seed': rep + 1})
],
)
strategy1 = ALEBOStrategy(D=100, d=4, init_size=10)
strategy2 = REMBOStrategy(D=100, d=2, init_per_proj=2)
strategy3 = HeSBOStrategy(D=100,
d=4,
init_per_proj=10,
name=f"HeSBO, d=2d")
all_benchmarks = full_benchmark_run(
num_replications=1,
num_trials=50,
batch_size=1,
methods=[strategy0, strategy1, strategy2, strategy3],
problems=[branin_100, gramacy_100],
)
with open(
f'results/branin_gramacy_100_alebo_rembo_hesbo_sobol_rep_{rep}.json',
"w") as fout:
json.dump(object_to_json(all_benchmarks), fout)
def test_sobol_GPEI_strategy_keep_generating(self, mock_GPEI_gen,
mock_GPEI_update,
mock_GPEI_init):
exp = get_branin_experiment()
sobol_GPEI_generation_strategy = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-1),
])
self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI")
self.assertEqual(sobol_GPEI_generation_strategy.generator_changes, [5])
exp.new_trial(
generator_run=sobol_GPEI_generation_strategy.gen(exp)).run()
for i in range(1, 15):
# Passing in all experiment data should cause an error as only
# new data should be passed into `gen`.
if i > 1:
with self.assertRaisesRegex(ValueError, "Data for arm"):
g = sobol_GPEI_generation_strategy.gen(
exp, exp.fetch_data())
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1))
exp.new_trial(generator_run=g).run()
if i > 4:
self.assertIsInstance(sobol_GPEI_generation_strategy.model,
TorchModelBridge)
def make_basic_generation_strategy(
name: str,
acquisition: str,
num_initial_trials: int = 14,
surrogate_model_constructor: Callable = singletask_gp_model_constructor,
) -> GenerationStrategy:
if acquisition not in ACQF_MODEL_MAP:
acquisition = "Sobol"
logger.warning(f"{acquisition} is not a supported "
"acquisition function. Defaulting to Sobol.")
return GenerationStrategy(
name=name,
steps=[
GenerationStep(
model=Models.SOBOL,
num_trials=num_initial_trials,
min_trials_observed=num_initial_trials,
),
GenerationStep(
model=ACQF_MODEL_MAP[acquisition],
num_trials=-1,
model_kwargs={
"model_constructor": surrogate_model_constructor,
"transforms": Cont_X_trans + Y_trans,
},
),
],
)
def test_raise_all_exceptions(self):
"""Checks that an exception nested in the benchmarking stack is raised
when `raise_all_exceptions` is True.
"""
def broken_benchmark_replication(*args, **kwargs) -> Experiment:
raise ValueError("Oh, exception!")
with self.assertRaisesRegex(ValueError, "Oh, exception!"):
full_benchmark_run(
problem_groups={
self.CATEGORY_NAME:
[SimpleBenchmarkProblem(branin, noise_sd=0.4)]
},
method_groups={
self.CATEGORY_NAME: [
GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_trials=-1)
])
]
},
num_replications=3,
num_trials=5,
raise_all_exceptions=True,
benchmark_replication=broken_benchmark_replication,
)
def test_optimize_graceful_exit_on_exception(self) -> None:
"""Tests optimization as a single call, with exception during
candidate generation.
"""
best, vals, exp, model = optimize(
parameters=[ # pyre-fixme[6]
{"name": "x1", "type": "range", "bounds": [-10.0, 10.0]},
{"name": "x2", "type": "range", "bounds": [-10.0, 10.0]},
],
# Booth function.
evaluation_function=lambda p: (
(p["x1"] + 2 * p["x2"] - 7) ** 2 + (2 * p["x1"] + p["x2"] - 5) ** 2,
None,
),
minimize=True,
total_trials=6,
generation_strategy=GenerationStrategy(
name="Sobol", steps=[GenerationStep(model=Models.SOBOL, num_trials=3)]
),
)
self.assertEqual(len(exp.trials), 3) # Check that we stopped at 3 trials.
# All the regular return values should still be present.
self.assertIn("x1", best)
self.assertIn("x2", best)
self.assertIsNotNone(vals)
self.assertIn("objective", vals[0])
self.assertIn("objective", vals[1])
self.assertIn("objective", vals[1]["objective"])
def test_sobol(self):
suite = BOBenchmarkingSuite()
runner = suite.run(
num_runs=1,
total_iterations=5,
batch_size=2,
bo_strategies=[
GenerationStrategy(
[GenerationStep(model=Models.SOBOL, num_arms=10)])
],
bo_problems=[branin],
)
# If run_benchmarking_trial fails, corresponding trial in '_runs' is None.
self.assertTrue(all(x is not None for x in runner._runs.values()))
# Make sure no errors came up in running trials.
self.assertEqual(len(runner.errors), 0)
report = suite.generate_report()
self.assertIsInstance(report, str)
# Add a trial
setup = BenchmarkSetup(problem=branin,
total_iterations=10,
batch_size=1)
suite.add_run(setup=setup, strategy_name="strategy_name")
self.assertTrue(("Branin", "strategy_name", 0) in suite._runner._runs)
suite.add_run(setup=setup, strategy_name="strategy_name")
self.assertTrue(("Branin", "strategy_name", 1) in suite._runner._runs)
def test_factorial_thompson_strategy(self, _):
exp = get_branin_experiment()
factorial_thompson_generation_strategy = GenerationStrategy(steps=[
GenerationStep(
model=Models.FACTORIAL,
num_trials=1,
model_kwargs=self.step_model_kwargs,
),
GenerationStep(
model=Models.THOMPSON,
num_trials=-1,
model_kwargs=self.step_model_kwargs,
),
])
self.assertEqual(factorial_thompson_generation_strategy.name,
"Factorial+Thompson")
self.assertEqual(
factorial_thompson_generation_strategy.model_transitions, [1])
mock_model_bridge = self.mock_discrete_model_bridge.return_value
# Initial factorial batch.
exp.new_batch_trial(
factorial_thompson_generation_strategy.gen(experiment=exp))
args, kwargs = mock_model_bridge._set_kwargs_to_save.call_args
self.assertEqual(kwargs.get("model_key"), "Factorial")
# Subsequent Thompson sampling batch.
exp.new_batch_trial(
factorial_thompson_generation_strategy.gen(experiment=exp))
args, kwargs = mock_model_bridge._set_kwargs_to_save.call_args
self.assertEqual(kwargs.get("model_key"), "Thompson")
def test_sobol_GPEI_strategy_batches(self, mock_GPEI_gen, mock_GPEI_update,
mock_GPEI_init):
exp = get_branin_experiment()
sobol_GPEI_generation_strategy = GenerationStrategy(
name="Sobol+GPEI",
steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=8),
],
)
self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI")
self.assertEqual(sobol_GPEI_generation_strategy.generator_changes, [5])
exp.new_batch_trial(
generator_run=sobol_GPEI_generation_strategy.gen(exp, n=2)).run()
for i in range(1, 8):
if i == 2:
with self.assertRaisesRegex(ValueError,
"Cannot generate 2 new"):
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1), n=2)
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1))
elif i == 7:
# Check completeness error message.
with self.assertRaisesRegex(ValueError, "Generation strategy"):
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1), n=2)
else:
g = sobol_GPEI_generation_strategy.gen(
exp, exp._fetch_trial_data(trial_index=i - 1), n=2)
exp.new_batch_trial(generator_run=g).run()
with self.assertRaises(ValueError):
sobol_GPEI_generation_strategy.gen(exp, exp.fetch_data())
self.assertIsInstance(sobol_GPEI_generation_strategy.model,
TorchModelBridge)
def test_annotate_exception(self, _):
strategy0 = GenerationStrategy(
name="Sobol",
steps=[GenerationStep(model=Models.SOBOL, num_trials=-1)])
loop = OptimizationLoop.with_evaluation_function(
parameters=[
{
"name": "x1",
"type": "range",
"bounds": [-5.0, 10.0],
"value_type": "float",
"log_scale": False,
},
{
"name": "x2",
"type": "range",
"bounds": [0.0, 10.0]
},
],
experiment_name="test",
objective_name="branin",
minimize=True,
evaluation_function=_branin_evaluation_function,
total_trials=6,
generation_strategy=strategy0,
)
with self.assertRaisesRegex(
expected_exception=RuntimeError,
expected_regex="Cholesky errors typically occur",
):
loop.run_trial()
def test_factorial_thompson_strategy(self, mock_update, mock_gen,
mock_discrete):
exp = get_branin_experiment()
factorial_thompson_generation_strategy = GenerationStrategy(steps=[
GenerationStep(model=Models.FACTORIAL, num_arms=1),
GenerationStep(model=Models.THOMPSON, num_arms=-1),
])
self.assertEqual(factorial_thompson_generation_strategy.name,
"factorial+thompson")
self.assertEqual(
factorial_thompson_generation_strategy.generator_changes, [1])
for i in range(2):
data = get_data() if i > 0 else None
factorial_thompson_generation_strategy.gen(experiment=exp,
new_data=data)
exp.new_batch_trial().add_arm(Arm(parameters={"x1": i, "x2": i}))
if i < 1:
mock_discrete.assert_called()
args, kwargs = mock_discrete.call_args
self.assertIsInstance(kwargs.get("model"),
FullFactorialGenerator)
exp.new_batch_trial()
else:
mock_discrete.assert_called()
args, kwargs = mock_discrete.call_args
self.assertIsInstance(
kwargs.get("model"),
(ThompsonSampler, EmpiricalBayesThompsonSampler),
)
def test_custom_gs(self) -> None:
"""Managed loop with custom generation strategy"""
strategy0 = GenerationStrategy(
name="Sobol",
steps=[GenerationStep(model=Models.SOBOL, num_trials=-1)])
loop = OptimizationLoop.with_evaluation_function(
parameters=[
{
"name": "x1",
"type": "range",
"bounds": [-5.0, 10.0],
"value_type": "float",
"log_scale": False,
},
{
"name": "x2",
"type": "range",
"bounds": [0.0, 10.0]
},
],
experiment_name="test",
objective_name="branin",
minimize=True,
evaluation_function=_branin_evaluation_function,
total_trials=6,
generation_strategy=strategy0,
)
bp, _ = loop.full_run().get_best_point()
self.assertIn("x1", bp)
self.assertIn("x2", bp)
def generation_strategy_from_json(
generation_strategy_json: Dict[str, Any],
experiment: Optional[Experiment] = None) -> GenerationStrategy:
"""Load generation strategy from JSON."""
steps = object_from_json(generation_strategy_json.pop("steps"))
gs = GenerationStrategy(steps=steps,
name=generation_strategy_json.pop("name"))
gs._db_id = object_from_json(generation_strategy_json.pop("db_id"))
gs._experiment = experiment or object_from_json(
generation_strategy_json.pop("experiment"))
gs._curr = gs._steps[generation_strategy_json.pop("curr_index")]
gs._generator_runs = object_from_json(
generation_strategy_json.pop("generator_runs"))
if generation_strategy_json.pop(
"had_initialized_model"): # pragma: no cover
# If model in the current step was not directly from the `Models` enum,
# pass its type to `restore_model_from_generator_run`, which will then
# attempt to use this type to recreate the model.
if type(gs._curr.model) != Models:
models_enum = type(gs._curr.model)
assert issubclass(models_enum, ModelRegistryBase)
# pyre-ignore[6]: `models_enum` typing hackiness
gs._restore_model_from_generator_run(models_enum=models_enum)
return gs
gs._restore_model_from_generator_run()
return gs
def generation_strategy_from_sqa(
self,
gs_sqa: SQAGenerationStrategy,
experiment: Optional[Experiment] = None,
reduced_state: bool = False,
) -> GenerationStrategy:
"""Convert SQALchemy generation strategy to Ax `GenerationStrategy`."""
steps = object_from_json(
gs_sqa.steps,
decoder_registry=self.config.json_decoder_registry,
class_decoder_registry=self.config.json_class_decoder_registry,
)
gs = GenerationStrategy(name=gs_sqa.name, steps=steps)
gs._curr = gs._steps[gs_sqa.curr_index]
immutable_ss_and_oc = (experiment.immutable_search_space_and_opt_config
if experiment is not None else False)
if reduced_state and gs_sqa.generator_runs:
# Only fully load the last of the generator runs, load the rest with
# reduced state.
gs._generator_runs = [
self.generator_run_from_sqa(
generator_run_sqa=gr,
reduced_state=True,
immutable_search_space_and_opt_config=immutable_ss_and_oc,
) for gr in gs_sqa.generator_runs[:-1]
]
gs._generator_runs.append(
self.generator_run_from_sqa(
generator_run_sqa=gs_sqa.generator_runs[-1],
reduced_state=False,
immutable_search_space_and_opt_config=immutable_ss_and_oc,
))
else:
gs._generator_runs = [
self.generator_run_from_sqa(
generator_run_sqa=gr,
reduced_state=False,
immutable_search_space_and_opt_config=immutable_ss_and_oc,
) for gr in gs_sqa.generator_runs
]
if len(gs._generator_runs) > 0:
# Generation strategy had an initialized model.
if experiment is None:
raise SQADecodeError(
"Cannot decode a generation strategy with a non-zero number of "
"generator runs without an experiment.")
gs._experiment = experiment
# If model in the current step was not directly from the `Models` enum,
# pass its type to `restore_model_from_generator_run`, which will then
# attempt to use this type to recreate the model.
if type(gs._curr.model) != Models:
models_enum = type(gs._curr.model)
assert issubclass(models_enum, ModelRegistryBase)
# pyre-ignore[6]: `models_enum` typing hackiness
gs._restore_model_from_generator_run(models_enum=models_enum)
else:
gs._restore_model_from_generator_run()
gs.db_id = gs_sqa.id
return gs
def test_string_representation(self):
gs1 = GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-1),
])
self.assertEqual(
str(gs1),
("GenerationStrategy(name='Sobol+GPEI', steps=[Sobol for 5 arms,"
" GPEI for subsequent arms], generated 0 arm(s) so far)"),
)
gs2 = GenerationStrategy(
steps=[GenerationStep(model=Models.SOBOL, num_arms=-1)])
self.assertEqual(
str(gs2),
("GenerationStrategy(name='Sobol', steps=[Sobol for all arms], "
"generated 0 arm(s) so far)"),
)
