def get_sobol(
search_space: SearchSpace,
seed: Optional[int] = None,
deduplicate: bool = False,
init_position: int = 0,
scramble: bool = True,
fallback_to_sample_polytope: bool = False,
) -> RandomModelBridge:
"""Instantiates a Sobol sequence quasi-random generator.
Args:
search_space: Sobol generator search space.
kwargs: Custom args for sobol generator.
Returns:
RandomModelBridge, with SobolGenerator as model.
"""
return checked_cast(
RandomModelBridge,
Models.SOBOL(
search_space=search_space,
seed=seed,
deduplicate=deduplicate,
init_position=init_position,
scramble=scramble,
fallback_to_sample_polytope=fallback_to_sample_polytope,
),
)
def testContours(self):
exp = get_branin_experiment(with_str_choice_param=True,
with_batch=True)
exp.trials[0].run()
model = Models.BOTORCH(
# Model bridge kwargs
experiment=exp,
data=exp.fetch_data(),
)
# Assert that each type of plot can be constructed successfully
plot = plot_contour_plotly(model, model.parameters[0],
model.parameters[1],
list(model.metric_names)[0])
self.assertIsInstance(plot, go.Figure)
plot = interact_contour_plotly(model, list(model.metric_names)[0])
self.assertIsInstance(plot, go.Figure)
plot = interact_contour(model, list(model.metric_names)[0])
self.assertIsInstance(plot, AxPlotConfig)
plot = plot = plot_contour(model, model.parameters[0],
model.parameters[1],
list(model.metric_names)[0])
self.assertIsInstance(plot, AxPlotConfig)
# Make sure all parameters and metrics are displayed in tooltips
tooltips = list(exp.parameters.keys()) + list(exp.metrics.keys())
for d in plot.data["data"]:
# Only check scatter plots hoverovers
if d["type"] != "scatter":
continue
for text in d["text"]:
for tt in tooltips:
self.assertTrue(tt in text)
def setUp(self):
self.branin_experiment = get_branin_experiment()
sobol = Models.SOBOL(search_space=self.branin_experiment.search_space)
sobol_run = sobol.gen(n=20)
self.branin_experiment.new_batch_trial().add_generator_run(
sobol_run
).run().mark_completed()
data = self.branin_experiment.fetch_data()
ms_gpei = ModelSpec(model_enum=Models.GPEI)
ms_gpei.fit(experiment=self.branin_experiment, data=data)
ms_gpkg = ModelSpec(model_enum=Models.GPKG)
ms_gpkg.fit(experiment=self.branin_experiment, data=data)
self.fitted_model_specs = [ms_gpei, ms_gpkg]
self.model_selection_node = GenerationNode(
model_specs=self.fitted_model_specs,
best_model_selector=SingleDiagnosticBestModelSelector(
diagnostic="Fisher exact test p",
criterion=MetricAggregation.MEAN,
metric_aggregation=DiagnosticCriterion.MIN,
),
)
def test_enum_empirical_bayes_thompson(self):
"""Tests EB/TS instantiation through the Models enum."""
exp = get_factorial_experiment()
factorial = Models.FACTORIAL(exp.search_space)
self.assertIsInstance(factorial, DiscreteModelBridge)
factorial_run = factorial.gen(n=-1)
exp.new_batch_trial().add_generator_run(
factorial_run).run().mark_completed()
data = exp.fetch_data()
eb_thompson = Models.EMPIRICAL_BAYES_THOMPSON(experiment=exp,
data=data,
min_weight=0.0)
self.assertIsInstance(eb_thompson, DiscreteModelBridge)
self.assertIsInstance(eb_thompson.model, EmpiricalBayesThompsonSampler)
thompson_run = eb_thompson.gen(n=5)
self.assertEqual(len(thompson_run.arms), 5)
def get_GPMES(
experiment: Experiment,
data: Data,
search_space: Optional[SearchSpace] = None,
cost_intercept: float = 0.01,
dtype: torch.dtype = torch.double,
device: torch.device = DEFAULT_TORCH_DEVICE,
transforms: List[Type[Transform]] = Cont_X_trans + Y_trans,
transform_configs: Optional[Dict[str, TConfig]] = None,
**kwargs: Any,
) -> TorchModelBridge:
"""Instantiates a GP model that generates points with MES."""
if search_space is None:
search_space = experiment.search_space
if data.df.empty: # pragma: no cover
raise ValueError("GP + MES BotorchModel requires non-empty data.")
inputs = {
"search_space": search_space,
"experiment": experiment,
"data": data,
"cost_intercept": cost_intercept,
"torch_dtype": dtype,
"torch_device": device,
"transforms": transforms,
"transform_configs": transform_configs,
}
if any(p.is_fidelity for k, p in experiment.parameters.items()):
inputs["linear_truncated"] = kwargs.get("linear_truncated", True)
return checked_cast(TorchModelBridge, Models.GPMES(**inputs)) # pyre-ignore: [16]
def test_enum_uniform(self):
"""Tests uniform random instantiation through the Models enum."""
exp = get_branin_experiment()
uniform = Models.UNIFORM(exp.search_space)
self.assertIsInstance(uniform, RandomModelBridge)
uniform_run = uniform.gen(n=5)
self.assertEqual(len(uniform_run.arms), 5)
def get_empirical_bayes_thompson(
experiment: Experiment,
data: Data,
search_space: Optional[SearchSpace] = None,
num_samples: int = 10000,
min_weight: Optional[float] = None,
uniform_weights: bool = False,
) -> DiscreteModelBridge:
"""Instantiates an empirical Bayes / Thompson sampling model."""
if data.df.empty: # pragma: no cover
raise ValueError(
"Empirical Bayes Thompson sampler requires non-empty data.")
logger.info(
"Factory functions (like `get_empirical_bayes_thompson`) will soon be "
"deprecated). Use the model registry instead (`Models.EMPIRICAL_BAYES"
"(...)`).")
return checked_cast(
DiscreteModelBridge,
Models.EMPIRICAL_BAYES_THOMPSON(
experiment=experiment,
data=data,
search_space=search_space or experiment.search_space,
num_samples=num_samples,
min_weight=min_weight,
uniform_weights=uniform_weights,
),
)
def test_enum_factorial(self):
"""Tests factorial instantiation through the Models enum."""
exp = get_factorial_experiment()
factorial = Models.FACTORIAL(exp.search_space)
self.assertIsInstance(factorial, DiscreteModelBridge)
factorial_run = factorial.gen(n=-1)
self.assertEqual(len(factorial_run.arms), 24)
def get_sobol(
search_space: SearchSpace,
seed: Optional[int] = None,
deduplicate: bool = False,
init_position: int = 0,
scramble: bool = True,
) -> RandomModelBridge:
"""Instantiates a Sobol sequence quasi-random generator.
Args:
search_space: Sobol generator search space.
kwargs: Custom args for sobol generator.
Returns:
RandomModelBridge, with SobolGenerator as model.
"""
logger.info(
"Factory functions (like `get_sobol`) will soon be deprecated. Use "
"the model registry instead (`Models.SOBOL(...)`).")
return checked_cast(
RandomModelBridge,
Models.SOBOL(
search_space=search_space,
seed=seed,
deduplicate=deduplicate,
init_position=init_position,
scramble=scramble,
),
)
def get_factorial(search_space: SearchSpace) -> DiscreteModelBridge:
"""Instantiates a factorial generator."""
logger.info(
"Factory functions (like `get_factorial`) will soon be deprecated). Use "
"the model registry instead (`Models.FACTORIAL(...)`).")
return checked_cast(DiscreteModelBridge,
Models.FACTORIAL(search_space=search_space))
def test_ST_MTGP_NEHVI(self):
"""Tests single type MTGP NEHVI instantiation."""
multi_obj_exp = get_branin_experiment_with_multi_objective(
with_batch=True,
with_status_quo=True,
)
metrics = multi_obj_exp.optimization_config.objective.metrics
multi_objective_thresholds = [
ObjectiveThreshold(metric=metrics[0],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ),
ObjectiveThreshold(metric=metrics[1],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ),
]
sobol = Models.SOBOL(search_space=multi_obj_exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(2):
sobol_run = sobol.gen(n=1)
t = multi_obj_exp.new_batch_trial().add_generator_run(sobol_run)
t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5)
t.run().mark_completed()
status_quo_features = ObservationFeatures(
parameters=multi_obj_exp.trials[0].status_quo.parameters,
trial_index=0,
)
mtgp = Models.ST_MTGP_NEHVI(
experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data(),
status_quo_features=status_quo_features,
objective_thresholds=multi_objective_thresholds,
)
self.assertIsInstance(mtgp, TorchModelBridge)
self.assertIsInstance(mtgp.model, MultiObjectiveBotorchModel)
# test it can generate
mtgp_run = mtgp.gen(n=1,
fixed_features=ObservationFeatures(parameters={},
trial_index=1))
self.assertEqual(len(mtgp_run.arms), 1)
# test a generated trial can be completed
t = multi_obj_exp.new_batch_trial().add_generator_run(mtgp_run)
t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5)
t.run().mark_completed()
def get_modelbridge() -> ModelBridge:
exp = get_branin_experiment(with_batch=True)
exp.trials[0].run()
return Models.BOTORCH(
# Model bridge kwargs
experiment=exp,
data=exp.fetch_data(),
)
def setUp(self) -> None:
self.experiment = get_branin_experiment()
sobol = Models.SOBOL(search_space=self.experiment.search_space)
sobol_run = sobol.gen(n=20)
self.experiment.new_batch_trial().add_generator_run(
sobol_run
).run().mark_completed()
self.data = self.experiment.fetch_data()
def setUp(self):
experiment = get_branin_experiment()
experiment.add_tracking_metric(
BraninMetric(name="m2", param_names=["x1", "x2"]))
sobol = Models.SOBOL(experiment.search_space)
a = sobol.gen(5)
experiment.new_batch_trial(generator_run=a).run()
self.experiment = experiment
self.metrics = list(experiment.metrics.values())
def test_enum_sobol_GPEI(self):
"""Tests Sobol instantiation through the Models enum."""
exp = get_branin_experiment()
# Check that factory generates a valid sobol modelbridge.
sobol = Models.SOBOL(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(n=1)
self.assertEqual(sobol_run._model_key, "Sobol")
exp.new_batch_trial().add_generator_run(sobol_run).run()
# Check that factory generates a valid GP+EI modelbridge.
exp.optimization_config = get_branin_optimization_config()
gpei = Models.GPEI(experiment=exp, data=exp.fetch_data())
self.assertIsInstance(gpei, TorchModelBridge)
gpei = Models.GPEI(experiment=exp,
data=exp.fetch_data(),
search_space=exp.search_space)
self.assertIsInstance(gpei, TorchModelBridge)
def plot_progress(self, ax_client):
model = Models.GPEI(experiment=ax_client.experiment, data=ax_client.experiment.fetch_data())
html_elements = [plot_config_to_html(ax_client.get_optimization_trace())]
model_params = get_range_parameters(model)
if len(model_params) > 1:
html_elements.append(plot_config_to_html(interact_contour(model=model, metric_name=self.YR.args.eval_primary_metric)))
else:
html_elements.append(plot_config_to_html(interact_slice(model=model, param_name=model_params[0].name, metric_name=self.YR.args.eval_primary_metric)))
with open(os.path.join(self.bayes_opt_root_experiment_folder, "optimization_plots.html"), 'w') as f:
f.write(render_report_elements(self.experiment_name, html_elements))
def test_cross_validate_raises_not_implemented_error_for_non_cv_model_with_data(
self,
):
exp = get_branin_experiment(with_batch=True)
exp.trials[0].run().complete()
sobol = Models.SOBOL(
experiment=exp, search_space=exp.search_space, data=exp.fetch_data()
)
with self.assertRaises(NotImplementedError):
cross_validate(model=sobol)
def setUp(self):
super().setUp()
search_space = get_search_space()
gr = Models.SOBOL(search_space=search_space).gen(n=1)
self.model = Mock(
search_space=search_space,
status_quo=Mock(
features=ObservationFeatures(parameters=gr.arms[0].parameters)
),
)
def testPlotMultipleParetoFrontiers(self):
experiment = get_branin_experiment_with_multi_objective(
has_objective_thresholds=True, )
sobol = Models.SOBOL(experiment.search_space)
a = sobol.gen(5)
experiment.new_batch_trial(generator_run=a).run()
pfrs = get_observed_pareto_frontiers(experiment=experiment)
pfrs2 = copy.deepcopy(pfrs)
pfr_lists = {"pfrs 1": pfrs, "pfrs 2": pfrs2}
self.assertIsNotNone(interact_multiple_pareto_frontier(pfr_lists))
def plot_progress(self, ax_client):
model = Models.GPEI(experiment=ax_client.experiment, data=ax_client.experiment.fetch_data())
html_elements = []
html_elements.append(plot_config_to_html(ax_client.get_optimization_trace()))
try:
html_elements.append(plot_config_to_html(interact_contour(model=model, metric_name=self.YR.args.eval_primary_metric)))
except:
pass
with open(os.path.join(self.bayes_opt_root_experiment_folder, "optimization_plots.html"), 'w') as f:
f.write(render_report_elements(self.experiment_name, html_elements))
def test_enum_model_kwargs(self):
"""Tests that kwargs are passed correctly when instantiating through the
Models enum."""
exp = get_branin_experiment()
sobol = Models.SOBOL(
search_space=exp.search_space, init_position=2, scramble=False, seed=239
)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(1)
exp.new_batch_trial().add_generator_run(sobol_run).run()
def test_ST_MTGP(self):
"""Tests single type MTGP instantiation."""
# Test Single-type MTGP
exp = get_branin_experiment()
sobol = Models.SOBOL(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(n=1)
t = exp.new_batch_trial().add_generator_run(sobol_run)
t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5)
t.run().mark_completed()
status_quo_features = ObservationFeatures(
parameters=exp.trials[0].status_quo.parameters,
trial_index=0,
)
mtgp = Models.ST_MTGP(
experiment=exp,
data=exp.fetch_data(),
status_quo_features=status_quo_features,
)
self.assertIsInstance(mtgp, TorchModelBridge)
exp = get_branin_experiment()
sobol = Models.SOBOL(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
sobol_run = sobol.gen(n=1)
t = exp.new_batch_trial().add_generator_run(sobol_run)
t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5)
t.run().mark_completed()
with self.assertRaises(ValueError):
status_quo_features = ObservationFeatures(
parameters=exp.trials[0].status_quo.parameters,
trial_index=0,
)
Models.ST_MTGP(
experiment=exp,
data=exp.fetch_data(),
status_quo_features=status_quo_features,
)
def test_get_standard_plots(self):
exp = get_branin_experiment()
self.assertEqual(
len(
get_standard_plots(experiment=exp,
model=get_generation_strategy().model)),
0,
)
exp = get_branin_experiment(with_batch=True, minimize=True)
exp.trials[0].run()
gs = choose_generation_strategy(search_space=exp.search_space)
gs._model = Models.BOTORCH(experiment=exp, data=exp.fetch_data())
plots = get_standard_plots(experiment=exp, model=gs.model)
self.assertEqual(len(plots), 5)
self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots))
exp = get_branin_experiment_with_multi_objective(with_batch=True)
exp.trials[0].run()
gs = choose_generation_strategy(
search_space=exp.search_space,
optimization_config=exp.optimization_config)
gs._model = Models.BOTORCH(experiment=exp, data=exp.fetch_data())
plots = get_standard_plots(experiment=exp, model=gs.model)
self.assertEqual(len(plots), 6)
def test_ALEBO_Initializer(self):
"""Tests Alebo Initializer generations"""
experiment = get_branin_experiment(with_batch=True)
B = np.array([[1.0, 2.0]])
m = Models.ALEBO_INITIALIZER(
experiment=experiment,
search_space=None,
B=B,
)
self.assertIsInstance(m, RandomModelBridge)
self.assertIsInstance(m.model, ALEBOInitializer)
gr = m.gen(n=2)
self.assertEqual(len(gr.arms), 2)
def test_cross_validation(self):
exp = get_branin_experiment(with_batch=True)
exp.trials[0].run()
model = Models.BOTORCH(
# Model bridge kwargs
experiment=exp,
data=exp.fetch_data(),
)
cv = cross_validate(model)
# Assert that each type of plot can be constructed successfully
plot = interact_cross_validation_plotly(cv)
self.assertIsInstance(plot, go.Figure)
plot = interact_cross_validation(cv)
self.assertIsInstance(plot, AxPlotConfig)
def get_MOO_EHVI(
experiment: Experiment,
data: Data,
objective_thresholds: Optional[TRefPoint] = None,
search_space: Optional[SearchSpace] = None,
dtype: torch.dtype = torch.double,
device: Optional[torch.device] = None,
) -> TorchModelBridge:
"""Instantiates a multi-objective model that generates points with EHVI.
Requires `objective_thresholds`,
a list of `ax.core.ObjectiveThresholds`, for every objective being optimized.
An arm only improves hypervolume if it is strictly better than all
objective thresholds.
`objective_thresholds` can be passed in the optimization_config or
passed directly here.
"""
device = device or (torch.device("cuda")
if torch.cuda.is_available() else torch.device("cpu"))
# pyre-ignore: [16] `Optional` has no attribute `objective`.
if not isinstance(experiment.optimization_config.objective,
MultiObjective):
raise ValueError(
"Multi-objective optimization requires multiple objectives.")
if data.df.empty: # pragma: no cover
raise ValueError("MultiObjectiveOptimization requires non-empty data.")
return checked_cast(
TorchModelBridge,
Models.MOO(
experiment=experiment,
data=data,
objective_thresholds=objective_thresholds,
search_space=search_space or experiment.search_space,
torch_dtype=dtype,
torch_device=device,
acqf_constructor=get_EHVI,
default_model_gen_options={
"acquisition_function_kwargs": {
"sequential": True
},
"optimizer_kwargs": {
# having a batch limit is very important for avoiding
# memory issues in the initialization
"batch_limit": DEFAULT_EHVI_BATCH_LIMIT
},
},
),
)
def get_uniform(
search_space: SearchSpace, deduplicate: bool = False, seed: Optional[int] = None
) -> RandomModelBridge:
"""Instantiate uniform generator.
Args:
search_space: Uniform generator search space.
kwargs: Custom args for uniform generator.
Returns:
RandomModelBridge, with UniformGenerator as model.
"""
return checked_cast(
RandomModelBridge,
Models.UNIFORM(search_space=search_space, seed=seed, deduplicate=deduplicate),
)
def get_model_from_generator_run(generator_run: GeneratorRun,
experiment: Experiment,
data: Data) -> ModelBridge:
"""Reinstantiate a model from model key and kwargs stored on a given generator
run, with the given experiment and the data to initialize the model with.
"""
if not generator_run._model_key: # pragma: no cover
raise ValueError(
"Cannot restore model from generator run as no model key was "
"on the generator run stored.")
model = Models(generator_run._model_key)
return model(
experiment=experiment,
data=data,
**(generator_run._model_kwargs or {}),
**(generator_run._bridge_kwargs or {}),
)
def get_MOO_EHVI(
experiment: Experiment,
data: Data,
ref_point: Dict[str, float],
search_space: Optional[SearchSpace] = None,
dtype: torch.dtype = torch.double,
device: torch.device = (torch.device("cuda") if torch.cuda.is_available()
else torch.device("cpu")),
) -> MultiObjectiveTorchModelBridge:
"""Instantiates a multi-objective model that generates points with EHVI.
Requires a `ref_point`, a dictionary of the metric name to the reference point value
for every objective being optimized. An arm only improves hypervolume if it is
strictly better than this point in all metrics.
"""
# pyre-ignore: [16] `Optional` has no attribute `objective`.
if not isinstance(experiment.optimization_config.objective,
MultiObjective):
raise ValueError(
"Multi-objective optimization requires multiple objectives.")
if data.df.empty: # pragma: no cover
raise ValueError("MultiObjectiveOptimization requires non-empty data.")
return checked_cast(
MultiObjectiveTorchModelBridge,
Models.MOO(
experiment=experiment,
data=data,
ref_point=ref_point,
search_space=search_space or experiment.search_space,
torch_dtype=dtype,
torch_device=device,
default_model_gen_options={
"acquisition_function_kwargs": {
"sequential": True
},
"optimizer_kwargs": {
# having a batch limit is very important for avoiding
# memory issues in the initialization
"batch_limit": DEFAULT_EHVI_BATCH_LIMIT
},
},
),
)
def testSlices(self):
exp = get_branin_experiment(with_batch=True)
exp.trials[0].run()
model = Models.BOTORCH(
# Model bridge kwargs
experiment=exp,
data=exp.fetch_data(),
)
# Assert that each type of plot can be constructed successfully
plot = plot_slice_plotly(model, model.parameters[0],
list(model.metric_names)[0])
self.assertIsInstance(plot, go.Figure)
plot = interact_slice_plotly(model)
self.assertIsInstance(plot, go.Figure)
plot = plot_slice(model, model.parameters[0],
list(model.metric_names)[0])
self.assertIsInstance(plot, AxPlotConfig)
plot = interact_slice(model)
self.assertIsInstance(plot, AxPlotConfig)
