def test_transform_ref_point(self, _mock_fit, _mock_predict, _mock_unwrap):
exp = get_branin_experiment_with_multi_objective(
has_optimization_config=True, with_batch=False)
metrics = exp.optimization_config.objective.metrics
ref_point = {metrics[0].name: 0.0, metrics[1].name: 0.0}
modelbridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=exp.optimization_config,
transforms=[t1, t2],
experiment=exp,
data=exp.fetch_data(),
ref_point=ref_point,
)
self.assertIsNone(modelbridge._transformed_ref_point)
exp = get_branin_experiment_with_multi_objective(
has_optimization_config=True, with_batch=True)
exp.attach_data(
get_branin_data_multi_objective(trial_indices=exp.trials))
modelbridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=exp.optimization_config,
transforms=[t1, t2],
experiment=exp,
data=exp.fetch_data(),
ref_point=ref_point,
)
self.assertIsNotNone(modelbridge._transformed_ref_point)
self.assertEqual(2, len(modelbridge._transformed_ref_point))
mixed_objective_constraints_optimization_config = OptimizationConfig(
objective=MultiObjective(
metrics=[get_branin_metric(name="branin_b")], minimize=False),
outcome_constraints=[
OutcomeConstraint(metric=Metric(name="branin_a"),
op=ComparisonOp.LEQ,
bound=1)
],
)
modelbridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=mixed_objective_constraints_optimization_config,
transforms=[t1, t2],
experiment=exp,
data=exp.fetch_data(),
ref_point={"branin_b": 0.0},
)
self.assertEqual({"branin_a", "branin_b"}, modelbridge._metric_names)
self.assertEqual(["branin_b"], modelbridge._objective_metric_names)
self.assertIsNotNone(modelbridge._transformed_ref_point)
self.assertEqual(1, len(modelbridge._transformed_ref_point))
def test_is_moo_problem(self):
exp = get_branin_experiment()
self.assertFalse(exp.is_moo_problem)
exp = get_branin_experiment_with_multi_objective()
self.assertTrue(exp.is_moo_problem)
exp._optimization_config = None
self.assertFalse(exp.is_moo_problem)
def test_MOO_RS(self):
single_obj_exp = get_branin_experiment(with_batch=True)
with self.assertRaises(ValueError):
get_MOO_RS(experiment=single_obj_exp,
data=single_obj_exp.fetch_data())
multi_obj_exp = get_branin_experiment_with_multi_objective(
with_batch=True)
with self.assertRaises(ValueError):
get_MOO_RS(experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data())
multi_obj_exp.trials[0].run()
moo_rs = get_MOO_RS(experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data())
self.assertIsInstance(moo_rs, TorchModelBridge)
self.assertEqual(
{
"acquisition_function_kwargs": {
"random_scalarization": True,
"sequential": True,
}
},
moo_rs._default_model_gen_options,
)
moo_rs_run = moo_rs.gen(n=5)
self.assertEqual(len(moo_rs_run.arms), 5)
def test_status_quo_for_non_monolithic_data(self, mock_gen):
mock_gen.return_value = (
[
ObservationFeatures(parameters={
"x1": float(i),
"x2": float(i)
},
trial_index=np.int64(1)) for i in range(5)
],
[1] * 5,
None,
{},
)
exp = get_branin_experiment_with_multi_objective(with_status_quo=True)
sobol = Models.SOBOL(search_space=exp.search_space)
exp.new_batch_trial(sobol.gen(5)).set_status_quo_and_optimize_power(
status_quo=exp.status_quo).run()
# create data where metrics vary in start and end times
data = get_non_monolithic_branin_moo_data()
with warnings.catch_warnings(record=True) as ws:
bridge = ModelBridge(
experiment=exp,
data=data,
model=Model(),
search_space=exp.search_space,
)
# just testing it doesn't error
bridge.gen(5)
self.assertTrue(any("start_time" in str(w.message) for w in ws))
self.assertTrue(any("end_time" in str(w.message) for w in ws))
self.assertEqual(bridge.status_quo.arm_name, "status_quo")
def setUp(self):
self.branin_experiment = get_branin_experiment_with_multi_objective()
sobol = Models.SOBOL(search_space=self.branin_experiment.search_space)
sobol_run = sobol.gen(n=20)
self.branin_experiment.new_batch_trial().add_generator_run(
sobol_run
).run().mark_completed()
data = self.branin_experiment.fetch_data()
ms_gpei = ModelSpec(model_enum=Models.GPEI)
ms_gpei.fit(experiment=self.branin_experiment, data=data)
ms_gpkg = ModelSpec(model_enum=Models.GPKG)
ms_gpkg.fit(experiment=self.branin_experiment, data=data)
self.fitted_model_specs = [ms_gpei, ms_gpkg]
self.model_selection_node = GenerationNode(
model_specs=self.fitted_model_specs,
best_model_selector=SingleDiagnosticBestModelSelector(
diagnostic="Fisher exact test p",
criterion=MetricAggregation.MEAN,
metric_aggregation=DiagnosticCriterion.MIN,
),
)
def test_MOO_PAREGO(self):
single_obj_exp = get_branin_experiment(with_batch=True)
with self.assertRaises(ValueError):
get_MOO_PAREGO(experiment=single_obj_exp, data=single_obj_exp.fetch_data())
multi_obj_exp = get_branin_experiment_with_multi_objective(with_batch=True)
with self.assertRaises(ValueError):
get_MOO_PAREGO(experiment=multi_obj_exp, data=multi_obj_exp.fetch_data())
multi_obj_exp.trials[0].run().mark_completed()
moo_parego = get_MOO_PAREGO(
experiment=multi_obj_exp, data=multi_obj_exp.fetch_data()
)
self.assertIsInstance(moo_parego, TorchModelBridge)
self.assertEqual(
{
"acquisition_function_kwargs": {
"chebyshev_scalarization": True,
"sequential": True,
}
},
moo_parego._default_model_gen_options,
)
moo_parego_run = moo_parego.gen(n=2)
self.assertEqual(len(moo_parego_run.arms), 2)
def test_pareto_frontier(self, _):
exp = get_branin_experiment_with_multi_objective(
has_optimization_config=True, with_batch=True
)
for trial in exp.trials.values():
trial.mark_running(no_runner_required=True).mark_completed()
metrics_dict = exp.optimization_config.metrics
objective_thresholds = [
ObjectiveThreshold(
metric=metrics_dict["branin_a"],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ,
),
ObjectiveThreshold(
metric=metrics_dict["branin_b"],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ,
),
]
exp.optimization_config = exp.optimization_config.clone_with_args(
objective_thresholds=objective_thresholds
)
exp.attach_data(
get_branin_data_multi_objective(trial_indices=exp.trials.keys())
)
modelbridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=exp.optimization_config,
transforms=[t1, t2],
experiment=exp,
data=exp.fetch_data(),
objective_thresholds=objective_thresholds,
)
with patch(
PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator
) as wrapped_frontier_evaluator:
modelbridge.model.frontier_evaluator = wrapped_frontier_evaluator
observed_frontier_data = modelbridge.observed_pareto_frontier(
objective_thresholds=objective_thresholds
)
wrapped_frontier_evaluator.assert_called_once()
self.assertEqual(1, len(observed_frontier_data))
with self.assertRaises(ValueError):
modelbridge.predicted_pareto_frontier(
objective_thresholds=objective_thresholds, observation_features=[]
)
observation_features = [
ObservationFeatures(parameters={"x1": 0.0, "x2": 1.0}),
ObservationFeatures(parameters={"x1": 1.0, "x2": 0.0}),
]
predicted_frontier_data = modelbridge.predicted_pareto_frontier(
objective_thresholds=objective_thresholds,
observation_features=observation_features,
)
self.assertTrue(len(predicted_frontier_data) <= 2)
def test_MOO_EHVI(self):
single_obj_exp = get_branin_experiment(with_batch=True)
with self.assertRaises(ValueError):
get_MOO_EHVI(
experiment=single_obj_exp,
data=single_obj_exp.fetch_data(),
ref_point=[0, 0],
)
multi_obj_exp = get_branin_experiment_with_multi_objective(
with_batch=True)
metrics = multi_obj_exp.optimization_config.objective.metrics
with self.assertRaises(ValueError):
get_MOO_EHVI(
experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data(),
ref_point={
metrics[0].name: 0.0,
metrics[1].name: 0.0
},
)
multi_obj_exp.trials[0].run()
moo_ehvi = get_MOO_EHVI(
experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data(),
ref_point={
metrics[0].name: 0.0,
metrics[1].name: 0.0
},
)
self.assertIsInstance(moo_ehvi, MultiObjectiveTorchModelBridge)
moo_ehvi_run = moo_ehvi.gen(n=1)
self.assertEqual(len(moo_ehvi_run.arms), 1)
def testPlotMultipleParetoFrontiers(self):
experiment = get_branin_experiment_with_multi_objective(
has_objective_thresholds=True, )
sobol = Models.SOBOL(experiment.search_space)
a = sobol.gen(5)
experiment.new_batch_trial(generator_run=a).run()
pfrs = get_observed_pareto_frontiers(experiment=experiment)
pfrs2 = copy.deepcopy(pfrs)
pfr_lists = {"pfrs 1": pfrs, "pfrs 2": pfrs2}
self.assertIsNotNone(interact_multiple_pareto_frontier(pfr_lists))
def test_MOO_with_more_outcomes_than_thresholds(self):
experiment = get_branin_experiment_with_multi_objective(
has_optimization_config=False)
metric_c = Metric(name="c", lower_is_better=False)
metric_a = Metric(name="a", lower_is_better=False)
objective_thresholds = [
ObjectiveThreshold(
metric=metric_c,
bound=2.0,
relative=False,
),
ObjectiveThreshold(
metric=metric_a,
bound=1.0,
relative=False,
),
]
experiment.optimization_config = MultiObjectiveOptimizationConfig(
objective=MultiObjective(objectives=[
Objective(metric=metric_a),
Objective(metric=metric_c),
]),
objective_thresholds=objective_thresholds,
)
experiment.add_tracking_metric(Metric(name="b", lower_is_better=False))
sobol = get_sobol(search_space=experiment.search_space, )
sobol_run = sobol.gen(1)
experiment.new_batch_trial().add_generator_run(
sobol_run).run().mark_completed()
data = Data(
pd.DataFrame(
data={
"arm_name": ["0_0", "0_0", "0_0"],
"metric_name": ["a", "b", "c"],
"mean": [1.0, 2.0, 3.0],
"trial_index": [0, 0, 0],
"sem": [0, 0, 0],
}))
test_names_to_fns = {
"MOO_NEHVI": get_MOO_NEHVI,
"MOO_EHVI": get_MOO_NEHVI,
"MOO_PAREGO": get_MOO_PAREGO,
"MOO_RS": get_MOO_RS,
}
for test_name, factory_fn in test_names_to_fns.items():
with self.subTest(test_name):
moo_model = factory_fn(
experiment=experiment,
data=data,
)
moo_gr = moo_model.gen(n=1)
obj_t = moo_gr.gen_metadata["objective_thresholds"]
self.assertEqual(obj_t[0], objective_thresholds[1])
self.assertEqual(obj_t[1], objective_thresholds[0])
self.assertEqual(len(obj_t), 2)
def test_ST_MTGP_NEHVI(self):
"""Tests single type MTGP NEHVI instantiation."""
multi_obj_exp = get_branin_experiment_with_multi_objective(
with_batch=True,
with_status_quo=True,
)
metrics = multi_obj_exp.optimization_config.objective.metrics
multi_objective_thresholds = [
ObjectiveThreshold(metric=metrics[0],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ),
ObjectiveThreshold(metric=metrics[1],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ),
]
sobol = Models.SOBOL(search_space=multi_obj_exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(2):
sobol_run = sobol.gen(n=1)
t = multi_obj_exp.new_batch_trial().add_generator_run(sobol_run)
t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5)
t.run().mark_completed()
status_quo_features = ObservationFeatures(
parameters=multi_obj_exp.trials[0].status_quo.parameters,
trial_index=0,
)
mtgp = Models.ST_MTGP_NEHVI(
experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data(),
status_quo_features=status_quo_features,
objective_thresholds=multi_objective_thresholds,
)
self.assertIsInstance(mtgp, TorchModelBridge)
self.assertIsInstance(mtgp.model, MultiObjectiveBotorchModel)
# test it can generate
mtgp_run = mtgp.gen(n=1,
fixed_features=ObservationFeatures(parameters={},
trial_index=1))
self.assertEqual(len(mtgp_run.arms), 1)
# test a generated trial can be completed
t = multi_obj_exp.new_batch_trial().add_generator_run(mtgp_run)
t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5)
t.run().mark_completed()
def test_MOO_EHVI(self):
single_obj_exp = get_branin_experiment(with_batch=True)
metrics = single_obj_exp.optimization_config.objective.metrics
objective_thresholds = [
ObjectiveThreshold(metric=metrics[0],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ)
]
# ValueError: Multi-objective optimization requires multiple objectives.
with self.assertRaises(ValueError):
get_MOO_EHVI(
experiment=single_obj_exp,
data=single_obj_exp.fetch_data(),
objective_thresholds=objective_thresholds,
)
multi_obj_exp = get_branin_experiment_with_multi_objective(
with_batch=True)
metrics = multi_obj_exp.optimization_config.objective.metrics
multi_objective_thresholds = [
ObjectiveThreshold(metric=metrics[0],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ),
ObjectiveThreshold(metric=metrics[1],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ),
]
# ValueError: MultiObjectiveOptimization requires non-empty data.
with self.assertRaises(ValueError):
get_MOO_EHVI(
experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data(),
objective_thresholds=multi_objective_thresholds,
)
multi_obj_exp.trials[0].run().mark_completed()
moo_ehvi = get_MOO_EHVI(
experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data(),
objective_thresholds=multi_objective_thresholds,
)
self.assertIsInstance(moo_ehvi, MultiObjectiveTorchModelBridge)
moo_ehvi_run = moo_ehvi.gen(n=1)
self.assertEqual(len(moo_ehvi_run.arms), 1)
def test_status_quo_for_non_monolithic_data(self):
exp = get_branin_experiment_with_multi_objective(with_status_quo=True)
sobol_generator = get_sobol(search_space=exp.search_space, )
sobol_run = sobol_generator.gen(n=5)
exp.new_batch_trial(sobol_run).set_status_quo_and_optimize_power(
status_quo=exp.status_quo).run()
# create data where metrics vary in start and end times
data = get_non_monolithic_branin_moo_data()
bridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=exp.optimization_config,
experiment=exp,
data=data,
transforms=[],
)
self.assertEqual(bridge.status_quo.arm_name, "status_quo")
def test_MOO_EHVI(self):
single_obj_exp = get_branin_experiment(with_batch=True)
metrics = single_obj_exp.optimization_config.objective.metrics
metrics[0].lower_is_better = True
objective_thresholds = [
ObjectiveThreshold(metric=metrics[0],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ)
]
with self.assertRaises(ValueError):
get_MOO_EHVI(
experiment=single_obj_exp,
data=single_obj_exp.fetch_data(),
objective_thresholds=objective_thresholds,
)
multi_obj_exp = get_branin_experiment_with_multi_objective(
with_batch=True)
metrics = multi_obj_exp.optimization_config.objective.metrics
metrics[0].lower_is_better = False
metrics[1].lower_is_better = True
multi_objective_thresholds = [
ObjectiveThreshold(metric=metrics[0], bound=0.0, relative=False),
ObjectiveThreshold(metric=metrics[1], bound=0.0, relative=False),
]
with self.assertRaises(ValueError):
get_MOO_EHVI(
experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data(),
objective_thresholds=multi_objective_thresholds,
)
multi_obj_exp.trials[0].run()
moo_ehvi = get_MOO_EHVI(
experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data(),
objective_thresholds=multi_objective_thresholds,
)
self.assertIsInstance(moo_ehvi, MultiObjectiveTorchModelBridge)
moo_ehvi_run = moo_ehvi.gen(n=1)
self.assertEqual(len(moo_ehvi_run.arms), 1)
def test_get_standard_plots(self):
exp = get_branin_experiment()
self.assertEqual(
len(
get_standard_plots(experiment=exp,
model=get_generation_strategy().model)),
0,
)
exp = get_branin_experiment(with_batch=True, minimize=True)
exp.trials[0].run()
gs = choose_generation_strategy(search_space=exp.search_space)
gs._model = Models.BOTORCH(experiment=exp, data=exp.fetch_data())
plots = get_standard_plots(experiment=exp, model=gs.model)
self.assertEqual(len(plots), 5)
self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots))
exp = get_branin_experiment_with_multi_objective(with_batch=True)
exp.trials[0].run()
gs = choose_generation_strategy(
search_space=exp.search_space,
optimization_config=exp.optimization_config)
gs._model = Models.BOTORCH(experiment=exp, data=exp.fetch_data())
plots = get_standard_plots(experiment=exp, model=gs.model)
self.assertEqual(len(plots), 6)
def test_get_best_trial_moo(self):
experiment = get_branin_experiment_with_multi_objective()
experiment.runner = self.runner
scheduler = Scheduler(
experiment=experiment,
generation_strategy=self.sobol_GPEI_GS,
options=SchedulerOptions(init_seconds_between_polls=0.1),
)
scheduler.run_n_trials(max_trials=1)
with self.assertRaisesRegex(
NotImplementedError,
"Please use `get_pareto_optimal_parameters`"):
scheduler.get_best_trial()
with self.assertRaisesRegex(
NotImplementedError,
"Please use `get_pareto_optimal_parameters`"):
scheduler.get_best_parameters()
self.assertIsNotNone(scheduler.get_pareto_optimal_parameters())
def test_pareto_frontier(self, _):
exp = get_branin_experiment_with_multi_objective(
has_optimization_config=True, with_batch=True)
for trial in exp.trials.values():
trial.mark_running(no_runner_required=True).mark_completed()
metrics_dict = exp.optimization_config.metrics
objective_thresholds = [
ObjectiveThreshold(
metric=metrics_dict["branin_a"],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ,
),
ObjectiveThreshold(
metric=metrics_dict["branin_b"],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ,
),
]
exp.optimization_config = exp.optimization_config.clone_with_args(
objective_thresholds=objective_thresholds)
exp.attach_data(
get_branin_data_multi_objective(trial_indices=exp.trials.keys()))
modelbridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=exp.optimization_config,
transforms=[t1, t2],
experiment=exp,
data=exp.fetch_data(),
objective_thresholds=objective_thresholds,
)
with patch(
PARETO_FRONTIER_EVALUATOR_PATH,
wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator:
modelbridge.model.frontier_evaluator = wrapped_frontier_evaluator
observed_frontier = observed_pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds)
wrapped_frontier_evaluator.assert_called_once()
self.assertIsNone(wrapped_frontier_evaluator.call_args.kwargs["X"])
self.assertEqual(1, len(observed_frontier))
self.assertEqual(observed_frontier[0].arm_name, "0_0")
with self.assertRaises(ValueError):
predicted_pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=[],
)
predicted_frontier = predicted_pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=None,
)
self.assertEqual(predicted_frontier[0].arm_name, "0_0")
observation_features = [
ObservationFeatures(parameters={
"x1": 0.0,
"x2": 1.0
}),
ObservationFeatures(parameters={
"x1": 1.0,
"x2": 0.0
}),
]
observation_data = [
ObservationData(
metric_names=["branin_b", "branin_a"],
means=np.array([1.0, 2.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
),
ObservationData(
metric_names=["branin_a", "branin_b"],
means=np.array([3.0, 4.0]),
covariance=np.array([[1.0, 2.0], [3.0, 4.0]]),
),
]
predicted_frontier = predicted_pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=observation_features,
)
self.assertTrue(len(predicted_frontier) <= 2)
self.assertIsNone(predicted_frontier[0].arm_name, None)
with patch(
PARETO_FRONTIER_EVALUATOR_PATH,
wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator:
observed_frontier = pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=observation_features,
observation_data=observation_data,
)
wrapped_frontier_evaluator.assert_called_once()
self.assertTrue(
torch.equal(
wrapped_frontier_evaluator.call_args.kwargs["X"],
torch.tensor([[1.0, 4.0], [4.0, 1.0]]),
))
with patch(
PARETO_FRONTIER_EVALUATOR_PATH,
wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator:
observed_frontier = pareto_frontier(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=observation_features,
observation_data=observation_data,
use_model_predictions=False,
)
wrapped_frontier_evaluator.assert_called_once()
self.assertIsNone(wrapped_frontier_evaluator.call_args.kwargs["X"])
self.assertTrue(
torch.equal(
wrapped_frontier_evaluator.call_args.kwargs["Y"],
torch.tensor([[9.0, 4.0], [16.0, 25.0]]),
))
def test_hypervolume(self, _, cuda=False):
for num_objectives in (2, 3):
exp = get_branin_experiment_with_multi_objective(
has_optimization_config=True,
with_batch=True,
num_objectives=num_objectives,
)
for trial in exp.trials.values():
trial.mark_running(no_runner_required=True).mark_completed()
metrics_dict = exp.optimization_config.metrics
objective_thresholds = [
ObjectiveThreshold(
metric=metrics_dict["branin_a"],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ,
),
ObjectiveThreshold(
metric=metrics_dict["branin_b"],
bound=1.0,
relative=False,
op=ComparisonOp.GEQ,
),
]
if num_objectives == 3:
objective_thresholds.append(
ObjectiveThreshold(
metric=metrics_dict["branin_c"],
bound=2.0,
relative=False,
op=ComparisonOp.GEQ,
)
)
optimization_config = exp.optimization_config.clone_with_args(
objective_thresholds=objective_thresholds
)
exp.attach_data(
get_branin_data_multi_objective(
trial_indices=exp.trials.keys(), num_objectives=num_objectives
)
)
modelbridge = TorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=optimization_config,
transforms=[],
experiment=exp,
data=exp.fetch_data(),
torch_device=torch.device("cuda" if cuda else "cpu"),
objective_thresholds=objective_thresholds,
)
with patch(
PARETO_FRONTIER_EVALUATOR_PATH, wraps=pareto_frontier_evaluator
) as wrapped_frontier_evaluator:
modelbridge.model.frontier_evaluator = wrapped_frontier_evaluator
hv = observed_hypervolume(
modelbridge=modelbridge, objective_thresholds=objective_thresholds
)
expected_hv = 20 if num_objectives == 2 else 60 # 5 * 4 (* 3)
wrapped_frontier_evaluator.assert_called_once()
self.assertEqual(expected_hv, hv)
if num_objectives == 3:
# Test selected_metrics
hv = observed_hypervolume(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
selected_metrics=["branin_a", "branin_c"],
)
expected_hv = 15 # (5 - 0) * (5 - 2)
self.assertEqual(expected_hv, hv)
# test that non-objective outcome raises value error
with self.assertRaises(ValueError):
hv = observed_hypervolume(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
selected_metrics=["tracking"],
)
with self.assertRaises(ValueError):
predicted_hypervolume(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=[],
)
observation_features = [
ObservationFeatures(parameters={"x1": 1.0, "x2": 2.0}),
ObservationFeatures(parameters={"x1": 2.0, "x2": 1.0}),
]
predicted_hv = predicted_hypervolume(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=observation_features,
)
self.assertTrue(predicted_hv >= 0)
if num_objectives == 3:
# Test selected_metrics
predicted_hv = predicted_hypervolume(
modelbridge=modelbridge,
objective_thresholds=objective_thresholds,
observation_features=observation_features,
selected_metrics=["branin_a", "branin_c"],
)
self.assertTrue(predicted_hv >= 0)
def test_MTGP_NEHVI(self):
single_obj_exp = get_branin_experiment(with_batch=True)
metrics = single_obj_exp.optimization_config.objective.metrics
metrics[0].lower_is_better = True
objective_thresholds = [
ObjectiveThreshold(metric=metrics[0], bound=0.0, relative=False)
]
with self.assertRaises(ValueError):
get_MTGP_NEHVI(
experiment=single_obj_exp,
data=single_obj_exp.fetch_data(),
objective_thresholds=objective_thresholds,
)
multi_obj_exp = get_branin_experiment_with_multi_objective(with_batch=True)
metrics = multi_obj_exp.optimization_config.objective.metrics
multi_objective_thresholds = [
ObjectiveThreshold(
metric=metrics[0], bound=0.0, relative=False, op=ComparisonOp.GEQ
),
ObjectiveThreshold(
metric=metrics[1], bound=0.0, relative=False, op=ComparisonOp.GEQ
),
]
with self.assertRaises(ValueError):
get_MTGP_NEHVI(
experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data(),
objective_thresholds=multi_objective_thresholds,
)
multi_obj_exp.trials[0].run()
sobol_generator = get_sobol(search_space=multi_obj_exp.search_space)
sobol_run = sobol_generator.gen(n=3)
multi_obj_exp.new_batch_trial(optimize_for_power=False).add_generator_run(
sobol_run
)
multi_obj_exp.trials[1].run()
mt_ehvi = get_MTGP_NEHVI(
experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data(),
objective_thresholds=multi_objective_thresholds,
trial_index=1,
)
self.assertIsInstance(mt_ehvi, TorchModelBridge)
self.assertIsInstance(mt_ehvi.model.model.models[0], MultiTaskGP)
task_covar_factor = mt_ehvi.model.model.models[0].task_covar_module.covar_factor
self.assertEqual(task_covar_factor.shape, torch.Size([2, 2]))
mt_ehvi_run = mt_ehvi.gen(
n=1, fixed_features=ObservationFeatures(parameters={}, trial_index=1)
)
self.assertEqual(len(mt_ehvi_run.arms), 1)
# Bad index given
with self.assertRaises(ValueError):
get_MTGP_NEHVI(
experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data(),
objective_thresholds=multi_objective_thresholds,
trial_index=999,
)
# Multi-type + multi-objective experiment
multi_type_multi_obj_exp = get_multi_type_experiment_with_multi_objective(
add_trials=True
)
data = multi_type_multi_obj_exp.fetch_data()
mt_ehvi = get_MTGP_NEHVI(
experiment=multi_type_multi_obj_exp,
data=data,
objective_thresholds=multi_objective_thresholds,
)
def test_infer_objective_thresholds(self, _, cuda=False):
# lightweight test
exp = get_branin_experiment_with_multi_objective(
has_optimization_config=True,
with_batch=True,
with_status_quo=True,
)
for trial in exp.trials.values():
trial.mark_running(no_runner_required=True).mark_completed()
exp.attach_data(
get_branin_data_multi_objective(trial_indices=exp.trials.keys())
)
data = exp.fetch_data()
modelbridge = TorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=exp.optimization_config,
transforms=Cont_X_trans + Y_trans,
torch_device=torch.device("cuda" if cuda else "cpu"),
experiment=exp,
data=data,
)
fixed_features = ObservationFeatures(parameters={"x1": 0.0})
search_space = exp.search_space.clone()
param_constraints = [
ParameterConstraint(constraint_dict={"x1": 1.0}, bound=10.0)
]
search_space.add_parameter_constraints(param_constraints)
oc = exp.optimization_config.clone()
oc.objective._objectives[0].minimize = True
expected_base_gen_args = modelbridge._get_transformed_gen_args(
search_space=search_space.clone(),
optimization_config=oc,
fixed_features=fixed_features,
)
with ExitStack() as es:
mock_model_infer_obj_t = es.enter_context(
patch(
"ax.modelbridge.torch.infer_objective_thresholds",
wraps=infer_objective_thresholds,
)
)
mock_get_transformed_gen_args = es.enter_context(
patch.object(
modelbridge,
"_get_transformed_gen_args",
wraps=modelbridge._get_transformed_gen_args,
)
)
mock_get_transformed_model_gen_args = es.enter_context(
patch.object(
modelbridge,
"_get_transformed_model_gen_args",
wraps=modelbridge._get_transformed_model_gen_args,
)
)
mock_untransform_objective_thresholds = es.enter_context(
patch.object(
modelbridge,
"_untransform_objective_thresholds",
wraps=modelbridge._untransform_objective_thresholds,
)
)
obj_thresholds = modelbridge.infer_objective_thresholds(
search_space=search_space,
optimization_config=oc,
fixed_features=fixed_features,
)
expected_obj_weights = torch.tensor([-1.0, 1.0])
ckwargs = mock_model_infer_obj_t.call_args[1]
self.assertTrue(
torch.equal(ckwargs["objective_weights"], expected_obj_weights)
)
# check that transforms have been applied (at least UnitX)
self.assertEqual(ckwargs["bounds"], [(0.0, 1.0), (0.0, 1.0)])
lc = ckwargs["linear_constraints"]
self.assertTrue(torch.equal(lc[0], torch.tensor([[15.0, 0.0]])))
self.assertTrue(torch.equal(lc[1], torch.tensor([[15.0]])))
self.assertEqual(ckwargs["fixed_features"], {0: 1.0 / 3.0})
mock_get_transformed_gen_args.assert_called_once()
mock_get_transformed_model_gen_args.assert_called_once_with(
search_space=expected_base_gen_args.search_space,
fixed_features=expected_base_gen_args.fixed_features,
pending_observations=expected_base_gen_args.pending_observations,
optimization_config=expected_base_gen_args.optimization_config,
)
mock_untransform_objective_thresholds.assert_called_once()
ckwargs = mock_untransform_objective_thresholds.call_args[1]
self.assertTrue(
torch.equal(ckwargs["objective_weights"], expected_obj_weights)
)
self.assertEqual(ckwargs["bounds"], [(0.0, 1.0), (0.0, 1.0)])
self.assertEqual(ckwargs["fixed_features"], {0: 1.0 / 3.0})
self.assertEqual(obj_thresholds[0].metric.name, "branin_a")
self.assertEqual(obj_thresholds[1].metric.name, "branin_b")
self.assertEqual(obj_thresholds[0].op, ComparisonOp.LEQ)
self.assertEqual(obj_thresholds[1].op, ComparisonOp.GEQ)
self.assertFalse(obj_thresholds[0].relative)
self.assertFalse(obj_thresholds[1].relative)
df = exp_to_df(exp)
Y = np.stack([df.branin_a.values, df.branin_b.values]).T
Y = torch.from_numpy(Y)
Y[:, 0] *= -1
pareto_Y = Y[is_non_dominated(Y)]
nadir = pareto_Y.min(dim=0).values
self.assertTrue(
np.all(
np.array([-obj_thresholds[0].bound, obj_thresholds[1].bound])
< nadir.numpy()
)
)
# test using MTGP
sobol_generator = get_sobol(
search_space=exp.search_space,
seed=TEST_SOBOL_SEED,
# set initial position equal to the number of sobol arms generated
# so far. This means that new sobol arms will complement the previous
# arms in a space-filling fashion
init_position=len(exp.arms_by_name) - 1,
)
sobol_run = sobol_generator.gen(n=2)
trial = exp.new_batch_trial(optimize_for_power=True)
trial.add_generator_run(sobol_run)
trial.mark_running(no_runner_required=True).mark_completed()
data = exp.fetch_data()
torch.manual_seed(0) # make model fitting deterministic
modelbridge = TorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=exp.optimization_config,
transforms=ST_MTGP_trans,
experiment=exp,
data=data,
)
fixed_features = ObservationFeatures(parameters={}, trial_index=1)
expected_base_gen_args = modelbridge._get_transformed_gen_args(
search_space=search_space.clone(),
optimization_config=exp.optimization_config,
fixed_features=fixed_features,
)
with ExitStack() as es:
mock_model_infer_obj_t = es.enter_context(
patch(
"ax.modelbridge.torch.infer_objective_thresholds",
wraps=infer_objective_thresholds,
)
)
mock_untransform_objective_thresholds = es.enter_context(
patch.object(
modelbridge,
"_untransform_objective_thresholds",
wraps=modelbridge._untransform_objective_thresholds,
)
)
obj_thresholds = modelbridge.infer_objective_thresholds(
search_space=search_space,
optimization_config=exp.optimization_config,
fixed_features=fixed_features,
)
ckwargs = mock_model_infer_obj_t.call_args[1]
self.assertEqual(ckwargs["fixed_features"], {2: 1.0})
mock_untransform_objective_thresholds.assert_called_once()
ckwargs = mock_untransform_objective_thresholds.call_args[1]
self.assertEqual(ckwargs["fixed_features"], {2: 1.0})
self.assertEqual(obj_thresholds[0].metric.name, "branin_a")
self.assertEqual(obj_thresholds[1].metric.name, "branin_b")
self.assertEqual(obj_thresholds[0].op, ComparisonOp.GEQ)
self.assertEqual(obj_thresholds[1].op, ComparisonOp.GEQ)
self.assertFalse(obj_thresholds[0].relative)
self.assertFalse(obj_thresholds[1].relative)
df = exp_to_df(exp)
trial_mask = df.trial_index == 1
Y = np.stack([df.branin_a.values[trial_mask], df.branin_b.values[trial_mask]]).T
Y = torch.from_numpy(Y)
pareto_Y = Y[is_non_dominated(Y)]
nadir = pareto_Y.min(dim=0).values
self.assertTrue(
np.all(
np.array([obj_thresholds[0].bound, obj_thresholds[1].bound])
< nadir.numpy()
)
)
def testObservedParetoFrontiers(self):
experiment = get_branin_experiment_with_multi_objective(
with_batch=True,
has_optimization_config=False,
with_status_quo=True)
# Optimization config is not optional
with self.assertRaises(ValueError):
get_observed_pareto_frontiers(experiment=experiment, data=Data())
objectives = [
Objective(
metric=BraninMetric(name="m1",
param_names=["x1", "x2"],
lower_is_better=True),
minimize=True,
),
Objective(
metric=NegativeBraninMetric(name="m2",
param_names=["x1", "x2"],
lower_is_better=True),
minimize=True,
),
Objective(
metric=BraninMetric(name="m3",
param_names=["x1", "x2"],
lower_is_better=True),
minimize=True,
),
]
bounds = [0, -100, 0]
objective_thresholds = [
ObjectiveThreshold(
metric=objective.metric,
bound=bounds[i],
relative=True,
op=ComparisonOp.LEQ,
) for i, objective in enumerate(objectives)
]
objective = MultiObjective(objectives=objectives)
optimization_config = MultiObjectiveOptimizationConfig(
objective=objective,
objective_thresholds=objective_thresholds,
)
experiment.optimization_config = optimization_config
experiment.trials[0].run()
# For the check below, compute which arms are better than SQ
df = experiment.fetch_data().df
df["sem"] = np.nan
data = Data(df)
sq_val = df[(df["arm_name"] == "status_quo")
& (df["metric_name"] == "m1")]["mean"].values[0]
pareto_arms = sorted(
df[(df["mean"] <= sq_val)
& (df["metric_name"] == "m1")]["arm_name"].unique().tolist())
pfrs = get_observed_pareto_frontiers(experiment=experiment, data=data)
# We have all pairs of metrics
self.assertEqual(len(pfrs), 3)
true_pairs = [("m1", "m2"), ("m1", "m3"), ("m2", "m3")]
for i, pfr in enumerate(pfrs):
self.assertEqual(pfr.primary_metric, true_pairs[i][0])
self.assertEqual(pfr.secondary_metric, true_pairs[i][1])
self.assertEqual(pfr.absolute_metrics, [])
self.assertEqual(list(pfr.means.keys()), ["m1", "m2", "m3"])
self.assertEqual(len(pfr.means["m1"]), len(pareto_arms))
self.assertTrue(np.isnan(pfr.sems["m1"]).all())
self.assertEqual(len(pfr.arm_names), len(pareto_arms))
arm_idx = np.argsort(pfr.arm_names)
for i, idx in enumerate(arm_idx):
name = pareto_arms[i]
self.assertEqual(pfr.arm_names[idx], name)
self.assertEqual(pfr.param_dicts[idx],
experiment.arms_by_name[name].parameters)
def test_get_standard_plots(self):
exp = get_branin_experiment()
self.assertEqual(
len(
get_standard_plots(experiment=exp,
model=get_generation_strategy().model)),
0,
)
exp = get_branin_experiment(with_batch=True, minimize=True)
exp.trials[0].run()
plots = get_standard_plots(
experiment=exp,
model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()),
)
self.assertEqual(len(plots), 6)
self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots))
exp = get_branin_experiment_with_multi_objective(with_batch=True)
exp.optimization_config.objective.objectives[0].minimize = False
exp.optimization_config.objective.objectives[1].minimize = True
exp.optimization_config._objective_thresholds = [
ObjectiveThreshold(metric=exp.metrics["branin_a"],
op=ComparisonOp.GEQ,
bound=-100.0),
ObjectiveThreshold(metric=exp.metrics["branin_b"],
op=ComparisonOp.LEQ,
bound=100.0),
]
exp.trials[0].run()
plots = get_standard_plots(experiment=exp,
model=Models.MOO(experiment=exp,
data=exp.fetch_data()))
self.assertEqual(len(plots), 7)
# All plots are successfully created when objective thresholds are absent
exp.optimization_config._objective_thresholds = []
plots = get_standard_plots(experiment=exp,
model=Models.MOO(experiment=exp,
data=exp.fetch_data()))
self.assertEqual(len(plots), 7)
exp = get_branin_experiment_with_timestamp_map_metric(
with_status_quo=True)
exp.new_trial().add_arm(exp.status_quo)
exp.trials[0].run()
exp.new_trial(generator_run=Models.SOBOL(
search_space=exp.search_space).gen(n=1))
exp.trials[1].run()
plots = get_standard_plots(
experiment=exp,
model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()),
true_objective_metric_name="branin",
)
self.assertEqual(len(plots), 9)
self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots))
self.assertIn(
"Objective branin_map vs. True Objective Metric branin",
[p.layout.title.text for p in plots],
)
with self.assertRaisesRegex(
ValueError, "Please add a valid true_objective_metric_name"):
plots = get_standard_plots(
experiment=exp,
model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()),
true_objective_metric_name="not_present",
)
def test_hypervolume(self):
exp = get_branin_experiment_with_multi_objective(
has_optimization_config=True, with_batch=False)
metrics_dict = exp.optimization_config.metrics
objective_thresholds = [
ObjectiveThreshold(
metric=metrics_dict["branin_a"],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ,
),
ObjectiveThreshold(
metric=metrics_dict["branin_b"],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ,
),
]
exp = get_branin_experiment_with_multi_objective(
has_optimization_config=True, with_batch=True)
optimization_config = exp.optimization_config.clone_with_args(
objective_thresholds=objective_thresholds)
exp.attach_data(
get_branin_data_multi_objective(trial_indices=exp.trials))
modelbridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=optimization_config,
transforms=[t1, t2],
experiment=exp,
data=exp.fetch_data(),
objective_thresholds=objective_thresholds,
)
with patch(
PARETO_FRONTIER_EVALUATOR_PATH,
wraps=pareto_frontier_evaluator) as wrapped_frontier_evaluator:
modelbridge.model.frontier_evaluator = wrapped_frontier_evaluator
hv = modelbridge.observed_hypervolume(
objective_thresholds=objective_thresholds)
expected_hv = 25 # (5 - 0) * (5 - 0)
wrapped_frontier_evaluator.assert_called_once()
self.assertEqual(expected_hv, hv)
with self.assertRaises(ValueError):
modelbridge.predicted_hypervolume(
objective_thresholds=objective_thresholds,
observation_features=[])
observation_features = [
ObservationFeatures(parameters={
"x1": 1.0,
"x2": 2.0
}),
ObservationFeatures(parameters={
"x1": 2.0,
"x2": 1.0
}),
]
predicted_hv = modelbridge.predicted_hypervolume(
objective_thresholds=objective_thresholds,
observation_features=observation_features,
)
self.assertTrue(predicted_hv >= 0)
def test_infer_objective_thresholds(self, _, cuda=False):
# lightweight test
exp = get_branin_experiment_with_multi_objective(
has_optimization_config=True,
with_batch=True,
with_status_quo=True,
)
for trial in exp.trials.values():
trial.mark_running(no_runner_required=True).mark_completed()
exp.attach_data(
get_branin_data_multi_objective(trial_indices=exp.trials.keys()))
data = exp.fetch_data()
modelbridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=exp.optimization_config,
transforms=Cont_X_trans + Y_trans,
torch_device=torch.device("cuda" if cuda else "cpu"),
experiment=exp,
data=data,
)
fixed_features = ObservationFeatures(parameters={"x1": 0.0})
search_space = exp.search_space.clone()
param_constraints = [
ParameterConstraint(constraint_dict={"x1": 1.0}, bound=10.0)
]
outcome_constraints = [
OutcomeConstraint(
metric=exp.metrics["branin_a"],
op=ComparisonOp.GEQ,
bound=-40.0,
relative=False,
)
]
search_space.add_parameter_constraints(param_constraints)
exp.optimization_config.outcome_constraints = outcome_constraints
oc = exp.optimization_config.clone()
oc.objective._objectives[0].minimize = True
expected_base_gen_args = modelbridge._get_transformed_gen_args(
search_space=search_space.clone(),
optimization_config=oc,
fixed_features=fixed_features,
)
with ExitStack() as es:
mock_model_infer_obj_t = es.enter_context(
patch(
"ax.modelbridge.multi_objective_torch.infer_objective_thresholds",
wraps=infer_objective_thresholds,
))
mock_get_transformed_gen_args = es.enter_context(
patch.object(
modelbridge,
"_get_transformed_gen_args",
wraps=modelbridge._get_transformed_gen_args,
))
mock_get_transformed_model_gen_args = es.enter_context(
patch.object(
modelbridge,
"_get_transformed_model_gen_args",
wraps=modelbridge._get_transformed_model_gen_args,
))
mock_untransform_objective_thresholds = es.enter_context(
patch.object(
modelbridge,
"untransform_objective_thresholds",
wraps=modelbridge.untransform_objective_thresholds,
))
obj_thresholds = modelbridge.infer_objective_thresholds(
search_space=search_space,
optimization_config=oc,
fixed_features=fixed_features,
)
expected_obj_weights = torch.tensor([-1.0, 1.0])
ckwargs = mock_model_infer_obj_t.call_args[1]
self.assertTrue(
torch.equal(ckwargs["objective_weights"],
expected_obj_weights))
# check that transforms have been applied (at least UnitX)
self.assertEqual(ckwargs["bounds"], [(0.0, 1.0), (0.0, 1.0)])
oc = ckwargs["outcome_constraints"]
self.assertTrue(torch.equal(oc[0], torch.tensor([[-1.0, 0.0]])))
self.assertTrue(torch.equal(oc[1], torch.tensor([[45.0]])))
lc = ckwargs["linear_constraints"]
self.assertTrue(torch.equal(lc[0], torch.tensor([[15.0, 0.0]])))
self.assertTrue(torch.equal(lc[1], torch.tensor([[15.0]])))
self.assertEqual(ckwargs["fixed_features"], {0: 1.0 / 3.0})
mock_get_transformed_gen_args.assert_called_once()
mock_get_transformed_model_gen_args.assert_called_once_with(
search_space=expected_base_gen_args.search_space,
fixed_features=expected_base_gen_args.fixed_features,
pending_observations=expected_base_gen_args.
pending_observations,
optimization_config=expected_base_gen_args.optimization_config,
)
mock_untransform_objective_thresholds.assert_called_once()
ckwargs = mock_untransform_objective_thresholds.call_args[1]
self.assertTrue(
torch.equal(ckwargs["objective_weights"],
expected_obj_weights))
self.assertEqual(ckwargs["bounds"], [(0.0, 1.0), (0.0, 1.0)])
self.assertEqual(ckwargs["fixed_features"], {0: 1.0 / 3.0})
self.assertEqual(obj_thresholds[0].metric.name, "branin_a")
self.assertEqual(obj_thresholds[1].metric.name, "branin_b")
self.assertEqual(obj_thresholds[0].op, ComparisonOp.LEQ)
self.assertEqual(obj_thresholds[1].op, ComparisonOp.GEQ)
self.assertFalse(obj_thresholds[0].relative)
self.assertFalse(obj_thresholds[1].relative)
df = exp_to_df(exp)
Y = np.stack([df.branin_a.values, df.branin_b.values]).T
Y = torch.from_numpy(Y)
Y[:, 0] *= -1
pareto_Y = Y[is_non_dominated(Y)]
nadir = pareto_Y.min(dim=0).values
self.assertTrue(
np.all(
np.array([-obj_thresholds[0].bound, obj_thresholds[1].bound]) <
nadir.numpy()))
# test using MTGP
sobol_generator = get_sobol(search_space=exp.search_space)
sobol_run = sobol_generator.gen(n=5)
trial = exp.new_batch_trial(optimize_for_power=True)
trial.add_generator_run(sobol_run)
trial.mark_running(no_runner_required=True).mark_completed()
data = exp.fetch_data()
modelbridge = MultiObjectiveTorchModelBridge(
search_space=exp.search_space,
model=MultiObjectiveBotorchModel(),
optimization_config=exp.optimization_config,
transforms=ST_MTGP_trans,
experiment=exp,
data=data,
)
fixed_features = ObservationFeatures(parameters={}, trial_index=1)
expected_base_gen_args = modelbridge._get_transformed_gen_args(
search_space=search_space.clone(),
optimization_config=exp.optimization_config,
fixed_features=fixed_features,
)
with self.assertRaises(ValueError):
# Check that a ValueError is raised when MTGP is being used
# and trial_index is not specified as a fixed features.
# Note: this error is raised by StratifiedStandardizeY
modelbridge.infer_objective_thresholds(
search_space=search_space,
optimization_config=exp.optimization_config,
)
with ExitStack() as es:
mock_model_infer_obj_t = es.enter_context(
patch(
"ax.modelbridge.multi_objective_torch.infer_objective_thresholds",
wraps=infer_objective_thresholds,
))
mock_untransform_objective_thresholds = es.enter_context(
patch.object(
modelbridge,
"untransform_objective_thresholds",
wraps=modelbridge.untransform_objective_thresholds,
))
obj_thresholds = modelbridge.infer_objective_thresholds(
search_space=search_space,
optimization_config=exp.optimization_config,
fixed_features=fixed_features,
)
ckwargs = mock_model_infer_obj_t.call_args[1]
self.assertEqual(ckwargs["fixed_features"], {2: 1.0})
mock_untransform_objective_thresholds.assert_called_once()
ckwargs = mock_untransform_objective_thresholds.call_args[1]
self.assertEqual(ckwargs["fixed_features"], {2: 1.0})
self.assertEqual(obj_thresholds[0].metric.name, "branin_a")
self.assertEqual(obj_thresholds[1].metric.name, "branin_b")
self.assertEqual(obj_thresholds[0].op, ComparisonOp.GEQ)
self.assertEqual(obj_thresholds[1].op, ComparisonOp.GEQ)
self.assertFalse(obj_thresholds[0].relative)
self.assertFalse(obj_thresholds[1].relative)
df = exp_to_df(exp)
trial_mask = df.trial_index == 1
Y = np.stack(
[df.branin_a.values[trial_mask], df.branin_b.values[trial_mask]]).T
Y = torch.from_numpy(Y)
pareto_Y = Y[is_non_dominated(Y)]
nadir = pareto_Y.min(dim=0).values
self.assertTrue(
np.all(
np.array([obj_thresholds[0].bound, obj_thresholds[1].bound]) <
nadir.numpy()))
