def test_BotorchMOOModel_with_random_scalarization_and_outcome_constraints(
self, dtype=torch.float, cuda=False
):
tkwargs = {
"device": torch.device("cuda") if cuda else torch.device("cpu"),
"dtype": dtype,
}
Xs1, Ys1, Yvars1, bounds, tfs, fns, mns = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True
)
Xs2, Ys2, Yvars2, _, _, _, _ = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True
)
n = 2
objective_weights = torch.tensor([1.0, 1.0], **tkwargs)
obj_t = torch.tensor([1.0, 1.0], **tkwargs)
model = MultiObjectiveBotorchModel(acqf_constructor=get_NEI)
X_dummy = torch.tensor([[[1.0, 2.0, 3.0]]], **tkwargs)
acqfv_dummy = torch.tensor([[[1.0, 2.0, 3.0]]], **tkwargs)
with mock.patch(FIT_MODEL_MO_PATH) as _mock_fit_model:
model.fit(
Xs=Xs1 + Xs2,
Ys=Ys1 + Ys2,
Yvars=Yvars1 + Yvars2,
search_space_digest=SearchSpaceDigest(
feature_names=fns,
bounds=bounds,
task_features=tfs,
),
metric_names=mns,
)
_mock_fit_model.assert_called_once()
with mock.patch(
SAMPLE_SIMPLEX_UTIL_PATH,
autospec=True,
return_value=torch.tensor([0.7, 0.3], **tkwargs),
) as _mock_sample_simplex, mock.patch(
"ax.models.torch.botorch_moo_defaults.optimize_acqf_list",
return_value=(X_dummy, acqfv_dummy),
) as _:
model.gen(
n,
bounds,
objective_weights,
outcome_constraints=(
torch.tensor([[1.0, 1.0]], **tkwargs),
torch.tensor([[10.0]], **tkwargs),
),
model_gen_options={
"acquisition_function_kwargs": {"random_scalarization": True},
"optimizer_kwargs": _get_optimizer_kwargs(),
},
objective_thresholds=obj_t,
)
self.assertEqual(n, _mock_sample_simplex.call_count)
def test_BotorchMOOModel_with_chebyshev_scalarization_and_outcome_constraints(
self, dtype=torch.float, cuda=False
):
tkwargs = {
"device": torch.device("cuda") if cuda else torch.device("cpu"),
"dtype": torch.float,
}
Xs1, Ys1, Yvars1, bounds, tfs, fns, mns = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True
)
Xs2, Ys2, Yvars2, _, _, _, _ = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True
)
n = 2
objective_weights = torch.tensor([1.0, 1.0], **tkwargs)
model = MultiObjectiveBotorchModel(acqf_constructor=get_NEI)
X_dummy = torch.tensor([[[1.0, 2.0, 3.0]]], **tkwargs)
acqfv_dummy = torch.tensor([[[1.0, 2.0, 3.0]]], **tkwargs)
with mock.patch(FIT_MODEL_MO_PATH) as _mock_fit_model:
model.fit(
Xs=Xs1 + Xs2,
Ys=Ys1 + Ys2,
Yvars=Yvars1 + Yvars2,
search_space_digest=SearchSpaceDigest(
feature_names=fns,
bounds=bounds,
task_features=tfs,
),
metric_names=mns,
)
_mock_fit_model.assert_called_once()
with mock.patch(
CHEBYSHEV_SCALARIZATION_PATH, wraps=get_chebyshev_scalarization
) as _mock_chebyshev_scalarization, mock.patch(
"ax.models.torch.botorch_defaults.optimize_acqf",
return_value=(X_dummy, acqfv_dummy),
) as _:
model.gen(
n,
bounds,
objective_weights,
outcome_constraints=(
torch.tensor([[1.0, 1.0]], **tkwargs),
torch.tensor([[10.0]], **tkwargs),
),
model_gen_options={
"acquisition_function_kwargs": {"chebyshev_scalarization": True},
"optimizer_kwargs": _get_optimizer_kwargs(),
},
)
# get_chebyshev_scalarization should be called once for generated candidate.
self.assertEqual(n, _mock_chebyshev_scalarization.call_count)
def test_BotorchMOOModel_with_ehvi_and_outcome_constraints(
self, dtype=torch.float, cuda=False
):
tkwargs = {
"device": torch.device("cuda") if cuda else torch.device("cpu"),
"dtype": dtype,
}
Xs1, Ys1, Yvars1, bounds, tfs, fns, mns = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True
)
Xs2, Ys2, Yvars2, _, _, _, _ = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True
)
n = 3
objective_weights = torch.tensor([1.0, 1.0], **tkwargs)
model = MultiObjectiveBotorchModel(acqf_constructor=get_EHVI)
X_dummy = torch.tensor([[[1.0, 2.0, 3.0]]], **tkwargs)
acqfv_dummy = torch.tensor([[[1.0, 2.0, 3.0]]], **tkwargs)
with mock.patch(FIT_MODEL_MO_PATH) as _mock_fit_model:
model.fit(
Xs=Xs1 + Xs2,
Ys=Ys1 + Ys2,
Yvars=Yvars1 + Yvars2,
search_space_digest=SearchSpaceDigest(
feature_names=fns,
bounds=bounds,
task_features=tfs,
),
metric_names=mns,
)
_mock_fit_model.assert_called_once()
with mock.patch(
EHVI_ACQF_PATH, wraps=moo_monte_carlo.qExpectedHypervolumeImprovement
) as _mock_ehvi_acqf, mock.patch(
"ax.models.torch.botorch_defaults.optimize_acqf",
return_value=(X_dummy, acqfv_dummy),
) as _:
model.gen(
n,
bounds,
objective_weights,
outcome_constraints=(
torch.tensor([[1.0, 1.0]], **tkwargs),
torch.tensor([[10.0]], **tkwargs),
),
model_gen_options={"optimizer_kwargs": _get_optimizer_kwargs()},
objective_thresholds=torch.tensor([1.0, 1.0]),
)
# the EHVI acquisition function should be created only once.
self.assertEqual(1, _mock_ehvi_acqf.call_count)
def test_BotorchMOOModel_with_qehvi_and_outcome_constraints(
self, dtype=torch.float, cuda=False, use_qnehvi=False):
acqf_constructor = get_NEHVI if use_qnehvi else get_EHVI
tkwargs = {
"device": torch.device("cuda") if cuda else torch.device("cpu"),
"dtype": dtype,
}
Xs1, Ys1, Yvars1, bounds, tfs, fns, mns = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True)
Xs2, Ys2, Yvars2, _, _, _, _ = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True)
Xs3, Ys3, Yvars3, _, _, _, _ = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True)
n = 3
objective_weights = torch.tensor([1.0, 1.0, 0.0], **tkwargs)
obj_t = torch.tensor([1.0, 1.0, 1.0], **tkwargs)
model = MultiObjectiveBotorchModel(acqf_constructor=acqf_constructor)
with mock.patch(FIT_MODEL_MO_PATH) as _mock_fit_model:
model.fit(
Xs=Xs1 + Xs2 + Xs3,
Ys=Ys1 + Ys2 + Ys3,
Yvars=Yvars1 + Yvars2 + Yvars3,
search_space_digest=SearchSpaceDigest(
feature_names=fns,
bounds=bounds,
task_features=tfs,
),
metric_names=mns,
)
_mock_fit_model.assert_called_once()
# test wrong number of objective thresholds
with self.assertRaises(AxError):
model.gen(
n,
bounds,
objective_weights,
objective_thresholds=torch.tensor([1.0, 1.0], **tkwargs),
)
# test that objective thresholds and weights are properly subsetted
obj_t = torch.tensor([1.0, 1.0, 1.0], **tkwargs)
with mock.patch.object(
model,
"acqf_constructor",
wraps=botorch_moo_defaults.get_NEHVI,
) as mock_get_nehvi:
model.gen(
n,
bounds,
objective_weights,
model_gen_options={
"optimizer_kwargs": _get_optimizer_kwargs()
},
objective_thresholds=obj_t,
)
mock_get_nehvi.assert_called_once()
_, ckwargs = mock_get_nehvi.call_args
self.assertEqual(ckwargs["model"].num_outputs, 2)
self.assertTrue(
torch.equal(ckwargs["objective_weights"],
objective_weights[:-1]))
self.assertTrue(
torch.equal(ckwargs["objective_thresholds"], obj_t[:-1]))
self.assertIsNone(ckwargs["outcome_constraints"])
# the second datapoint is out of bounds
self.assertTrue(torch.equal(ckwargs["X_observed"], Xs1[0][:1]))
self.assertIsNone(ckwargs["X_pending"])
# test that outcome constraints are passed properly
oc = (
torch.tensor([[0.0, 0.0, 1.0]], **tkwargs),
torch.tensor([[10.0]], **tkwargs),
)
with mock.patch.object(
model,
"acqf_constructor",
wraps=botorch_moo_defaults.get_NEHVI,
) as mock_get_nehvi:
model.gen(
n,
bounds,
objective_weights,
outcome_constraints=oc,
model_gen_options={
"optimizer_kwargs": _get_optimizer_kwargs()
},
objective_thresholds=obj_t,
)
mock_get_nehvi.assert_called_once()
_, ckwargs = mock_get_nehvi.call_args
self.assertEqual(ckwargs["model"].num_outputs, 3)
self.assertTrue(
torch.equal(ckwargs["objective_weights"], objective_weights))
self.assertTrue(torch.equal(ckwargs["objective_thresholds"],
obj_t))
self.assertTrue(
torch.equal(ckwargs["outcome_constraints"][0], oc[0]))
self.assertTrue(
torch.equal(ckwargs["outcome_constraints"][1], oc[1]))
# the second datapoint is out of bounds
self.assertTrue(torch.equal(ckwargs["X_observed"], Xs1[0][:1]))
self.assertIsNone(ckwargs["X_pending"])
def test_BotorchMOOModel_with_qehvi(self,
dtype=torch.float,
cuda=False,
use_qnehvi=False):
if use_qnehvi:
acqf_constructor = get_NEHVI
partitioning_path = NEHVI_PARTITIONING_PATH
else:
acqf_constructor = get_EHVI
partitioning_path = EHVI_PARTITIONING_PATH
tkwargs = {
"device": torch.device("cuda") if cuda else torch.device("cpu"),
"dtype": dtype,
}
Xs1, Ys1, Yvars1, bounds, tfs, fns, mns = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True)
Xs2, Ys2, Yvars2, _, _, _, _ = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True)
n = 3
objective_weights = torch.tensor([1.0, 1.0], **tkwargs)
obj_t = torch.tensor([1.0, 1.0], **tkwargs)
model = MultiObjectiveBotorchModel(acqf_constructor=acqf_constructor)
X_dummy = torch.tensor([[[1.0, 2.0, 3.0]]], **tkwargs)
acqfv_dummy = torch.tensor([[[1.0, 2.0, 3.0]]], **tkwargs)
with mock.patch(FIT_MODEL_MO_PATH) as _mock_fit_model:
model.fit(
Xs=Xs1 + Xs2,
Ys=Ys1 + Ys2,
Yvars=Yvars1 + Yvars2,
search_space_digest=SearchSpaceDigest(
feature_names=fns,
bounds=bounds,
task_features=tfs,
),
metric_names=mns,
)
_mock_fit_model.assert_called_once()
with ExitStack() as es:
_mock_acqf = es.enter_context(
mock.patch(
NEHVI_ACQF_PATH,
wraps=moo_monte_carlo.qNoisyExpectedHypervolumeImprovement,
))
if use_qnehvi:
_mock_acqf = es.enter_context(
mock.patch(
NEHVI_ACQF_PATH,
wraps=moo_monte_carlo.
qNoisyExpectedHypervolumeImprovement,
))
else:
_mock_acqf = es.enter_context(
mock.patch(
EHVI_ACQF_PATH,
wraps=moo_monte_carlo.qExpectedHypervolumeImprovement,
))
es.enter_context(
mock.patch(
"ax.models.torch.botorch_defaults.optimize_acqf",
return_value=(X_dummy, acqfv_dummy),
))
_mock_partitioning = es.enter_context(
mock.patch(
partitioning_path,
wraps=moo_monte_carlo.FastNondominatedPartitioning,
))
_, _, gen_metadata, _ = model.gen(
n,
bounds,
objective_weights,
objective_thresholds=obj_t,
model_gen_options={
"optimizer_kwargs": _get_optimizer_kwargs()
},
)
# the NEHVI acquisition function should be created only once.
self.assertEqual(1, _mock_acqf.call_count)
# check partitioning strategy
# NEHVI should call FastNondominatedPartitioning 1 time
# since a batched partitioning is used for 2 objectives
_mock_partitioning.assert_called_once()
self.assertTrue(
torch.equal(gen_metadata["objective_thresholds"], obj_t.cpu()))
_mock_fit_model = es.enter_context(mock.patch(FIT_MODEL_MO_PATH))
# 3 objective
model.fit(
Xs=Xs1 + Xs2 + Xs2,
Ys=Ys1 + Ys2 + Ys2,
Yvars=Yvars1 + Yvars2 + Yvars2,
search_space_digest=SearchSpaceDigest(
feature_names=fns,
bounds=bounds,
task_features=tfs,
),
metric_names=mns,
)
model.gen(
n,
bounds,
torch.tensor([1.0, 1.0, 1.0], **tkwargs),
model_gen_options={
"optimizer_kwargs": _get_optimizer_kwargs()
},
objective_thresholds=torch.tensor([1.0, 1.0, 1.0], **tkwargs),
)
# check partitioning strategy
# NEHVI should call FastNondominatedPartitioning 129 times because
# we have called gen twice: The first time, a batch partitioning is used
# so there is one call to _mock_partitioning. The second time gen() is
# called with three objectives so 128 calls are made to _mock_partitioning
# because a BoxDecompositionList is used. qEHVI will only make 2 calls.
self.assertEqual(len(_mock_partitioning.mock_calls),
129 if use_qnehvi else 2)
# test inferred objective thresholds in gen()
# create several data points
Xs1 = [torch.cat([Xs1[0], Xs1[0] - 0.1], dim=0)]
Ys1 = [torch.cat([Ys1[0], Ys1[0] - 0.5], dim=0)]
Ys2 = [torch.cat([Ys2[0], Ys2[0] + 0.5], dim=0)]
Yvars1 = [torch.cat([Yvars1[0], Yvars1[0] + 0.2], dim=0)]
Yvars2 = [torch.cat([Yvars2[0], Yvars2[0] + 0.1], dim=0)]
model.fit(
Xs=Xs1 + Xs1,
Ys=Ys1 + Ys2,
Yvars=Yvars1 + Yvars2,
search_space_digest=SearchSpaceDigest(
feature_names=fns,
bounds=bounds,
task_features=tfs,
),
metric_names=mns + ["dummy_metric"],
)
_mock_model_infer_objective_thresholds = es.enter_context(
mock.patch(
"ax.models.torch.botorch_moo.infer_objective_thresholds",
wraps=infer_objective_thresholds,
))
_mock_infer_reference_point = es.enter_context(
mock.patch(
"ax.models.torch.botorch_moo_defaults.infer_reference_point",
wraps=infer_reference_point,
))
# after subsetting, the model will only have two outputs
_mock_num_outputs = es.enter_context(
mock.patch(
"botorch.utils.testing.MockModel.num_outputs",
new_callable=mock.PropertyMock,
))
_mock_num_outputs.return_value = 3
preds = torch.tensor(
[
[11.0, 2.0],
[9.0, 3.0],
],
**tkwargs,
)
model.model = MockModel(MockPosterior(
mean=preds,
samples=preds,
), )
subset_mock_model = MockModel(
MockPosterior(
mean=preds,
samples=preds,
), )
es.enter_context(
mock.patch.object(
model.model,
"subset_output",
return_value=subset_mock_model,
))
outcome_constraints = (
torch.tensor([[1.0, 0.0, 0.0]], **tkwargs),
torch.tensor([[10.0]], **tkwargs),
)
_, _, gen_metadata, _ = model.gen(
n,
bounds,
objective_weights=torch.tensor([-1.0, -1.0, 0.0], **tkwargs),
outcome_constraints=outcome_constraints,
model_gen_options={
"optimizer_kwargs": _get_optimizer_kwargs(),
# do not used cached root decomposition since
# MockPosterior does not have an mvn attribute
"acquisition_function_kwargs": {
"cache_root": False
},
},
)
# the NEHVI acquisition function should be created only once.
self.assertEqual(_mock_acqf.call_count, 3)
ckwargs = _mock_model_infer_objective_thresholds.call_args[1]
X_observed = ckwargs["X_observed"]
sorted_idcs = X_observed[:, 0].argsort()
expected_X_observed = torch.tensor(
[[1.0, 2.0, 3.0], [0.9, 1.9, 2.9]], **tkwargs)
sorted_idcs2 = expected_X_observed[:, 0].argsort()
self.assertTrue(
torch.equal(
X_observed[sorted_idcs],
expected_X_observed[sorted_idcs2],
))
self.assertTrue(
torch.equal(
ckwargs["objective_weights"],
torch.tensor([-1.0, -1.0, 0.0], **tkwargs),
))
oc = ckwargs["outcome_constraints"]
self.assertTrue(torch.equal(oc[0], outcome_constraints[0]))
self.assertTrue(torch.equal(oc[1], outcome_constraints[1]))
self.assertIs(ckwargs["model"], subset_mock_model)
self.assertTrue(
torch.equal(
ckwargs["subset_idcs"],
torch.tensor([0, 1], device=tkwargs["device"]),
))
_mock_infer_reference_point.assert_called_once()
ckwargs = _mock_infer_reference_point.call_args[1]
self.assertEqual(ckwargs["scale"], 0.1)
self.assertTrue(
torch.equal(ckwargs["pareto_Y"],
torch.tensor([[-9.0, -3.0]], **tkwargs)))
self.assertIn("objective_thresholds", gen_metadata)
obj_t = gen_metadata["objective_thresholds"]
self.assertTrue(
torch.equal(obj_t[:2],
torch.tensor([9.9, 3.3], dtype=tkwargs["dtype"])))
self.assertTrue(np.isnan(obj_t[2]))
# test providing model with extra tracking metrics and objective thresholds
provided_obj_t = torch.tensor([10.0, 4.0, float("nan")], **tkwargs)
_, _, gen_metadata, _ = model.gen(
n,
bounds,
objective_weights=torch.tensor([-1.0, -1.0, 0.0], **tkwargs),
outcome_constraints=outcome_constraints,
model_gen_options={
"optimizer_kwargs": _get_optimizer_kwargs(),
# do not used cached root decomposition since
# MockPosterior does not have an mvn attribute
"acquisition_function_kwargs": {
"cache_root": False
},
},
objective_thresholds=provided_obj_t,
)
self.assertIn("objective_thresholds", gen_metadata)
obj_t = gen_metadata["objective_thresholds"]
self.assertTrue(torch.equal(obj_t[:2], provided_obj_t[:2].cpu()))
self.assertTrue(np.isnan(obj_t[2]))
def test_BotorchMOOModel_with_random_scalarization(self,
dtype=torch.float,
cuda=False):
tkwargs = {
"device": torch.device("cuda") if cuda else torch.device("cpu"),
"dtype": dtype,
}
Xs1, Ys1, Yvars1, bounds, tfs, fns, mns = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True)
Xs2, Ys2, Yvars2, _, _, _, _ = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True)
n = 3
objective_weights = torch.tensor([1.0, 1.0], **tkwargs)
obj_t = torch.tensor([1.0, 1.0], **tkwargs)
model = MultiObjectiveBotorchModel(acqf_constructor=get_NEI)
with mock.patch(FIT_MODEL_MO_PATH) as _mock_fit_model:
model.fit(
Xs=Xs1 + Xs2,
Ys=Ys1 + Ys2,
Yvars=Yvars1 + Yvars2,
search_space_digest=SearchSpaceDigest(
feature_names=fns,
bounds=bounds,
task_features=tfs,
),
metric_names=mns,
)
_mock_fit_model.assert_called_once()
with mock.patch(
SAMPLE_SIMPLEX_UTIL_PATH,
autospec=True,
return_value=torch.tensor([0.7, 0.3], **tkwargs),
) as _mock_sample_simplex:
model.gen(
n,
bounds,
objective_weights,
objective_thresholds=obj_t,
model_gen_options={
"acquisition_function_kwargs": {
"random_scalarization": True
},
"optimizer_kwargs": _get_optimizer_kwargs(),
},
)
# Sample_simplex should be called once for generated candidate.
self.assertEqual(n, _mock_sample_simplex.call_count)
with mock.patch(
SAMPLE_HYPERSPHERE_UTIL_PATH,
autospec=True,
return_value=torch.tensor([0.6, 0.8], **tkwargs),
) as _mock_sample_hypersphere:
model.gen(
n,
bounds,
objective_weights,
objective_thresholds=obj_t,
model_gen_options={
"acquisition_function_kwargs": {
"random_scalarization": True,
"random_scalarization_distribution": HYPERSPHERE,
},
"optimizer_kwargs": _get_optimizer_kwargs(),
},
)
# Sample_simplex should be called once per generated candidate.
self.assertEqual(n, _mock_sample_hypersphere.call_count)
# test input warping
self.assertFalse(model.use_input_warping)
model = MultiObjectiveBotorchModel(acqf_constructor=get_NEI,
use_input_warping=True)
model.fit(
Xs=Xs1 + Xs2,
Ys=Ys1 + Ys2,
Yvars=Yvars1 + Yvars2,
search_space_digest=SearchSpaceDigest(
feature_names=fns,
bounds=bounds,
task_features=tfs,
),
metric_names=mns,
)
self.assertTrue(model.use_input_warping)
self.assertIsInstance(model.model, ModelListGP)
for m in model.model.models:
self.assertTrue(hasattr(m, "input_transform"))
self.assertIsInstance(m.input_transform, Warp)
self.assertFalse(hasattr(model.model, "input_transform"))
# test loocv pseudo likelihood
self.assertFalse(model.use_loocv_pseudo_likelihood)
model = MultiObjectiveBotorchModel(acqf_constructor=get_NEI,
use_loocv_pseudo_likelihood=True)
model.fit(
Xs=Xs1 + Xs2,
Ys=Ys1 + Ys2,
Yvars=Yvars1 + Yvars2,
search_space_digest=SearchSpaceDigest(
feature_names=fns,
bounds=bounds,
task_features=tfs,
),
metric_names=mns,
)
self.assertTrue(model.use_loocv_pseudo_likelihood)
def test_BotorchMOOModel_with_ehvi(self, dtype=torch.float, cuda=False):
tkwargs = {
"device": torch.device("cuda") if cuda else torch.device("cpu"),
"dtype": dtype,
}
Xs1, Ys1, Yvars1, bounds, tfs, fns, mns = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True)
Xs2, Ys2, Yvars2, _, _, _, _ = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True)
n = 3
objective_weights = torch.tensor([1.0, 1.0], **tkwargs)
model = MultiObjectiveBotorchModel(acqf_constructor=get_EHVI)
X_dummy = torch.tensor([[[1.0, 2.0, 3.0]]], **tkwargs)
acqfv_dummy = torch.tensor([[[1.0, 2.0, 3.0]]], **tkwargs)
with mock.patch(FIT_MODEL_MO_PATH) as _mock_fit_model:
model.fit(
Xs=Xs1 + Xs2,
Ys=Ys1 + Ys2,
Yvars=Yvars1 + Yvars2,
bounds=bounds,
task_features=tfs,
feature_names=fns,
metric_names=mns,
fidelity_features=[],
)
_mock_fit_model.assert_called_once()
with mock.patch(EHVI_ACQF_PATH,
wraps=moo_monte_carlo.qExpectedHypervolumeImprovement
) as _mock_ehvi_acqf, mock.patch(
"ax.models.torch.botorch_defaults.optimize_acqf",
return_value=(X_dummy, acqfv_dummy),
) as _, mock.patch(
PARTITIONING_PATH,
wraps=moo_monte_carlo.NondominatedPartitioning
) as _mock_partitioning:
model.gen(
n,
bounds,
objective_weights,
model_gen_options={
"optimizer_kwargs": _get_optimizer_kwargs()
},
objective_thresholds=torch.tensor([1.0, 1.0]),
)
# the EHVI acquisition function should be created only once.
self.assertEqual(1, _mock_ehvi_acqf.call_count)
# check partitioning strategy
self.assertEqual(_mock_partitioning.call_args[1]["alpha"], 0.0)
# 3 objective
with mock.patch(FIT_MODEL_MO_PATH) as _mock_fit_model:
model.fit(
Xs=Xs1 + Xs2 + Xs2,
Ys=Ys1 + Ys2 + Ys2,
Yvars=Yvars1 + Yvars2 + Yvars2,
bounds=bounds,
task_features=tfs,
feature_names=fns,
metric_names=mns,
fidelity_features=[],
)
with mock.patch(EHVI_ACQF_PATH,
wraps=moo_monte_carlo.qExpectedHypervolumeImprovement
) as _mock_ehvi_acqf, mock.patch(
"ax.models.torch.botorch_defaults.optimize_acqf",
return_value=(X_dummy, acqfv_dummy),
) as _, mock.patch(
PARTITIONING_PATH,
wraps=moo_monte_carlo.NondominatedPartitioning
) as _mock_partitioning:
model.gen(
n,
bounds,
torch.tensor([1.0, 1.0, 1.0], **tkwargs),
model_gen_options={
"optimizer_kwargs": _get_optimizer_kwargs()
},
objective_thresholds=torch.tensor([1.0, 1.0, 1.0]),
)
# check partitioning strategy
self.assertEqual(_mock_partitioning.call_args[1]["alpha"], 1e-5)
class FrontierEvaluatorTest(TestCase):
def setUp(self):
self.X = torch.tensor([[1.0, 0.0], [1.0, 1.0], [1.0, 3.0], [2.0, 2.0],
[3.0, 1.0]])
self.Y = torch.tensor([
[1.0, 0.0, 0.0],
[1.0, 1.0, 1.0],
[1.0, 3.0, 3.0],
[2.0, 2.0, 4.0],
[3.0, 1.0, 3.0],
])
self.Yvar = torch.zeros(5, 3)
self.outcome_constraints = (
torch.tensor([[0.0, 0.0, 1.0]]),
torch.tensor([[3.5]]),
)
self.objective_thresholds = torch.tensor([0.5, 1.5])
self.objective_weights = torch.tensor([1.0, 1.0])
bounds = [(0.0, 4.0), (0.0, 4.0)]
self.model = MultiObjectiveBotorchModel(model_predictor=dummy_predict)
with mock.patch(FIT_MODEL_MO_PATH) as _mock_fit_model:
self.model.fit(
Xs=[self.X],
Ys=[self.Y],
Yvars=[self.Yvar],
bounds=bounds,
task_features=[],
feature_names=["x1", "x2"],
metric_names=["a", "b", "c"],
fidelity_features=[],
)
_mock_fit_model.assert_called_once()
def test_pareto_frontier_raise_error_when_missing_data(self):
with self.assertRaises(ValueError):
pareto_frontier_evaluator(
model=self.model,
objective_thresholds=self.objective_thresholds,
objective_weights=self.objective_weights,
Yvar=self.Yvar,
)
def test_pareto_frontier_evaluator_raw(self):
Y, cov = pareto_frontier_evaluator(
model=self.model,
objective_weights=self.objective_weights,
objective_thresholds=self.objective_thresholds,
Y=self.Y,
Yvar=self.Yvar,
)
pred = self.Y[2:4]
self.assertTrue(torch.allclose(Y, pred), f"{Y} does not match {pred}")
# Omit objective_thresholds
Y, cov = pareto_frontier_evaluator(
model=self.model,
objective_weights=self.objective_weights,
Y=self.Y,
Yvar=self.Yvar,
)
pred = self.Y[2:]
self.assertTrue(torch.allclose(Y, pred), f"{Y} does not match {pred}")
# Change objective_weights so goal is to minimize b
Y, cov = pareto_frontier_evaluator(
model=self.model,
objective_weights=torch.tensor([1.0, -1.0]),
objective_thresholds=self.objective_thresholds,
Y=self.Y,
Yvar=self.Yvar,
)
pred = self.Y[[0, 4]]
self.assertTrue(torch.allclose(Y, pred),
f"actual {Y} does not match pred {pred}")
def test_pareto_frontier_evaluator_predict(self):
Y, cov = pareto_frontier_evaluator(
model=self.model,
objective_weights=self.objective_weights,
objective_thresholds=self.objective_thresholds,
X=self.X,
)
pred = self.Y[2:4]
self.assertTrue(torch.allclose(Y, pred),
f"actual {Y} does not match pred {pred}")
def test_pareto_frontier_evaluator_with_outcome_constraints(self):
Y, cov = pareto_frontier_evaluator(
model=self.model,
objective_weights=self.objective_weights,
objective_thresholds=self.objective_thresholds,
Y=self.Y,
Yvar=self.Yvar,
outcome_constraints=self.outcome_constraints,
)
pred = self.Y[2, :]
self.assertTrue(torch.allclose(Y, pred),
f"actual {Y} does not match pred {pred}")
class FrontierEvaluatorTest(TestCase):
def setUp(self):
self.X = torch.tensor([[1.0, 0.0], [1.0, 1.0], [1.0, 3.0], [2.0, 2.0],
[3.0, 1.0]])
self.Y = torch.tensor([
[1.0, 0.0, 0.0],
[1.0, 1.0, 1.0],
[1.0, 3.0, 3.0],
[2.0, 2.0, 4.0],
[3.0, 1.0, 3.0],
])
self.Yvar = torch.zeros(5, 3)
self.outcome_constraints = (
torch.tensor([[0.0, 0.0, 1.0]]),
torch.tensor([[3.5]]),
)
self.objective_thresholds = torch.tensor([0.5, 1.5])
self.objective_weights = torch.tensor([1.0, 1.0])
bounds = [(0.0, 4.0), (0.0, 4.0)]
self.model = MultiObjectiveBotorchModel(model_predictor=dummy_predict)
with mock.patch(FIT_MODEL_MO_PATH) as _mock_fit_model:
self.model.fit(
Xs=[self.X],
Ys=[self.Y],
Yvars=[self.Yvar],
search_space_digest=SearchSpaceDigest(
feature_names=["x1", "x2"],
bounds=bounds,
),
metric_names=["a", "b", "c"],
)
_mock_fit_model.assert_called_once()
def test_pareto_frontier_raise_error_when_missing_data(self):
with self.assertRaises(ValueError):
pareto_frontier_evaluator(
model=self.model,
objective_thresholds=self.objective_thresholds,
objective_weights=self.objective_weights,
Yvar=self.Yvar,
)
def test_pareto_frontier_evaluator_raw(self):
Yvar = torch.diag_embed(self.Yvar)
Y, cov, indx = pareto_frontier_evaluator(
model=self.model,
objective_weights=self.objective_weights,
objective_thresholds=self.objective_thresholds,
Y=self.Y,
Yvar=Yvar,
)
pred = self.Y[2:4]
self.assertTrue(torch.allclose(Y, pred), f"{Y} does not match {pred}")
expected_cov = Yvar[2:4]
self.assertTrue(torch.allclose(expected_cov, cov))
self.assertTrue(torch.equal(torch.arange(2, 4), indx))
# Omit objective_thresholds
Y, cov, indx = pareto_frontier_evaluator(
model=self.model,
objective_weights=self.objective_weights,
Y=self.Y,
Yvar=Yvar,
)
pred = self.Y[2:]
self.assertTrue(torch.allclose(Y, pred), f"{Y} does not match {pred}")
expected_cov = Yvar[2:]
self.assertTrue(torch.allclose(expected_cov, cov))
self.assertTrue(torch.equal(torch.arange(2, 5), indx))
# Change objective_weights so goal is to minimize b
Y, cov, indx = pareto_frontier_evaluator(
model=self.model,
objective_weights=torch.tensor([1.0, -1.0]),
objective_thresholds=self.objective_thresholds,
Y=self.Y,
Yvar=Yvar,
)
pred = self.Y[[0, 4]]
self.assertTrue(torch.allclose(Y, pred),
f"actual {Y} does not match pred {pred}")
expected_cov = Yvar[[0, 4]]
self.assertTrue(torch.allclose(expected_cov, cov))
# test no points better than reference point
Y, cov, indx = pareto_frontier_evaluator(
model=self.model,
objective_weights=self.objective_weights,
objective_thresholds=torch.full_like(self.objective_thresholds,
100.0),
Y=self.Y,
Yvar=Yvar,
)
self.assertTrue(torch.equal(Y, self.Y[:0]))
self.assertTrue(torch.equal(cov, torch.zeros(0, 3, 3)))
self.assertTrue(torch.equal(torch.tensor([], dtype=torch.long), indx))
def test_pareto_frontier_evaluator_predict(self):
Y, cov, indx = pareto_frontier_evaluator(
model=self.model,
objective_weights=self.objective_weights,
objective_thresholds=self.objective_thresholds,
X=self.X,
)
pred = self.Y[2:4]
self.assertTrue(torch.allclose(Y, pred),
f"actual {Y} does not match pred {pred}")
self.assertTrue(torch.equal(torch.arange(2, 4), indx))
def test_pareto_frontier_evaluator_with_outcome_constraints(self):
Y, cov, indx = pareto_frontier_evaluator(
model=self.model,
objective_weights=self.objective_weights,
objective_thresholds=self.objective_thresholds,
Y=self.Y,
Yvar=self.Yvar,
outcome_constraints=self.outcome_constraints,
)
pred = self.Y[2, :]
self.assertTrue(torch.allclose(Y, pred),
f"actual {Y} does not match pred {pred}")
self.assertTrue(torch.equal(torch.tensor([2], dtype=torch.long), indx))
def test_BotorchMOOModel_with_random_scalarization(self,
dtype=torch.float,
cuda=False):
tkwargs = {
"device": torch.device("cuda") if cuda else torch.device("cpu"),
"dtype": dtype,
}
Xs1, Ys1, Yvars1, bounds, tfs, fns, mns = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True)
Xs2, Ys2, Yvars2, _, _, _, _ = _get_torch_test_data(
dtype=dtype, cuda=cuda, constant_noise=True)
n = 3
objective_weights = torch.tensor([1.0, 1.0], **tkwargs)
X_dummy = torch.tensor([[[1.0, 2.0, 3.0]]], **tkwargs)
acqfv_dummy = torch.tensor([[[1.0, 2.0, 3.0]]], **tkwargs)
model = MultiObjectiveBotorchModel(acqf_constructor=get_NEI)
with mock.patch(FIT_MODEL_MO_PATH) as _mock_fit_model:
model.fit(
Xs=Xs1 + Xs2,
Ys=Ys1 + Ys2,
Yvars=Yvars1 + Yvars2,
bounds=bounds,
task_features=tfs,
feature_names=fns,
metric_names=mns,
fidelity_features=[],
)
_mock_fit_model.assert_called_once()
with mock.patch(
SAMPLE_SIMPLEX_UTIL_PATH,
autospec=True,
return_value=torch.tensor([0.7, 0.3], **tkwargs),
) as _mock_sample_simplex, mock.patch(
"ax.models.torch.botorch_defaults.optimize_acqf",
return_value=(X_dummy, acqfv_dummy),
) as _:
model.gen(
n,
bounds,
objective_weights,
model_gen_options={
"acquisition_function_kwargs": {
"random_scalarization": True
},
"optimizer_kwargs": _get_optimizer_kwargs(),
},
)
# Sample_simplex should be called once for generated candidate.
self.assertEqual(n, _mock_sample_simplex.call_count)
with mock.patch(
SAMPLE_HYPERSPHERE_UTIL_PATH,
autospec=True,
return_value=torch.tensor([0.6, 0.8], **tkwargs),
) as _mock_sample_hypersphere, mock.patch(
"ax.models.torch.botorch_defaults.optimize_acqf",
return_value=(X_dummy, acqfv_dummy),
) as _:
model.gen(
n,
bounds,
objective_weights,
model_gen_options={
"acquisition_function_kwargs": {
"random_scalarization": True,
"random_scalarization_distribution": HYPERSPHERE,
},
"optimizer_kwargs": _get_optimizer_kwargs(),
},
)
# Sample_simplex should be called once per generated candidate.
self.assertEqual(n, _mock_sample_hypersphere.call_count)