def test_MockModel(self):
mp = MockPosterior()
mm = MockModel(mp)
X = torch.empty(0)
self.assertEqual(mm.posterior(X), mp)
self.assertEqual(mm.num_outputs, 0)
mm.state_dict()
mm.load_state_dict()
class TestGetAcquisitionFunction(BotorchTestCase):
def setUp(self):
super().setUp()
self.model = mock.MagicMock()
self.objective = DummyMCObjective()
self.X_observed = torch.tensor([[1.0, 2.0, 3.0], [2.0, 3.0, 4.0]])
self.X_pending = torch.tensor([[1.0, 3.0, 4.0]])
self.mc_samples = 250
self.qmc = True
self.ref_point = [0.0, 0.0]
self.mo_objective = DummyMCMultiOutputObjective()
self.Y = torch.tensor([[1.0, 2.0]])
self.seed = 1
@mock.patch(f"{monte_carlo.__name__}.qExpectedImprovement")
def test_GetQEI(self, mock_acqf):
self.model = MockModel(MockPosterior(mean=torch.zeros(1, 2)))
acqf = get_acquisition_function(
acquisition_function_name="qEI",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=self.seed,
marginalize_dim=0,
)
self.assertTrue(acqf == mock_acqf.return_value)
best_f = self.objective(self.model.posterior(self.X_observed).mean).max().item()
mock_acqf.assert_called_once_with(
model=self.model,
best_f=best_f,
sampler=mock.ANY,
objective=self.objective,
X_pending=self.X_pending,
)
# test batched model
self.model = MockModel(MockPosterior(mean=torch.zeros(1, 2, 1)))
acqf = get_acquisition_function(
acquisition_function_name="qEI",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=self.seed,
)
self.assertTrue(acqf == mock_acqf.return_value)
# test batched model without marginalize dim
args, kwargs = mock_acqf.call_args
self.assertEqual(args, ())
sampler = kwargs["sampler"]
self.assertIsInstance(sampler, SobolQMCNormalSampler)
self.assertEqual(sampler.sample_shape, torch.Size([self.mc_samples]))
self.assertEqual(sampler.seed, 1)
self.assertTrue(torch.equal(kwargs["X_pending"], self.X_pending))
@mock.patch(f"{monte_carlo.__name__}.qProbabilityOfImprovement")
def test_GetQPI(self, mock_acqf):
# basic test
self.model = MockModel(MockPosterior(mean=torch.zeros(1, 2)))
acqf = get_acquisition_function(
acquisition_function_name="qPI",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=self.seed,
)
self.assertTrue(acqf == mock_acqf.return_value)
best_f = self.objective(self.model.posterior(self.X_observed).mean).max().item()
mock_acqf.assert_called_once_with(
model=self.model,
best_f=best_f,
sampler=mock.ANY,
objective=self.objective,
X_pending=self.X_pending,
tau=1e-3,
)
args, kwargs = mock_acqf.call_args
self.assertEqual(args, ())
sampler = kwargs["sampler"]
self.assertIsInstance(sampler, SobolQMCNormalSampler)
self.assertEqual(sampler.sample_shape, torch.Size([self.mc_samples]))
self.assertEqual(sampler.seed, 1)
self.assertTrue(torch.equal(kwargs["X_pending"], self.X_pending))
# test with different tau, non-qmc
acqf = get_acquisition_function(
acquisition_function_name="qPI",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
qmc=False,
seed=2,
tau=1.0,
)
self.assertTrue(mock_acqf.call_count, 2)
args, kwargs = mock_acqf.call_args
self.assertEqual(args, ())
self.assertEqual(kwargs["tau"], 1.0)
sampler = kwargs["sampler"]
self.assertIsInstance(sampler, IIDNormalSampler)
self.assertEqual(sampler.sample_shape, torch.Size([self.mc_samples]))
self.assertEqual(sampler.seed, 2)
self.assertTrue(torch.equal(kwargs["X_pending"], self.X_pending))
acqf = get_acquisition_function(
acquisition_function_name="qPI",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
qmc=False,
seed=2,
tau=1.0,
)
# test batched model
self.model = MockModel(MockPosterior(mean=torch.zeros(1, 2, 1)))
acqf = get_acquisition_function(
acquisition_function_name="qPI",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=self.seed,
)
self.assertTrue(acqf == mock_acqf.return_value)
@mock.patch(f"{monte_carlo.__name__}.qNoisyExpectedImprovement")
def test_GetQNEI(self, mock_acqf):
# basic test
acqf = get_acquisition_function(
acquisition_function_name="qNEI",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=self.seed,
marginalize_dim=0,
)
self.assertTrue(acqf == mock_acqf.return_value)
self.assertTrue(mock_acqf.call_count, 1)
args, kwargs = mock_acqf.call_args
self.assertEqual(args, ())
self.assertTrue(torch.equal(kwargs["X_baseline"], self.X_observed))
self.assertTrue(torch.equal(kwargs["X_pending"], self.X_pending))
sampler = kwargs["sampler"]
self.assertIsInstance(sampler, SobolQMCNormalSampler)
self.assertEqual(sampler.sample_shape, torch.Size([self.mc_samples]))
self.assertEqual(sampler.seed, 1)
self.assertEqual(kwargs["marginalize_dim"], 0)
# test with non-qmc, no X_pending
acqf = get_acquisition_function(
acquisition_function_name="qNEI",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=None,
mc_samples=self.mc_samples,
qmc=False,
seed=2,
)
self.assertTrue(mock_acqf.call_count, 2)
args, kwargs = mock_acqf.call_args
self.assertEqual(args, ())
self.assertTrue(torch.equal(kwargs["X_baseline"], self.X_observed))
self.assertEqual(kwargs["X_pending"], None)
sampler = kwargs["sampler"]
self.assertIsInstance(sampler, IIDNormalSampler)
self.assertEqual(sampler.sample_shape, torch.Size([self.mc_samples]))
self.assertEqual(sampler.seed, 2)
self.assertTrue(torch.equal(kwargs["X_baseline"], self.X_observed))
@mock.patch(f"{monte_carlo.__name__}.qSimpleRegret")
def test_GetQSR(self, mock_acqf):
# basic test
acqf = get_acquisition_function(
acquisition_function_name="qSR",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=self.seed,
)
self.assertTrue(acqf == mock_acqf.return_value)
mock_acqf.assert_called_once_with(
model=self.model,
sampler=mock.ANY,
objective=self.objective,
X_pending=self.X_pending,
)
args, kwargs = mock_acqf.call_args
self.assertEqual(args, ())
sampler = kwargs["sampler"]
self.assertIsInstance(sampler, SobolQMCNormalSampler)
self.assertEqual(sampler.sample_shape, torch.Size([self.mc_samples]))
self.assertEqual(sampler.seed, 1)
self.assertTrue(torch.equal(kwargs["X_pending"], self.X_pending))
# test with non-qmc
acqf = get_acquisition_function(
acquisition_function_name="qSR",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
qmc=False,
seed=2,
)
self.assertTrue(mock_acqf.call_count, 2)
args, kwargs = mock_acqf.call_args
self.assertEqual(args, ())
sampler = kwargs["sampler"]
self.assertIsInstance(sampler, IIDNormalSampler)
self.assertEqual(sampler.sample_shape, torch.Size([self.mc_samples]))
self.assertEqual(sampler.seed, 2)
self.assertTrue(torch.equal(kwargs["X_pending"], self.X_pending))
@mock.patch(f"{monte_carlo.__name__}.qUpperConfidenceBound")
def test_GetQUCB(self, mock_acqf):
# make sure beta is specified
with self.assertRaises(ValueError):
acqf = get_acquisition_function(
acquisition_function_name="qUCB",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=self.seed,
)
acqf = get_acquisition_function(
acquisition_function_name="qUCB",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=self.seed,
beta=0.3,
)
self.assertTrue(acqf == mock_acqf.return_value)
mock_acqf.assert_called_once_with(
model=self.model,
beta=0.3,
sampler=mock.ANY,
objective=self.objective,
X_pending=self.X_pending,
)
args, kwargs = mock_acqf.call_args
self.assertEqual(args, ())
sampler = kwargs["sampler"]
self.assertIsInstance(sampler, SobolQMCNormalSampler)
self.assertEqual(sampler.sample_shape, torch.Size([self.mc_samples]))
self.assertEqual(sampler.seed, 1)
self.assertTrue(torch.equal(kwargs["X_pending"], self.X_pending))
# test with different tau, non-qmc
acqf = get_acquisition_function(
acquisition_function_name="qUCB",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
qmc=False,
seed=2,
beta=0.2,
)
self.assertTrue(mock_acqf.call_count, 2)
args, kwargs = mock_acqf.call_args
self.assertEqual(args, ())
self.assertEqual(kwargs["beta"], 0.2)
sampler = kwargs["sampler"]
self.assertIsInstance(sampler, IIDNormalSampler)
self.assertEqual(sampler.sample_shape, torch.Size([self.mc_samples]))
self.assertEqual(sampler.seed, 2)
self.assertTrue(torch.equal(kwargs["X_pending"], self.X_pending))
@mock.patch(f"{moo_monte_carlo.__name__}.qExpectedHypervolumeImprovement")
def test_GetQEHVI(self, mock_acqf):
# make sure ref_point is specified
with self.assertRaises(ValueError):
acqf = get_acquisition_function(
acquisition_function_name="qEHVI",
model=self.model,
objective=self.mo_objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=self.seed,
Y=self.Y,
)
# make sure Y is specified
with self.assertRaises(ValueError):
acqf = get_acquisition_function(
acquisition_function_name="qEHVI",
model=self.model,
objective=self.mo_objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=self.seed,
ref_point=self.ref_point,
)
acqf = get_acquisition_function(
acquisition_function_name="qEHVI",
model=self.model,
objective=self.mo_objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=self.seed,
ref_point=self.ref_point,
Y=self.Y,
)
self.assertTrue(acqf == mock_acqf.return_value)
mock_acqf.assert_called_once_with(
constraints=None,
model=self.model,
objective=self.mo_objective,
ref_point=self.ref_point,
partitioning=mock.ANY,
sampler=mock.ANY,
X_pending=self.X_pending,
)
args, kwargs = mock_acqf.call_args
self.assertEqual(args, ())
sampler = kwargs["sampler"]
self.assertIsInstance(sampler, SobolQMCNormalSampler)
self.assertEqual(sampler.sample_shape, torch.Size([self.mc_samples]))
self.assertEqual(sampler.seed, 1)
# test with non-qmc
acqf = get_acquisition_function(
acquisition_function_name="qEHVI",
model=self.model,
objective=self.mo_objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=2,
qmc=False,
ref_point=self.ref_point,
Y=self.Y,
)
self.assertTrue(mock_acqf.call_count, 2)
args, kwargs = mock_acqf.call_args
self.assertEqual(args, ())
self.assertEqual(kwargs["ref_point"], self.ref_point)
sampler = kwargs["sampler"]
self.assertIsInstance(sampler, IIDNormalSampler)
self.assertIsInstance(kwargs["objective"], DummyMCMultiOutputObjective)
partitioning = kwargs["partitioning"]
self.assertIsInstance(partitioning, NondominatedPartitioning)
self.assertEqual(sampler.sample_shape, torch.Size([self.mc_samples]))
self.assertEqual(sampler.seed, 2)
# test constraints
acqf = get_acquisition_function(
acquisition_function_name="qEHVI",
model=self.model,
objective=self.mo_objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
constraints=[lambda Y: Y[..., -1]],
seed=2,
qmc=False,
ref_point=self.ref_point,
Y=self.Y,
)
_, kwargs = mock_acqf.call_args
partitioning = kwargs["partitioning"]
self.assertEqual(partitioning.pareto_Y.shape[0], 0)
def test_GetUnknownAcquisitionFunction(self):
with self.assertRaises(NotImplementedError):
get_acquisition_function(
acquisition_function_name="foo",
model=self.model,
objective=self.objective,
X_observed=self.X_observed,
X_pending=self.X_pending,
mc_samples=self.mc_samples,
seed=self.seed,
)
def test_get_f_X_samples(self):
tkwargs = {"device": self.device}
for dtype in (torch.float, torch.double):
tkwargs["dtype"] = dtype
mean = torch.zeros(5, 1, **tkwargs)
variance = torch.ones(5, 1, **tkwargs)
mm = MockModel(
MockPosterior(mean=mean,
variance=variance,
samples=torch.rand(5, 1, **tkwargs)))
# basic test
sampler = IIDNormalSampler(1)
acqf = DummyCachedCholeskyAcqf(model=mm, sampler=sampler)
acqf._setup(model=mm, sampler=sampler, cache_root=True)
q = 3
baseline_L = torch.eye(5 - q, **tkwargs)
acqf._baseline_L = baseline_L
posterior = mm.posterior(torch.rand(5, 1, **tkwargs))
# basic test
rv = torch.rand(1, 5, 1, **tkwargs)
with mock.patch(
"botorch.acquisition.cached_cholesky.sample_cached_cholesky",
return_value=rv,
) as mock_sample_cached_cholesky:
samples = acqf._get_f_X_samples(posterior=posterior, q_in=q)
mock_sample_cached_cholesky.assert_called_once_with(
posterior=posterior,
baseline_L=acqf._baseline_L,
q=q,
base_samples=acqf.sampler.base_samples,
sample_shape=acqf.sampler.sample_shape,
)
self.assertTrue(torch.equal(rv, samples))
# test fall back when sampling from cached cholesky fails
for error_cls in (NanError, NotPSDError):
base_samples = torch.rand(1, 5, 1, **tkwargs)
acqf.sampler.base_samples = base_samples
acqf._baseline_L = baseline_L
with mock.patch(
"botorch.acquisition.cached_cholesky.sample_cached_cholesky",
side_effect=error_cls,
) as mock_sample_cached_cholesky:
with warnings.catch_warnings(
record=True) as ws, settings.debug(True):
samples = acqf._get_f_X_samples(posterior=posterior,
q_in=q)
mock_sample_cached_cholesky.assert_called_once_with(
posterior=posterior,
baseline_L=acqf._baseline_L,
q=q,
base_samples=base_samples,
sample_shape=acqf.sampler.sample_shape,
)
self.assertTrue(
issubclass(ws[0].category, BotorchWarning))
self.assertTrue(samples.shape, torch.Size([1, q, 1]))
# test HOGP
hogp = HigherOrderGP(torch.zeros(2, 1), torch.zeros(2, 1,
1)).eval()
acqf = DummyCachedCholeskyAcqf(model=hogp, sampler=sampler)
acqf._setup(model=hogp, sampler=sampler, cache_root=True)
mock_samples = torch.rand(5, 1, 1, **tkwargs)
posterior = MockPosterior(mean=mean,
variance=variance,
samples=mock_samples)
samples = acqf._get_f_X_samples(posterior=posterior, q_in=q)
self.assertTrue(torch.equal(samples,
mock_samples[2:].unsqueeze(0)))
def test_sample_points_around_best(self):
tkwargs = {"device": self.device}
_bounds = torch.ones(2, 2)
_bounds[1] = 2
for dtype in (torch.float, torch.double):
tkwargs["dtype"] = dtype
bounds = _bounds.to(**tkwargs)
X_train = 1 + torch.rand(20, 2, **tkwargs)
model = MockModel(
MockPosterior(mean=(2 * X_train + 1).sum(dim=-1, keepdim=True))
)
# test NEI with X_baseline
acqf = qNoisyExpectedImprovement(model, X_baseline=X_train)
with mock.patch(
"botorch.optim.initializers.sample_perturbed_subset_dims"
) as mock_subset_dims:
X_rnd = sample_points_around_best(
acq_function=acqf,
n_discrete_points=4,
sigma=1e-3,
bounds=bounds,
)
mock_subset_dims.assert_not_called()
self.assertTrue(X_rnd.shape, torch.Size([4, 2]))
self.assertTrue((X_rnd >= 1).all())
self.assertTrue((X_rnd <= 2).all())
# test model that returns a batched mean
model = MockModel(
MockPosterior(
mean=(2 * X_train + 1).sum(dim=-1, keepdim=True).unsqueeze(0)
)
)
acqf = qNoisyExpectedImprovement(model, X_baseline=X_train)
X_rnd = sample_points_around_best(
acq_function=acqf,
n_discrete_points=4,
sigma=1e-3,
bounds=bounds,
)
self.assertTrue(X_rnd.shape, torch.Size([4, 2]))
self.assertTrue((X_rnd >= 1).all())
self.assertTrue((X_rnd <= 2).all())
# test EI without X_baseline
acqf = qExpectedImprovement(model, best_f=0.0)
with warnings.catch_warnings(record=True) as w, settings.debug(True):
X_rnd = sample_points_around_best(
acq_function=acqf,
n_discrete_points=4,
sigma=1e-3,
bounds=bounds,
)
self.assertEqual(len(w), 1)
self.assertTrue(issubclass(w[-1].category, BotorchWarning))
self.assertIsNone(X_rnd)
# set train inputs
model.train_inputs = (X_train,)
X_rnd = sample_points_around_best(
acq_function=acqf,
n_discrete_points=4,
sigma=1e-3,
bounds=bounds,
)
self.assertTrue(X_rnd.shape, torch.Size([4, 2]))
self.assertTrue((X_rnd >= 1).all())
self.assertTrue((X_rnd <= 2).all())
# test an acquisition function that has objective=None
# and maximize=False
pm = PosteriorMean(model, maximize=False)
self.assertIsNone(pm.objective)
self.assertFalse(pm.maximize)
X_rnd = sample_points_around_best(
acq_function=pm,
n_discrete_points=4,
sigma=0,
bounds=bounds,
best_pct=1e-8, # ensures that we only use best value
)
idx = (-model.posterior(X_train).mean).argmax()
self.assertTrue((X_rnd == X_train[idx : idx + 1]).all(dim=-1).all())
# test acquisition function that has no model
ff = FixedFeatureAcquisitionFunction(pm, d=2, columns=[0], values=[0])
# set X_baseline for testing purposes
ff.X_baseline = X_train
with warnings.catch_warnings(record=True) as w, settings.debug(True):
X_rnd = sample_points_around_best(
acq_function=ff,
n_discrete_points=4,
sigma=1e-3,
bounds=bounds,
)
self.assertEqual(len(w), 1)
self.assertTrue(issubclass(w[-1].category, BotorchWarning))
self.assertIsNone(X_rnd)
# test constraints with NEHVI
constraints = [lambda Y: Y[..., 0]]
ref_point = torch.zeros(2, **tkwargs)
# test cases when there are and are not any feasible points
for any_feas in (True, False):
Y_train = torch.stack(
[
torch.linspace(-0.5, 0.5, X_train.shape[0], **tkwargs)
if any_feas
else torch.ones(X_train.shape[0], **tkwargs),
X_train.sum(dim=-1),
],
dim=-1,
)
moo_model = MockModel(MockPosterior(mean=Y_train, samples=Y_train))
acqf = qNoisyExpectedHypervolumeImprovement(
moo_model,
ref_point=ref_point,
X_baseline=X_train,
constraints=constraints,
cache_root=False,
)
X_rnd = sample_points_around_best(
acq_function=acqf,
n_discrete_points=4,
sigma=0.0,
bounds=bounds,
)
self.assertTrue(X_rnd.shape, torch.Size([4, 2]))
# this should be true since sigma=0
# and we should only be returning feasible points
violation = constraints[0](Y_train)
neg_violation = -violation.clamp_min(0.0)
feas = neg_violation == 0
eq_mask = (X_train.unsqueeze(1) == X_rnd.unsqueeze(0)).all(dim=-1)
if feas.any():
# determine
# create n_train x n_rnd tensor of booleans
eq_mask = (X_train.unsqueeze(1) == X_rnd.unsqueeze(0)).all(dim=-1)
# check that all X_rnd correspond to feasible points
self.assertEqual(eq_mask[feas].sum(), 4)
else:
idcs = torch.topk(neg_violation, k=2).indices
self.assertEqual(eq_mask[idcs].sum(), 4)
self.assertTrue((X_rnd >= 1).all())
self.assertTrue((X_rnd <= 2).all())
# test that subset_dims is called if d>=21
X_train = 1 + torch.rand(20, 21, **tkwargs)
model = MockModel(
MockPosterior(mean=(2 * X_train + 1).sum(dim=-1, keepdim=True))
)
bounds = torch.ones(2, 21, **tkwargs)
bounds[1] = 2
# test NEI with X_baseline
acqf = qNoisyExpectedImprovement(model, X_baseline=X_train)
with mock.patch(
"botorch.optim.initializers.sample_perturbed_subset_dims",
wraps=sample_perturbed_subset_dims,
) as mock_subset_dims:
X_rnd = sample_points_around_best(
acq_function=acqf, n_discrete_points=5, sigma=1e-3, bounds=bounds
)
self.assertTrue(X_rnd.shape, torch.Size([5, 2]))
self.assertTrue((X_rnd >= 1).all())
self.assertTrue((X_rnd <= 2).all())
mock_subset_dims.assert_called_once()
# test tiny prob_perturb to make sure we perturb at least one dimension
X_rnd = sample_points_around_best(
acq_function=acqf,
n_discrete_points=5,
sigma=1e-3,
bounds=bounds,
prob_perturb=1e-8,
)
self.assertTrue(
((X_rnd.unsqueeze(0) == X_train.unsqueeze(1)).all(dim=-1)).sum() == 0
)