def test_gp(self):
for (iteration_fidelity, data_fidelity) in self.FIDELITY_TEST_PAIRS:
num_dim = 1 + (iteration_fidelity is not None) + (data_fidelity
is not None)
for batch_shape, num_outputs, dtype, lin_trunc in itertools.product(
(torch.Size(), torch.Size([2])),
(1, 2),
(torch.float, torch.double),
(False, True),
):
tkwargs = {"device": self.device, "dtype": dtype}
model, _ = _get_model_and_data(
iteration_fidelity=iteration_fidelity,
data_fidelity=data_fidelity,
batch_shape=batch_shape,
num_outputs=num_outputs,
lin_truncated=lin_trunc,
**tkwargs,
)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
mll.to(**tkwargs)
with warnings.catch_warnings():
warnings.filterwarnings("ignore",
category=OptimizationWarning)
fit_gpytorch_model(mll,
sequential=False,
options={"maxiter": 1})
# test init
self.assertIsInstance(model.mean_module, ConstantMean)
self.assertIsInstance(model.covar_module, ScaleKernel)
# test param sizes
params = dict(model.named_parameters())
for p in params:
self.assertEqual(
params[p].numel(),
num_outputs * torch.tensor(batch_shape).prod().item(),
)
# test posterior
# test non batch evaluation
X = torch.rand(batch_shape + torch.Size([3, num_dim]),
**tkwargs)
posterior = model.posterior(X)
self.assertIsInstance(posterior, GPyTorchPosterior)
self.assertEqual(posterior.mean.shape,
batch_shape + torch.Size([3, num_outputs]))
# test batch evaluation
X = torch.rand(
torch.Size([2]) + batch_shape + torch.Size([3, num_dim]),
**tkwargs)
posterior = model.posterior(X)
self.assertIsInstance(posterior, GPyTorchPosterior)
self.assertEqual(
posterior.mean.shape,
torch.Size([2]) + batch_shape +
torch.Size([3, num_outputs]),
)
def test_sample_all_priors(self, cuda=False):
device = torch.device("cuda" if cuda else "cpu")
for dtype in (torch.float, torch.double):
train_X = torch.rand(3, 5, device=device, dtype=dtype)
train_Y = torch.rand(3, 1, device=device, dtype=dtype)
model = SingleTaskGP(train_X=train_X, train_Y=train_Y)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
mll.to(device=device, dtype=dtype)
original_state_dict = dict(deepcopy(mll.model.state_dict()))
sample_all_priors(model)
# make sure one of the hyperparameters changed
self.assertTrue(
dict(model.state_dict())["likelihood.noise_covar.raw_noise"] !=
original_state_dict["likelihood.noise_covar.raw_noise"])
# check that lengthscales are all different
ls = model.covar_module.base_kernel.raw_lengthscale.view(
-1).tolist()
self.assertTrue(all(ls[0] != ls[i]) for i in range(1, len(ls)))
# change one of the priors to SmoothedBoxPrior
model.covar_module = ScaleKernel(
MaternKernel(
nu=2.5,
ard_num_dims=model.train_inputs[0].shape[-1],
batch_shape=model._aug_batch_shape,
lengthscale_prior=SmoothedBoxPrior(3.0, 6.0),
),
batch_shape=model._aug_batch_shape,
outputscale_prior=GammaPrior(2.0, 0.15),
)
original_state_dict = dict(deepcopy(mll.model.state_dict()))
with warnings.catch_warnings(
record=True) as ws, settings.debug(True):
sample_all_priors(model)
self.assertEqual(len(ws), 1)
self.assertTrue("rsample" in str(ws[0].message))
# the lengthscale should not have changed because sampling is
# not implemented for SmoothedBoxPrior
self.assertTrue(
torch.equal(
dict(model.state_dict())
["covar_module.base_kernel.raw_lengthscale"],
original_state_dict[
"covar_module.base_kernel.raw_lengthscale"],
))
# set setting_closure to None and make sure RuntimeError is raised
prior_tuple = model.likelihood.noise_covar._priors["noise_prior"]
model.likelihood.noise_covar._priors["noise_prior"] = (
prior_tuple[0],
prior_tuple[1],
None,
)
with self.assertRaises(RuntimeError):
sample_all_priors(model)
def test_fit_gpytorch_model_singular(self, cuda=False):
options = {"disp": False, "maxiter": 5}
device = torch.device("cuda") if cuda else torch.device("cpu")
for dtype in (torch.float, torch.double):
X_train = torch.rand(2, 2, device=device, dtype=dtype)
Y_train = torch.zeros(2, device=device, dtype=dtype)
test_likelihood = GaussianLikelihood(noise_constraint=GreaterThan(
-1.0, transform=None, initial_value=0.0))
gp = SingleTaskGP(X_train, Y_train, likelihood=test_likelihood)
mll = ExactMarginalLogLikelihood(gp.likelihood, gp)
mll.to(device=device, dtype=dtype)
# this will do multiple retries (and emit warnings, which is desired)
fit_gpytorch_model(mll, options=options, max_retries=2)
def _getBatchedModel(
self, kind="SingleTaskGP", double=False, outcome_transform=False
):
dtype = torch.double if double else torch.float
train_x = torch.linspace(0, 1, 10, device=self.device, dtype=dtype).unsqueeze(
-1
)
noise = torch.tensor(NOISE, device=self.device, dtype=dtype)
train_y1 = torch.sin(train_x * (2 * math.pi)) + noise
train_y2 = torch.sin(train_x * (2 * math.pi)) + noise
train_y = torch.cat([train_y1, train_y2], dim=-1)
kwargs = {}
if outcome_transform:
kwargs["outcome_transform"] = Standardize(m=2)
if kind == "SingleTaskGP":
model = SingleTaskGP(train_x, train_y, **kwargs)
elif kind == "FixedNoiseGP":
model = FixedNoiseGP(
train_x, train_y, 0.1 * torch.ones_like(train_y), **kwargs
)
elif kind == "HeteroskedasticSingleTaskGP":
model = HeteroskedasticSingleTaskGP(
train_x, train_y, 0.1 * torch.ones_like(train_y), **kwargs
)
else:
raise NotImplementedError
mll = ExactMarginalLogLikelihood(model.likelihood, model)
return mll.to(device=self.device, dtype=dtype)
def test_fit_gpytorch_model_singular(self):
options = {"disp": False, "maxiter": 5}
for dtype in (torch.float, torch.double):
X_train = torch.rand(2, 2, device=self.device, dtype=dtype)
Y_train = torch.zeros(2, 1, device=self.device, dtype=dtype)
test_likelihood = GaussianLikelihood(
noise_constraint=GreaterThan(-1.0, transform=None, initial_value=0.0)
)
gp = SingleTaskGP(X_train, Y_train, likelihood=test_likelihood)
mll = ExactMarginalLogLikelihood(gp.likelihood, gp)
mll.to(device=self.device, dtype=dtype)
# this will do multiple retries (and emit warnings, which is desired)
with warnings.catch_warnings(record=True) as ws, settings.debug(True):
fit_gpytorch_model(mll, options=options, max_retries=2)
self.assertTrue(
any(issubclass(w.category, OptimizationWarning) for w in ws)
)
def get_fitted_model(train_x, train_obj, state_dict=None):
# initialize and fit model
model = SingleTaskGP(train_X=train_x, train_Y=train_obj)
# # initialize likelihood and model
# likelihood = gpytorch.likelihoods.GaussianLikelihood()
# model = ExactGPModel(train_x, train_obj, likelihood)
# model.train()
# likelihood.train()
if state_dict is not None:
model.load_state_dict(state_dict)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
mll.to(train_x)
fit_gpytorch_model(mll)
return model
def _getModel(self, double=False):
dtype = torch.double if double else torch.float
train_x = torch.linspace(0, 1, 10, device=self.device,
dtype=dtype).unsqueeze(-1)
noise = torch.tensor(NOISE, device=self.device, dtype=dtype)
train_y = torch.sin(train_x * (2 * math.pi)) + noise
model = SingleTaskGP(train_x, train_y)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
return mll.to(device=self.device, dtype=dtype)
def _getModel(self, double=False, cuda=False):
device = torch.device("cuda") if cuda else torch.device("cpu")
dtype = torch.double if double else torch.float
train_x = torch.linspace(0, 1, 10, device=device, dtype=dtype).unsqueeze(-1)
noise = torch.tensor(NOISE, device=device, dtype=dtype)
train_y = torch.sin(train_x.view(-1) * (2 * math.pi)) + noise
model = SingleTaskGP(train_x, train_y)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
return mll.to(device=device, dtype=dtype)
def test_fit_gpytorch_model_singular(self):
options = {"disp": False, "maxiter": 5}
for dtype in (torch.float, torch.double):
X_train = torch.ones(2, 2, device=self.device, dtype=dtype)
Y_train = torch.zeros(2, 1, device=self.device, dtype=dtype)
test_likelihood = GaussianLikelihood(
noise_constraint=GreaterThan(-1e-7, transform=None, initial_value=0.0)
)
gp = SingleTaskGP(X_train, Y_train, likelihood=test_likelihood)
mll = ExactMarginalLogLikelihood(gp.likelihood, gp)
mll.to(device=self.device, dtype=dtype)
# this will do multiple retries (and emit warnings, which is desired)
with warnings.catch_warnings(record=True) as ws, settings.debug(True):
fit_gpytorch_model(mll, options=options, max_retries=2)
self.assertTrue(
any(issubclass(w.category, NumericalWarning) for w in ws)
)
# ensure that we fail if noise ensures that jitter does not help
gp.likelihood = GaussianLikelihood(
noise_constraint=Interval(-2, -1, transform=None, initial_value=-1.5)
)
mll = ExactMarginalLogLikelihood(gp.likelihood, gp)
mll.to(device=self.device, dtype=dtype)
with self.assertRaises(NotPSDError):
fit_gpytorch_model(mll, options=options, max_retries=2)
# ensure we can handle NaNErrors in the optimizer
with mock.patch.object(SingleTaskGP, "__call__", side_effect=NanError):
gp = SingleTaskGP(X_train, Y_train, likelihood=test_likelihood)
mll = ExactMarginalLogLikelihood(gp.likelihood, gp)
mll.to(device=self.device, dtype=dtype)
fit_gpytorch_model(
mll, options={"disp": False, "maxiter": 1}, max_retries=1
)
def test_fit_gpytorch_model_singular(self):
options = {"disp": False, "maxiter": 5}
for dtype in (torch.float, torch.double):
X_train = torch.ones(2, 2, device=self.device, dtype=dtype)
Y_train = torch.zeros(2, 1, device=self.device, dtype=dtype)
test_likelihood = GaussianLikelihood(
noise_constraint=GreaterThan(-1e-7, transform=None, initial_value=0.0)
)
gp = SingleTaskGP(X_train, Y_train, likelihood=test_likelihood)
mll = ExactMarginalLogLikelihood(gp.likelihood, gp)
mll.to(device=self.device, dtype=dtype)
# this will do multiple retries (and emit warnings, which is desired)
with warnings.catch_warnings(record=True) as ws, settings.debug(True):
fit_gpytorch_model(mll, options=options, max_retries=2)
self.assertTrue(
any(issubclass(w.category, NumericalWarning) for w in ws)
)
# ensure that we fail if noise ensures that jitter does not help
gp.likelihood = GaussianLikelihood(
noise_constraint=Interval(-2, -1, transform=None, initial_value=-1.5)
)
mll = ExactMarginalLogLikelihood(gp.likelihood, gp)
mll.to(device=self.device, dtype=dtype)
with self.assertLogs(level="DEBUG") as logs:
fit_gpytorch_model(mll, options=options, max_retries=2)
self.assertTrue(any("NotPSDError" in log for log in logs.output))
# ensure we can handle NaNErrors in the optimizer
with mock.patch.object(SingleTaskGP, "__call__", side_effect=NanError):
gp = SingleTaskGP(X_train, Y_train, likelihood=test_likelihood)
mll = ExactMarginalLogLikelihood(gp.likelihood, gp)
mll.to(device=self.device, dtype=dtype)
fit_gpytorch_model(
mll, options={"disp": False, "maxiter": 1}, max_retries=1
)
# ensure we catch NotPSDErrors
with mock.patch.object(SingleTaskGP, "__call__", side_effect=NotPSDError):
mll = self._getModel()
with self.assertLogs(level="DEBUG") as logs:
fit_gpytorch_model(mll, max_retries=2)
for retry in [1, 2]:
self.assertTrue(
any(
f"Fitting failed on try {retry} due to a NotPSDError."
in log
for log in logs.output
)
)
# Failure due to optimization warning
def optimize_w_warning(mll, **kwargs):
warnings.warn("Dummy warning.", OptimizationWarning)
return mll, None
mll = self._getModel()
with self.assertLogs(level="DEBUG") as logs, settings.debug(True):
fit_gpytorch_model(mll, optimizer=optimize_w_warning, max_retries=2)
self.assertTrue(
any("Fitting failed on try 1." in log for log in logs.output)
)
def _getBatchedModel(self, kind="SingleTaskGP", double=False, cuda=False):
device = torch.device("cuda") if cuda else torch.device("cpu")
dtype = torch.double if double else torch.float
train_x = torch.linspace(0, 1, 10, device=device,
dtype=dtype).unsqueeze(-1)
noise = torch.tensor(NOISE, device=device, dtype=dtype)
train_y1 = torch.sin(train_x * (2 * math.pi)) + noise
train_y2 = torch.sin(train_x * (2 * math.pi)) + noise
train_y = torch.cat([train_y1, train_y2], dim=-1)
if kind == "SingleTaskGP":
model = SingleTaskGP(train_x, train_y)
elif kind == "FixedNoiseGP":
model = FixedNoiseGP(train_x, train_y,
0.1 * torch.ones_like(train_y))
elif kind == "HeteroskedasticSingleTaskGP":
model = HeteroskedasticSingleTaskGP(train_x, train_y,
0.1 * torch.ones_like(train_y))
else:
raise NotImplementedError
mll = ExactMarginalLogLikelihood(model.likelihood, model)
return mll.to(device=device, dtype=dtype)
def test_gp(self):
for (iteration_fidelity, data_fidelity) in self.FIDELITY_TEST_PAIRS:
num_dim = 1 + (iteration_fidelity is not None) + (data_fidelity
is not None)
for batch_shape, m, dtype, lin_trunc, use_octf in itertools.product(
(torch.Size(), torch.Size([2])),
(1, 2),
(torch.float, torch.double),
(False, True),
(False, True),
):
tkwargs = {"device": self.device, "dtype": dtype}
octf = Standardize(
m=m, batch_shape=batch_shape) if use_octf else None
model, _ = _get_model_and_data(
iteration_fidelity=iteration_fidelity,
data_fidelity=data_fidelity,
batch_shape=batch_shape,
m=m,
lin_truncated=lin_trunc,
outcome_transform=octf,
**tkwargs,
)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
mll.to(**tkwargs)
with warnings.catch_warnings():
warnings.filterwarnings("ignore",
category=OptimizationWarning)
fit_gpytorch_model(mll,
sequential=False,
options={"maxiter": 1})
# test init
self.assertIsInstance(model.mean_module, ConstantMean)
self.assertIsInstance(model.covar_module, ScaleKernel)
if use_octf:
self.assertIsInstance(model.outcome_transform, Standardize)
# test param sizes
params = dict(model.named_parameters())
for p in params:
self.assertEqual(
params[p].numel(),
m * torch.tensor(batch_shape).prod().item())
# test posterior
# test non batch evaluation
X = torch.rand(*batch_shape, 3, num_dim, **tkwargs)
expected_shape = batch_shape + torch.Size([3, m])
posterior = model.posterior(X)
self.assertIsInstance(posterior, GPyTorchPosterior)
self.assertEqual(posterior.mean.shape, expected_shape)
self.assertEqual(posterior.variance.shape, expected_shape)
if use_octf:
# ensure un-transformation is applied
tmp_tf = model.outcome_transform
del model.outcome_transform
pp_tf = model.posterior(X)
model.outcome_transform = tmp_tf
expected_var = tmp_tf.untransform_posterior(pp_tf).variance
self.assertTrue(
torch.allclose(posterior.variance, expected_var))
# test batch evaluation
X = torch.rand(2, *batch_shape, 3, num_dim, **tkwargs)
expected_shape = torch.Size([2]) + batch_shape + torch.Size(
[3, m])
posterior = model.posterior(X)
self.assertIsInstance(posterior, GPyTorchPosterior)
self.assertEqual(posterior.mean.shape, expected_shape)
self.assertEqual(posterior.variance.shape, expected_shape)
if use_octf:
# ensure un-transformation is applied
tmp_tf = model.outcome_transform
del model.outcome_transform
pp_tf = model.posterior(X)
model.outcome_transform = tmp_tf
expected_var = tmp_tf.untransform_posterior(pp_tf).variance
self.assertTrue(
torch.allclose(posterior.variance, expected_var))
