def test_set_transformed_inputs(self):
for dtype in (torch.float, torch.double):
train_x = torch.rand(5, 1, dtype=dtype, device=self.device)
train_y = torch.rand(5, 1, dtype=dtype, device=self.device)
tf = Normalize(
d=1,
bounds=torch.tensor([[0.0], [2.0]], dtype=dtype, device=self.device),
transform_on_preprocess=False,
)
model = SingleTaskGP(train_x, train_y, input_transform=tf)
self.assertTrue(torch.equal(model.train_inputs[0], train_x))
mll = ExactMarginalLogLikelihood(model.likelihood, model)
# check that input transform is only applied when the transform
# is a transform_on_preprocess is True
self.assertTrue(torch.equal(model.train_inputs[0], train_x))
tf.transform_on_preprocess = True
_set_transformed_inputs(mll)
self.assertTrue(torch.equal(model.train_inputs[0], tf(train_x)))
model.eval()
# test no set_train_data method
mock_model = MockGP(MockPosterior())
mock_model.train_inputs = (train_x,)
mock_model.likelihood = model.likelihood
mock_model.input_transform = tf
mll = ExactMarginalLogLikelihood(mock_model.likelihood, mock_model)
with self.assertRaises(BotorchError):
_set_transformed_inputs(mll)
def test_chained_input_transform(self):
ds = (1, 2)
batch_shapes = (torch.Size(), torch.Size([2]))
dtypes = (torch.float, torch.double)
for d, batch_shape, dtype in itertools.product(ds, batch_shapes,
dtypes):
bounds = torch.tensor([[-2.0] * d, [2.0] * d],
device=self.device,
dtype=dtype)
tf1 = Normalize(d=d, bounds=bounds, batch_shape=batch_shape)
tf2 = Normalize(d=d, batch_shape=batch_shape)
tf = ChainedInputTransform(stz_fixed=tf1, stz_learned=tf2)
tf1_, tf2_ = deepcopy(tf1), deepcopy(tf2)
self.assertTrue(tf.training)
self.assertEqual(sorted(tf.keys()), ["stz_fixed", "stz_learned"])
self.assertEqual(tf["stz_fixed"], tf1)
self.assertEqual(tf["stz_learned"], tf2)
# make copies for validation below
tf1_, tf2_ = deepcopy(tf1), deepcopy(tf2)
X = torch.rand(*batch_shape, 4, d, device=self.device, dtype=dtype)
X_tf = tf(X)
X_tf_ = tf2_(tf1_(X))
self.assertTrue(tf1.training)
self.assertTrue(tf2.training)
self.assertTrue(torch.equal(X_tf, X_tf_))
X_utf = tf.untransform(X_tf)
self.assertTrue(torch.allclose(X_utf, X, atol=1e-4, rtol=1e-4))
def test_transforms(self):
train_x = torch.rand(10, 3, device=self.device)
train_y = torch.randn(10, 4, 5, device=self.device)
# test handling of Standardize
with self.assertWarns(RuntimeWarning):
model = HigherOrderGP(train_X=train_x,
train_Y=train_y,
outcome_transform=Standardize(m=5))
self.assertIsInstance(model.outcome_transform, FlattenedStandardize)
self.assertEqual(model.outcome_transform.output_shape,
train_y.shape[1:])
self.assertEqual(model.outcome_transform.batch_shape, torch.Size())
model = HigherOrderGP(
train_X=train_x,
train_Y=train_y,
input_transform=Normalize(d=3),
outcome_transform=FlattenedStandardize(train_y.shape[1:]),
)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
fit_gpytorch_torch(mll, options={"maxiter": 1, "disp": False})
test_x = torch.rand(2, 5, 3, device=self.device)
test_y = torch.randn(2, 5, 4, 5, device=self.device)
posterior = model.posterior(test_x)
self.assertIsInstance(posterior, TransformedPosterior)
conditioned_model = model.condition_on_observations(test_x, test_y)
self.assertIsInstance(conditioned_model, HigherOrderGP)
self.check_transform_forward(model)
self.check_transform_untransform(model)
def load_gp(log_dir):
try:
# The GP state, i.e., hyperparameters, normalization, etc.
model_file = os.path.join(log_dir, "model_state.pth")
with open(model_file, "rb") as f:
state_dict = torch.load(f)
# Get the evaluated data points
eval_dict = load_eval(log_dir)
train_X = eval_dict["train_inputs"]
train_Y = eval_dict["train_targets"]
# The bounds of the domain
config_dict = load_config(log_dir)
lb = torch.tensor(config_dict["lower_bound"])
ub = torch.tensor(config_dict["upper_bound"])
bounds = torch.stack((lb, ub))
# Create GP instance and load respective parameters
gp = SingleTaskGP(
train_X=train_X,
train_Y=train_Y,
outcome_transform=Standardize(m=1),
input_transform=Normalize(d=1, bounds=bounds),
)
gp.load_state_dict(state_dict=state_dict)
except FileNotFoundError:
print(f"The model file could not be found in: {log_dir}")
exit(1)
return gp
def _get_model(fixed_noise=False, use_octf=False, use_intf=False, **tkwargs):
train_x1, train_y1 = _get_random_data(batch_shape=torch.Size(),
m=1,
n=10,
**tkwargs)
train_x2, train_y2 = _get_random_data(batch_shape=torch.Size(),
m=1,
n=11,
**tkwargs)
octfs = [Standardize(m=1), Standardize(m=1)] if use_octf else [None, None]
intfs = [Normalize(d=1), Normalize(d=1)] if use_intf else [None, None]
if fixed_noise:
train_y1_var = 0.1 + 0.1 * torch.rand_like(train_y1, **tkwargs)
train_y2_var = 0.1 + 0.1 * torch.rand_like(train_y2, **tkwargs)
model1 = FixedNoiseGP(
train_X=train_x1,
train_Y=train_y1,
train_Yvar=train_y1_var,
outcome_transform=octfs[0],
input_transform=intfs[0],
)
model2 = FixedNoiseGP(
train_X=train_x2,
train_Y=train_y2,
train_Yvar=train_y2_var,
outcome_transform=octfs[1],
input_transform=intfs[1],
)
else:
model1 = SingleTaskGP(
train_X=train_x1,
train_Y=train_y1,
outcome_transform=octfs[0],
input_transform=intfs[0],
)
model2 = SingleTaskGP(
train_X=train_x2,
train_Y=train_y2,
outcome_transform=octfs[1],
input_transform=intfs[1],
)
model = ModelListGP(model1, model2)
return model.to(**tkwargs)
def test_batched_to_model_list(self):
for dtype in (torch.float, torch.double):
# test SingleTaskGP
train_X = torch.rand(10, 2, device=self.device, dtype=dtype)
train_Y1 = train_X.sum(dim=-1)
train_Y2 = train_X[:, 0] - train_X[:, 1]
train_Y = torch.stack([train_Y1, train_Y2], dim=-1)
batch_gp = SingleTaskGP(train_X, train_Y)
list_gp = batched_to_model_list(batch_gp)
self.assertIsInstance(list_gp, ModelListGP)
# test FixedNoiseGP
batch_gp = FixedNoiseGP(train_X, train_Y, torch.rand_like(train_Y))
list_gp = batched_to_model_list(batch_gp)
self.assertIsInstance(list_gp, ModelListGP)
# test SingleTaskMultiFidelityGP
for lin_trunc in (False, True):
batch_gp = SingleTaskMultiFidelityGP(
train_X,
train_Y,
iteration_fidelity=1,
linear_truncated=lin_trunc)
list_gp = batched_to_model_list(batch_gp)
self.assertIsInstance(list_gp, ModelListGP)
# test HeteroskedasticSingleTaskGP
batch_gp = HeteroskedasticSingleTaskGP(train_X, train_Y,
torch.rand_like(train_Y))
with self.assertRaises(NotImplementedError):
batched_to_model_list(batch_gp)
# test with transforms
input_tf = Normalize(
d=2,
bounds=torch.tensor([[0.0, 0.0], [1.0, 1.0]],
device=self.device,
dtype=dtype),
)
octf = Standardize(m=2)
batch_gp = SingleTaskGP(train_X,
train_Y,
outcome_transform=octf,
input_transform=input_tf)
list_gp = batched_to_model_list(batch_gp)
for i, m in enumerate(list_gp.models):
self.assertIsInstance(m.input_transform, Normalize)
self.assertTrue(
torch.equal(m.input_transform.bounds, input_tf.bounds))
self.assertIsInstance(m.outcome_transform, Standardize)
self.assertEqual(m.outcome_transform._m, 1)
expected_octf = octf.subset_output(idcs=[i])
for attr_name in ["means", "stdvs", "_stdvs_sq"]:
self.assertTrue(
torch.equal(
m.outcome_transform.__getattr__(attr_name),
expected_octf.__getattr__(attr_name),
))
def test_initializations(self):
train_X = torch.rand(15, 1, device=self.device)
train_Y = torch.rand(15, 1, device=self.device)
stacked_train_X = torch.cat((train_X, train_X), dim=0)
for X, num_ind in [[train_X, 5], [stacked_train_X, 20], [stacked_train_X, 5]]:
model = SingleTaskVariationalGP(train_X=X, inducing_points=num_ind)
if num_ind == 5:
self.assertLessEqual(
model.model.variational_strategy.inducing_points.shape,
torch.Size((5, 1)),
)
else:
# should not have 20 inducing points when 15 singular dimensions
# are passed
self.assertLess(
model.model.variational_strategy.inducing_points.shape[-2], num_ind
)
test_X = torch.rand(5, 1, device=self.device)
# test transforms
for inp_trans, out_trans in itertools.product(
[None, Normalize(d=1)], [None, Log()]
):
model = SingleTaskVariationalGP(
train_X=train_X,
train_Y=train_Y,
outcome_transform=out_trans,
input_transform=inp_trans,
)
if inp_trans is not None:
self.assertIsInstance(model.input_transform, Normalize)
else:
self.assertFalse(hasattr(model, "input_transform"))
if out_trans is not None:
self.assertIsInstance(model.outcome_transform, Log)
posterior = model.posterior(test_X)
self.assertIsInstance(posterior, TransformedPosterior)
else:
self.assertFalse(hasattr(model, "outcome_transform"))
def test_batched_to_model_list(self):
for dtype in (torch.float, torch.double):
# test SingleTaskGP
train_X = torch.rand(10, 2, device=self.device, dtype=dtype)
train_Y1 = train_X.sum(dim=-1)
train_Y2 = train_X[:, 0] - train_X[:, 1]
train_Y = torch.stack([train_Y1, train_Y2], dim=-1)
batch_gp = SingleTaskGP(train_X, train_Y)
list_gp = batched_to_model_list(batch_gp)
self.assertIsInstance(list_gp, ModelListGP)
# test FixedNoiseGP
batch_gp = FixedNoiseGP(train_X, train_Y, torch.rand_like(train_Y))
list_gp = batched_to_model_list(batch_gp)
self.assertIsInstance(list_gp, ModelListGP)
# test SingleTaskMultiFidelityGP
for lin_trunc in (False, True):
batch_gp = SingleTaskMultiFidelityGP(
train_X,
train_Y,
iteration_fidelity=1,
linear_truncated=lin_trunc)
list_gp = batched_to_model_list(batch_gp)
self.assertIsInstance(list_gp, ModelListGP)
# test HeteroskedasticSingleTaskGP
batch_gp = HeteroskedasticSingleTaskGP(train_X, train_Y,
torch.rand_like(train_Y))
with self.assertRaises(NotImplementedError):
batched_to_model_list(batch_gp)
# test input transform
input_tf = Normalize(
d=2,
bounds=torch.tensor([[0.0, 0.0], [1.0, 1.0]],
device=self.device,
dtype=dtype),
)
batch_gp = SingleTaskGP(train_X, train_Y, input_transform=input_tf)
list_gp = batched_to_model_list(batch_gp)
for m in list_gp.models:
self.assertIsInstance(m.input_transform, Normalize)
self.assertTrue(
torch.equal(m.input_transform.bounds, input_tf.bounds))
def test_with_transforms(self):
dim = 2
bounds = torch.stack([torch.zeros(dim), torch.ones(dim) * 3])
intf = Normalize(d=dim, bounds=bounds)
octf = Standardize(m=1)
# update octf state with dummy data
octf(torch.rand(5, 1) * 7)
octf.eval()
model = DummyDeterministicModel(octf, intf)
# check that the posterior output agrees with the manually transformed one
test_X = torch.rand(3, dim)
expected_Y, _ = octf.untransform(model.forward(intf(test_X)))
with warnings.catch_warnings(record=True) as ws:
posterior = model.posterior(test_X)
msg = "does not have a `train_inputs` attribute"
self.assertTrue(any(msg in str(w.message) for w in ws))
self.assertTrue(torch.allclose(expected_Y, posterior.mean))
# check that model.train/eval works and raises the warning
model.train()
with self.assertWarns(RuntimeWarning):
model.eval()
def test_transforms(self):
train_x = rand(10, 3, device=self.device)
train_y = randn(10, 4, 5, device=self.device)
model = HigherOrderGP(
train_x,
train_y,
input_transform=Normalize(d=3),
outcome_transform=FlattenedStandardize(train_y.shape[1:]),
)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
fit_gpytorch_torch(mll, options={"maxiter": 1, "disp": False})
test_x = rand(2, 5, 3, device=self.device)
test_y = randn(2, 5, 4, 5, device=self.device)
posterior = model.posterior(test_x)
self.assertIsInstance(posterior, TransformedPosterior)
conditioned_model = model.condition_on_observations(test_x, test_y)
self.assertIsInstance(conditioned_model, HigherOrderGP)
self.check_transform_forward(model)
self.check_transform_untransform(model)
def test_normalize(self):
for dtype in (torch.float, torch.double):
# basic init, learned bounds
nlz = Normalize(d=2)
self.assertTrue(nlz.learn_bounds)
self.assertTrue(nlz.training)
self.assertEqual(nlz._d, 2)
self.assertEqual(nlz.mins.shape, torch.Size([1, 2]))
self.assertEqual(nlz.ranges.shape, torch.Size([1, 2]))
nlz = Normalize(d=2, batch_shape=torch.Size([3]))
self.assertTrue(nlz.learn_bounds)
self.assertTrue(nlz.training)
self.assertEqual(nlz._d, 2)
self.assertEqual(nlz.mins.shape, torch.Size([3, 1, 2]))
self.assertEqual(nlz.ranges.shape, torch.Size([3, 1, 2]))
# basic init, fixed bounds
bounds = torch.zeros(2, 2, device=self.device, dtype=dtype)
nlz = Normalize(d=2, bounds=bounds)
self.assertFalse(nlz.learn_bounds)
self.assertTrue(nlz.training)
self.assertEqual(nlz._d, 2)
self.assertTrue(torch.equal(nlz.mins, bounds[..., 0:1, :]))
self.assertTrue(
torch.equal(nlz.mins,
bounds[..., 1:2, :] - bounds[..., 0:1, :]))
# test .to
other_dtype = torch.float if dtype == torch.double else torch.double
nlz.to(other_dtype)
self.assertTrue(nlz.mins.dtype == other_dtype)
# test incompatible dimensions of specified bounds
with self.assertRaises(BotorchTensorDimensionError):
bounds = torch.zeros(2, 3, device=self.device, dtype=dtype)
Normalize(d=2, bounds=bounds)
# basic usage
for batch_shape in (torch.Size(), torch.Size([3])):
# learned bounds
nlz = Normalize(d=2, batch_shape=batch_shape)
X = torch.randn(*batch_shape,
4,
2,
device=self.device,
dtype=dtype)
X_nlzd = nlz(X)
self.assertEqual(X_nlzd.min().item(), 0.0)
self.assertEqual(X_nlzd.max().item(), 1.0)
nlz.eval()
X_unnlzd = nlz.untransform(X_nlzd)
self.assertTrue(
torch.allclose(X, X_unnlzd, atol=1e-4, rtol=1e-4))
expected_bounds = torch.cat(
[
X.min(dim=-2, keepdim=True)[0],
X.max(dim=-2, keepdim=True)[0]
],
dim=-2,
)
self.assertTrue(torch.allclose(nlz.bounds, expected_bounds))
# test errors on wrong shape
nlz = Normalize(d=2, batch_shape=batch_shape)
X = torch.randn(*batch_shape,
2,
1,
device=self.device,
dtype=dtype)
with self.assertRaises(BotorchTensorDimensionError):
nlz(X)
# fixed bounds
bounds = torch.tensor([[-2.0, -1], [1, 2.0]],
device=self.device,
dtype=dtype).expand(*batch_shape, 2, 2)
nlz = Normalize(d=2, bounds=bounds)
X = torch.rand(*batch_shape,
4,
2,
device=self.device,
dtype=dtype)
X_nlzd = nlz(X)
self.assertTrue(torch.equal(nlz.bounds, bounds))
X_unnlzd = nlz.untransform(X_nlzd)
self.assertTrue(
torch.allclose(X, X_unnlzd, atol=1e-4, rtol=1e-4))
# test no normalization on eval
nlz = Normalize(d=2,
bounds=bounds,
batch_shape=batch_shape,
transform_on_eval=False)
X_nlzd = nlz(X)
X_unnlzd = nlz.untransform(X_nlzd)
self.assertTrue(
torch.allclose(X, X_unnlzd, atol=1e-4, rtol=1e-4))
nlz.eval()
self.assertTrue(torch.equal(nlz(X), X))
# test no normalization on train
nlz = Normalize(
d=2,
bounds=bounds,
batch_shape=batch_shape,
transform_on_train=False,
)
X_nlzd = nlz(X)
self.assertTrue(torch.equal(nlz(X), X))
nlz.eval()
X_nlzd = nlz(X)
X_unnlzd = nlz.untransform(X_nlzd)
self.assertTrue(
torch.allclose(X, X_unnlzd, atol=1e-4, rtol=1e-4))
# test reverse
nlz = Normalize(d=2,
bounds=bounds,
batch_shape=batch_shape,
reverse=True)
X2 = nlz(X_nlzd)
self.assertTrue(torch.allclose(X2, X, atol=1e-4, rtol=1e-4))
X_nlzd2 = nlz.untransform(X2)
self.assertTrue(
torch.allclose(X_nlzd, X_nlzd2, atol=1e-4, rtol=1e-4))
# test equals
nlz2 = Normalize(d=2,
bounds=bounds,
batch_shape=batch_shape,
reverse=False)
self.assertFalse(nlz.equals(nlz2))
nlz3 = Normalize(d=2,
bounds=bounds,
batch_shape=batch_shape,
reverse=True)
self.assertTrue(nlz.equals(nlz3))
new_bounds = bounds + 1
nlz4 = Normalize(d=2,
bounds=new_bounds,
batch_shape=batch_shape,
reverse=True)
self.assertFalse(nlz.equals(nlz4))
nlz5 = Normalize(d=2, batch_shape=batch_shape)
self.assertFalse(nlz.equals(nlz5))
def test_FixedNoiseMultiTaskGP(self):
bounds = torch.tensor([[-1.0, 0.0], [1.0, 1.0]])
for dtype, use_intf in itertools.product(
(torch.float, torch.double), (False, True)
):
tkwargs = {"device": self.device, "dtype": dtype}
intf = (
Normalize(
d=2, bounds=bounds.to(**tkwargs), transform_on_preprocess=True
)
if use_intf
else None
)
model, train_X, _, _ = _get_fixed_noise_model_and_training_data(
input_transform=intf, **tkwargs
)
self.assertIsInstance(model, FixedNoiseMultiTaskGP)
self.assertEqual(model.num_outputs, 2)
self.assertIsInstance(model.likelihood, FixedNoiseGaussianLikelihood)
self.assertIsInstance(model.mean_module, ConstantMean)
self.assertIsInstance(model.covar_module, ScaleKernel)
matern_kernel = model.covar_module.base_kernel
self.assertIsInstance(matern_kernel, MaternKernel)
self.assertIsInstance(matern_kernel.lengthscale_prior, GammaPrior)
self.assertIsInstance(model.task_covar_module, IndexKernel)
self.assertEqual(model._rank, 2)
self.assertEqual(
model.task_covar_module.covar_factor.shape[-1], model._rank
)
if use_intf:
self.assertIsInstance(model.input_transform, Normalize)
# test model fitting
mll = ExactMarginalLogLikelihood(model.likelihood, model)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=OptimizationWarning)
mll = fit_gpytorch_model(mll, options={"maxiter": 1}, max_retries=1)
# check that training data has input transform applied
# check that the train inputs have been transformed and set on the model
if use_intf:
self.assertTrue(
torch.equal(model.train_inputs[0], model.input_transform(train_X))
)
# test posterior
test_x = torch.rand(2, 1, **tkwargs)
posterior_f = model.posterior(test_x)
self.assertIsInstance(posterior_f, GPyTorchPosterior)
self.assertIsInstance(posterior_f.mvn, MultitaskMultivariateNormal)
self.assertEqual(posterior_f.mean.shape, torch.Size([2, 2]))
self.assertEqual(posterior_f.variance.shape, torch.Size([2, 2]))
# test that posterior w/ observation noise raises appropriate error
with self.assertRaises(NotImplementedError):
model.posterior(test_x, observation_noise=True)
with self.assertRaises(NotImplementedError):
model.posterior(test_x, observation_noise=torch.rand(2, **tkwargs))
# test posterior w/ single output index
posterior_f = model.posterior(test_x, output_indices=[0])
self.assertIsInstance(posterior_f, GPyTorchPosterior)
self.assertIsInstance(posterior_f.mvn, MultivariateNormal)
self.assertEqual(posterior_f.mean.shape, torch.Size([2, 1]))
self.assertEqual(posterior_f.variance.shape, torch.Size([2, 1]))
# test posterior w/ bad output index
with self.assertRaises(ValueError):
model.posterior(test_x, output_indices=[2])
# test posterior (batch eval)
test_x = torch.rand(3, 2, 1, **tkwargs)
posterior_f = model.posterior(test_x)
self.assertIsInstance(posterior_f, GPyTorchPosterior)
self.assertIsInstance(posterior_f.mvn, MultitaskMultivariateNormal)
# test that unsupported batch shape MTGPs throw correct error
with self.assertRaises(ValueError):
FixedNoiseMultiTaskGP(
torch.rand(2, 2, 2), torch.rand(2, 2, 1), torch.rand(2, 2, 1), 0
)
# test that bad feature index throws correct error
train_X, train_Y = _get_random_mt_data(**tkwargs)
train_Yvar = torch.full_like(train_Y, 0.05)
with self.assertRaises(ValueError):
FixedNoiseMultiTaskGP(train_X, train_Y, train_Yvar, 2)
# test that bad output task throws correct error
with self.assertRaises(RuntimeError):
FixedNoiseMultiTaskGP(train_X, train_Y, train_Yvar, 0, output_tasks=[2])
def test_model_list_to_batched(self):
for dtype in (torch.float, torch.double):
# basic test
train_X = torch.rand(10, 2, device=self.device, dtype=dtype)
train_Y1 = train_X.sum(dim=-1, keepdim=True)
train_Y2 = (train_X[:, 0] - train_X[:, 1]).unsqueeze(-1)
gp1 = SingleTaskGP(train_X, train_Y1)
gp2 = SingleTaskGP(train_X, train_Y2)
list_gp = ModelListGP(gp1, gp2)
batch_gp = model_list_to_batched(list_gp)
self.assertIsInstance(batch_gp, SingleTaskGP)
# test degenerate (single model)
batch_gp = model_list_to_batched(ModelListGP(gp1))
self.assertEqual(batch_gp._num_outputs, 1)
# test different model classes
gp2 = FixedNoiseGP(train_X, train_Y1, torch.ones_like(train_Y1))
with self.assertRaises(UnsupportedError):
model_list_to_batched(ModelListGP(gp1, gp2))
# test non-batched models
gp1_ = SimpleGPyTorchModel(train_X, train_Y1)
gp2_ = SimpleGPyTorchModel(train_X, train_Y2)
with self.assertRaises(UnsupportedError):
model_list_to_batched(ModelListGP(gp1_, gp2_))
# test list of multi-output models
train_Y = torch.cat([train_Y1, train_Y2], dim=-1)
gp2 = SingleTaskGP(train_X, train_Y)
with self.assertRaises(UnsupportedError):
model_list_to_batched(ModelListGP(gp1, gp2))
# test different training inputs
gp2 = SingleTaskGP(2 * train_X, train_Y2)
with self.assertRaises(UnsupportedError):
model_list_to_batched(ModelListGP(gp1, gp2))
# check scalar agreement
gp2 = SingleTaskGP(train_X, train_Y2)
gp2.likelihood.noise_covar.noise_prior.rate.fill_(1.0)
with self.assertRaises(UnsupportedError):
model_list_to_batched(ModelListGP(gp1, gp2))
# check tensor shape agreement
gp2 = SingleTaskGP(train_X, train_Y2)
gp2.covar_module.raw_outputscale = torch.nn.Parameter(
torch.tensor([0.0], device=self.device, dtype=dtype))
with self.assertRaises(UnsupportedError):
model_list_to_batched(ModelListGP(gp1, gp2))
# test HeteroskedasticSingleTaskGP
gp2 = HeteroskedasticSingleTaskGP(train_X, train_Y1,
torch.ones_like(train_Y1))
with self.assertRaises(NotImplementedError):
model_list_to_batched(ModelListGP(gp2))
# test custom likelihood
gp2 = SingleTaskGP(train_X,
train_Y2,
likelihood=GaussianLikelihood())
with self.assertRaises(NotImplementedError):
model_list_to_batched(ModelListGP(gp2))
# test FixedNoiseGP
train_X = torch.rand(10, 2, device=self.device, dtype=dtype)
train_Y1 = train_X.sum(dim=-1, keepdim=True)
train_Y2 = (train_X[:, 0] - train_X[:, 1]).unsqueeze(-1)
gp1_ = FixedNoiseGP(train_X, train_Y1, torch.rand_like(train_Y1))
gp2_ = FixedNoiseGP(train_X, train_Y2, torch.rand_like(train_Y2))
list_gp = ModelListGP(gp1_, gp2_)
batch_gp = model_list_to_batched(list_gp)
# test SingleTaskMultiFidelityGP
gp1_ = SingleTaskMultiFidelityGP(train_X,
train_Y1,
iteration_fidelity=1)
gp2_ = SingleTaskMultiFidelityGP(train_X,
train_Y2,
iteration_fidelity=1)
list_gp = ModelListGP(gp1_, gp2_)
batch_gp = model_list_to_batched(list_gp)
gp2_ = SingleTaskMultiFidelityGP(train_X,
train_Y2,
iteration_fidelity=2)
list_gp = ModelListGP(gp1_, gp2_)
with self.assertRaises(UnsupportedError):
model_list_to_batched(list_gp)
# test input transform
input_tf = Normalize(
d=2,
bounds=torch.tensor([[0.0, 0.0], [1.0, 1.0]],
device=self.device,
dtype=dtype),
)
gp1_ = SingleTaskGP(train_X, train_Y1, input_transform=input_tf)
gp2_ = SingleTaskGP(train_X, train_Y2, input_transform=input_tf)
list_gp = ModelListGP(gp1_, gp2_)
batch_gp = model_list_to_batched(list_gp)
self.assertIsInstance(batch_gp.input_transform, Normalize)
self.assertTrue(
torch.equal(batch_gp.input_transform.bounds, input_tf.bounds))
# test different input transforms
input_tf2 = Normalize(
d=2,
bounds=torch.tensor([[-1.0, -1.0], [1.0, 1.0]],
device=self.device,
dtype=dtype),
)
gp1_ = SingleTaskGP(train_X, train_Y1, input_transform=input_tf)
gp2_ = SingleTaskGP(train_X, train_Y2, input_transform=input_tf2)
list_gp = ModelListGP(gp1_, gp2_)
with self.assertRaises(UnsupportedError):
model_list_to_batched(list_gp)
# test batched input transform
input_tf2 = Normalize(
d=2,
bounds=torch.tensor([[-1.0, -1.0], [1.0, 1.0]],
device=self.device,
dtype=dtype),
batch_shape=torch.Size([3]),
)
gp1_ = SingleTaskGP(train_X, train_Y1, input_transform=input_tf2)
gp2_ = SingleTaskGP(train_X, train_Y2, input_transform=input_tf2)
list_gp = ModelListGP(gp1_, gp2_)
with self.assertRaises(UnsupportedError):
model_list_to_batched(list_gp)
# test outcome transform
octf = Standardize(m=1)
gp1_ = SingleTaskGP(train_X, train_Y1, outcome_transform=octf)
gp2_ = SingleTaskGP(train_X, train_Y2, outcome_transform=octf)
list_gp = ModelListGP(gp1_, gp2_)
with self.assertRaises(UnsupportedError):
model_list_to_batched(list_gp)
def test_batched_multi_output_to_single_output(self):
for dtype in (torch.float, torch.double):
# basic test
train_X = torch.rand(10, 2, device=self.device, dtype=dtype)
train_Y = torch.stack(
[
train_X.sum(dim=-1),
(train_X[:, 0] - train_X[:, 1]),
],
dim=1,
)
batched_mo_model = SingleTaskGP(train_X, train_Y)
batched_so_model = batched_multi_output_to_single_output(
batched_mo_model)
self.assertIsInstance(batched_so_model, SingleTaskGP)
self.assertEqual(batched_so_model.num_outputs, 1)
# test non-batched models
non_batch_model = SimpleGPyTorchModel(train_X, train_Y[:, :1])
with self.assertRaises(UnsupportedError):
batched_multi_output_to_single_output(non_batch_model)
gp2 = HeteroskedasticSingleTaskGP(train_X, train_Y,
torch.ones_like(train_Y))
with self.assertRaises(NotImplementedError):
batched_multi_output_to_single_output(gp2)
# test custom likelihood
gp2 = SingleTaskGP(train_X,
train_Y,
likelihood=GaussianLikelihood())
with self.assertRaises(NotImplementedError):
batched_multi_output_to_single_output(gp2)
# test FixedNoiseGP
train_X = torch.rand(10, 2, device=self.device, dtype=dtype)
batched_mo_model = FixedNoiseGP(train_X, train_Y,
torch.rand_like(train_Y))
batched_so_model = batched_multi_output_to_single_output(
batched_mo_model)
self.assertIsInstance(batched_so_model, FixedNoiseGP)
self.assertEqual(batched_so_model.num_outputs, 1)
# test SingleTaskMultiFidelityGP
batched_mo_model = SingleTaskMultiFidelityGP(train_X,
train_Y,
iteration_fidelity=1)
batched_so_model = batched_multi_output_to_single_output(
batched_mo_model)
self.assertIsInstance(batched_so_model, SingleTaskMultiFidelityGP)
self.assertEqual(batched_so_model.num_outputs, 1)
# test input transform
input_tf = Normalize(
d=2,
bounds=torch.tensor([[0.0, 0.0], [1.0, 1.0]],
device=self.device,
dtype=dtype),
)
batched_mo_model = SingleTaskGP(train_X,
train_Y,
input_transform=input_tf)
batch_so_model = batched_multi_output_to_single_output(
batched_mo_model)
self.assertIsInstance(batch_so_model.input_transform, Normalize)
self.assertTrue(
torch.equal(batch_so_model.input_transform.bounds,
input_tf.bounds))
# test batched input transform
input_tf2 = Normalize(
d=2,
bounds=torch.tensor([[-1.0, -1.0], [1.0, 1.0]],
device=self.device,
dtype=dtype),
batch_shape=torch.Size([2]),
)
batched_mo_model = SingleTaskGP(train_X,
train_Y,
input_transform=input_tf2)
batched_so_model = batched_multi_output_to_single_output(
batched_mo_model)
self.assertIsInstance(batch_so_model.input_transform, Normalize)
self.assertTrue(
torch.equal(batch_so_model.input_transform.bounds,
input_tf.bounds))
# test outcome transform
batched_mo_model = SingleTaskGP(train_X,
train_Y,
outcome_transform=Standardize(m=2))
with self.assertRaises(NotImplementedError):
batched_multi_output_to_single_output(batched_mo_model)
def test_pairwise_gp(self):
for batch_shape, dtype in itertools.product(
(torch.Size(), torch.Size([2])), (torch.float, torch.double)):
tkwargs = {"device": self.device, "dtype": dtype}
X_dim = 2
model, model_kwargs = self._get_model_and_data(
batch_shape=batch_shape, X_dim=X_dim, **tkwargs)
train_X = model_kwargs["datapoints"]
train_comp = model_kwargs["comparisons"]
# test training
# regular training
mll = PairwiseLaplaceMarginalLogLikelihood(model).to(**tkwargs)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=OptimizationWarning)
fit_gpytorch_model(mll, options={"maxiter": 2}, max_retries=1)
# prior training
prior_m = PairwiseGP(None, None).to(**tkwargs)
with self.assertRaises(RuntimeError):
prior_m(train_X)
# forward in training mode with non-training data
custom_m = PairwiseGP(**model_kwargs)
other_X = torch.rand(batch_shape + torch.Size([3, X_dim]),
**tkwargs)
other_comp = train_comp.clone()
with self.assertRaises(RuntimeError):
custom_m(other_X)
custom_mll = PairwiseLaplaceMarginalLogLikelihood(custom_m).to(
**tkwargs)
post = custom_m(train_X)
with self.assertRaises(RuntimeError):
custom_mll(post, other_comp)
# setting jitter = 0 with a singular covar will raise error
sing_train_X = torch.ones(batch_shape + torch.Size([10, X_dim]),
**tkwargs)
with self.assertRaises(RuntimeError):
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=RuntimeWarning)
custom_m = PairwiseGP(sing_train_X, train_comp, jitter=0)
custom_m.posterior(sing_train_X)
# test init
self.assertIsInstance(model.mean_module, ConstantMean)
self.assertIsInstance(model.covar_module, ScaleKernel)
self.assertIsInstance(model.covar_module.base_kernel, RBFKernel)
self.assertIsInstance(
model.covar_module.base_kernel.lengthscale_prior, GammaPrior)
self.assertIsInstance(model.covar_module.outputscale_prior,
SmoothedBoxPrior)
self.assertEqual(model.num_outputs, 1)
self.assertEqual(model.batch_shape, batch_shape)
# test custom models
custom_m = PairwiseGP(**model_kwargs, covar_module=LinearKernel())
self.assertIsInstance(custom_m.covar_module, LinearKernel)
# prior prediction
prior_m = PairwiseGP(None, None).to(**tkwargs)
prior_m.eval()
post = prior_m.posterior(train_X)
self.assertIsInstance(post, GPyTorchPosterior)
# test trying adding jitter
pd_mat = torch.eye(2, 2)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=RuntimeWarning)
jittered_pd_mat = model._add_jitter(pd_mat)
diag_diff = (jittered_pd_mat - pd_mat).diagonal(dim1=-2, dim2=-1)
self.assertTrue(
torch.allclose(
diag_diff,
torch.full_like(diag_diff, model._jitter),
atol=model._jitter / 10,
))
# test initial utility val
util_comp = torch.topk(model.utility, k=2,
dim=-1).indices.unsqueeze(-2)
self.assertTrue(torch.all(util_comp == train_comp))
# test posterior
# test non batch evaluation
X = torch.rand(batch_shape + torch.Size([3, X_dim]), **tkwargs)
expected_shape = batch_shape + torch.Size([3, 1])
posterior = model.posterior(X)
self.assertIsInstance(posterior, GPyTorchPosterior)
self.assertEqual(posterior.mean.shape, expected_shape)
self.assertEqual(posterior.variance.shape, expected_shape)
# expect to raise error when output_indices is not None
with self.assertRaises(RuntimeError):
model.posterior(X, output_indices=[0])
# test re-evaluating utility when it's None
model.utility = None
posterior = model.posterior(X)
self.assertIsInstance(posterior, GPyTorchPosterior)
# test batch evaluation
X = torch.rand(2, *batch_shape, 3, X_dim, **tkwargs)
expected_shape = torch.Size([2]) + batch_shape + torch.Size([3, 1])
posterior = model.posterior(X)
self.assertIsInstance(posterior, GPyTorchPosterior)
self.assertEqual(posterior.mean.shape, expected_shape)
# test input_transform
# the untransfomed one should be stored
normalize_tf = Normalize(d=2,
bounds=torch.tensor([[0, 0], [0.5, 1.5]]))
model = PairwiseGP(**model_kwargs, input_transform=normalize_tf)
self.assertTrue(torch.all(model.datapoints == train_X))
# test set_train_data strict mode
model = PairwiseGP(**model_kwargs)
changed_train_X = train_X.unsqueeze(0)
changed_train_comp = train_comp.unsqueeze(0)
# expect to raise error when set data to something different
with self.assertRaises(RuntimeError):
model.set_train_data(changed_train_X,
changed_train_comp,
strict=True)
# the same datapoints but changed comparison will also raise error
with self.assertRaises(RuntimeError):
model.set_train_data(train_X, changed_train_comp, strict=True)
def test_KroneckerMultiTaskGP_default(self):
bounds = torch.tensor([[-1.0, 0.0], [1.0, 1.0]])
for batch_shape, dtype, use_intf, use_octf in itertools.product(
(torch.Size(),
), # torch.Size([3])), TODO: Fix and test batch mode
(torch.float, torch.double),
(False, True),
(False, True),
):
tkwargs = {"device": self.device, "dtype": dtype}
octf = Standardize(m=2) if use_octf else None
intf = (Normalize(
d=2, bounds=bounds.to(
**tkwargs), transform_on_train=True) if use_intf else None)
# initialization with default settings
model, train_X, _ = _get_kronecker_model_and_training_data(
model_kwargs={
"outcome_transform": octf,
"input_transform": intf
},
batch_shape=batch_shape,
**tkwargs,
)
self.assertIsInstance(model, KroneckerMultiTaskGP)
self.assertEqual(model.num_outputs, 2)
self.assertIsInstance(model.likelihood,
MultitaskGaussianLikelihood)
self.assertEqual(model.likelihood.rank, 0)
self.assertIsInstance(model.mean_module, MultitaskMean)
self.assertIsInstance(model.covar_module, MultitaskKernel)
base_kernel = model.covar_module
self.assertIsInstance(base_kernel.data_covar_module, MaternKernel)
self.assertIsInstance(base_kernel.task_covar_module, IndexKernel)
task_covar_prior = base_kernel.task_covar_module.IndexKernelPrior
self.assertIsInstance(task_covar_prior, LKJCovariancePrior)
self.assertEqual(task_covar_prior.correlation_prior.eta, 1.5)
self.assertIsInstance(task_covar_prior.sd_prior, SmoothedBoxPrior)
lengthscale_prior = base_kernel.data_covar_module.lengthscale_prior
self.assertIsInstance(lengthscale_prior, GammaPrior)
self.assertEqual(lengthscale_prior.concentration, 3.0)
self.assertEqual(lengthscale_prior.rate, 6.0)
self.assertEqual(
base_kernel.task_covar_module.covar_factor.shape[-1], 2)
# test model fitting
mll = ExactMarginalLogLikelihood(model.likelihood, model)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=OptimizationWarning)
mll = fit_gpytorch_model(mll,
options={"maxiter": 1},
max_retries=1)
# test posterior
test_x = torch.rand(2, 2, **tkwargs)
posterior_f = model.posterior(test_x)
if not use_octf:
self.assertIsInstance(posterior_f, GPyTorchPosterior)
self.assertIsInstance(posterior_f.mvn,
MultitaskMultivariateNormal)
else:
self.assertIsInstance(posterior_f, TransformedPosterior)
self.assertIsInstance(posterior_f._posterior.mvn,
MultitaskMultivariateNormal)
self.assertEqual(posterior_f.mean.shape, torch.Size([2, 2]))
self.assertEqual(posterior_f.variance.shape, torch.Size([2, 2]))
if use_octf:
# ensure un-transformation is applied
tmp_tf = model.outcome_transform
del model.outcome_transform
p_tf = model.posterior(test_x)
model.outcome_transform = tmp_tf
expected_var = tmp_tf.untransform_posterior(p_tf).variance
self.assertTrue(
torch.allclose(posterior_f.variance, expected_var))
else:
# test observation noise
# TODO: outcome transform + likelihood noise?
posterior_noisy = model.posterior(test_x,
observation_noise=True)
self.assertTrue(
torch.allclose(
posterior_noisy.variance,
model.likelihood(posterior_f.mvn).variance,
))
# test posterior (batch eval)
test_x = torch.rand(3, 2, 2, **tkwargs)
posterior_f = model.posterior(test_x)
if not use_octf:
self.assertIsInstance(posterior_f, GPyTorchPosterior)
self.assertIsInstance(posterior_f.mvn,
MultitaskMultivariateNormal)
else:
self.assertIsInstance(posterior_f, TransformedPosterior)
self.assertIsInstance(posterior_f._posterior.mvn,
MultitaskMultivariateNormal)
self.assertEqual(posterior_f.mean.shape, torch.Size([3, 2, 2]))
self.assertEqual(posterior_f.variance.shape, torch.Size([3, 2, 2]))