def __init__(self):
super(GPClassificationModel, self).__init__(grid_size=32, grid_bounds=[(0, 1)])
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-5, 5))
self.covar_module = ScaleKernel(
RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True)),
log_outputscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True),
)
def __init__(self, train_x, train_y, likelihood):
super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
self.base_covar_module = ScaleKernel(
RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True))
)
self.covar_module = GridInterpolationKernel(self.base_covar_module, grid_size=50, grid_bounds=[(0, 1)])
self.feature_extractor = feature_extractor
def __init__(self, train_inputs, train_targets, likelihood):
super(ExactGPModel, self).__init__(train_inputs, train_targets, likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
self.rbf_covar_module = RBFKernel(
log_lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
)
self.noise_covar_module = WhiteNoiseKernel(variances=torch.ones(11) * 0.001)
self.covar_module = ScaleKernel(self.rbf_covar_module + self.noise_covar_module)
def __init__(self, train_x, train_y, likelihood):
super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
self.base_covar_module = RBFKernel(
log_lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)
)
self.covar_module = MultiplicativeGridInterpolationKernel(
self.base_covar_module, grid_size=100, grid_bounds=[(0, 1)], n_components=2
)
def __init__(self, train_x, train_y, likelihood):
super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
self.base_covar_module = ScaleKernel(
RBFKernel(lengthscale_prior=SmoothedBoxPrior(
exp(-5), exp(6), sigma=0.1)))
self.covar_module = GridInterpolationKernel(self.base_covar_module,
grid_size=50,
num_dims=1)
def __init__(self, train_x, train_y, likelihood):
super(MultitaskGPModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
self.covar_module = RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(
exp(-6), exp(6), sigma=0.1, log_transform=True))
self.task_covar_module = IndexKernel(n_tasks=2,
rank=1,
prior=InverseWishartPrior(
nu=2, K=torch.eye(2)))
def __init__(self):
super(GPClassificationModel, self).__init__(grid_size=8,
grid_bounds=[(0, 3),
(0, 3)])
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
self.covar_module = ScaleKernel(
RBFKernel(ard_num_dims=2,
log_lengthscale_prior=SmoothedBoxPrior(
exp(-2.5), exp(3), sigma=0.1, log_transform=True)))
def __init__(self, train_x, train_y, likelihood):
super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
self.base_covar_module = ScaleKernel(
RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(
exp(-3), exp(3), sigma=0.1, log_transform=True)))
self.covar_module = ProductStructureKernel(GridInterpolationKernel(
self.base_covar_module, grid_size=100, num_dims=2),
num_dims=2)
def __init__(self, train_x, train_y, likelihood):
super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
self.base_covar_module = RBFKernel(
log_lengthscale_prior=SmoothedBoxPrior(
exp(-5), exp(6), sigma=0.1, log_transform=True))
self.covar_module = InducingPointKernel(self.base_covar_module,
inducing_points=torch.linspace(
0, 1, 32))
def __init__(self, train_x, train_y, likelihood):
super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
self.base_covar_module = RBFKernel(
log_lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True)
)
self.grid_covar_module = GridInterpolationKernel(self.base_covar_module, grid_size=50, grid_bounds=[(0, 1)])
self.noise_covar_module = WhiteNoiseKernel(variances=torch.ones(100) * 0.001)
self.covar_module = self.grid_covar_module + self.noise_covar_module
def __init__(self, train_x):
super(GPClassificationModel, self).__init__(train_x)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
self.covar_module = ScaleKernel(
RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(
exp(-5), exp(6), sigma=0.1, log_transform=True)),
log_outputscale_prior=SmoothedBoxPrior(exp(-5),
exp(6),
sigma=0.1,
log_transform=True),
)
def __init__(self, grid_size=6, grid_bounds=[(-0.33, 1.33), (-0.33, 1.33)]):
variational_distribution = gpytorch.variational.CholeskyVariationalDistribution(
num_inducing_points=int(pow(grid_size, len(grid_bounds)))
)
variational_strategy = gpytorch.variational.GridInterpolationVariationalStrategy(
self, grid_size=grid_size, grid_bounds=grid_bounds, variational_distribution=variational_distribution
)
super(GPClassificationModel, self).__init__(variational_strategy)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
self.covar_module = ScaleKernel(
RBFKernel(ard_num_dims=2, lengthscale_prior=SmoothedBoxPrior(exp(-2.5), exp(3), sigma=0.1))
)
def __init__(self, train_x):
super(GPClassificationModel, self).__init__(train_x)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
self.covar_module = RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(
exp(-5), exp(6), sigma=0.1, log_transform=True))
self.register_parameter(
name="log_outputscale",
parameter=torch.nn.Parameter(torch.Tensor([0])),
prior=SmoothedBoxPrior(exp(-5),
exp(6),
sigma=0.1,
log_transform=True),
)
def __init__(self, grid_size=20, grid_bounds=[(-0.1, 1.1)]):
variational_distribution = gpytorch.variational.CholeskyVariationalDistribution(
num_inducing_points=int(pow(grid_size, len(grid_bounds))))
variational_strategy = gpytorch.variational.GridInterpolationVariationalStrategy(
self,
grid_size=grid_size,
grid_bounds=grid_bounds,
variational_distribution=variational_distribution)
super(GPRegressionModel, self).__init__(variational_strategy)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-10, 10))
self.covar_module = ScaleKernel(
RBFKernel(lengthscale_prior=SmoothedBoxPrior(
exp(-3), exp(6), sigma=0.1)))
def __init__(self):
super(GPRegressionModel, self).__init__(grid_size=20,
grid_bounds=[(-0.05, 1.05)])
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-10, 10))
self.covar_module = RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(
exp(-3), exp(6), sigma=0.1, log_transform=True))
self.register_parameter(
name="log_outputscale",
parameter=torch.nn.Parameter(torch.Tensor([0])),
prior=SmoothedBoxPrior(exp(-5),
exp(1),
sigma=0.1,
log_transform=True),
)
def __init__(self, grid_size=16, grid_bounds=([-1, 1],)):
variational_distribution = CholeskyVariationalDistribution(num_inducing_points=16, batch_size=2)
variational_strategy = AdditiveGridInterpolationVariationalStrategy(
self,
grid_size=grid_size,
grid_bounds=grid_bounds,
num_dim=2,
variational_distribution=variational_distribution,
)
super(GPClassificationModel, self).__init__(variational_strategy)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5))
self.covar_module = ScaleKernel(
RBFKernel(ard_num_dims=1, lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1)),
outputscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1),
)
def __init__(self, train_x, train_y, likelihood):
super(SpectralMixtureGPModel, self).__init__(train_x, train_y,
likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
self.covar_module = SpectralMixtureKernel(num_mixtures=4,
ard_num_dims=1)
self.covar_module.initialize_from_data(train_x, train_y)
def test_lkj_covariance_prior_validate_args(self):
sd_prior = SmoothedBoxPrior(exp(-1), exp(1), validate_args=True)
LKJCovariancePrior(2, 1.0, sd_prior)
with self.assertRaises(ValueError):
LKJCovariancePrior(1.5, 1.0, sd_prior, validate_args=True)
with self.assertRaises(ValueError):
LKJCovariancePrior(2, -1.0, sd_prior, validate_args=True)
def __init__(self,
grid_size,
grid_bounds,
n_components,
mixing_params=False,
sum_output=True):
super(AdditiveGridInducingVariationalGP,
self).__init__(grid_size, grid_bounds)
self.n_components = n_components
self.sum_output = sum_output
# Resize variational parameters to have one size per component
variational_mean = self.variational_mean
chol_variational_covar = self.chol_variational_covar
variational_mean.data.resize_(*([n_components] +
list(variational_mean.size())))
chol_variational_covar.data.resize_(
*([n_components] + list(chol_variational_covar.size())))
# Mixing parameters
if mixing_params:
self.register_parameter(
name="mixing_params",
parameter=torch.nn.Parameter(
torch.ones(n_components) / n_components),
prior=SmoothedBoxPrior(-2, 2, sigma=0.01, size=n_components),
)
def test_lkj_covariance_prior_to_gpu(self):
if torch.cuda.is_available():
sd_prior = SmoothedBoxPrior(exp(-1), exp(1))
prior = LKJCovariancePrior(2, 1.0, sd_prior).cuda()
self.assertEqual(prior.correlation_prior.eta.device.type, "cuda")
self.assertEqual(prior.correlation_prior.C.device.type, "cuda")
self.assertEqual(prior.sd_prior.a.device.type, "cuda")
def __init__(self, train_x, train_y, likelihood):
super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
self.base_covar_module = RBFKernel(ard_num_dims=2)
self.covar_module = GridInterpolationKernel(self.base_covar_module,
grid_size=16,
num_dims=2)
def test_lkj_covariance_prior_log_prob_hetsd(self, cuda=False):
device = torch.device("cuda") if cuda else torch.device("cpu")
a = torch.tensor([exp(-1), exp(-2)], device=device)
b = torch.tensor([exp(1), exp(2)], device=device)
sd_prior = SmoothedBoxPrior(a, b, log_transform=True)
prior = LKJCovariancePrior(2, torch.tensor(0.5, device=device),
sd_prior)
self.assertFalse(prior.log_transform)
S = torch.eye(2, device=device)
self.assertAlmostEqual(prior.log_prob(S).item(), -4.71958, places=4)
S = torch.stack(
[S, torch.tensor([[1.0, 0.5], [0.5, 1]], device=S.device)])
self.assertTrue(
approx_equal(prior.log_prob(S),
torch.tensor([-4.71958, -4.57574], device=S.device)))
with self.assertRaises(ValueError):
prior.log_prob(torch.eye(3, device=device))
# For eta=1.0 log_prob is flat over all covariance matrices
prior = LKJCovariancePrior(2, torch.tensor(1.0, device=device),
sd_prior)
marginal_sd = torch.diagonal(S, dim1=-2, dim2=-1).sqrt()
log_prob_expected = prior.correlation_prior.C + prior.sd_prior.log_prob(
marginal_sd)
self.assertTrue(approx_equal(prior.log_prob(S), log_prob_expected))
def __init__(self, train_x, train_y, likelihood):
super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
self.base_covar_module = ScaleKernel(RBFKernel())
self.covar_module = ProductStructureKernel(GridInterpolationKernel(
self.base_covar_module, grid_size=100, num_dims=1),
num_dims=2)
def __init__(self, train_x, train_y, likelihood):
super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ZeroMean()
self.base_covar_module = ScaleKernel(
RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True))
)
self.covar_module = AdditiveGridInterpolationKernel(
self.base_covar_module, grid_size=100, grid_bounds=[(-0.5, 1.5)], n_components=2
)
def test_posterior_latent_gp_and_likelihood_fast_pred_var(
self, cuda=False):
train_x, test_x, train_y, test_y = self._get_data(cuda=cuda)
with gpytorch.settings.fast_pred_var(), gpytorch.settings.debug(False):
# We're manually going to set the hyperparameters to
# something they shouldn't be
likelihood = GaussianLikelihood(
noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1))
gp_model = ExactGPModel(train_x, train_y, likelihood)
mll = gpytorch.mlls.ExactMarginalLogLikelihood(
likelihood, gp_model)
gp_model.covar_module.base_kernel.initialize(lengthscale=exp(1))
gp_model.mean_module.initialize(constant=0)
likelihood.initialize(noise=exp(1))
if cuda:
gp_model.cuda()
likelihood.cuda()
# Find optimal model hyperparameters
gp_model.train()
likelihood.train()
optimizer = optim.Adam(list(gp_model.parameters()) +
list(likelihood.parameters()),
lr=0.1)
optimizer.n_iter = 0
for _ in range(50):
optimizer.zero_grad()
output = gp_model(train_x)
loss = -mll(output, train_y)
loss.backward()
optimizer.n_iter += 1
optimizer.step()
for param in gp_model.parameters():
self.assertTrue(param.grad is not None)
self.assertGreater(param.grad.norm().item(), 0)
for param in likelihood.parameters():
self.assertTrue(param.grad is not None)
self.assertGreater(param.grad.norm().item(), 0)
optimizer.step()
# Test the model
gp_model.eval()
likelihood.eval()
# Set the cache
test_function_predictions = likelihood(gp_model(train_x))
# Now bump up the likelihood to something huge
# This will make it easy to calculate the variance
likelihood.noise_covar.raw_noise.data.fill_(3)
test_function_predictions = likelihood(gp_model(train_x))
noise = likelihood.noise_covar.noise
var_diff = (test_function_predictions.variance - noise).abs()
self.assertLess(torch.max(var_diff / noise), 0.05)
def test_posterior_latent_gp_and_likelihood_without_optimization(
self, cuda=False):
train_x, test_x, train_y, test_y = self._get_data(cuda=cuda)
with gpytorch.settings.debug(False):
# We're manually going to set the hyperparameters to be ridiculous
likelihood = GaussianLikelihood(
noise_prior=SmoothedBoxPrior(exp(-10), exp(10), sigma=0.25),
noise_constraint=Positive(),
)
gp_model = ExactGPModel(train_x, train_y, likelihood)
# Update lengthscale prior to accommodate extreme parameters
gp_model.rbf_covar_module.register_prior(
"lengthscale_prior",
SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5),
"raw_lengthscale")
gp_model.rbf_covar_module.initialize(lengthscale=exp(-10))
gp_model.mean_module.initialize(constant=0)
likelihood.initialize(noise=exp(-10))
if cuda:
gp_model.cuda()
likelihood.cuda()
# Compute posterior distribution
gp_model.eval()
likelihood.eval()
# Let's see how our model does, conditioned with weird hyperparams
# The posterior should fit all the data
function_predictions = likelihood(gp_model(train_x))
self.assertLess(torch.norm(function_predictions.mean - train_y),
1e-3)
self.assertLess(torch.norm(function_predictions.variance), 5e-3)
# It shouldn't fit much else though
test_function_predictions = gp_model(
torch.tensor([1.1]).type_as(test_x))
self.assertLess(torch.norm(test_function_predictions.mean - 0),
1e-4)
self.assertLess(
torch.norm(test_function_predictions.variance -
gp_model.covar_module.outputscale), 1e-4)
def __init__(self, train_x, train_y, likelihood):
super(GPRegressionModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ZeroMean()
self.base_covar_module = ScaleKernel(
RBFKernel(ard_num_dims=2,
lengthscale_prior=SmoothedBoxPrior(exp(-3),
exp(3),
sigma=0.1)))
self.covar_module = AdditiveStructureKernel(GridInterpolationKernel(
self.base_covar_module, grid_size=100, num_dims=2),
num_dims=2)
def test_lkj_covariance_prior_batch_log_prob(self, cuda=False):
device = torch.device("cuda") if cuda else torch.device("cpu")
v = torch.ones(2, 1, device=device)
sd_prior = SmoothedBoxPrior(exp(-1) * v, exp(1) * v)
prior = LKJCovariancePrior(2, torch.tensor([0.5, 1.5], device=device),
sd_prior)
corr_dist = LKJCholesky(2, torch.tensor([0.5, 1.5], device=device))
S = torch.eye(2, device=device)
dist_log_prob = corr_dist.log_prob(S) + sd_prior.log_prob(S.diag())
self.assertLessEqual((prior.log_prob(S) - dist_log_prob).abs().sum(),
1e-4)
S = torch.stack(
[S, torch.tensor([[1.0, 0.5], [0.5, 1]], device=S.device)])
S_chol = torch.linalg.cholesky(S)
dist_log_prob = corr_dist.log_prob(S_chol) + sd_prior.log_prob(
torch.diagonal(S, dim1=-2, dim2=-1))
self.assertLessEqual((prior.log_prob(S) - dist_log_prob).abs().sum(),
1e-4)
def test_smoothed_box_prior_to_gpu(self):
if torch.cuda.is_available():
prior = SmoothedBoxPrior(torch.zeros(2), torch.ones(2)).cuda()
self.assertEqual(prior.a.device.type, "cuda")
self.assertEqual(prior.b.device.type, "cuda")
self.assertEqual(prior.sigma.device.type, "cuda")
self.assertEqual(prior._c.device.type, "cuda")
self.assertEqual(prior._r.device.type, "cuda")
self.assertEqual(prior._M.device.type, "cuda")
self.assertEqual(prior.tails.loc.device.type, "cuda")
self.assertEqual(prior.tails.scale.device.type, "cuda")
def __init__(self):
variational_distribution = gpytorch.variational.CholeskyVariationalDistribution(
16)
variational_strategy = gpytorch.variational.VariationalStrategy(
self,
torch.randn(16, 1),
variational_distribution,
learn_inducing_locations=True)
super(GPClassificationModel, self).__init__(variational_strategy)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-5, 5))
self.covar_module = ScaleKernel(
RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(
exp(-5), exp(6), sigma=0.1, log_transform=True)),
log_outputscale_prior=SmoothedBoxPrior(exp(-5),
exp(6),
sigma=0.1,
log_transform=True),
)
self.covar_module.base_kernel.initialize(log_lengthscale=-1)
