def test_toeplitz_mvn_kl_divergence_forward():
x = Variable(torch.linspace(0, 1, 5))
rbf_covar = RBFKernel()
rbf_covar.initialize(log_lengthscale=-4)
covar_module = GridInterpolationKernel(rbf_covar)
covar_module.initialize_interpolation_grid(10, grid_bounds=(0, 1))
covar_x = covar_module.forward(x.unsqueeze(1), x.unsqueeze(1))
c = Variable(covar_x.c.data, requires_grad=True)
mu1 = Variable(torch.randn(10), requires_grad=True)
mu2 = Variable(torch.randn(10), requires_grad=True)
T = Variable(torch.zeros(len(c), len(c)))
for i in range(len(c)):
for j in range(len(c)):
T[i, j] = utils.toeplitz.toeplitz_getitem(c, c, i, j)
U = torch.randn(10, 10).triu()
U = Variable(U.mul(U.diag().sign().unsqueeze(1).expand_as(U).triu()),
requires_grad=True)
actual = gpytorch.mvn_kl_divergence(mu1, U, mu2, T, num_samples=1000)
res = gpytorch.mvn_kl_divergence(mu1, U, mu2, covar_x, num_samples=1000)
assert all(torch.abs((res.data - actual.data) / actual.data) < 0.15)
def marginal_log_likelihood(self, output, train_y, num_samples=10):
chol_var_covar = self.chol_variational_covar.triu()
# Negate each row with a negative diagonal (the Cholesky decomposition
# of a matrix requires that the diagonal elements be positive).
chol_var_covar = chol_var_covar.mul(
chol_var_covar.diag().sign().unsqueeze(1).expand_as(
chol_var_covar).triu())
_, train_covar = output.representation()
inducing_output = self.forward(*self.inducing_points)
inducing_mean = inducing_output.mean()
train_covar = gpytorch.add_jitter(train_covar)
log_likelihood = gpytorch.monte_carlo_log_likelihood(
self.prior_model.likelihood.log_probability, train_y,
self.variational_mean, chol_var_covar, train_covar, num_samples)
kl_divergence = gpytorch.mvn_kl_divergence(self.variational_mean,
chol_var_covar,
inducing_mean, train_covar,
num_samples)
return log_likelihood.squeeze() - kl_divergence
def test_toeplitz_mvn_kl_divergence_backward():
x = Variable(torch.linspace(0, 1, 5))
rbf_covar = RBFKernel()
rbf_covar.initialize(log_lengthscale=-4)
covar_module = GridInterpolationKernel(rbf_covar)
covar_module.initialize_interpolation_grid(4, grid_bounds=(0, 1))
covar_x = covar_module.forward(x.unsqueeze(1), x.unsqueeze(1))
covar_x.c = Variable(covar_x.c.data, requires_grad=True)
c = covar_x.c
mu1 = Variable(torch.randn(4), requires_grad=True)
mu2 = Variable(torch.randn(4), requires_grad=True)
mu_diff = mu2 - mu1
T = Variable(torch.zeros(len(c), len(c)))
for i in range(len(c)):
for j in range(len(c)):
T[i, j] = utils.toeplitz.toeplitz_getitem(c, c, i, j)
U = torch.randn(4, 4).triu()
U = Variable(U.mul(U.diag().sign().unsqueeze(1).expand_as(U).triu()),
requires_grad=True)
actual = 0.5 * (_det(T).log() + mu_diff.dot(T.inverse().mv(mu_diff)) +
T.inverse().mm(U.t().mm(U)).trace() -
U.diag().log().sum(0) * 2 - len(mu_diff))
actual.backward()
actual_c_grad = c.grad.data.clone()
actual_mu1_grad = mu1.grad.data.clone()
actual_mu2_grad = mu2.grad.data.clone()
actual_U_grad = U.grad.data.clone()
c.grad.data.fill_(0)
mu1.grad.data.fill_(0)
mu2.grad.data.fill_(0)
U.grad.data.fill_(0)
res = gpytorch.mvn_kl_divergence(mu1, U, mu2, covar_x, num_samples=1000)
res.backward()
res_c_grad = c.grad.data
res_mu1_grad = mu1.grad.data
res_mu2_grad = mu2.grad.data
res_U_grad = U.grad.data
assert torch.abs(
(res_c_grad - actual_c_grad)).sum() / actual_c_grad.abs().sum() < 1e-1
assert torch.abs(
(res_mu1_grad -
actual_mu1_grad)).sum() / actual_mu1_grad.abs().sum() < 1e-5
assert torch.abs(
(res_mu2_grad -
actual_mu2_grad)).sum() / actual_mu2_grad.abs().sum() < 1e-5
assert torch.abs(
(res_U_grad - actual_U_grad)).sum() / actual_U_grad.abs().sum() < 1e-2
def marginal_log_likelihood(self, output, target):
"""
Returns the marginal log likelihood of the data
Args:
- output: (GaussianRandomVariable) - the output of the model
- target: (Variable) - target
"""
mean, covar = output.representation()
# Exact inference
if self.exact_inference:
return gpytorch.exact_gp_marginal_log_likelihood(
covar, target - mean)
# Approximate inference
else:
# Get inducing points
if not hasattr(self, 'train_inputs'):
raise RuntimeError('Must condition on data.')
train_x = self.train_inputs[0]
if hasattr(self, 'inducing_points'):
inducing_points = Variable(self.inducing_points)
else:
inducing_points = train_x
chol_var_covar = self.chol_variational_covar.triu()
# Negate each row with a negative diagonal (the Cholesky decomposition
# of a matrix requires that the diagonal elements be positive).
inside = chol_var_covar.diag().sign().unsqueeze(1).expand_as(
chol_var_covar).triu()
chol_var_covar = chol_var_covar.mul(inside)
_, train_covar = output.representation()
inducing_output = super(GPModel, self).__call__(inducing_points)
inducing_mean, inducing_covar = inducing_output.representation()
train_covar = gpytorch.add_jitter(train_covar)
log_likelihood = gpytorch.monte_carlo_log_likelihood(
self.likelihood.log_probability, target, self.variational_mean,
chol_var_covar, train_covar)
inducing_covar = gpytorch.add_jitter(inducing_covar)
kl_divergence = gpytorch.mvn_kl_divergence(self.variational_mean,
chol_var_covar,
inducing_mean,
inducing_covar)
res = log_likelihood.squeeze() - kl_divergence
return res
def test_mvn_kl_divergence_backward():
x = Variable(torch.linspace(0, 1, 4))
rbf_covar = RBFKernel()
rbf_covar.initialize(log_lengthscale=-4)
K = Variable(rbf_covar.forward(x.unsqueeze(1), x.unsqueeze(1)).data,
requires_grad=True)
mu1 = Variable(torch.randn(4), requires_grad=True)
mu2 = Variable(torch.randn(4), requires_grad=True)
U = torch.randn(4, 4).triu()
U = Variable(U.mul(U.diag().sign().unsqueeze(1).expand_as(U).triu()),
requires_grad=True)
mu_diff = mu2 - mu1
actual = 0.5 * (_det(K).log() + mu_diff.dot(K.inverse().mv(mu_diff)) +
K.inverse().mm(U.t().mm(U)).trace() -
U.diag().log().sum(0) * 2 - len(mu_diff))
actual.backward()
actual_K_grad = K.grad.data.clone()
actual_mu1_grad = mu1.grad.data.clone()
actual_mu2_grad = mu2.grad.data.clone()
actual_U_grad = U.grad.data.clone()
K.grad.data.fill_(0)
mu1.grad.data.fill_(0)
mu2.grad.data.fill_(0)
U.grad.data.fill_(0)
res = gpytorch.mvn_kl_divergence(mu1, U, mu2, K, num_samples=10000)
res.backward()
res_K_grad = K.grad.data
res_mu1_grad = mu1.grad.data
res_mu2_grad = mu2.grad.data
res_U_grad = U.grad.data
assert torch.abs(
(res_K_grad - actual_K_grad)).sum() / actual_K_grad.abs().sum() < 1e-1
assert torch.abs(
(res_mu1_grad -
actual_mu1_grad)).sum() / actual_mu1_grad.abs().sum() < 1e-5
assert torch.abs(
(res_mu2_grad -
actual_mu2_grad)).sum() / actual_mu2_grad.abs().sum() < 1e-5
assert torch.abs(
(res_U_grad - actual_U_grad)).sum() / actual_U_grad.abs().sum() < 1e-2
def test_mvn_kl_divergence_forward():
x = Variable(torch.linspace(0, 1, 4))
rbf_covar = RBFKernel()
rbf_covar.initialize(log_lengthscale=-4)
K = rbf_covar.forward(x.unsqueeze(1), x.unsqueeze(1))
mu1 = Variable(torch.randn(4), requires_grad=True)
mu2 = Variable(torch.randn(4), requires_grad=True)
U = torch.randn(4, 4).triu()
U = Variable(U.mul(U.diag().sign().unsqueeze(1).expand_as(U).triu()),
requires_grad=True)
mu_diff = mu2 - mu1
actual = 0.5 * (_det(K).log() + mu_diff.dot(K.inverse().mv(mu_diff)) +
K.inverse().mm(U.t().mm(U)).trace() -
U.diag().log().sum(0) * 2 - len(mu_diff))
res = gpytorch.mvn_kl_divergence(mu1, U, mu2, K, num_samples=1000)
assert all(torch.abs((res.data - actual.data) / actual.data) < 0.15)