def test_batch_matmul_mat_with_two_matrices(self):
mat1 = make_random_mat(20, rank=4, batch_size=5)
mat2 = make_random_mat(20, rank=4, batch_size=5)
vec = Variable(torch.randn(5, 20, 7), requires_grad=True)
mat1_copy = Variable(mat1.data, requires_grad=True)
mat2_copy = Variable(mat2.data, requires_grad=True)
vec_copy = Variable(vec.data, requires_grad=True)
# Forward
res = MulLazyVariable(RootLazyVariable(mat1),
RootLazyVariable(mat2)).matmul(vec)
actual = prod([
mat1_copy.matmul(mat1_copy.transpose(-1, -2)),
mat2_copy.matmul(mat2_copy.transpose(-1, -2))
]).matmul(vec_copy)
self.assertLess(
torch.max(((res.data - actual.data) / actual.data).abs()), 0.01)
# Backward
res.sum().backward()
actual.sum().backward()
self.assertLess(
torch.max(((mat1.grad.data - mat1_copy.grad.data) /
mat1_copy.grad.data).abs()), 0.01)
self.assertLess(
torch.max(((mat2.grad.data - mat2_copy.grad.data) /
mat2_copy.grad.data).abs()), 0.01)
self.assertLess(
torch.max(((vec.grad.data - vec_copy.grad.data) /
vec_copy.grad.data).abs()), 0.01)
def test_matmul_mat_with_two_matrices():
mat1 = make_random_mat(20, 5)
mat2 = make_random_mat(20, 5)
vec = Variable(torch.randn(20, 7), requires_grad=True)
mat1_copy = Variable(mat1.data, requires_grad=True)
mat2_copy = Variable(mat2.data, requires_grad=True)
vec_copy = Variable(vec.data, requires_grad=True)
# Forward
res = MulLazyVariable(RootLazyVariable(mat1),
RootLazyVariable(mat2)).matmul(vec)
actual = prod([
mat1_copy.matmul(mat1_copy.transpose(-1, -2)),
mat2_copy.matmul(mat2_copy.transpose(-1, -2)),
]).matmul(vec_copy)
assert torch.max(((res.data - actual.data) / actual.data).abs()) < 0.01
# Backward
res.sum().backward()
actual.sum().backward()
assert torch.max(((mat1.grad.data - mat1_copy.grad.data) /
mat1_copy.grad.data).abs()) < 0.01
assert torch.max(((mat2.grad.data - mat2_copy.grad.data) /
mat2_copy.grad.data).abs()) < 0.01
assert torch.max(((vec.grad.data - vec_copy.grad.data) /
vec_copy.grad.data).abs()) < 0.01
def test_batch_getitem():
mat1 = make_random_mat(20, rank=4, batch_size=5)
mat2 = make_random_mat(20, rank=4, batch_size=5)
mat3 = make_random_mat(20, rank=4, batch_size=5)
mat1_copy = Variable(mat1.data, requires_grad=True)
mat2_copy = Variable(mat2.data, requires_grad=True)
mat3_copy = Variable(mat3.data, requires_grad=True)
# Forward
res = MulLazyVariable(RootLazyVariable(mat1), RootLazyVariable(mat2),
RootLazyVariable(mat3))
actual = prod([
mat1_copy.matmul(mat1_copy.transpose(-1, -2)),
mat2_copy.matmul(mat2_copy.transpose(-1, -2)),
mat3_copy.matmul(mat3_copy.transpose(-1, -2)),
])
assert torch.max(((res[0].evaluate().data - actual[0].data) /
actual[0].data).abs()) < 0.01
assert torch.max(((res[0:2, 5, 3:5].data - actual[0:2, 5, 3:5].data) /
actual[0:2, 5, 3:5].data).abs()) < 0.01
assert torch.max(
((res[:, 3:5, 2:].evaluate().data - actual[:, 3:5, 2:].data) /
actual[:, 3:5, 2:].data).abs()) < 0.01
assert torch.max(
((res[:, 2:, 3:5].evaluate().data - actual[:, 2:, 3:5].data) /
actual[:, 2:, 3:5].data).abs()) < 0.01
def test_batch_diag(self):
mat1 = make_random_mat(20, rank=4, batch_size=5)
mat2 = make_random_mat(20, rank=4, batch_size=5)
mat3 = make_random_mat(20, rank=4, batch_size=5)
mat1_copy = Variable(mat1.data, requires_grad=True)
mat2_copy = Variable(mat2.data, requires_grad=True)
mat3_copy = Variable(mat3.data, requires_grad=True)
# Forward
res = MulLazyVariable(
RootLazyVariable(mat1),
RootLazyVariable(mat2),
RootLazyVariable(mat3),
).diag()
actual = prod([
mat1_copy.matmul(mat1_copy.transpose(-1, -2)),
mat2_copy.matmul(mat2_copy.transpose(-1, -2)),
mat3_copy.matmul(mat3_copy.transpose(-1, -2)),
])
actual = torch.cat([actual[i].diag().unsqueeze(0) for i in range(5)])
self.assertLess(
torch.max(((res.data - actual.data) / actual.data).abs()),
0.01,
)
def test_getitem(self):
mat1 = make_random_mat(20, rank=4)
mat2 = make_random_mat(20, rank=4)
mat3 = make_random_mat(20, rank=4)
mat1_copy = Variable(mat1.data, requires_grad=True)
mat2_copy = Variable(mat2.data, requires_grad=True)
mat3_copy = Variable(mat3.data, requires_grad=True)
# Forward
res = MulLazyVariable(RootLazyVariable(mat1), RootLazyVariable(mat2),
RootLazyVariable(mat3))
actual = prod([
mat1_copy.matmul(mat1_copy.transpose(-1, -2)),
mat2_copy.matmul(mat2_copy.transpose(-1, -2)),
mat3_copy.matmul(mat3_copy.transpose(-1, -2)),
])
self.assertLess(
torch.max(((res[5, 3:5].data - actual[5, 3:5].data) /
actual[5, 3:5].data).abs()),
0.01,
)
self.assertLess(
torch.max(((res[3:5, 2:].evaluate().data - actual[3:5, 2:].data) /
actual[3:5, 2:].data).abs()),
0.01,
)
self.assertLess(
torch.max(((res[2:, 3:5].evaluate().data - actual[2:, 3:5].data) /
actual[2:, 3:5].data).abs()),
0.01,
)
def test_matmul_vec_with_five_matrices(self):
mat1 = make_random_mat(20, 5)
mat2 = make_random_mat(20, 5)
mat3 = make_random_mat(20, 5)
mat4 = make_random_mat(20, 5)
mat5 = make_random_mat(20, 5)
vec = Variable(torch.randn(20), requires_grad=True)
mat1_copy = Variable(mat1.data, requires_grad=True)
mat2_copy = Variable(mat2.data, requires_grad=True)
mat3_copy = Variable(mat3.data, requires_grad=True)
mat4_copy = Variable(mat4.data, requires_grad=True)
mat5_copy = Variable(mat5.data, requires_grad=True)
vec_copy = Variable(vec.data, requires_grad=True)
# Forward
res = MulLazyVariable(
RootLazyVariable(mat1),
RootLazyVariable(mat2),
RootLazyVariable(mat3),
RootLazyVariable(mat4),
RootLazyVariable(mat5),
).matmul(vec)
actual = prod([
mat1_copy.matmul(mat1_copy.transpose(-1, -2)),
mat2_copy.matmul(mat2_copy.transpose(-1, -2)),
mat3_copy.matmul(mat3_copy.transpose(-1, -2)),
mat4_copy.matmul(mat4_copy.transpose(-1, -2)),
mat5_copy.matmul(mat5_copy.transpose(-1, -2)),
]).matmul(vec_copy)
self.assertLess(
torch.max(((res.data - actual.data) / actual.data).abs()), 0.01)
# Backward
res.sum().backward()
actual.sum().backward()
self.assertLess(
torch.max(((mat1.grad.data - mat1_copy.grad.data) /
mat1_copy.grad.data).abs()), 0.01)
self.assertLess(
torch.max(((mat2.grad.data - mat2_copy.grad.data) /
mat2_copy.grad.data).abs()), 0.01)
self.assertLess(
torch.max(((mat3.grad.data - mat3_copy.grad.data) /
mat3_copy.grad.data).abs()), 0.01)
self.assertLess(
torch.max(((mat4.grad.data - mat4_copy.grad.data) /
mat4_copy.grad.data).abs()), 0.01)
self.assertLess(
torch.max(((mat5.grad.data - mat5_copy.grad.data) /
mat5_copy.grad.data).abs()), 0.01)
self.assertLess(
torch.max(((vec.grad.data - vec_copy.grad.data) /
vec_copy.grad.data).abs()), 0.01)
def test_batch_get_indices(self):
root = torch.randn(2, 5, 1)
actual = root.matmul(root.transpose(-1, -2))
res = RootLazyVariable(root)
batch_indices = torch.LongTensor([0, 1, 0, 1])
left_indices = torch.LongTensor([1, 2, 4, 0])
right_indices = torch.LongTensor([0, 1, 3, 2])
self.assertTrue(
approx_equal(
actual[batch_indices, left_indices, right_indices],
res._batch_get_indices(batch_indices, left_indices,
right_indices)))
batch_indices = torch.LongTensor(
[0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1])
left_indices = torch.LongTensor(
[1, 2, 4, 0, 1, 2, 3, 1, 2, 2, 1, 1, 0, 0, 4, 4, 4, 4])
right_indices = torch.LongTensor(
[0, 1, 3, 2, 3, 4, 2, 2, 1, 1, 2, 1, 2, 4, 4, 3, 3, 0])
self.assertTrue(
approx_equal(
actual[batch_indices, left_indices, right_indices],
res._batch_get_indices(batch_indices, left_indices,
right_indices)))
def test_batch_mode_matmul_batch_mat_with_five_matrices():
mats = make_random_mat(6, rank=4, batch_size=30)
vec = Variable(torch.randn(5, 6, 7), requires_grad=True)
mats_copy = Variable(mats.data, requires_grad=True)
vec_copy = Variable(vec.data, requires_grad=True)
# Forward
res = RootLazyVariable(mats).mul_batch(mul_batch_size=6).matmul(vec)
reshaped_mats_copy = mats_copy.view(5, 6, 6, 4)
actual = prod([
reshaped_mats_copy[:, 0].matmul(reshaped_mats_copy[:, 0].transpose(
-1, -2)).view(5, 6, 6),
reshaped_mats_copy[:, 1].matmul(reshaped_mats_copy[:, 1].transpose(
-1, -2)).view(5, 6, 6),
reshaped_mats_copy[:, 2].matmul(reshaped_mats_copy[:, 2].transpose(
-1, -2)).view(5, 6, 6),
reshaped_mats_copy[:, 3].matmul(reshaped_mats_copy[:, 3].transpose(
-1, -2)).view(5, 6, 6),
reshaped_mats_copy[:, 4].matmul(reshaped_mats_copy[:, 4].transpose(
-1, -2)).view(5, 6, 6),
reshaped_mats_copy[:, 5].matmul(reshaped_mats_copy[:, 5].transpose(
-1, -2)).view(5, 6, 6),
]).matmul(vec_copy)
assert torch.max(((res.data - actual.data) / actual.data).abs()) < 0.01
# Backward
res.sum().backward()
actual.sum().backward()
assert torch.max(((mats.grad.data - mats_copy.grad.data) /
mats_copy.grad.data).abs()) < 0.05
assert torch.max(((vec.grad.data - vec_copy.grad.data) /
vec_copy.grad.data).abs()) < 0.05
def test_batch_mode_matmul_mat_with_five_matrices(self):
mats = make_random_mat(6, rank=4, batch_size=6)
vec = Variable(torch.randn(6, 7), requires_grad=True)
mats_copy = Variable(mats.data, requires_grad=True)
vec_copy = Variable(vec.data, requires_grad=True)
# Forward
res = RootLazyVariable(mats).mul_batch().matmul(vec)
actual = prod([
mats_copy[0].matmul(mats_copy[0].transpose(-1, -2)),
mats_copy[1].matmul(mats_copy[1].transpose(-1, -2)),
mats_copy[2].matmul(mats_copy[2].transpose(-1, -2)),
mats_copy[3].matmul(mats_copy[3].transpose(-1, -2)),
mats_copy[4].matmul(mats_copy[4].transpose(-1, -2)),
mats_copy[5].matmul(mats_copy[5].transpose(-1, -2)),
]).matmul(vec_copy)
self.assertLess(
torch.max(((res.data - actual.data) / actual.data).abs()), 0.01)
# Backward
res.sum().backward()
actual.sum().backward()
self.assertLess(
torch.max(((mats.grad.data - mats_copy.grad.data) /
mats_copy.grad.data).abs()),
0.01,
)
self.assertLess(
torch.max(((vec.grad.data - vec_copy.grad.data) /
vec_copy.grad.data).abs()),
0.01,
)
def forward(self, x1, x2):
if x1.size() == x2.size() and torch.equal(x1, x2):
# Use RootLazyVariable when x1 == x2 for efficiency when composing
# with other kernels
prod = RootLazyVariable(x1 - self.offset)
else:
prod = MatmulLazyVariable(x1 - self.offset, (x2 - self.offset).transpose(2, 1))
return prod + self.variance.expand(prod.size())
def test_diag(self):
mat1 = make_random_mat(20, rank=4)
mat2 = make_random_mat(20, rank=4)
mat3 = make_random_mat(20, rank=4)
mat1_copy = Variable(mat1.data, requires_grad=True)
mat2_copy = Variable(mat2.data, requires_grad=True)
mat3_copy = Variable(mat3.data, requires_grad=True)
# Forward
res = MulLazyVariable(RootLazyVariable(mat1), RootLazyVariable(mat2),
RootLazyVariable(mat3)).diag()
actual = prod([
mat1_copy.matmul(mat1_copy.transpose(-1, -2)),
mat2_copy.matmul(mat2_copy.transpose(-1, -2)),
mat3_copy.matmul(mat3_copy.transpose(-1, -2)),
]).diag()
assert torch.max(((res.data - actual.data) / actual.data).abs()) < 0.01
def test_mul_adding_another_variable():
mat1 = make_random_mat(20, rank=4, batch_size=5)
mat2 = make_random_mat(20, rank=4, batch_size=5)
mat3 = make_random_mat(20, rank=4, batch_size=5)
mat1_copy = Variable(mat1.data, requires_grad=True)
mat2_copy = Variable(mat2.data, requires_grad=True)
mat3_copy = Variable(mat3.data, requires_grad=True)
# Forward
res = MulLazyVariable(RootLazyVariable(mat1), RootLazyVariable(mat2))
res = res * RootLazyVariable(mat3)
actual = prod([
mat1_copy.matmul(mat1_copy.transpose(-1, -2)),
mat2_copy.matmul(mat2_copy.transpose(-1, -2)),
mat3_copy.matmul(mat3_copy.transpose(-1, -2)),
])
assert torch.max(
((res.evaluate().data - actual.data) / actual.data).abs()) < 0.01
def test_batch_matmul_mat_with_five_matrices():
mat1 = make_random_mat(20, rank=4, batch_size=5)
mat2 = make_random_mat(20, rank=4, batch_size=5)
mat3 = make_random_mat(20, rank=4, batch_size=5)
mat4 = make_random_mat(20, rank=4, batch_size=5)
mat5 = make_random_mat(20, rank=4, batch_size=5)
vec = Variable(torch.randn(5, 20, 7), requires_grad=True)
mat1_copy = Variable(mat1.data, requires_grad=True)
mat2_copy = Variable(mat2.data, requires_grad=True)
mat3_copy = Variable(mat3.data, requires_grad=True)
mat4_copy = Variable(mat4.data, requires_grad=True)
mat5_copy = Variable(mat5.data, requires_grad=True)
vec_copy = Variable(vec.data, requires_grad=True)
# Forward
res = MulLazyVariable(RootLazyVariable(mat1), RootLazyVariable(mat2),
RootLazyVariable(mat3), RootLazyVariable(mat4),
RootLazyVariable(mat5)).matmul(vec)
actual = prod([
mat1_copy.matmul(mat1_copy.transpose(-1, -2)),
mat2_copy.matmul(mat2_copy.transpose(-1, -2)),
mat3_copy.matmul(mat3_copy.transpose(-1, -2)),
mat4_copy.matmul(mat4_copy.transpose(-1, -2)),
mat5_copy.matmul(mat5_copy.transpose(-1, -2)),
]).matmul(vec_copy)
assert torch.max(((res.data - actual.data) / actual.data).abs()) < 0.01
# Backward
res.sum().backward()
actual.sum().backward()
assert torch.max(((mat1.grad.data - mat1_copy.grad.data) /
mat1_copy.grad.data).abs()) < 0.01
assert torch.max(((mat2.grad.data - mat2_copy.grad.data) /
mat2_copy.grad.data).abs()) < 0.01
assert torch.max(((mat3.grad.data - mat3_copy.grad.data) /
mat3_copy.grad.data).abs()) < 0.01
assert torch.max(((mat4.grad.data - mat4_copy.grad.data) /
mat4_copy.grad.data).abs()) < 0.01
assert torch.max(((mat5.grad.data - mat5_copy.grad.data) /
mat5_copy.grad.data).abs()) < 0.01
assert torch.max(((vec.grad.data - vec_copy.grad.data) /
vec_copy.grad.data).abs()) < 0.01
def _get_covariance(self, x1, x2):
k_ux1 = self.base_kernel_module(x1, self.inducing_points).evaluate()
if torch.equal(x1, x2):
covar = RootLazyVariable(k_ux1.matmul(self._inducing_inv_root))
else:
k_ux2 = self.base_kernel_module(x2, self.inducing_points).evaluate()
covar = MatmulLazyVariable(
k_ux1.matmul(self._inducing_inv_root), k_ux2.matmul(self._inducing_inv_root).transpose(-1, -2)
)
return covar
def test_batch_diag():
root = Variable(torch.randn(4, 5, 3))
actual = root.matmul(root.transpose(-1, -2))
actual_diag = torch.cat([
actual[0].diag().unsqueeze(0),
actual[1].diag().unsqueeze(0),
actual[2].diag().unsqueeze(0),
actual[3].diag().unsqueeze(0),
])
res = RootLazyVariable(root)
assert approx_equal(actual_diag.data, res.diag().data)
def test_batch_diag(self):
root = torch.randn(4, 5, 3)
actual = root.matmul(root.transpose(-1, -2))
actual_diag = torch.cat([
actual[0].diag().unsqueeze(0),
actual[1].diag().unsqueeze(0),
actual[2].diag().unsqueeze(0),
actual[3].diag().unsqueeze(0),
])
res = RootLazyVariable(root)
self.assertTrue(approx_equal(actual_diag, res.diag()))
def test_mul_adding_constant_mul():
mat1 = make_random_mat(20, rank=4, batch_size=5)
mat2 = make_random_mat(20, rank=4, batch_size=5)
mat3 = make_random_mat(20, rank=4, batch_size=5)
const = Variable(torch.ones(1), requires_grad=True)
mat1_copy = Variable(mat1.data, requires_grad=True)
mat2_copy = Variable(mat2.data, requires_grad=True)
mat3_copy = Variable(mat3.data, requires_grad=True)
const_copy = Variable(const.data, requires_grad=True)
# Forward
res = MulLazyVariable(RootLazyVariable(mat1), RootLazyVariable(mat2),
RootLazyVariable(mat3))
res = res * const
actual = prod([
mat1_copy.matmul(mat1_copy.transpose(-1, -2)),
mat2_copy.matmul(mat2_copy.transpose(-1, -2)),
mat3_copy.matmul(mat3_copy.transpose(-1, -2)),
]) * const_copy
assert torch.max(
((res.evaluate().data - actual.data) / actual.data).abs()) < 0.01
# Forward
res = MulLazyVariable(RootLazyVariable(mat1), RootLazyVariable(mat2),
RootLazyVariable(mat3))
res = res * 2.5
actual = prod([
mat1_copy.matmul(mat1_copy.transpose(-1, -2)),
mat2_copy.matmul(mat2_copy.transpose(-1, -2)),
mat3_copy.matmul(mat3_copy.transpose(-1, -2)),
]) * 2.5
assert torch.max(
((res.evaluate().data - actual.data) / actual.data).abs()) < 0.01
def test_matmul():
root = Variable(torch.randn(5, 3), requires_grad=True)
covar = RootLazyVariable(root)
mat = Variable(torch.eye(5))
res = covar.matmul(mat)
root_clone = Variable(root.data.clone(), requires_grad=True)
mat_clone = Variable(mat.data.clone())
actual = root_clone.matmul(root_clone.transpose(-1, -2)).matmul(mat_clone)
assert approx_equal(res.data, actual.data)
gradient = torch.randn(5, 5)
actual.backward(gradient=Variable(gradient))
res.backward(gradient=Variable(gradient))
assert approx_equal(root.grad.data, root_clone.grad.data)
def test_matmul(self):
root = torch.randn(5, 3, requires_grad=True)
covar = RootLazyVariable(root)
mat = torch.eye(5)
res = covar.matmul(mat)
root_clone = root.clone().detach()
root_clone.requires_grad = True
mat_clone = mat.clone().detach()
mat_clone.requires_grad = True
actual = root_clone.matmul(root_clone.transpose(-1, -2)).matmul(mat_clone)
self.assertTrue(approx_equal(res, actual))
gradient = torch.randn(5, 5)
actual.backward(gradient=gradient)
res.backward(gradient=gradient)
self.assertTrue(approx_equal(root.grad, root_clone.grad))
def forward(self, input):
"""
Adds the log task noises to the diagonal of the covariance matrix of the supplied
:obj:`gpytorch.random_variables.GaussianRandomVariable` or
:obj:`gpytorch.random_variables.MultitaskGaussianRandomVariable`, in case of
`rank` == 0. Otherwise, adds a rank `rank` covariance matrix to it.
To accomplish this, we form a new :obj:`gpytorch.lazy.KroneckerProductLazyVariable` between :math:`I_{n}`,
an identity matrix with size equal to the data and a (not necessarily diagonal) matrix containing the task
noises :math:`D_{t}`.
We also incorporate a shared `log_noise` parameter from the base
:class:`gpytorch.likelihoods.GaussianLikelihood` that we extend.
The final covariance matrix after this method is then :math:`K + D_{t} \otimes I_{n} + \sigma^{2}I_{nt}`.
Args:
input (:obj:`gpytorch.random_variables.MultitaskGaussianRandomVariable`): Random variable whose covariance
matrix is a :obj:`gpytorch.lazy.LazyVariable` we intend to augment.
Returns:
:obj:`gpytorch.random_variables.MultitaskGaussianRandomVariable`: A new random variable whose covariance
matrix is a :obj:`gpytorch.lazy.LazyVariable` with :math:`D_{t} \otimes I_{n}` and :math:`\sigma^{2}I_{nt}`
added.
"""
mean, covar = input.representation()
eye_lv = DiagLazyVariable(
torch.ones(covar.size(-1) // self.n_tasks,
device=self.log_noise.device))
if hasattr(self, "log_task_noises"):
task_var_lv = DiagLazyVariable(self.log_task_noises.exp())
else:
task_var_lv = RootLazyVariable(self.task_noise_covar_factor)
covar_kron_lv = KroneckerProductLazyVariable(task_var_lv, eye_lv)
noise = covar + covar_kron_lv
noise = add_diag(noise, self.log_noise.exp())
return input.__class__(mean, noise)
def test_diag():
root = Variable(torch.randn(5, 3))
actual = root.matmul(root.transpose(-1, -2))
res = RootLazyVariable(root)
assert approx_equal(actual.diag().data, res.diag().data)
def test_evaluate(self):
root = Variable(torch.randn(5, 3))
actual = root.matmul(root.transpose(-1, -2))
res = RootLazyVariable(root)
self.assertTrue(approx_equal(actual.data, res.evaluate().data))
def test_diag(self):
root = torch.randn(5, 3)
actual = root.matmul(root.transpose(-1, -2))
res = RootLazyVariable(root)
self.assertTrue(approx_equal(actual.diag(), res.diag()))