def test_matmul_vec(self): # Forward res = NonLazyTensor(self.mat).matmul(self.vec) actual = self.mat_copy.matmul(self.vec_copy) self.assertTrue(approx_equal(res, actual)) # Backward grad_output = torch.randn(3) res.backward(gradient=grad_output) actual.backward(gradient=grad_output) self.assertTrue(approx_equal(self.mat_copy.grad, self.mat.grad)) self.assertTrue(approx_equal(self.vec_copy.grad, self.vec.grad))
def test_matmul_multiple_vecs(self): # Forward res = NonLazyTensor(self.mats).matmul(self.vecs) actual = self.mats_copy.matmul(self.vecs_copy) self.assertTrue(approx_equal(res, actual)) # Backward grad_output = torch.randn(3, 4, 5, 2) res.backward(gradient=grad_output) actual.backward(gradient=grad_output) self.assertTrue(approx_equal(self.mats_copy.grad, self.mats.grad)) self.assertTrue(approx_equal(self.vecs_copy.grad, self.vecs.grad))
def test_log_det_only(self): # Forward pass with gpytorch.settings.num_trace_samples(1000): res = NonLazyTensor(self.mat).log_det() actual = self.mat_clone.logdet() self.assertAlmostEqual(res.item(), actual.item(), places=1) # Backward actual.backward() res.backward() self.assertLess( torch.max((self.mat_clone.grad - self.mat.grad).abs()).item(), 1e-1)
def test_inv_matmul_multiple_vecs(self): # Forward res = NonLazyTensor(self.mat_var).inv_matmul(self.vecs_var) actual = self.mat_var_copy.inverse().matmul(self.vecs_var_copy) self.assertTrue(approx_equal(res, actual)) # Backward grad_output = torch.randn(3, 4) res.backward(gradient=grad_output) actual.backward(gradient=grad_output) self.assertTrue(approx_equal(self.mat_var_copy.grad, self.mat_var.grad)) self.assertTrue( approx_equal(self.vecs_var_copy.grad, self.vecs_var.grad))
def test_log_det_only(self): # Forward pass with gpytorch.settings.num_trace_samples(1000): res = NonLazyTensor(self.mat_var).log_det() actual = self.mat_var_clone.det().log() self.assertAlmostEqual(res.item(), actual.item(), places=1) # Backward actual.backward() res.backward() self.assertTrue( approx_equal(self.mat_var_clone.grad, self.mat_var.grad, epsilon=1e-1))
def test_log_det_only(self): # Forward pass with gpytorch.settings.num_trace_samples(2000): res = NonLazyTensor(self.mats).log_det() actual = torch.cat( [mat.logdet().unsqueeze(0) for mat in self.mats_clone]) self.assertEqual(res.shape, actual.shape) self.assertLess(torch.max((res - actual).abs()).item(), 1e-1) # Backward grad_output = torch.randn(5) actual.backward(gradient=grad_output) res.backward(gradient=grad_output) self.assertLess( torch.max((self.mats_clone.grad - self.mats.grad).abs()).item(), 1e-1)
def test_inv_quad_only_vector(self): # Forward pass res = NonLazyTensor(self.mat_var).inv_quad(self.vec_var) actual = self.mat_var_clone.inverse().matmul(self.vec_var_clone).mul( self.vec_var_clone).sum() self.assertAlmostEqual(res.item(), actual.item(), places=1) # Backward actual.backward() res.backward() self.assertTrue( approx_equal(self.mat_var_clone.grad, self.mat_var.grad, epsilon=1e-1)) self.assertTrue( approx_equal(self.vec_var_clone.grad, self.vec_var.grad))
def test_inv_quad_only_vector(self): # Forward pass res = NonLazyTensor(self.mat).inv_quad(self.vec) actual = self.mat_clone.inverse().matmul(self.vec_clone).mul( self.vec_clone).sum() self.assertAlmostEqual(res.item(), actual.item(), places=1) # Backward actual.backward() res.backward() self.assertLess( torch.max((self.mat_clone.grad - self.mat.grad).abs()).item(), 1e-3) self.assertLess( torch.max((self.vec_clone.grad - self.vec.grad).abs()).item(), 1e-3)
def test_inv_matmul_multiple_vecs(self): # Forward with settings.terminate_cg_by_size(False): res = NonLazyTensor(self.mat).inv_matmul(self.vecs) actual = self.mat_copy.inverse().matmul(self.vecs_copy) self.assertLess( torch.max((res - actual).abs() / actual.abs()).item(), 1e-3) # Backward grad_output = torch.randn(8, 4) res.backward(gradient=grad_output) actual.backward(gradient=grad_output) self.assertLess( torch.max((self.mat_copy.grad - self.mat.grad).abs()).item(), 1e-3) self.assertLess( torch.max((self.vecs_copy.grad - self.vecs.grad).abs()).item(), 1e-3)
def test_log_det_only(self): # Forward pass with gpytorch.settings.num_trace_samples(1000): res = NonLazyTensor(self.mats_var).log_det() actual = torch.cat([ self.mats_var_clone[0].det().log().unsqueeze(0), self.mats_var_clone[1].det().log().unsqueeze(0) ]) self.assertTrue(approx_equal(res, actual, epsilon=1e-1)) # Backward grad_output = torch.tensor([3, 4], dtype=torch.float) actual.backward(gradient=grad_output) res.backward(gradient=grad_output) self.assertTrue( approx_equal(self.mats_var_clone.grad, self.mats_var.grad, epsilon=1e-1))
def test_inv_matmul_multiple_vecs(self): mat = self._create_mat().detach().requires_grad_(True) mat_copy = mat.detach().clone().requires_grad_(True) mat_copy.register_hook(_ensure_symmetric_grad) vecs = torch.randn(*mat.shape[:-2], mat.size(-1), 4).detach().requires_grad_(True) vecs_copy = vecs.detach().clone().requires_grad_(True) # Forward with settings.terminate_cg_by_size(False): res = NonLazyTensor(mat).inv_matmul(vecs) actual = mat_copy.inverse().matmul(vecs_copy) self.assertAllClose(res, actual) # Backward grad_output = torch.randn_like(vecs) res.backward(gradient=grad_output) actual.backward(gradient=grad_output) self.assertAllClose(mat.grad, mat_copy.grad) self.assertAllClose(vecs.grad, vecs_copy.grad)
def test_inv_quad_only_many_vectors(self): # Forward pass res = NonLazyTensor(self.mats_var).inv_quad(self.vecs_var).sum() actual = (torch.cat([ self.mats_var_clone[0].inverse().unsqueeze(0), self.mats_var_clone[1].inverse().unsqueeze(0) ]).matmul(self.vecs_var_clone).mul( self.vecs_var_clone).sum(2).sum(1)).sum() self.assertTrue(approx_equal(res, actual, epsilon=1e-1)) # Backward actual.backward() res.backward() self.assertTrue( approx_equal(self.mats_var_clone.grad, self.mats_var.grad, epsilon=1e-1)) self.assertTrue( approx_equal(self.vecs_var_clone.grad, self.vecs_var.grad))
def test_inv_matmul_vec(self): mat = self._create_mat().detach().requires_grad_(True) if mat.dim() > 2: # This isn't a feature for batch mode return mat_copy = mat.detach().clone().requires_grad_(True) mat_copy.register_hook(_ensure_symmetric_grad) vec = torch.randn(mat.size(-1)).detach().requires_grad_(True) vec_copy = vec.detach().clone().requires_grad_(True) # Forward with settings.terminate_cg_by_size(False): res = NonLazyTensor(mat).inv_matmul(vec) actual = mat_copy.inverse().matmul(vec_copy) self.assertAllClose(res, actual) # Backward grad_output = torch.randn_like(vec) res.backward(gradient=grad_output) actual.backward(gradient=grad_output) self.assertAllClose(mat.grad, mat_copy.grad) self.assertAllClose(vec.grad, vec_copy.grad)
def test_inv_matmul_multiple_vecs(self): # Forward with settings.terminate_cg_by_size(False): res = NonLazyTensor(self.mats).inv_matmul(self.vecs) flattened_mats_copy = self.mats_copy.view(-1, *self.mats.shape[-2:]) flatened_mats_inverse = torch.cat( [mat.inverse().unsqueeze(0) for mat in flattened_mats_copy]) mats_inverse = flatened_mats_inverse.view_as(self.mats) actual = mats_inverse.matmul(self.vecs_copy) self.assertLess(torch.max((res - actual).abs()).item(), 1e-3) # Backward grad_output = torch.randn(3, 4, 8, 2) res.backward(gradient=grad_output) actual.backward(gradient=grad_output) self.assertLess( torch.max((self.mats_copy.grad - self.mats.grad).abs()).item(), 1e-3) self.assertLess( torch.max((self.vecs_copy.grad - self.vecs.grad).abs()).item(), 1e-3)