def test_get_item_on_interpolated_variable_no_diagonal(): no_diag_toeplitz = ToeplitzLazyVariable(lazy_toeplitz_var.c, lazy_toeplitz_var.J_left, lazy_toeplitz_var.C_left, lazy_toeplitz_var.J_right, lazy_toeplitz_var.C_right) evaluated = no_diag_toeplitz.evaluate().data assert utils.approx_equal(no_diag_toeplitz[4:6].evaluate().data, evaluated[4:6]) assert utils.approx_equal(no_diag_toeplitz[4:6, 2:6].evaluate().data, evaluated[4:6, 2:6])
def test_exact_gp_mll(self): labels_var = Variable(torch.randn(4)) # Test case c1_var = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) c2_var = Variable(torch.Tensor([6, 0, 1, -1]), requires_grad=True) actual = ToeplitzLazyVariable(c1_var + c2_var) # Actual case sum_lv = make_sum_lazy_var() t1, t2 = sum_lv.lazy_vars # Test forward mll_res = sum_lv.exact_gp_marginal_log_likelihood(labels_var) mll_actual = actual.exact_gp_marginal_log_likelihood(labels_var) self.assertLess( math.fabs(mll_res.data.squeeze()[0] - mll_actual.data.squeeze()[0]), 5e-1, ) # Test backwards mll_res.backward() mll_actual.backward() self.assertLess( math.fabs(c1_var.grad.data[0] - t1.column.grad.data[0]), 1e-1, ) self.assertLess( math.fabs(c2_var.grad.data[0] - t2.column.grad.data[0]), 1e-1, )
def test_get_item_on_batch(self): toeplitz_var = ToeplitzLazyVariable( Variable(self.batch_toeplitz_column)) evaluated = toeplitz_var.evaluate().data self.assertTrue( utils.approx_equal(toeplitz_var[0, 1:3].evaluate().data, evaluated[0, 1:3]))
def test_trace_log_det_quad_form(): mu_diffs_var = Variable(torch.randn(4)) chol_covar_1_var = Variable(torch.eye(4)) # Test case c1_var = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) c2_var = Variable(torch.Tensor([6, 0, 1, -1]), requires_grad=True) actual = ToeplitzLazyVariable(c1_var + c2_var) # Actual case sum_lv = make_sum_lazy_var() t1, t2 = sum_lv.lazy_vars # Test forward tldqf_res = sum_lv.trace_log_det_quad_form(mu_diffs_var, chol_covar_1_var) tldqf_actual = actual.trace_log_det_quad_form(mu_diffs_var, chol_covar_1_var) assert (math.fabs(tldqf_res.data.squeeze()[0] - tldqf_actual.data.squeeze()[0]) < 1.5) # Test backwards tldqf_res.backward() tldqf_actual.backward() assert (math.fabs(c1_var.grad.data[0] - t1.column.grad.data[0]) < 1e-1) assert (math.fabs(c2_var.grad.data[0] - t2.column.grad.data[0]) < 1e-1)
def test_exact_posterior(): train_mean = Variable(torch.randn(4)) train_y = Variable(torch.randn(4)) test_mean = Variable(torch.randn(4)) # Test case c1_var = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) c2_var = Variable(torch.Tensor([6, 0, 1, -1]), requires_grad=True) indices = Variable(torch.arange(0, 4).long().view(4, 1)) values = Variable(torch.ones(4).view(4, 1)) toeplitz_1 = InterpolatedLazyVariable(ToeplitzLazyVariable(c1_var), indices, values, indices, values) toeplitz_2 = InterpolatedLazyVariable(ToeplitzLazyVariable(c2_var), indices, values, indices, values) sum_lv = toeplitz_1 + toeplitz_2 # Actual case actual = sum_lv.evaluate() # Test forward actual_alpha = gpytorch.posterior_strategy(actual).exact_posterior_alpha( train_mean, train_y) actual_mean = gpytorch.posterior_strategy(actual).exact_posterior_mean( test_mean, actual_alpha) sum_lv_alpha = sum_lv.posterior_strategy().exact_posterior_alpha( train_mean, train_y) sum_lv_mean = sum_lv.posterior_strategy().exact_posterior_mean( test_mean, sum_lv_alpha) assert (torch.norm(actual_mean.data - sum_lv_mean.data) < 1e-4)
def test_getitem(): c1_var = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) c2_var = Variable(torch.Tensor([12.5, 2.5, 5, 0]), requires_grad=True) toeplitz_lazy_var = ToeplitzLazyVariable(c1_var) * 2.5 actual = ToeplitzLazyVariable(c2_var) assert torch.norm(actual[2:, 2:].evaluate().data - toeplitz_lazy_var[2:, 2:].evaluate().data) < 1e-3
def test_get_item_square_on_variable(self): toeplitz_var = ToeplitzLazyVariable( Variable(torch.Tensor([1, 2, 3, 4]))) evaluated = toeplitz_var.evaluate().data self.assertTrue( utils.approx_equal(toeplitz_var[2:4, 2:4].evaluate().data, evaluated[2:4, 2:4]))
def make_mul_lazy_var(): diag = Variable(torch.Tensor([1]), requires_grad=True) c1 = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) t1 = ToeplitzLazyVariable(c1) c2 = Variable(torch.Tensor([[6, 0], [1, -1]]), requires_grad=True) t2 = KroneckerProductLazyVariable(c2) c3 = Variable(torch.Tensor([7, 2, 1, 0]), requires_grad=True) t3 = ToeplitzLazyVariable(c3) return (t1 * t2 * t3).add_diag(diag), diag
def forward(self, x1, x2, **kwargs): if not torch.equal(x1.data, self._inducing_points) or \ not torch.equal(x1.data, self._inducing_points): raise RuntimeError( 'The kernel should only receive the inducing points as input') if not self.training and hasattr(self, '_cached_kernel_mat'): return self._cached_kernel_mat else: d = x1.size(1) grid_var = Variable(self.grid) if d > 1: k_UUs = Variable(x1.data.new(d, self.grid_size).zero_()) for i in range(d): k_UUs[i] = self.base_kernel_module(grid_var[i, 0], grid_var[i], **kwargs).squeeze() K_XX = KroneckerProductLazyVariable(k_UUs) else: if gpytorch.functions.use_toeplitz: k_UU = self.base_kernel_module(grid_var[0, 0], grid_var[0], **kwargs).squeeze() K_XX = ToeplitzLazyVariable(k_UU) else: for i in range(100): k_UU = self.base_kernel_module(grid_var[0], grid_var[0], **kwargs).squeeze() K_XX = NonLazyVariable(k_UU) if not self.training: self._cached_kernel_mat = K_XX return K_XX
def forward(self, x1, x2, **kwargs): n, d = x1.size() grid_size = self.grid_size if self.conditioning: J1, C1, J2, C2 = self._compute_grid(x1, x2) self.train_J1 = J1 self.train_C1 = C1 self.train_J2 = J2 self.train_C2 = C2 else: train_data = self.train_inputs[0].data if hasattr( self, 'train_inputs') else None if train_data is not None and torch.equal( x1.data, train_data) and torch.equal(x2.data, train_data): J1 = self.train_J1 C1 = self.train_C1 J2 = self.train_J2 C2 = self.train_C2 else: J1, C1, J2, C2 = self._compute_grid(x1, x2) grid_var = Variable(self.grid) if d > 1: k_UUs = Variable(x1.data.new(d, grid_size).zero_()) for i in range(d): k_UUs[i] = self.base_kernel_module(grid_var[i, 0], grid_var[i], **kwargs).squeeze() K_XX = KroneckerProductLazyVariable(k_UUs, J1, C1, J2, C2) else: k_UU = self.base_kernel_module(grid_var[0, 0], grid_var[0], **kwargs).squeeze() K_XX = ToeplitzLazyVariable(k_UU, J1, C1, J2, C2) return K_XX
def test_inv_matmul(self): labels_var = Variable(torch.randn(4)) grad_output = torch.randn(4) # Test case c1_var = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) c2_var = Variable(torch.Tensor([12.5, 2.5, 5, 0]), requires_grad=True) toeplitz_lazy_var = ToeplitzLazyVariable(c1_var) * 2.5 actual = ToeplitzLazyVariable(c2_var) # Test forward with gpytorch.settings.max_cg_iterations(1000): res = toeplitz_lazy_var.inv_matmul(labels_var) actual = gpytorch.inv_matmul(actual, labels_var) # Test backwards res.backward(grad_output) actual.backward(grad_output) self.assertLess( math.fabs(res.data.squeeze()[0] - actual.data.squeeze()[0]), 6e-1, ) self.assertLess(math.fabs(c1_var.grad.data[0] - c2_var.grad.data[0]), 1)
def test_exact_posterior(): train_mean = Variable(torch.randn(4)) train_y = Variable(torch.randn(4)) test_mean = Variable(torch.randn(4)) # Test case c1_var = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) c2_var = Variable(torch.Tensor([[6, 0], [1, -1]]), requires_grad=True) c3_var = Variable(torch.Tensor([7, 2, 1, 0]), requires_grad=True) indices_1 = torch.arange(0, 4).long().view(4, 1) values_1 = torch.ones(4).view(4, 1) indices_2 = torch.arange(0, 2).expand(4, 2).long().view(2, 4, 1) values_2 = torch.ones(8).view(2, 4, 1) indices_3 = torch.arange(0, 4).long().view(4, 1) values_3 = torch.ones(4).view(4, 1) toeplitz_1 = InterpolatedLazyVariable(ToeplitzLazyVariable(c1_var), Variable(indices_1), Variable(values_1), Variable(indices_1), Variable(values_1)) kronecker_product = KroneckerProductLazyVariable(c2_var, indices_2, values_2, indices_2, values_2) toeplitz_2 = InterpolatedLazyVariable(ToeplitzLazyVariable(c3_var), Variable(indices_3), Variable(values_3), Variable(indices_3), Variable(values_3)) mul_lv = toeplitz_1 * kronecker_product * toeplitz_2 # Actual case actual = mul_lv.evaluate() # Test forward actual_alpha = gpytorch.posterior_strategy(actual).exact_posterior_alpha( train_mean, train_y) actual_mean = gpytorch.posterior_strategy(actual).exact_posterior_mean( test_mean, actual_alpha) mul_lv_alpha = mul_lv.posterior_strategy().exact_posterior_alpha( train_mean, train_y) mul_lv_mean = mul_lv.posterior_strategy().exact_posterior_mean( test_mean, mul_lv_alpha) assert (torch.norm(actual_mean.data - mul_lv_mean.data) < 1e-3)
def test_evaluate(): lazy_toeplitz_var = ToeplitzLazyVariable(Variable(toeplitz_column)) res = lazy_toeplitz_var.evaluate() actual = torch.Tensor([ [2, 0, 4, 1], [0, 2, 0, 4], [4, 0, 2, 0], [1, 4, 0, 2], ]) assert utils.approx_equal(res, actual) lazy_toeplitz_var = ToeplitzLazyVariable(Variable(batch_toeplitz_column)) res = lazy_toeplitz_var.evaluate() actual = torch.Tensor([ [ [2, 0, 4, 1], [0, 2, 0, 4], [4, 0, 2, 0], [1, 4, 0, 2], ], [ [1, 1, -1, 3], [1, 1, 1, -1], [-1, 1, 1, 1], [3, -1, 1, 1], ], ]) assert utils.approx_equal(res, actual)
def test_diag(self): c1_var = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) c2_var = Variable(torch.Tensor([12.5, 2.5, 5, 0]), requires_grad=True) toeplitz_lazy_var = ToeplitzLazyVariable(c1_var) * 2.5 actual = ToeplitzLazyVariable(c2_var) diff = torch.norm(actual.diag() - toeplitz_lazy_var.diag()) self.assertLess(diff, 1e-3)
def test_trace_log_det_quad_form(): mu_diffs_var = Variable(torch.arange(1, 5, 1)) chol_covar_1_var = Variable(torch.eye(4)) # Test case c1_var = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) c2_var = Variable(torch.Tensor([[6, 0], [1, -1]]), requires_grad=True) c3_var = Variable(torch.Tensor([7, 2, 1, 0]), requires_grad=True) diag_var = Variable(torch.Tensor([1]), requires_grad=True) diag_var_expand = diag_var.expand(4) toeplitz_1 = ToeplitzLazyVariable(c1_var).evaluate() kronecker_product = KroneckerProductLazyVariable(c2_var).evaluate() toeplitz_2 = ToeplitzLazyVariable(c3_var).evaluate() actual = toeplitz_1 * kronecker_product * toeplitz_2 + diag_var_expand.diag( ) # Actual case mul_lv, diag = make_mul_lazy_var() t1, t2, t3 = mul_lv.lazy_vars # Test forward tldqf_res = mul_lv.trace_log_det_quad_form(mu_diffs_var, chol_covar_1_var) tldqf_actual = gpytorch._trace_logdet_quad_form_factory_class()( mu_diffs_var, chol_covar_1_var, actual) assert (math.fabs(tldqf_res.data.squeeze()[0] - tldqf_actual.data.squeeze()[0]) < 1.5) # Test backwards tldqf_res.backward() tldqf_actual.backward() assert ((c1_var.grad.data - t1.column.grad.data).abs().norm() / c1_var.grad.data.abs().norm() < 1e-1) assert ((c2_var.grad.data - t2.columns.grad.data).abs().norm() / c2_var.grad.data.abs().norm() < 1e-1) assert ((c3_var.grad.data - t3.column.grad.data).abs().norm() / c3_var.grad.data.abs().norm() < 1e-1) assert ((diag_var.grad.data - diag.grad.data).abs().norm() / diag_var.grad.data.abs().norm() < 1e-1)
def test_diag(): left_interp_indices = Variable( torch.LongTensor([[2, 3], [3, 4], [4, 5]]).repeat(5, 1, 1)) left_interp_values = Variable( torch.Tensor([[1, 1], [1, 1], [1, 1]]).repeat(5, 1, 1)) right_interp_indices = Variable( torch.LongTensor([[0, 1], [1, 2], [2, 3]]).repeat(5, 1, 1)) right_interp_values = Variable( torch.Tensor([[1, 1], [1, 1], [1, 1]]).repeat(5, 1, 1)) # Non-lazy variable base_lazy_variable_mat = torch.randn(5, 6, 6) base_lazy_variable_mat = base_lazy_variable_mat.transpose( 1, 2).matmul(base_lazy_variable_mat) base_lazy_variable = NonLazyVariable( Variable(base_lazy_variable_mat, requires_grad=True)) interp_lazy_var = SumInterpolatedLazyVariable(base_lazy_variable, left_interp_indices, left_interp_values, right_interp_indices, right_interp_values) res = interp_lazy_var.diag() actual = interp_lazy_var.evaluate().diag() assert approx_equal(res, actual) # Toeplitz base_lazy_variable = ToeplitzLazyVariable(Variable(torch.randn(5, 6))) interp_lazy_var = SumInterpolatedLazyVariable(base_lazy_variable, left_interp_indices, left_interp_values, right_interp_indices, right_interp_values) res = interp_lazy_var.diag() actual = interp_lazy_var.evaluate().diag() assert approx_equal(res, actual) # Constant mul base_lazy_variable = base_lazy_variable * Variable(torch.ones(1) * 1.3) interp_lazy_var = SumInterpolatedLazyVariable(base_lazy_variable, left_interp_indices, left_interp_values, right_interp_indices, right_interp_values) res = interp_lazy_var.diag() actual = interp_lazy_var.evaluate().diag() assert approx_equal(res, actual)
def test_exact_gp_mll(): labels_var = Variable(torch.arange(1, 5, 1)) # Test case c1_var = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) c2_var = Variable(torch.Tensor([[6, 0], [1, -1]]), requires_grad=True) c3_var = Variable(torch.Tensor([7, 2, 1, 0]), requires_grad=True) diag_var = Variable(torch.Tensor([1]), requires_grad=True) diag_var_expand = diag_var.expand(4) toeplitz_1 = ToeplitzLazyVariable(c1_var).evaluate() kronecker_product = KroneckerProductLazyVariable(c2_var).evaluate() toeplitz_2 = ToeplitzLazyVariable(c3_var).evaluate() actual = toeplitz_1 * kronecker_product * toeplitz_2 + diag_var_expand.diag( ) # Actual case mul_lv, diag = make_mul_lazy_var() t1, t2, t3 = mul_lv.lazy_vars # Test forward mll_res = mul_lv.exact_gp_marginal_log_likelihood(labels_var) mll_actual = gpytorch.exact_gp_marginal_log_likelihood(actual, labels_var) assert (math.fabs(mll_res.data.squeeze()[0] - mll_actual.data.squeeze()[0]) < 1) # Test backwards mll_res.backward() mll_actual.backward() assert ((c1_var.grad.data - t1.column.grad.data).abs().norm() / c1_var.grad.data.abs().norm() < 1e-1) assert ((c2_var.grad.data - t2.columns.grad.data).abs().norm() / c2_var.grad.data.abs().norm() < 1e-1) assert ((c3_var.grad.data - t3.column.grad.data).abs().norm() / c3_var.grad.data.abs().norm() < 1e-1) assert ((diag_var.grad.data - diag.grad.data).abs().norm() / diag_var.grad.data.abs().norm() < 1e-1)
def forward(self, x1, x2, **kwargs): n, d = x1.size() m, _ = x2.size() if d > 1: raise RuntimeError(' '.join([ 'The grid interpolation kernel can only be applied to inputs of a single dimension at this time \ until Kronecker structure is implemented.' ])) if self.grid is None: raise RuntimeError(' '.join([ 'This GridInterpolationKernel has no grid. Call initialize_interpolation_grid \ on a GPModel first.' ])) both_min = torch.min(x1.min(0)[0].data, x2.min(0)[0].data)[0] both_max = torch.max(x1.max(0)[0].data, x2.max(0)[0].data)[0] if both_min < self.grid_bounds[0] or both_max > self.grid_bounds[1]: # Out of bounds data is still ok if we are specifically computing kernel values for grid entries. if torch.abs(both_min - self.grid[0].data)[0] > 1e-7 or torch.abs( both_max - self.grid[-1].data)[0] > 1e-7: raise RuntimeError( 'Received data that was out of bounds for the specified grid. \ Grid bounds were ({}, {}), but min = {}, max = {}' .format(self.grid_bounds[0], self.grid_bounds[1], both_min, both_max)) J1, C1 = Interpolation().interpolate(self.grid.data, x1.data.squeeze()) J2, C2 = Interpolation().interpolate(self.grid.data, x2.data.squeeze()) k_UU = self.base_kernel_module(self.grid[0], self.grid, **kwargs).squeeze() K_XX = ToeplitzLazyVariable(k_UU, J1, C1, J2, C2) return K_XX
def test_exact_gp_mll(): labels_var = Variable(torch.randn(4)) # Test case c1_var = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) c2_var = Variable(torch.Tensor([12.5, 2.5, 5, 0]), requires_grad=True) toeplitz_lazy_var = ToeplitzLazyVariable(c1_var) * 2.5 actual = ToeplitzLazyVariable(c2_var) # Test forward mll_res = toeplitz_lazy_var.exact_gp_marginal_log_likelihood(labels_var) mll_actual = actual.exact_gp_marginal_log_likelihood(labels_var) assert (math.fabs(mll_res.data.squeeze()[0] - mll_actual.data.squeeze()[0]) < 5e-1) # Test backwards mll_res.backward() mll_actual.backward() assert (math.fabs(c1_var.grad.data[0] - c2_var.grad.data[0]) < 1)
def test_exact_gp_mll(self): labels_var = Variable(torch.randn(4)) # Test case c1_var = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) c2_var = Variable(torch.Tensor([12.5, 2.5, 5, 0]), requires_grad=True) toeplitz_lazy_var = ToeplitzLazyVariable(c1_var) * 2.5 actual = ToeplitzLazyVariable(c2_var) # Test forward with gpytorch.settings.num_trace_samples(1000): mll_res = toeplitz_lazy_var.exact_gp_marginal_log_likelihood(labels_var) mll_actual = actual.exact_gp_marginal_log_likelihood(labels_var) # Test backwards mll_res.backward() mll_actual.backward() self.assertLess( math.fabs(mll_res.data.squeeze()[0] - mll_actual.data.squeeze()[0]), 6e-1, ) self.assertLess(math.fabs(c1_var.grad.data[0] - c2_var.grad.data[0]), 1)
def make_sum_lazy_var(): c1 = Variable(torch.Tensor([5, 1, 2, 0]), requires_grad=True) t1 = ToeplitzLazyVariable(c1) c2 = Variable(torch.Tensor([6, 0, 1, -1]), requires_grad=True) t2 = ToeplitzLazyVariable(c2) return t1 + t2
def test_get_item_scalar_on_batch(): toeplitz_var = ToeplitzLazyVariable(Variable(torch.Tensor([[1, 2, 3, 4]]))) evaluated = toeplitz_var.evaluate().data assert utils.approx_equal(toeplitz_var[0].evaluate().data, evaluated[0])
def test_get_item_square_on_variable(): toeplitz_var = ToeplitzLazyVariable(Variable(torch.Tensor([1, 2, 3, 4])), added_diag=Variable(torch.ones(4) * 3)) evaluated = toeplitz_var.evaluate().data assert utils.approx_equal(toeplitz_var[2:4, 2:4].evaluate().data, evaluated[2:4, 2:4])