def __init__(self): super(GPClassificationModel, self).__init__(grid_size=32, grid_bounds=[(0, 1)]) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-5, 5)) self.covar_module = ScaleKernel( RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True)), log_outputscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True), )
def __init__(self, train_x, train_y, likelihood): super(GPRegressionModel, self).__init__(train_x, train_y, likelihood) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5)) self.base_covar_module = ScaleKernel( RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True)) ) self.covar_module = GridInterpolationKernel(self.base_covar_module, grid_size=50, grid_bounds=[(0, 1)]) self.feature_extractor = feature_extractor
def __init__(self, train_inputs, train_targets, likelihood): super(ExactGPModel, self).__init__(train_inputs, train_targets, likelihood) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1)) self.rbf_covar_module = RBFKernel( log_lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True) ) self.noise_covar_module = WhiteNoiseKernel(variances=torch.ones(11) * 0.001) self.covar_module = ScaleKernel(self.rbf_covar_module + self.noise_covar_module)
def __init__(self, train_x, train_y, likelihood): super(GPRegressionModel, self).__init__(train_x, train_y, likelihood) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1)) self.base_covar_module = RBFKernel( log_lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True) ) self.covar_module = MultiplicativeGridInterpolationKernel( self.base_covar_module, grid_size=100, grid_bounds=[(0, 1)], n_components=2 )
def __init__(self, train_x, train_y, likelihood): super(GPRegressionModel, self).__init__(train_x, train_y, likelihood) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5)) self.base_covar_module = ScaleKernel( RBFKernel(lengthscale_prior=SmoothedBoxPrior( exp(-5), exp(6), sigma=0.1))) self.covar_module = GridInterpolationKernel(self.base_covar_module, grid_size=50, num_dims=1)
def __init__(self, train_x, train_y, likelihood): super(MultitaskGPModel, self).__init__(train_x, train_y, likelihood) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1)) self.covar_module = RBFKernel(log_lengthscale_prior=SmoothedBoxPrior( exp(-6), exp(6), sigma=0.1, log_transform=True)) self.task_covar_module = IndexKernel(n_tasks=2, rank=1, prior=InverseWishartPrior( nu=2, K=torch.eye(2)))
def __init__(self): super(GPClassificationModel, self).__init__(grid_size=8, grid_bounds=[(0, 3), (0, 3)]) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5)) self.covar_module = ScaleKernel( RBFKernel(ard_num_dims=2, log_lengthscale_prior=SmoothedBoxPrior( exp(-2.5), exp(3), sigma=0.1, log_transform=True)))
def __init__(self, train_x, train_y, likelihood): super(GPRegressionModel, self).__init__(train_x, train_y, likelihood) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1)) self.base_covar_module = ScaleKernel( RBFKernel(log_lengthscale_prior=SmoothedBoxPrior( exp(-3), exp(3), sigma=0.1, log_transform=True))) self.covar_module = ProductStructureKernel(GridInterpolationKernel( self.base_covar_module, grid_size=100, num_dims=2), num_dims=2)
def __init__(self, train_x, train_y, likelihood): super(GPRegressionModel, self).__init__(train_x, train_y, likelihood) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5)) self.base_covar_module = RBFKernel( log_lengthscale_prior=SmoothedBoxPrior( exp(-5), exp(6), sigma=0.1, log_transform=True)) self.covar_module = InducingPointKernel(self.base_covar_module, inducing_points=torch.linspace( 0, 1, 32))
def __init__(self, train_x, train_y, likelihood): super(GPRegressionModel, self).__init__(train_x, train_y, likelihood) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5)) self.base_covar_module = RBFKernel( log_lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True) ) self.grid_covar_module = GridInterpolationKernel(self.base_covar_module, grid_size=50, grid_bounds=[(0, 1)]) self.noise_covar_module = WhiteNoiseKernel(variances=torch.ones(100) * 0.001) self.covar_module = self.grid_covar_module + self.noise_covar_module
def __init__(self, train_x): super(GPClassificationModel, self).__init__(train_x) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5)) self.covar_module = ScaleKernel( RBFKernel(log_lengthscale_prior=SmoothedBoxPrior( exp(-5), exp(6), sigma=0.1, log_transform=True)), log_outputscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True), )
def __init__(self, grid_size=6, grid_bounds=[(-0.33, 1.33), (-0.33, 1.33)]): variational_distribution = gpytorch.variational.CholeskyVariationalDistribution( num_inducing_points=int(pow(grid_size, len(grid_bounds))) ) variational_strategy = gpytorch.variational.GridInterpolationVariationalStrategy( self, grid_size=grid_size, grid_bounds=grid_bounds, variational_distribution=variational_distribution ) super(GPClassificationModel, self).__init__(variational_strategy) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5)) self.covar_module = ScaleKernel( RBFKernel(ard_num_dims=2, lengthscale_prior=SmoothedBoxPrior(exp(-2.5), exp(3), sigma=0.1)) )
def __init__(self, train_x): super(GPClassificationModel, self).__init__(train_x) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5)) self.covar_module = RBFKernel(log_lengthscale_prior=SmoothedBoxPrior( exp(-5), exp(6), sigma=0.1, log_transform=True)) self.register_parameter( name="log_outputscale", parameter=torch.nn.Parameter(torch.Tensor([0])), prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True), )
def __init__(self, grid_size=20, grid_bounds=[(-0.1, 1.1)]): variational_distribution = gpytorch.variational.CholeskyVariationalDistribution( num_inducing_points=int(pow(grid_size, len(grid_bounds)))) variational_strategy = gpytorch.variational.GridInterpolationVariationalStrategy( self, grid_size=grid_size, grid_bounds=grid_bounds, variational_distribution=variational_distribution) super(GPRegressionModel, self).__init__(variational_strategy) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-10, 10)) self.covar_module = ScaleKernel( RBFKernel(lengthscale_prior=SmoothedBoxPrior( exp(-3), exp(6), sigma=0.1)))
def __init__(self): super(GPRegressionModel, self).__init__(grid_size=20, grid_bounds=[(-0.05, 1.05)]) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-10, 10)) self.covar_module = RBFKernel(log_lengthscale_prior=SmoothedBoxPrior( exp(-3), exp(6), sigma=0.1, log_transform=True)) self.register_parameter( name="log_outputscale", parameter=torch.nn.Parameter(torch.Tensor([0])), prior=SmoothedBoxPrior(exp(-5), exp(1), sigma=0.1, log_transform=True), )
def __init__(self, grid_size=16, grid_bounds=([-1, 1],)): variational_distribution = CholeskyVariationalDistribution(num_inducing_points=16, batch_size=2) variational_strategy = AdditiveGridInterpolationVariationalStrategy( self, grid_size=grid_size, grid_bounds=grid_bounds, num_dim=2, variational_distribution=variational_distribution, ) super(GPClassificationModel, self).__init__(variational_strategy) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1e-5, 1e-5)) self.covar_module = ScaleKernel( RBFKernel(ard_num_dims=1, lengthscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1)), outputscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1), )
def __init__(self, train_x, train_y, likelihood): super(SpectralMixtureGPModel, self).__init__(train_x, train_y, likelihood) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1)) self.covar_module = SpectralMixtureKernel(num_mixtures=4, ard_num_dims=1) self.covar_module.initialize_from_data(train_x, train_y)
def test_lkj_covariance_prior_validate_args(self): sd_prior = SmoothedBoxPrior(exp(-1), exp(1), validate_args=True) LKJCovariancePrior(2, 1.0, sd_prior) with self.assertRaises(ValueError): LKJCovariancePrior(1.5, 1.0, sd_prior, validate_args=True) with self.assertRaises(ValueError): LKJCovariancePrior(2, -1.0, sd_prior, validate_args=True)
def __init__(self, grid_size, grid_bounds, n_components, mixing_params=False, sum_output=True): super(AdditiveGridInducingVariationalGP, self).__init__(grid_size, grid_bounds) self.n_components = n_components self.sum_output = sum_output # Resize variational parameters to have one size per component variational_mean = self.variational_mean chol_variational_covar = self.chol_variational_covar variational_mean.data.resize_(*([n_components] + list(variational_mean.size()))) chol_variational_covar.data.resize_( *([n_components] + list(chol_variational_covar.size()))) # Mixing parameters if mixing_params: self.register_parameter( name="mixing_params", parameter=torch.nn.Parameter( torch.ones(n_components) / n_components), prior=SmoothedBoxPrior(-2, 2, sigma=0.01, size=n_components), )
def test_lkj_covariance_prior_to_gpu(self): if torch.cuda.is_available(): sd_prior = SmoothedBoxPrior(exp(-1), exp(1)) prior = LKJCovariancePrior(2, 1.0, sd_prior).cuda() self.assertEqual(prior.correlation_prior.eta.device.type, "cuda") self.assertEqual(prior.correlation_prior.C.device.type, "cuda") self.assertEqual(prior.sd_prior.a.device.type, "cuda")
def __init__(self, train_x, train_y, likelihood): super(GPRegressionModel, self).__init__(train_x, train_y, likelihood) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1)) self.base_covar_module = RBFKernel(ard_num_dims=2) self.covar_module = GridInterpolationKernel(self.base_covar_module, grid_size=16, num_dims=2)
def test_lkj_covariance_prior_log_prob_hetsd(self, cuda=False): device = torch.device("cuda") if cuda else torch.device("cpu") a = torch.tensor([exp(-1), exp(-2)], device=device) b = torch.tensor([exp(1), exp(2)], device=device) sd_prior = SmoothedBoxPrior(a, b, log_transform=True) prior = LKJCovariancePrior(2, torch.tensor(0.5, device=device), sd_prior) self.assertFalse(prior.log_transform) S = torch.eye(2, device=device) self.assertAlmostEqual(prior.log_prob(S).item(), -4.71958, places=4) S = torch.stack( [S, torch.tensor([[1.0, 0.5], [0.5, 1]], device=S.device)]) self.assertTrue( approx_equal(prior.log_prob(S), torch.tensor([-4.71958, -4.57574], device=S.device))) with self.assertRaises(ValueError): prior.log_prob(torch.eye(3, device=device)) # For eta=1.0 log_prob is flat over all covariance matrices prior = LKJCovariancePrior(2, torch.tensor(1.0, device=device), sd_prior) marginal_sd = torch.diagonal(S, dim1=-2, dim2=-1).sqrt() log_prob_expected = prior.correlation_prior.C + prior.sd_prior.log_prob( marginal_sd) self.assertTrue(approx_equal(prior.log_prob(S), log_prob_expected))
def __init__(self, train_x, train_y, likelihood): super(GPRegressionModel, self).__init__(train_x, train_y, likelihood) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1)) self.base_covar_module = ScaleKernel(RBFKernel()) self.covar_module = ProductStructureKernel(GridInterpolationKernel( self.base_covar_module, grid_size=100, num_dims=1), num_dims=2)
def __init__(self, train_x, train_y, likelihood): super(GPRegressionModel, self).__init__(train_x, train_y, likelihood) self.mean_module = ZeroMean() self.base_covar_module = ScaleKernel( RBFKernel(log_lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1, log_transform=True)) ) self.covar_module = AdditiveGridInterpolationKernel( self.base_covar_module, grid_size=100, grid_bounds=[(-0.5, 1.5)], n_components=2 )
def test_posterior_latent_gp_and_likelihood_fast_pred_var( self, cuda=False): train_x, test_x, train_y, test_y = self._get_data(cuda=cuda) with gpytorch.settings.fast_pred_var(), gpytorch.settings.debug(False): # We're manually going to set the hyperparameters to # something they shouldn't be likelihood = GaussianLikelihood( noise_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1)) gp_model = ExactGPModel(train_x, train_y, likelihood) mll = gpytorch.mlls.ExactMarginalLogLikelihood( likelihood, gp_model) gp_model.covar_module.base_kernel.initialize(lengthscale=exp(1)) gp_model.mean_module.initialize(constant=0) likelihood.initialize(noise=exp(1)) if cuda: gp_model.cuda() likelihood.cuda() # Find optimal model hyperparameters gp_model.train() likelihood.train() optimizer = optim.Adam(list(gp_model.parameters()) + list(likelihood.parameters()), lr=0.1) optimizer.n_iter = 0 for _ in range(50): optimizer.zero_grad() output = gp_model(train_x) loss = -mll(output, train_y) loss.backward() optimizer.n_iter += 1 optimizer.step() for param in gp_model.parameters(): self.assertTrue(param.grad is not None) self.assertGreater(param.grad.norm().item(), 0) for param in likelihood.parameters(): self.assertTrue(param.grad is not None) self.assertGreater(param.grad.norm().item(), 0) optimizer.step() # Test the model gp_model.eval() likelihood.eval() # Set the cache test_function_predictions = likelihood(gp_model(train_x)) # Now bump up the likelihood to something huge # This will make it easy to calculate the variance likelihood.noise_covar.raw_noise.data.fill_(3) test_function_predictions = likelihood(gp_model(train_x)) noise = likelihood.noise_covar.noise var_diff = (test_function_predictions.variance - noise).abs() self.assertLess(torch.max(var_diff / noise), 0.05)
def test_posterior_latent_gp_and_likelihood_without_optimization( self, cuda=False): train_x, test_x, train_y, test_y = self._get_data(cuda=cuda) with gpytorch.settings.debug(False): # We're manually going to set the hyperparameters to be ridiculous likelihood = GaussianLikelihood( noise_prior=SmoothedBoxPrior(exp(-10), exp(10), sigma=0.25), noise_constraint=Positive(), ) gp_model = ExactGPModel(train_x, train_y, likelihood) # Update lengthscale prior to accommodate extreme parameters gp_model.rbf_covar_module.register_prior( "lengthscale_prior", SmoothedBoxPrior(exp(-10), exp(10), sigma=0.5), "raw_lengthscale") gp_model.rbf_covar_module.initialize(lengthscale=exp(-10)) gp_model.mean_module.initialize(constant=0) likelihood.initialize(noise=exp(-10)) if cuda: gp_model.cuda() likelihood.cuda() # Compute posterior distribution gp_model.eval() likelihood.eval() # Let's see how our model does, conditioned with weird hyperparams # The posterior should fit all the data function_predictions = likelihood(gp_model(train_x)) self.assertLess(torch.norm(function_predictions.mean - train_y), 1e-3) self.assertLess(torch.norm(function_predictions.variance), 5e-3) # It shouldn't fit much else though test_function_predictions = gp_model( torch.tensor([1.1]).type_as(test_x)) self.assertLess(torch.norm(test_function_predictions.mean - 0), 1e-4) self.assertLess( torch.norm(test_function_predictions.variance - gp_model.covar_module.outputscale), 1e-4)
def __init__(self, train_x, train_y, likelihood): super(GPRegressionModel, self).__init__(train_x, train_y, likelihood) self.mean_module = ZeroMean() self.base_covar_module = ScaleKernel( RBFKernel(ard_num_dims=2, lengthscale_prior=SmoothedBoxPrior(exp(-3), exp(3), sigma=0.1))) self.covar_module = AdditiveStructureKernel(GridInterpolationKernel( self.base_covar_module, grid_size=100, num_dims=2), num_dims=2)
def test_lkj_covariance_prior_batch_log_prob(self, cuda=False): device = torch.device("cuda") if cuda else torch.device("cpu") v = torch.ones(2, 1, device=device) sd_prior = SmoothedBoxPrior(exp(-1) * v, exp(1) * v) prior = LKJCovariancePrior(2, torch.tensor([0.5, 1.5], device=device), sd_prior) corr_dist = LKJCholesky(2, torch.tensor([0.5, 1.5], device=device)) S = torch.eye(2, device=device) dist_log_prob = corr_dist.log_prob(S) + sd_prior.log_prob(S.diag()) self.assertLessEqual((prior.log_prob(S) - dist_log_prob).abs().sum(), 1e-4) S = torch.stack( [S, torch.tensor([[1.0, 0.5], [0.5, 1]], device=S.device)]) S_chol = torch.linalg.cholesky(S) dist_log_prob = corr_dist.log_prob(S_chol) + sd_prior.log_prob( torch.diagonal(S, dim1=-2, dim2=-1)) self.assertLessEqual((prior.log_prob(S) - dist_log_prob).abs().sum(), 1e-4)
def test_smoothed_box_prior_to_gpu(self): if torch.cuda.is_available(): prior = SmoothedBoxPrior(torch.zeros(2), torch.ones(2)).cuda() self.assertEqual(prior.a.device.type, "cuda") self.assertEqual(prior.b.device.type, "cuda") self.assertEqual(prior.sigma.device.type, "cuda") self.assertEqual(prior._c.device.type, "cuda") self.assertEqual(prior._r.device.type, "cuda") self.assertEqual(prior._M.device.type, "cuda") self.assertEqual(prior.tails.loc.device.type, "cuda") self.assertEqual(prior.tails.scale.device.type, "cuda")
def __init__(self): variational_distribution = gpytorch.variational.CholeskyVariationalDistribution( 16) variational_strategy = gpytorch.variational.VariationalStrategy( self, torch.randn(16, 1), variational_distribution, learn_inducing_locations=True) super(GPClassificationModel, self).__init__(variational_strategy) self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-5, 5)) self.covar_module = ScaleKernel( RBFKernel(log_lengthscale_prior=SmoothedBoxPrior( exp(-5), exp(6), sigma=0.1, log_transform=True)), log_outputscale_prior=SmoothedBoxPrior(exp(-5), exp(6), sigma=0.1, log_transform=True), ) self.covar_module.base_kernel.initialize(log_lengthscale=-1)