def __init__(self, train_x, train_y, likelihood):
super(SpectralMixtureGPModel, self).__init__(train_x, train_y,
likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
self.covar_module = SpectralMixtureKernel(num_mixtures=4,
ard_num_dims=1)
self.covar_module.initialize_from_data(train_x, train_y)
def __init__(self, train_x, train_y, num_mixtures=10):
kernel = SpectralMixtureKernel(num_mixtures)
kernel.initialize_from_data(train_x, train_y)
super(SMKernelGP, self).__init__(kernel, train_x, train_y)
self.mean = gp.means.ConstantMean()
self.cov = kernel
class SpectralMixtureGPModel(gpytorch.models.ExactGP):
def __init__(self, train_x, train_y, likelihood):
super(SpectralMixtureGPModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
self.covar_module = SpectralMixtureKernel(n_mixtures=4)
self.covar_module.initialize_from_data(train_x, train_y)
def forward(self, x):
mean_x = self.mean_module(x)
covar_x = self.covar_module(x)
return GaussianRandomVariable(mean_x, covar_x)
def __init__(self, train_x, train_y, num_mixtures=10):
smk = SpectralMixtureKernel(num_mixtures)
smk.initialize_from_data(train_x, train_y)
kernel = AdditiveKernel(
smk,
PolynomialKernel(2),
RBFKernel(),
)
super(CompositeKernelGP, self).__init__(kernel, train_x, train_y)
self.mean = gp.means.ConstantMean()
self.smk = smk
class SpectralMixtureGPModel(gpytorch.models.ExactGP):
def __init__(self, train_x, train_y, likelihood, empspect=False):
super(SpectralMixtureGPModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ConstantMean(prior=SmoothedBoxPrior(-1, 1))
self.covar_module = SpectralMixtureKernel(num_mixtures=4, ard_num_dims=1)
if empspect:
self.covar_module.initialize_from_data(train_x, train_y)
else:
self.covar_module.initialize_from_data_empspect(train_x, train_y)
def forward(self, x):
mean_x = self.mean_module(x)
covar_x = self.covar_module(x)
return MultivariateNormal(mean_x, covar_x)
def _init_covar_module(self, covar_module):
module = covar_module['type'] if covar_module is not None else 'rbf'
# Index kernel does some scaling, hence, scale kernel is not used
if module == 'rbf':
self.covar_module = RBFKernel(ard_num_dims=self.num_dims)
elif module == 'matern':
self.covar_module = MaternKernel(nu=1.5,
ard_num_dims=self.num_dims)
elif module == 'sm':
self.covar_module = SpectralMixtureKernel(
num_mixtures=covar_module['num_mixtures'],
ard_num_dims=self.num_dims)
elif module == 'kiss':
self.base_covar_module = RBFKernel()
self.covar_module = GridInterpolationKernel(self.base_covar_module,
grid_size=100,
num_dims=self.num_dims)
elif module == 'skip':
self.base_covar_module = RBFKernel()
self.covar_module = ProductStructureKernel(GridInterpolationKernel(
self.base_covar_module, grid_size=100, num_dims=1),
num_dims=self.num_dims)
else:
raise NotImplementedError
def __init__(self, train_x, train_y, likelihood):
super(SpectralMixtureGPModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = ConstantMean(constant_bounds=(-1, 1))
self.covar_module = SpectralMixtureKernel(
n_mixtures=3,
log_mixture_weight_bounds=(-5, 5),
log_mixture_mean_bounds=(-5, 5),
log_mixture_scale_bounds=(-5, 5),
)
def __init__(self):
super(SpectralMixtureGPModel, self).__init__(GaussianLikelihood())
self.mean_module = ConstantMean()
self.covar_module = SpectralMixtureKernel()
self.params = MLEParameterGroup(
log_noise=Parameter(torch.Tensor([-2])),
log_mixture_weights=Parameter(torch.zeros(3)),
log_mixture_means=Parameter(torch.zeros(3)),
log_mixture_scales=Parameter(torch.zeros(3)))
def __init__(self):
likelihood = GaussianLikelihood(log_noise_bounds=(-5, 5))
super(SpectralMixtureGPModel, self).__init__(likelihood)
self.mean_module = ConstantMean(constant_bounds=(-1, 1))
self.covar_module = SpectralMixtureKernel(
n_mixtures=3,
log_mixture_weight_bounds=(-5, 5),
log_mixture_mean_bounds=(-5, 5),
log_mixture_scale_bounds=(-5, 5),
)
def test_standard(self):
a = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1)
b = torch.tensor([0, 2], dtype=torch.float).view(2, 1)
means = [1, 2]
scales = [0.5, 0.25]
weights = [4, 2]
kernel = SpectralMixtureKernel(num_mixtures=2)
kernel.initialize(
log_mixture_weights=torch.tensor([[4, 2]],
dtype=torch.float).log(),
log_mixture_means=torch.tensor([[[[1]], [[2]]]],
dtype=torch.float).log(),
log_mixture_scales=torch.tensor([[[[0.5]], [[0.25]]]],
dtype=torch.float).log(),
)
kernel.eval()
actual = torch.zeros(3, 2)
for i in range(3):
for j in range(2):
for k in range(2):
new_term = torch.cos(2 * math.pi * (a[i] - b[j]) *
means[k])
new_term *= torch.exp(-2 * (math.pi * (a[i] - b[j]))**2 *
scales[k]**2)
new_term *= weights[k]
actual[i, j] += new_term.item()
res = kernel(a, b).evaluate()
self.assertLess(torch.norm(res - actual), 1e-5)
def __init__(self,
train_x,
train_y,
likelihood,
var=None,
latent=None,
kernel_params=None,
latent_params=None):
super(ExactGPModel, self).__init__(train_x, train_y, likelihood)
if latent_params is None:
latent_params = {'input_dim': train_x.size(-1)}
self._set_latent_function(latent, latent_params)
self.mean_module = ZeroMean()
ard_num_dims = self.latent_func.embed_dim if self.latent_func.embed_dim is not None else train_x.size(
-1)
kernel = kernel_params['type'] if kernel_params is not None else 'rbf'
if kernel is None or kernel == 'rbf':
self.kernel_covar_module = ScaleKernel(
RBFKernel(ard_num_dims=ard_num_dims))
elif kernel == 'matern':
self.kernel_covar_module = ScaleKernel(
MaternKernel(nu=1.5, ard_num_dims=ard_num_dims))
# without scale kernel: very poor performance
# matern 0.5, 1.5 and 2.5 all have similar performance
elif kernel == 'spectral_mixture':
self.kernel_covar_module = SpectralMixtureKernel(
num_mixtures=kernel_params['n_mixtures'],
ard_num_dims=train_x.size(-1))
self.kernel_covar_module.initialize_from_data(train_x, train_y)
else:
raise NotImplementedError
# set covariance module
if var is not None:
self.noise_covar_module = WhiteNoiseKernel(var)
self.covar_module = self.kernel_covar_module + self.noise_covar_module
else:
self.covar_module = self.kernel_covar_module
def test_batch_separate(self):
a = torch.tensor([[4, 2, 8], [1, 2, 3]], dtype=torch.float).view(2, 3, 1)
b = torch.tensor([[0, 2, 1], [-1, 2, 0]], dtype=torch.float).view(2, 3, 1)
means = torch.tensor([[1, 2], [2, 3]], dtype=torch.float).view(2, 2, 1, 1)
scales = torch.tensor([[0.5, 0.25], [0.25, 1]], dtype=torch.float).view(2, 2, 1, 1)
weights = torch.tensor([[4, 2], [1, 2]], dtype=torch.float).view(2, 2)
kernel = SpectralMixtureKernel(batch_shape=torch.Size([2]), num_mixtures=2)
kernel.initialize(mixture_weights=weights, mixture_means=means, mixture_scales=scales)
kernel.eval()
actual = torch.zeros(2, 3, 3)
for l in range(2):
for k in range(2):
for i in range(3):
for j in range(3):
new_term = torch.cos(2 * math.pi * (a[l, i] - b[l, j]) * means[l, k])
new_term *= torch.exp(-2 * (math.pi * (a[l, i] - b[l, j])) ** 2 * scales[l, k] ** 2)
new_term *= weights[l, k]
actual[l, i, j] += new_term.item()
res = kernel(a, b).evaluate()
self.assertLess(torch.norm(res - actual), 1e-5)
# diag
res = kernel(a, b).diag()
actual = torch.cat([actual[i].diag().unsqueeze(0) for i in range(actual.size(0))])
self.assertLess(torch.norm(res - actual), 1e-5)
def test_standard(self):
a = torch.tensor([[0, 1], [2, 2], [2, 0]], dtype=torch.float)
means = torch.tensor([[1, 2], [2, 1]], dtype=torch.float)
scales = torch.tensor([[0.5, 0.25], [0.25, 0.5]], dtype=torch.float)
scales = scales.unsqueeze(1)
means = means.unsqueeze(1)
weights = torch.tensor([4, 2], dtype=torch.float)
kernel = SpectralMixtureKernel(num_mixtures=2, ard_num_dims=2)
kernel.initialize(
mixture_weights=weights,
mixture_means=means,
mixture_scales=scales,
)
kernel.eval()
actual = torch.zeros(2, 3, 3, 2)
for i in range(3):
for j in range(3):
for k in range(2):
new_term = torch.cos(2 * math.pi * (a[i] - a[j]) *
means[k])
new_term *= torch.exp(-2 * (math.pi * (a[i] - a[j]))**2 *
scales[k]**2)
actual[k, i, j] = new_term
actual = actual.prod(-1)
actual[0].mul_(weights[0])
actual[1].mul_(weights[1])
actual = actual.sum(0)
res = kernel(a, a).evaluate()
self.assertLess(torch.norm(res - actual), 1e-5)
# diag
res = kernel(a, a).diag()
actual = actual.diag()
self.assertLess(torch.norm(res - actual), 1e-5)
# batch_dims
actual = torch.zeros(2, 3, 3, 2)
for i in range(3):
for j in range(3):
for k in range(2):
new_term = torch.cos(2 * math.pi * (a[i] - a[j]) *
means[k])
new_term *= torch.exp(-2 * (math.pi * (a[i] - a[j]))**2 *
scales[k]**2)
actual[k, i, j] = new_term
actual[0].mul_(weights[0])
actual[1].mul_(weights[1])
actual = actual.sum(0)
actual = actual.permute(2, 0, 1)
res = kernel(a, a, batch_dims=(0, 2)).evaluate()
self.assertLess(torch.norm(res - actual), 1e-5)
# batch_dims + diag
res = kernel(a, a, batch_dims=(0, 2)).diag()
actual = torch.cat(
[actual[i].diag().unsqueeze(0) for i in range(actual.size(0))])
self.assertLess(torch.norm(res - actual), 1e-5)
def __init__(self, train_x, train_y, likelihood):
super(SpectralMixtureGPModel, self).__init__(train_x, train_y,
likelihood)
self.mean_module = ConstantMean(constant_bounds=(-1, 1))
self.covar_module = SpectralMixtureKernel(n_mixtures=4)
self.covar_module.initialize_from_data(train_x, train_y)
def create_sm_2(sigma2, mu, dim):
sm = SpectralMixtureKernel(num_mixtures=1, ard_num_dims=dim)
sm.mixture_weight = 1.
sm.mixture_scales = sigma2 # rescale as in the paper
sm.mixture_means = mu # rescale as in the paper
return sm
class ExactGPModel(gpytorch.models.ExactGP):
def __init__(self,
train_x,
train_y,
likelihood,
var=None,
latent=None,
kernel_params=None,
latent_params=None):
super(ExactGPModel, self).__init__(train_x, train_y, likelihood)
if latent_params is None:
latent_params = {'input_dim': train_x.size(-1)}
self._set_latent_function(latent, latent_params)
self.mean_module = ZeroMean()
ard_num_dims = self.latent_func.embed_dim if self.latent_func.embed_dim is not None else train_x.size(
-1)
kernel = kernel_params['type'] if kernel_params is not None else 'rbf'
if kernel is None or kernel == 'rbf':
self.kernel_covar_module = ScaleKernel(
RBFKernel(ard_num_dims=ard_num_dims))
elif kernel == 'matern':
self.kernel_covar_module = ScaleKernel(
MaternKernel(nu=1.5, ard_num_dims=ard_num_dims))
# without scale kernel: very poor performance
# matern 0.5, 1.5 and 2.5 all have similar performance
elif kernel == 'spectral_mixture':
self.kernel_covar_module = SpectralMixtureKernel(
num_mixtures=kernel_params['n_mixtures'],
ard_num_dims=train_x.size(-1))
self.kernel_covar_module.initialize_from_data(train_x, train_y)
else:
raise NotImplementedError
# set covariance module
if var is not None:
self.noise_covar_module = WhiteNoiseKernel(var)
self.covar_module = self.kernel_covar_module + self.noise_covar_module
else:
self.covar_module = self.kernel_covar_module
def _set_latent_function(self, latent, latent_params):
if latent is None or latent == 'identity':
self.latent_func = IdentityLatentFunction()
elif latent == 'linear':
if 'embed_dim' not in latent_params:
latent_params['embed_dim'] = 6
self.latent_func = LinearLatentFunction(latent_params['input_dim'],
latent_params['embed_dim'])
elif latent == 'non_linear':
if 'embed_dim' not in latent_params:
latent_params['embed_dim'] = 6
self.latent_func = NonLinearLatentFunction(
latent_params['input_dim'], latent_params['embed_dim'],
latent_params['embed_dim'])
else:
raise NotImplementedError
def forward(self, inp):
x = self.latent_func(inp)
mean_x = self.mean_module(x)
covar_x = self.covar_module(x)
return MultivariateNormal(mean_x, covar_x)
def create_kernel(self, num_dims, **kwargs):
return SpectralMixtureKernel(num_mixtures=5,
ard_num_dims=num_dims,
**kwargs)