def _parse_mean(input_size: int,
dim_outputs: int = 1,
kind: str = 'zero') -> Mean:
"""Parse Mean string.
Parameters
----------
input_size: int.
Size of input to GP (needed for linear mean functions).
kind: str.
String that identifies mean function.
Returns
-------
mean: Mean.
Mean function.
"""
if kind.lower() == 'constant':
mean = ConstantMean()
elif kind.lower() == 'zero':
mean = ZeroMean()
elif kind.lower() == 'linear':
mean = LinearMean(input_size=input_size,
batch_shape=torch.Size([dim_outputs]))
else:
raise NotImplementedError(
'Mean function {} not implemented.'.format(kind))
return mean
def __init__(self,
input_dims,
output_dims,
num_inducing=128,
mean_type='constant'):
if output_dims is None:
inducing_points = torch.randn(num_inducing, input_dims)
batch_shape = torch.Size([])
else:
inducing_points = torch.randn(output_dims, num_inducing,
input_dims)
batch_shape = torch.Size([output_dims])
variational_distribution = CholeskyVariationalDistribution(
num_inducing_points=num_inducing, batch_shape=batch_shape)
variational_strategy = VariationalStrategy(
self,
inducing_points,
variational_distribution,
learn_inducing_locations=True)
super(ApproximateDeepGPHiddenLayer,
self).__init__(variational_strategy, input_dims, output_dims)
if mean_type == 'constant':
self.mean_module = ConstantMean(batch_shape=batch_shape)
else:
self.mean_module = LinearMean(input_dims)
self.covar_module = ScaleKernel(RBFKernel(batch_shape=batch_shape,
ard_num_dims=input_dims),
batch_shape=batch_shape,
ard_num_dims=None)
self.linear_layer = Linear(input_dims, 1)
def __init__(self,
input_dims,
output_dims,
num_inducing=128,
mean_type="constant"):
if output_dims is None:
inducing_points = torch.randn(num_inducing, input_dims)
batch_shape = torch.Size([])
else:
inducing_points = torch.randn(output_dims, num_inducing,
input_dims)
batch_shape = torch.Size([output_dims])
variational_distribution = CholeskyVariationalDistribution(
num_inducing_points=num_inducing, batch_shape=batch_shape)
variational_strategy = VariationalStrategy(
self,
inducing_points,
variational_distribution,
learn_inducing_locations=True)
super().__init__(variational_strategy,
input_dims=input_dims,
output_dims=output_dims)
if mean_type == "constant":
self.mean = ConstantMean(batch_shape=batch_shape)
else:
self.mean = LinearMean(input_dims)
self.covar = ScaleKernel(RBFKernel(ard_num_dims=input_dims,
batch_shape=batch_shape),
batch_shape=batch_shape,
ard_num_dims=None)
def create_mean(self,
input_size=1,
batch_shape=torch.Size(),
bias=True,
**kwargs):
return LinearMean(input_size=input_size,
batch_shape=batch_shape,
bias=bias)
def __init__(self,
mean_list,
kernel_list,
num_points=100,
num_samples=1000,
amplitude_range=(-5., 5.)):
self.mean_list = mean_list
self.kernel_list = kernel_list
self.num_config = len(mean_list) * len(kernel_list)
self.num_samples = num_samples
self.num_points = num_points
self.x_dim = 1 # x and y dim are fixed for this dataset.
self.y_dim = 1
self.amplitude_range = amplitude_range
self.data = []
# initialize likelihood and model
x = torch.linspace(self.amplitude_range[0], self.amplitude_range[1],
num_points).unsqueeze(1)
likelihood = gpytorch.likelihoods.GaussianLikelihood()
mean_dict = {'constant': ConstantMean(), 'linear': LinearMean(1)}
kernel_dict = {
'RBF': RBFKernel(),
'cosine': CosineKernel(),
'linear': LinearKernel(),
'periodic': PeriodicKernel(period_length=0.5),
'LCM': LCMKernel(base_kernels=[CosineKernel()], num_tasks=1),
'polynomial': PolynomialKernel(power=2),
'matern': MaternKernel()
}
# create a different GP from each possible configuration
for mean in self.mean_list:
for kernel in self.kernel_list:
# evaluate GP on prior distribution
with gpytorch.settings.prior_mode(True):
model = ExactGPModel(x,
None,
likelihood,
mean_module=mean_dict[mean],
kernel_module=kernel_dict[kernel])
gp = model(x)
# sample from current configuration
for i in range(num_samples // self.num_config + 1):
y = gp.sample()
self.data.append(
(x, y.unsqueeze(1))) #+torch.randn(y.shape)*0))
def __init__(self,
input_dims,
output_dims,
num_inducing=128,
mean_type='constant'):
if output_dims is None:
inducing_points = torch.randn(num_inducing, input_dims)
batch_shape = torch.Size([])
else:
inducing_points = torch.randn(output_dims, num_inducing,
input_dims)
batch_shape = torch.Size([output_dims])
variational_distribution = CholeskyVariationalDistribution(
num_inducing_points=num_inducing, batch_shape=batch_shape)
variational_strategy = VariationalStrategy(
self,
inducing_points,
variational_distribution,
learn_inducing_locations=True)
super(DGPHiddenLayer, self).__init__(variational_strategy, input_dims,
output_dims)
if mean_type == 'constant':
self.mean_module = ConstantMean(batch_shape=batch_shape)
else: # (if 'linear')
self.mean_module = LinearMean(input_dims)
#lengthscale_constraint = gpytorch.constraints.Interval(0.0001, 10.0) # needs to be floats
lengthscale_prior = gpytorch.priors.NormalPrior(0.1, 2.0)
outputscale_prior = gpytorch.priors.NormalPrior(1.0, 3.0)
lengthscale_constraint = None
#lengthscale_prior = None
self.covar_module = ScaleKernel(
RBFKernel(
batch_shape=batch_shape,
ard_num_dims=input_dims,
#active_dims=(0),
lengthscale_constraint=lengthscale_constraint,
lengthscale_prior=lengthscale_prior),
outputscale_prior=outputscale_prior,
batch_shape=batch_shape,
ard_num_dims=input_dims)
def __init__(self,
input_dims,
output_dims,
num_inducing=300,
inducing_points=None,
mean_type="constant",
Q=8):
if output_dims is None:
# An output_dims of None implies there is only one GP in this layer
# (e.g., the last layer for univariate regression).
inducing_points = torch.randn(num_inducing, input_dims)
else:
inducing_points = torch.randn(output_dims, num_inducing,
input_dims)
# Let's use mean field / diagonal covariance structure.
variational_distribution = MeanFieldVariationalDistribution(
num_inducing_points=num_inducing,
batch_shape=torch.Size([output_dims])
if output_dims is not None else torch.Size([]),
)
# Standard variational inference.
variational_strategy = VariationalStrategy(
self,
inducing_points,
variational_distribution,
learn_inducing_locations=True)
batch_shape = torch.Size([]) if output_dims is None else torch.Size(
[output_dims])
super().__init__(variational_strategy, input_dims, output_dims, Q)
if mean_type == "constant":
self.mean_module = ConstantMean(batch_shape=batch_shape)
elif mean_type == "linear":
self.mean_module = LinearMean(input_dims, batch_shape=batch_shape)
self.covar_module = ScaleKernel(MaternKernel(batch_shape=batch_shape,
ard_num_dims=input_dims),
batch_shape=batch_shape,
ard_num_dims=None)
def __init__(self, x_train, y_train, likelihood):
""" Takes training data and likelihood and constructs objects neccessary
for 'forward' module. Commonly mean and kernel module. """
super(GP, self).__init__(x_train, y_train, likelihood)
#self.mean_module = gpytorch.means.ConstantMean()
self.mean_module = LinearMean(2)
# prior_ls = gpytorch.priors.NormalPrior(3, 3)
# prior_os = gpytorch.priors.NormalPrior(4, 3)
# self.covar_module = gpytorch.kernels.ScaleKernel(
# gpytorch.kernels.RBFKernel(lengthscale_prior=prior_ls),
# outputscale_prior=prior_os)
lengthscale_prior = gpytorch.priors.NormalPrior(0.1, 2.0)
outputscale_prior = gpytorch.priors.NormalPrior(1.0, 3.0)
lengthscale_constraint = None
self.covar_module = ScaleKernel(RBFKernel(
lengthscale_constraint=lengthscale_constraint,
lengthscale_prior=lengthscale_prior),
outputscale_prior=outputscale_prior)
def get_mean_and_kernel(kernel,
mean_type,
shape,
input_size,
is_composite=False,
matern_nu=2.5):
"""Utility function to extract mean and kernel for GPR model.
Parameters:
kernel (str): Type of kernel to use for optimization. Defaults to "
"Matern kernel ('matern'). Other options include RBF
(Radial Basis Function)/SE (Squared Exponential) ('rbf'), and RQ
(Rational Quadratic) ('rq').
mean_type (str): Type of mean function to use for Gaussian Process.
Defaults to zero mean ('zero'). Other options: linear
('linear'), and constant ('constant').")
shape (int): The batch shape used for creating this BatchedGP model.
This corresponds to the number of samples we wish to interpolate.
input_size (int): If using a linear mean (else not applicable), this
is the number of X dimensions.
is_composite (bool): Whether we are constructing means and kernels for
a composite GPR kernel. If True, returns two kernel objects (whose
attributes are then later set). Defaults to False.
matern_nu (float): Value in set if {1/2, 3/2, 5/2} that denotes the power
to raise the matern kernel evaluation to. Smaller values allow for
greater discontinuity. Only relevant if kernel is matern. Defaults to
2.5.
Returns:
M (gpytorch.means.Mean): Mean function object for the GPyTorch model.
K (gptorch.kernels.Kernel): Kernel function object for the GPyTorch model.
"""
# Create tensor for size
batch_shape = torch.Size([shape])
# Determine mean type
if mean_type == "zero":
M = ZeroMean(batch_shape=batch_shape)
elif mean_type == "constant":
M = ConstantMean(batch_shape=batch_shape)
elif mean_type == "linear":
M = LinearMean(input_size, batch_shape=batch_shape)
else:
raise Exception("Please select a valid mean type for the GPR. "
"Choices are {'zero', 'constant', 'linear'}.")
# Determine kernel type
if kernel == "matern":
K = MaternKernel
elif kernel == "rbf":
K = RBFKernel
elif kernel == "rbf_grad":
K = RBFKernelGrad
elif kernel == "rq":
K = RQKernel
else:
raise Exception("Please select a valid kernel for the GPR. "
"Choices are {'matern', 'rbf', 'rq'}.")
# Determine what extra parameters to return
kwargs = {}
if kernel == "matern":
kwargs["nu"] = matern_nu
# Return means and kernels
if is_composite:
return M, K, K, kwargs
else:
return M, K, kwargs
def __init__(self,
input_dims,
output_dims,
num_inducing=128,
mean_type='constant'):
# FOR VARIATIONAL INFERENCE: CREATE INDUCING POINTS DRAWN FROM N(0,1)
if output_dims is None:
print("num_inducing:", num_inducing)
print("input_dims:", input_dims)
inducing_points = torch.randn(num_inducing, input_dims)
batch_shape = torch.Size([])
else:
inducing_points = torch.randn(output_dims, num_inducing,
input_dims)
batch_shape = torch.Size([output_dims])
# INITALIZE VARIATIONAL DISTRUBUTION
# The distrubution used for approximation of true posterior distrubution.
# Cholesky has a full mean vector of size num_induxing and a full covariance
# matrix of size num_inducing * num_inducing. These are learning during training.
variational_distribution = CholeskyVariationalDistribution(
num_inducing_points=num_inducing, batch_shape=batch_shape)
# INITIALIZE VARIATIONAL STRATEGY
# Variational strategy wrapper for variational distrubution above.
variational_strategy = VariationalStrategy(
self,
inducing_points,
variational_distribution,
learn_inducing_locations=True)
# Call the DeepGPLayer of GPyTorch do initalize the real class for DGPs.
super(DGPHiddenLayer, self).__init__(variational_strategy, input_dims,
output_dims)
# INITALIZE MEAN
# The mean module to be used. A true Gaussian is often times constant in it's output.
if mean_type == 'constant':
self.mean_module = ConstantMean(
batch_shape=batch_shape
) # batch_shape so it knows the dimensions
else: # (if 'linear')
self.mean_module = LinearMean(input_dims)
# INITIALIZE KERNEL
# RBF has no scaling, so wrap it with a ScaleKernel with constant k, that is
# kernel = k * kernel_rbf. Can make constraints and priors for parameters as well.
# It's probobly a good idea to set a prior since we normalize the data and have a
# prior belief about them since we can observe the training data that has a certain appearance.
# The question is what to set them to. One might have them free to begin with and note
# what lengthscales turn out good and then constrain to them to get faster convergence
# for future training.
#lengthscale_constraint = gpytorch.constraints.Interval(0.0001, 10.0) # needs to be floats
lengthscale_prior = gpytorch.priors.NormalPrior(0.5, 3.0)
lengthscale_constraint = None
#lengthscale_prior = None
self.covar_module = ScaleKernel(
RBFKernel(
batch_shape=
batch_shape, # to set separate lengthscale for each eventuall batch
ard_num_dims=input_dims,
#active_dims=(0), # set input dims to compute covariance for, tuple of ints corresponding to indices of dimensions
lengthscale_constraint=lengthscale_constraint,
lengthscale_prior=lengthscale_prior),
batch_shape=batch_shape, # for ScaleKernel
ard_num_dims=None) # for ScaleKernel