def __init__(self, train_x, train_y):
variational_distribution = CholeskyVariationalDistribution(train_x.size(0))
variational_strategy = UnwhitenedVariationalStrategy(
self, train_x, variational_distribution, learn_inducing_locations=False
)
super(GPClassificationModel, self).__init__(variational_strategy)
self.mean_module = gpytorch.means.ConstantMean()
kern = gpytorch.kernels.RBFKernel(ard_num_dims=2)
kern.lengthscale = torch.tensor([0.5, 0.1])
self.covar_module = gpytorch.kernels.ScaleKernel(kern)
self.likelihood = gpytorch.likelihoods.BernoulliLikelihood()
self.train()
self.likelihood.train()
def custom_auto_tune_params():
""" this is how we can remove certain params from optimization """
for param in self.named_parameters():
name = param[0]
if name != "covar_module.base_kernel.raw_lengthscale":
yield param[1]
optimizer = torch.optim.Adam(custom_auto_tune_params(), lr=0.1)
mll = gpytorch.mlls.VariationalELBO(self.likelihood, self, train_y.numel())
for i in range(30):
optimizer.zero_grad()
output = self.__call__(train_x)
loss = -mll(output, train_y)
loss.backward()
optimizer.step()
self.eval()
self.likelihood.eval()
def _choose_var_strat(model,
var_strat,
var_dist,
ind_pt,
learn_ind=True,
num_classes=None):
if var_strat == "multi_task":
try:
num_classes = int(num_classes)
except TypeError:
raise RuntimeError(
"Multi-task variational strategy must specify integer num_classes"
)
return gpytorch.variational.MultitaskVariationalStrategy(
VariationalStrategy(model,
ind_pt,
var_dist,
learn_inducing_locations=learn_ind),
num_tasks=num_classes,
task_dim=0,
)
else:
return UnwhitenedVariationalStrategy(
model, ind_pt, var_dist, learn_inducing_locations=learn_ind)
def __init__(self, train_x):
variational_distribution = CholeskyVariationalDistribution(train_x.size(0))
variational_strategy = UnwhitenedVariationalStrategy(
self, train_x, variational_distribution, learn_inducing_locations=False
)
super(GPClassificationModel, self).__init__(variational_strategy)
self.mean_module = gpytorch.means.ConstantMean()
self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel())
def __init__(self, inducing_points, use_fast_strategy):
inducing_points = torch.from_numpy(inducing_points).float()
variational_distribution = CholeskyVariationalDistribution(
inducing_points.size(0))
variational_strategy = UnwhitenedVariationalStrategy(
self,
inducing_points,
variational_distribution,
learn_inducing_locations=True)
super().__init__(variational_strategy)
self.mean_module = ConstantMean()
dims = inducing_points.shape[1]
# self.covar_module = ScaleKernel(MaternKernel(ard_num_dims=dims)) + ScaleKernel(LinearKernel(ard_num_dims=dims))
self.covar_module = ScaleKernel(RBFKernel())
def __init__(self, x_train):
'''
Since exact inference is intractable for GP classification, GPyTorch approximates the classification posterior
using variational inference. We believe that variational inference is ideal for a number of reasons.
Firstly, variational inference commonly relies on gradient descent techniques, which take full advantage
of PyTorch's autograd. This reduces the amount of code needed to develop complex variational models.
Additionally, variational inference can be performed with stochastic gradient decent,
which can be extremely scalable for large datasets.
'''
variational_distribution = CholeskyVariationalDistribution(
x_train.size(0))
# we're using an UnwhitenedVariationalStrategy because we are using the training data as inducing points
variational_strategy = UnwhitenedVariationalStrategy(
self,
inducing_points=x_train,
variational_distribution=variational_distribution,
learn_inducing_locations=True)
super().__init__(variational_strategy)
self.mean = ConstantMean()
self.covar = ScaleKernel(RBFKernel())
def __init__(self,
input_dims,
output_dims,
n_inducing=50,
mean=None,
kernel=None,
collapsed=False):
# Cast in case numpy-type
self.input_dims = int(input_dims)
self.output_dims = int(output_dims)
n_inducing = int(n_inducing)
if output_dims is None or output_dims == 1:
batch_shape = torch.Size([])
else:
batch_shape = torch.Size([self.output_dims])
x_u = torch.randn(n_inducing, self.input_dims)
if collapsed:
assert batch_shape == torch.Size([])
strategy = CollapsedStrategy(self, n_inducing)
else:
variational_dist = CholeskyVariationalDistribution(
n_inducing, batch_shape=batch_shape)
strategy = UnwhitenedVariationalStrategy(
self, x_u, variational_dist, learn_inducing_locations=True)
super().__init__(strategy)
if mean is None:
mean = ZeroMean()
self.mean = mean
if kernel is None:
rbf = RBFKernel(ard_num_dims=input_dims)
kernel = ScaleKernel(rbf)
self.kernel = kernel
self.prior_point_process = SquaredReducingPointProcess(n_inducing)
self.variational_point_process = PoissonPointProcess(n_inducing)
def __init__(self,
inducing_inputs,
kern,
mean=None,
learning_inducing=True,
fix_hyper=False):
variational_distribution = CholeskyVariationalDistribution(
inducing_inputs.size(0))
variational_strategy = UnwhitenedVariationalStrategy(
self,
inducing_inputs,
variational_distribution,
learn_inducing_locations=learning_inducing)
super(SGPClassModel, self).__init__(variational_strategy)
if mean is None:
self.mean_module = gpytorch.means.ConstantMean()
else:
self.mean_module = mean
self.covar_module = kern
if fix_hyper:
[p.requires_grad_(False) for p in self.covar_module.parameters()]
[p.requires_grad_(False) for p in self.mean_module.parameters()]
def __init__(self,
dim: int,
train_X: Tensor,
train_Y: Tensor,
options: dict,
which_type: Optional[str] = "obj") -> None:
variational_distribution = CholeskyVariationalDistribution(
train_X.size(0))
variational_strategy = UnwhitenedVariationalStrategy(
self,
train_X,
variational_distribution,
learn_inducing_locations=False)
super(GPClassifier, self).__init__(variational_strategy)
self.dim = dim
# pdb.set_trace()
if len(train_X) == 0: # No data case
train_X = None
train_Y = None
self.train_inputs = None
self.train_targets = None
self.train_x = None
self.train_yl = None
else:
# Error checking:
assert train_Y.dim() == 1, "train_Y is required to be 1D"
assert train_X.shape[
-1] == self.dim, "Input dimensions do not agree ... (!)"
self.train_inputs = [train_X.clone()]
self.train_targets = train_Y.clone()
self.train_x = train_X.clone()
self.train_yl = torch.cat(
[torch.zeros((len(train_Y)), 1),
train_Y.view(-1, 1)], dim=1)
print("\n")
logger.info("### Initializing GP classifier for constraint g(x) ###")
# Likelihood:
noise_std = options.hyperpars.noise_std.value
self.likelihood = BernoulliLikelihood()
# For compatibility:
self.threshold = torch.tensor([float("Inf")])
# Initialize hyperpriors using scipy because gpytorch's gamma and beta distributions do not have the inverse CDF
hyperpriors = dict(
lengthscales=eval(options.hyperpars.lenthscales.prior),
outputscale=eval(options.hyperpars.outputscale.prior))
# Index hyperparameters:
self.idx_hyperpars = dict(lengthscales=list(range(0, self.dim)),
outputscale=[self.dim])
self.dim_hyperpars = sum(
[len(val) for val in self.idx_hyperpars.values()])
# Get bounds:
self.hyperpars_bounds = self._get_hyperparameters_bounds(hyperpriors)
logger.info("hyperpars_bounds:" + str(self.hyperpars_bounds))
# Initialize prior mean:
# self.mean_module = ConstantMean()
self.mean_module = ZeroMean()
# Initialize covariance function:
base_kernel = MaternKernel(nu=2.5,
ard_num_dims=self.dim,
lengthscale=0.1 * torch.ones(self.dim))
self.covar_module = ScaleKernel(base_kernel=base_kernel)
self.disp_info_scipy_opti = True
# Get a hyperparameter sample within bounds (not the same as sampling from the corresponding priors):
hyperpars_sample = self._sample_hyperparameters_within_bounds(
Nsamples=1).squeeze(0)
self.covar_module.outputscale = hyperpars_sample[
self.idx_hyperpars["outputscale"]]
self.covar_module.base_kernel.lengthscale = hyperpars_sample[
self.idx_hyperpars["lengthscales"]]
self.noise_std = options.hyperpars.noise_std.value # The evaluation noise is fixed, and given by the user
self.Nrestarts = options.hyperpars.optimization.Nrestarts
self._update_hyperparameters()
self.eval()
self.likelihood.eval()