def GP_prob(K, X, Y, parallel_updates=True, method="EP"):
#lik = GPy.likelihoods.Bernoulli()
#m = GPy.models.GPClassification(X=X,
# Y=Y,
# kernel=CustomMatrix(X.shape[1],X,K),
# # inference_method=GPy.inference.latent_function_inference.PEP(alpha = 1), #only for regression apparently
# inference_method=GPy.inference.latent_function_inference.expectation_propagation.EP(),
# likelihood=lik)
lik = GPy.likelihoods.Bernoulli()
if method == "Laplace":
print("USING LAPLACE")
inference_method = GPy.inference.latent_function_inference.laplace.Laplace(
)
elif method == "EP":
print("USING EP")
inference_method = GPy.inference.latent_function_inference.expectation_propagation.EP(
parallel_updates=parallel_updates)
m = GPy.core.GP(
X=X,
Y=Y,
kernel=CustomMatrix(X.shape[1], X, K),
inference_method=inference_method,
#inference_method=GPy.inference.latent_function_inference.PEP(alpha = 0.5),
# inference_method=GPy.inference.latent_function_inference.expectation_propagation.EP(parallel_updates=True,epsilon=1e-5),
# inference_method=GPy.inference.latent_function_inference.expectation_propagation.EP(parallel_updates=False),
likelihood=lik)
# m.likelihood = lik
#m.inference_method = GPy.inference.latent_function_inference.PEP(alpha = 0.5)
return m.log_likelihood()
def nngp_mse_heaviside_posteror_params(Xtrain,Ytrain,Xtest,Kfull):
#find out the analytical posterior
Xfull = np.concatenate([Xtrain,Xtest])
inference_method = GPy.inference.latent_function_inference.exact_gaussian_inference.ExactGaussianInference()
lik = GPy.likelihoods.gaussian.Gaussian(variance=0.002)
kernel = CustomMatrix(Xfull.shape[1],Xfull,Kfull)
gp_model = GPy.core.GP(X=Xtrain,Y=Ytrain,kernel=kernel,inference_method=inference_method, likelihood=lik)
mean, cov = gp_model.predict(Xtest,full_cov=True)
return mean, cov
def GP_prob(K, X, Y):
m = gpflow.models.GPMC(
X.astype(np.float64),
Y,
kern=CustomMatrix(X.shape[1], X, K),
# kern=gpflow.kernels.RBF(28*28),
likelihood=gpflow.likelihoods.Bernoulli(),
)
m.compile()
# print(m)
return m.compute_log_likelihood()
def nngp_mse_heaviside_posteror_logp(Xtest,Ytest,mean,cov):
#use EP approximation to estimate probability of binary labelling on test set.
linkfun = GPy.likelihoods.link_functions.Heaviside()
lik = GPy.likelihoods.Bernoulli(linkfun)
inference_method = GPy.inference.latent_function_inference.expectation_propagation.EP(parallel_updates=False)
m = GPy.core.GP(X=Xtest,
Y=Ytest,
kernel=CustomMatrix(Xtest.shape[1],Xtest,cov),
inference_method=inference_method,
mean_function=CustomMean(Xtest,mean),
likelihood=lik)
# return m.log_prior()
return m.log_likelihood()
def GP_prob(K,X,Y,parallel_updates=True,method="EP", using_exactPB=False):
#lik = GPy.likelihoods.Bernoulli()
#m = GPy.models.GPClassification(X=X,
# Y=Y,
# kernel=CustomMatrix(X.shape[1],X,K),
# # inference_method=GPy.inference.latent_function_inference.PEP(alpha = 1), #only for regression apparently
# inference_method=GPy.inference.latent_function_inference.expectation_propagation.EP(),
# likelihood=lik)
lik = GPy.likelihoods.Bernoulli()
if method=="Laplace":
print("USING LAPLACE")
inference_method = GPy.inference.latent_function_inference.laplace.Laplace()
elif method == "EP":
print("USING EP")
inference_method = GPy.inference.latent_function_inference.expectation_propagation.EP(parallel_updates=parallel_updates)
m = GPy.core.GP(X=X,
Y=Y,
kernel=CustomMatrix(X.shape[1],X,K),
inference_method=inference_method,
#inference_method=GPy.inference.latent_function_inference.PEP(alpha = 0.5),
# inference_method=GPy.inference.latent_function_inference.expectation_propagation.EP(parallel_updates=True,epsilon=1e-5),
# inference_method=GPy.inference.latent_function_inference.expectation_propagation.EP(parallel_updates=False),
likelihood=lik)
# m.likelihood = lik
#m.inference_method = GPy.inference.latent_function_inference.PEP(alpha = 0.5)
if using_exactPB:
import numpy as np
mean, cov = m._raw_predict(X, full_cov=True)
alpha = np.linalg.solve(K,mean)
# m.log_likelihood()
# B = m.inference_method.B
n = X.shape[0]
# return m.log_likelihood()
# KL = 0.5*np.log(np.linalg.det(B)) + 0.5*np.trace(np.linalg.inv(B)) + 0.5*np.matmul(alpha.T,np.matmul(K,alpha)) -n/2
covi = np.linalg.inv(cov)
coviK = np.matmul(covi,K)
KL = 0.5*np.log(np.linalg.det(coviK)) + 0.5*np.trace(np.linalg.inv(coviK)) + 0.5*np.matmul(alpha.T,np.matmul(K,alpha)) - n/2
return -KL[0,0]
else:
return m.log_likelihood()
test_ys = h5f['test_ys'][:]
h5f.close()
input_dim = train_images.shape[1]
num_channels = train_images.shape[-1]
tp_order = np.concatenate([[0,len(train_images.shape)-1], np.arange(1, len(train_images.shape)-1)])
flat_train_images = np.transpose(train_images, tp_order) # NHWC -> NCHW # this is because the cnn GP kernels assume this
flat_train_images = np.array([train_image.flatten() for train_image in flat_train_images])
train_images = tf.constant(train_images)
K = np.load(open(kernel_folder+"K_"+str(total_samples)+"_"+dataset+"_"+network+"_"+str(number_layers)+("_whitening" if whitening else "")+".npy","rb"))
import GPy
from custom_kernel_matrix.custom_kernel_matrix import CustomMatrix
link_fun = GPy.likelihoods.link_functions.Heaviside()
lik = GPy.likelihoods.Bernoulli(gp_link=link_fun)
inference_method = GPy.inference.latent_function_inference.expectation_propagation.EP(parallel_updates=True)
X = flat_train_images
Y = np.array(ys)
# Y
m = GPy.core.GP(X=X,
Y=Y,
kernel=CustomMatrix(X.shape[1],X,K),
inference_method=inference_method,
likelihood=lik)
m.log_likelihood()
def GP_prob(K,X,Y, method="importance_sampling"):
# globals().update(FLAGS)
if method=="importance_sampling":
# link_fun = GPy.likelihoods.link_functions.Heaviside()
# lik = GPy.likelihoods.Bernoulli(gp_link=link_fun)
lik = GPy.likelihoods.Bernoulli()
inference_method = GPy.inference.latent_function_inference.expectation_propagation.EP(parallel_updates=True)
# inference_method = GPy.inference.latent_function_inference.laplace.Laplace()
model = GPy.core.GP(X=X,
Y=Y,
kernel=CustomMatrix(X.shape[1],X,K),
inference_method=inference_method,
likelihood=lik)
mean, cov = model._raw_predict(X, full_cov=True)
mean *= mean_mult
mean = mean.flatten()
cov *= cov_mult
log_norm_ratio = np.sum(np.log((np.linalg.eigh(cov)[0] / np.linalg.eigh(K)[0])))/2
covinv = np.linalg.inv(cov)
Kinv = np.linalg.inv(K)
tot = 0
shift = m*np.log(2)*0.3
#parallelizing tasks with mpi rank
num_tasks = num_post_samples
num_tasks_per_job = num_tasks//size
tasks = list(range(rank*num_tasks_per_job,(rank+1)*num_tasks_per_job))
if rank < num_tasks%size:
tasks.append(size*num_tasks_per_job+rank)
sample = np.random.multivariate_normal(mean, cov, len(tasks)).T
import sys
for i in range(len(tasks)):
# print(i,len(tasks))
# if (i%int(len(tasks)/100)) == (int(len(tasks)/100)-1):
# print(str(int(100*i/len(tasks)))+"%")
# # print(f)
# # print(Y.T)
# sys.stdout.flush()
f = sample[:,i]
PQratio = np.exp(shift-0.5*(np.matmul(f.T, np.matmul(Kinv, f)) - np.matmul((f-mean).T, np.matmul(covinv, (f-mean))) ) + log_norm_ratio)
if np.prod((f.T>0) == Y.T):
print(PQratio)
sys.stdout.flush()
tot += PQratio
tots = comm.gather(tot,root=0)
if rank == 0:
tot = sum(tots)
logPU = np.log(tot) - shift - np.log(num_post_samples)
return logPU
else:
return None
elif method == "HMC":
import gpflow
import custom_kernel_gpflow
from custom_kernel_gpflow import CustomMatrix
m = gpflow.models.GPMC(X.astype(np.float64), Y,
kern=CustomMatrix(X.shape[1],X,K),
# kern=gpflow.kernels.RBF(28*28),
likelihood=gpflow.likelihoods.Bernoulli(),)
m.compile()
o = gpflow.train.AdamOptimizer(0.01)
o.minimize(m, maxiter=1000) # start near MAP
s = gpflow.train.HMC()
samples = s.sample(m, 10, epsilon=1e-4, lmax=7, lmin=2, thin=1, logprobs=True)#, verbose=True)
n = samples.iloc[0][0].shape[0]
N = len(samples)
def hasZeroLikelihood(f):
return np.any(np.sign(f) != Y*2-1)
#Using MCMC estimate from https://stats.stackexchange.com/a/210196/32021 (with high prob region being the intersection of ball of radius
# sqrt(n) and the non-zero-likelihood quadrant.)
tot=0
for i, (f,logP) in samples.iterrows():
if not hasZeroLikelihood(f) and np.linalg.norm(f) <= np.sqrt(n):
tot += np.exp(-logP)
# !pip install -U scipy
import importlib; importlib.reload(scipy)
import scipy
from scipy import special
logV = (n/2)*np.log(n)+(n/2)*np.log(np.pi)-np.log(special.gamma(n/2+1))-n*np.log(2)
return logV - np.log(tot/N)
elif method == "Metropolis_EPproposal":
assert Y>=0 # not -1,1
Kinv = np.linalg.inv(K)
det = np.linalg.eigh(K)[0]
n = len(X)
normalization = (np.sqrt(np.power(2*np.pi,n) * det))
lognorm = 0.5*(len(X)*np.log(2*np.pi)+np.sum(np.log(det)))
def logPtilde(f):
return -0.5*(np.matmul(f.T, np.matmul(Kinv, f))) - lognorm
def logProposal(f2,f1):
return -0.5*(np.matmul(f.T, np.matmul(np.eye(n), f)))/sigma**2 - lognorm
def newProposal(f1):
#return np.random.multivariate_normal(f1,sigma*np.eye(n))
return np.random.multivariate_normal(f1,sigma*K)
def hasZeroLikelihood(f):
return np.any(np.sign(f) != Y*2-1)
def alpha(logPf2,logPf1):
if hasZeroLikelihood(f2):
return 0
else:
logRatio = logPf2-logPf1
return min(1,np.exp(logRatio))
sigma=0.05
import scipy
# V=np.power(n,n/2)*np.power(np.pi,n/2)/scipy.special.gamma(n/2+1)/np.power(2,n)
logV = (n/2)*np.log(n)+(n/2)*np.log(np.pi)-np.log(scipy.special.gamma(n/2+1))-n*np.log(2)
f1 = np.squeeze(Y*2-1)
tot = 0
N = 10000
accepted = 0
for i in range(N):
f2 = newProposal(f1)
logPf2 = logPtilde(f2)
logPf = logPf1 = logPtilde(f1)
if np.random.rand() <= alpha(logPf2,logPf1):
f1 = f2
logPf = logPf2
accepted += 1
if np.linalg.norm(f1) <= np.sqrt(n):
tot += np.exp(-logPf)
# print(i)
# print(i,",".join([str(x) for x in f1]))
return logV - np.log(tot/N)
else:
raise NotImplementedError()