def infExact(gp, X, y, Xstar, R=None, w=None, Rstar=None):
# Exact inference for a GP with Gaussian likelihood. Compute a parametrization
# of the posterior, the negative log marginal likelihood and its derivatives
# w.r.t. the hyperparameters. See also "help infMethods".
#
if R==None:
K = feval(gp['covfunc'], gp['covtheta'], X) # training covariances
Kss = feval(gp['covfunc'], gp['covtheta'], Xstar, 'diag') # test covariances (Kss = self covariances, Kstar = cov between train and test cases)
Kstar = feval(gp['covfunc'], gp['covtheta'], X, Xstar) # test covariances (Kss = self covariances, Kstar = cov between train and test cases)
else:
K = feval(gp['covfunc'], gp['covtheta'], X, R, w) # training covariances
Kss = feval(gp['covfunc'],gp['covtheta'], Xstar, R, w, 'diag', Rstar) # test covariances
Kstar = feval(gp['covfunc'],gp['covtheta'], X, R, w, Xstar, Rstar) # test covariances
ms = feval(gp['meanfunc'], gp['meantheta'], Xstar)
mean_y = feval(gp['meanfunc'], gp['meantheta'], X)
K += sn2*np.eye(X.shape[0]) # Hardcoded noise
try:
L = np.linalg.cholesky(K) # cholesky factorization of cov (lower triangular matrix)
except linalg.linalg.LinAlgError:
L = np.linalg.cholesky(nearPD(K)) # Find the "Nearest" covariance mattrix to K and do cholesky on that'
alpha = solve_chol(L.T,y-mean_y) # compute inv(K)*(y-mean(y))
fmu = ms + np.dot(Kstar.T,alpha) # predicted means
v = np.linalg.solve(L, Kstar)
tmp = v*v
fs2 = Kss - np.array([tmp.sum(axis=0)]).T # predicted variances
fs2[fs2 < 0.] = 0. # Remove numerical noise i.e. negative variances
return [fmu, fs2]
def get_W(theta, gp, X, y, R=None, w=None):
'''Precompute W for convenience.'''
n = X.shape[0]
mt = len(gp['meantheta'])
meantheta = theta[:mt]
covtheta = theta[mt:]
# compute training set covariance matrix
if R==None:
K = feval(gp['covfunc'], covtheta, X)
else:
K = feval(gp['covfunc'], covtheta, X, R, w)
K += sn2*np.eye(X.shape[0])
m = feval(gp['meanfunc'], meantheta, X)
# cholesky factorization of the covariance
try:
L = np.linalg.cholesky(K) # lower triangular matrix
except np.linalg.linalg.LinAlgError:
L = np.linalg.cholesky(nearPD(K)) # Find the "Nearest" covariance mattrix to K and do cholesky on that
alpha = solve_chol(L.T,y-m)
W = np.linalg.solve(L.T,np.linalg.solve(L,np.eye(n)))-np.dot(alpha,alpha.T)
return W
def dnlml(theta, gp, X, y, R=None, w=None):
mt = len(gp['meantheta'])
ct = len(gp['covtheta'])
out = np.zeros(mt+ct)
meantheta = theta[:mt]
covtheta = theta[mt:]
if gp['covfunc'][0][0] == 'kernels.covFITC':
# Do FITC Approximation
cov = gp['covfunc']
xu = cov[-1]
covfunc = cov[1:-1][0]
nu = len(xu)
[diagK,Kuu,Ku] = feval(cov[:-1], xu, covtheta, X) # evaluate covariance matrix
snu2 = 1.e-6 * sn2
Kuu += snu2*np.eye(nu)
m = feval(gp['meanfunc'], meantheta, X) # evaluate mean vector
n,D = X.shape
nu = Ku.shape[0]
try:
Luu = np.linalg.cholesky(Kuu) # Kuu = Luu'*Luu
except np.linalg.linalg.LinAlgError:
Luu = np.linalg.cholesky(nearPD(Kuu)) # Kuu = Luu'*Luu, or at least closest SDP Kuu
V = np.linalg.solve(Luu.T,Ku) # V = inv(Luu')*Ku => V'*V = Q
g_sn2 = diagK - (V*V).sum(axis=0).T # g = diag(K) - diag(Q)
diagG = np.reshape( np.diag(g_sn2),(g_sn2.shape[0],1))
V = V/np.tile(diagG.T,(nu,1))
Lu = np.linalg.cholesky( np.eye(nu) + np.dot(V,V.T) ) # Lu'*Lu = I + V*diag(1/g_sn2)*V'
r = (y-m)/np.sqrt(diagG)
#be = np.linalg.solve(Lu.T,np.dot(V,(r/np.sqrt(g_sn2))))
be = np.linalg.solve(Lu.T,np.dot(V,r))
V = np.linalg.solve(Luu.T,Ku) # V = inv(Luu')*Ku => V'*V = Q
iKuu = solve_chol(Luu,np.eye(nu)) # inv(Kuu + snu2*I) = iKuu
LuBe = np.linalg.solve(Lu,be)
alpha = np.linalg.solve(Luu,LuBe) # return the posterior parameters
L = solve_chol(np.dot(Lu,Luu),np.eye(nu)) - iKuu # Sigma-inv(Kuu)'''
#Q = np.dot(Ku.T,np.linalg.solve(Kuu,Ku))
#Lam = diagK - np.reshape( np.diag(Q), (diagK.shape[0],1) ) # diag(K) - diag(Q), Q = Kxu * Kuu^-1 * Kux
#K = Q + Lam* np.eye(len(Lam))
# cholesky factorization of the covariance
#try:
# L = np.linalg.cholesky(K) # lower triangular matrix
#except np.linalg.linalg.LinAlgError:
# L = np.linalg.cholesky(nearPD(K)) # Find the "Nearest" covariance mattrix to K and do cholesky on that
#alpha = solve_chol(L.T,y-m)
#alpha = np.linalg.solve(K,y-m)
#W = np.linalg.solve(L.T,np.linalg.solve(L,np.eye(n)))-np.dot(alpha,alpha.T)
#W = np.linalg.solve(K,np.eye(n))-np.dot(alpha,alpha.T)
W = Ku/np.tile(diagG.T,(nu,1))
W = np.linalg.solve((Kuu + np.dot(W,W.T) + snu2*np.eye(nu)).T, Ku)
al = ( (y-m) - np.dot(W.T,np.dot(W,(y-m)/diagG)) )/diagG
B = np.dot(iKuu,Ku)
Wdg = W/np.tile(diagG.T,(nu,1))
w = np.dot(B,al)
'''al = r/np.sqrt(g_sn2) - np.dot(V.T,LuBe)/g_sn2 # al = (Kt+sn2*eye(n))\y
B = np.dot(iKuu,Ku)
w = np.dot(B,al)
W = np.linalg.solve(Lu.T, (V/np.tile(g_sn2.T,(nu,1))))'''
if R==None:
for ii in range(len(meantheta)):
out[ii] = -1.*np.dot(feval(gp['meanfunc'], meantheta, X, ii).T,al)
kk = len(gp['meantheta'])
for ii in range(len(covtheta)):
[ddiagKi,dKuui,dKui] = feval(cov[:-1], covtheta, X, None, ii) # eval cov deriv
R = 2.*dKui - np.dot(dKuui,B)
v = ddiagKi - (R*B).sum(axis=0).T # diag part of cov deriv
out[ii+kk] = ( np.dot(ddiagKi.T,(1./g_sn2)) + np.dot(w.T,(np.dot(dKuui,w)-2.*np.dot(dKui,al))-np.dot(al.T,(v*al) \
- np.dot((Wdg*Wdg).sum(axis=0),v) - np.dot(R,Wdg.T)*np.dot(B,Wdg.T))) )/2.
#A = feval(covfunc, covtheta, X, None, ii)
#out[ii+kk] = ( W * feval(covfunc, covtheta, X, None, ii) ).sum()/2.
else:
for ii in range(len(meantheta)):
out[ii] = (W*feval(gp['meanfunc'], meantheta, X, R, w, ii)).sum()/2.
kk = len(meantheta)
for ii in range(len(covtheta)):
out[ii+kk] = (W*feval(covfunc, covtheta, X, R, w, ii)).sum()/2.
else:
# Do Exact inference
W = get_W(theta, gp, X, y, R, w)
if R==None:
for ii in range(len(meantheta)):
out[ii] = (W*feval(gp['meanfunc'], meantheta, X, ii)).sum()/2.
kk = len(meantheta)
for ii in range(len(covtheta)):
out[ii+kk] = (W*feval(gp['covfunc'], covtheta, X, None, ii)).sum()/2.
else:
for ii in range(len(meantheta)):
out[ii] = (W*feval(gp['meanfunc'], meantheta, X, R, w, ii)).sum()/2.
kk = len(meantheta)
for ii in range(len(gpcovtheta)):
out[ii+kk] = (W*feval(gp['covfunc'], covtheta, X, R, w, ii)).sum()/2.
return out
def nlml(theta, gp, X, y, R=None, w=None):
mt = len(gp['meantheta'])
ct = len(gp['covtheta'])
out = np.zeros(mt+ct)
meantheta = theta[:mt]
covtheta = theta[mt:]
n,D = X.shape
if gp['covfunc'][0][0] == 'kernels.covFITC':
# Do FITC Approximation
cov = gp['covfunc']
xu = cov[-1]
covfunc = cov[:-1]
[diagK,Kuu,Ku] = feval(covfunc, xu, covtheta, X) # evaluate covariance matrix
m = feval(gp['meanfunc'], meantheta, X) # evaluate mean vector
nu = Ku.shape[0]
snu2 = 1.e-6 * sn2
Kuu += snu2*np.eye(nu)
try:
Luu = np.linalg.cholesky(Kuu) # Kuu = Luu'*Luu
except np.linalg.linalg.LinAlgError:
Luu = np.linalg.cholesky(nearPD(Kuu)) # Kuu = Luu'*Luu, or at least closest SDP Kuu
V = np.linalg.solve(Luu.T,Ku) # V = inv(Luu')*Ku => V'*V = Q
g_sn2 = diagK - (V*V).sum(axis=0).T # g = diag(K) - diag(Q)
diagG = np.reshape( np.diag(g_sn2),(g_sn2.shape[0],1))
V = V/np.tile(diagG.T,(nu,1))
Lu = np.linalg.cholesky( np.eye(nu) + np.dot(V,V.T) ) # Lu'*Lu = I + V*diag(1/g_sn2)*V'
r = (y-m)/np.sqrt(diagG)
V = np.linalg.solve(Luu.T,Ku) # V = inv(Luu')*Ku => V'*V = Q
#be = np.linalg.solve(Lu.T,np.dot(V,(r/np.sqrt(g_sn2))))
be = np.linalg.solve(Lu.T,np.dot(V,r))
iKuu = solve_chol(Luu,np.eye(nu)) # inv(Kuu + snu2*I) = iKuu
LuBe = np.linalg.solve(Lu,be)
alpha = np.linalg.solve(Luu,LuBe) # return the posterior parameters
L = solve_chol(np.dot(Lu,Luu),np.eye(nu)) - iKuu # Sigma-inv(Kuu)
#Q = np.dot(Ku.T,np.linalg.solve(Kuu,Ku))
#Lam = diagK - np.reshape( np.diag(Q), (diagK.shape[0],1) ) # diag(K) - diag(Q), Q = Kxu * Kuu^-1 * Kux
#K = Q + Lam* np.eye(len(Lam))
ms = feval(gp['meanfunc'], meantheta, X)
# cholesky factorization of the covariance
#try:
# L = np.linalg.cholesky(K) # lower triangular matrix
#except np.linalg.linalg.LinAlgError:
# L = np.linalg.cholesky(nearPD(K)) # Find the "Nearest" covariance mattrix to K and do cholesky on that
# compute inv(K)*y
#alpha = solve_chol(L.T,y-m)
#alpha = np.linalg.solve(K,y-m)
# compute the negative log marginal likelihood
aa = ( np.log(np.diag(Lu)).sum() + np.log(g_sn2).sum() + n*np.log(2.*np.pi) + np.dot(r.T,r) - np.dot(be.T,be) )/2.
#aa = ( 0.5*np.dot((y-ms).T,alpha) + (np.log(np.diag(L))).sum(axis=0) + 0.5*n*np.log(2.*np.pi) )
#aa = ( 0.5*np.dot((y-ms).T,alpha) + (np.log(np.linalg.det(K))) + 0.5*n*np.log(2.*np.pi) )
#t1 = 0.5*np.dot((y-ms).T,alpha)
#t2 = (np.log(np.linalg.det(K)))
#t3 = 0.5*n*np.log(2.*np.pi)
#t4 = np.linalg.det(K)
#print t1, t2, t3, t4
#aa = t1 + t2 + t3
else:
# Do Exact inference
# compute training set covariance matrix
if R==None:
K = feval(gp['covfunc'], covtheta, X)
else:
K = feval(gp['covfunc'], covtheta, X, R, w)
K += sn2*np.eye(X.shape[0])
m = feval(gp['meanfunc'], meantheta, X)
# cholesky factorization of the covariance
try:
L = np.linalg.cholesky(K) # lower triangular matrix
except np.linalg.linalg.LinAlgError:
L = np.linalg.cholesky(nearPD(K)) # Find the "Nearest" covariance mattrix to K and do cholesky on that
# compute inv(K)*y
#alpha = solve_chol(L.T,y-m)
alpha = np.linalg.solve(K,y-m)
# compute the negative log marginal likelihood
aa = ( 0.5*np.dot((y-m).T,alpha) + (np.log(np.diag(L))).sum(axis=0) + 0.5*n*np.log(2.*np.pi) )
return aa[0]
def infFITC(gp, X, y, Xstar, R=None, w=None, Rstar=None):
# FITC approximation to the posterior Gaussian process. The function is
# equivalent to infExact with the covariance function:
# Kt = Q + G; G = diag(diag(K-Q); Q = Ku'*inv(Kuu + snu2*eye(nu))*Ku;
# where Ku and Kuu are covariances w.r.t. to inducing inputs xu and
# snu2 = sn2/1e6 is the noise of the inducing inputs. We fixed the standard
# deviation of the inducing inputs to be a one per mil of the measurement noise
# standard deviation.
# The implementation exploits the Woodbury matrix identity
# inv(Kt) = inv(G) - inv(G)*Ku'*inv(Kuu+Ku*inv(G)*Ku')*Ku*inv(G)
# in order to be applicable to large datasets. The computational complexity
# is O(n nu^2) where n is the number of data points x and nu the number of
# inducing inputs in xu.
cov = gp['covfunc']
assert( isinstance(cov[-1],np.ndarray) )
xu = cov[-1]
covfunc = cov[:-1]
m = feval(gp['meanfunc'], gp['meantheta'], X) # evaluate mean vector
ms = feval(gp['meanfunc'], gp['meantheta'], Xstar) # evaluate mean vector
n,D = X.shape
nu = len(xu)
[diagK,Kuu,Ku] = feval(covfunc, xu, gp['covtheta'], X) # evaluate covariance matrix
snu2 = 1.e-6 * sn2
Kuu += snu2*np.eye(nu)
try:
Luu = np.linalg.cholesky(Kuu) # Kuu = Luu'*Luu
except np.linalg.linalg.LinAlgError:
Luu = np.linalg.cholesky(nearPD(Kuu)) # Kuu = Luu'*Luu, or at least closest SDP Kuu
V = np.linalg.solve(Luu.T,Ku) # V = inv(Luu')*Ku => V'*V = Q
M = np.reshape( np.diag(np.dot(V.T,V)), (diagK.shape[0],1) )
g_sn20 = diagK + sn2 - (V*V).sum(axis=0).T # g = diag(K) + sn2 - diag(Q)
g_sn2 = diagK + sn2 - M # g = diag(K) + sn2 - diag(Q)
print "Values = ",g_sn20, g_sn2
diagG = np.reshape( np.diag(g_sn2),(g_sn2.shape[0],1))
V = V/np.tile(diagG.T,(nu,1))
Lu = np.linalg.cholesky( np.eye(nu) + np.dot(V,V.T) ) # Lu'*Lu = I + V*diag(1/g_sn2)*V'
r = (y-m)/np.sqrt(diagG)
#be = np.linalg.solve(Lu.T,np.dot(V,(r/np.sqrt(g_sn2))))
be = np.linalg.solve(Lu.T,np.dot(V,r))
iKuu = solve_chol(Luu,np.eye(nu)) # inv(Kuu + snu2*I) = iKuu
LuBe = np.linalg.solve(Lu,be)
alpha = np.linalg.solve(Luu,LuBe) # return the posterior parameters
L = solve_chol(np.dot(Lu,Luu),np.eye(nu)) - iKuu # Sigma-inv(Kuu)
#Q = np.dot(Ku.T,np.linalg.solve(Kuu,Ku))
#Lam = diagK - np.reshape( np.diag(Q), (diagK.shape[0],1) ) # diag(K) - diag(Q), Q = Kxu * Kuu^-1 * Kux
#K = Q + Lam* np.eye(len(Lam))
if R==None:
Kss = feval(covfunc[1], gp['covtheta'], Xstar, 'diag')
Kus = feval(covfunc[1], gp['covtheta'], xu, Xstar)
Kuf = feval(covfunc[1], gp['covtheta'], xu, X)
else:
Kss = feval(covfunc[1], gp['covtheta'], Xstar, R, w, 'diag', Rstar) # test covariances
Ku = feval(covfunc[1], gp['covtheta'], Xstar, R, w, xu, Rstar) # test covariances
#Qsf = np.dot(Kus.T,np.linalg.solve(Kuu, Kuf))
#fmu = ms + np.dot(Qsf,np.linalg.solve(K,y-m)) # predicted means
fmu = ms + np.dot(Kus.T,alpha) # predicted means
#QQ = np.dot(Qsf,np.linalg.solve(K,Qsf.T))
#fs2 = Kss - np.reshape( np.diag(QQ), (Kss.shape[0],1) ) # predicted variances
vv = np.linalg.solve(L.T, Kuf)
tmp = vv*vv
fs2 = Kss - np.array([tmp.sum(axis=0)]).T # predicted variances
fs2[fs2 < 0.] = 0. # Remove numerical noise i.e. negative variances
return [fmu, fs2]
