def _LMLgrad_lik(self,hyperparams):
"""
evaluates the gradient of the log marginal likelihood with respect to
the hyperparameters of the likelihood function
"""
try:
KV = self.get_covariances(hyperparams)
except LA.LinAlgError:
LG.error('linalg exception in _LML_covar')
return {'lik':SP.zeros(len(hyperparams['lik']))}
except ValueError:
LG.error('value error in _LML_covar')
return {'lik':SP.zeros(len(hyperparams['lik']))}
YtildeVec = ravel(KV['Ytilde'])
Kd_diag = self.likelihood.Kdiag_grad_theta(hyperparams['lik'],self.n,0)
temp = SP.kron(SP.dot(KV['U_c'].T, SP.dot(KV['Binv'], SP.dot(KV['Binv'].T, KV['U_c']))),SP.diag(Kd_diag))
LMLgrad_det = SP.diag(SP.dot(SP.diag(1./KV['S']),temp)).sum() # Needs more optimization
sigma_grad = 2 * SP.exp(2 * hyperparams['lik'])
LMLgrad_quad = - (sigma_grad * YtildeVec *
ravel(SP.dot(SP.eye(self.n), SP.dot(KV['Ytilde'],KV['UBinvBinvU'].T)))).sum()
LMLgrad = 0.5*(LMLgrad_det + LMLgrad_quad)
return {'lik':SP.array([LMLgrad])}
def _LMLgrad_lik(self,hyperparams,debugging=False):
"""
evaluates the gradient of the log marginal likelihood with respect to
the hyperparameters of the likelihood function
"""
try:
KV = self.get_covariances(hyperparams,debugging=debugging)
except LA.LinAlgError:
LG.error('linalg exception in _LML_covar')
return {'lik':SP.zeros(len(hyperparams['lik']))}
except ValueError:
LG.error('value error in _LML_covar')
return {'lik':SP.zeros(len(hyperparams['lik']))}
YtildeVec = ravel(KV['Ytilde'])
Kd_diag = self.likelihood.Kdiag_grad_theta(hyperparams['lik'],self.nt,0)
LMLgrad_det = ((1./KV['S'])*Kd_diag).sum()
LMLgrad_quad = -(YtildeVec**2 * Kd_diag).sum()
LMLgrad = 0.5*(LMLgrad_det + LMLgrad_quad)
if debugging:
W = KV['W']
Kd = self.likelihood.Kgrad_theta(hyperparams['lik'],self.nt,0)
_LMLgrad = 0.5 * (W*Kd).sum()
assert SP.allclose(LMLgrad,_LMLgrad), 'ouch, gradient is wrong for likelihood param'
return {'lik':SP.array([LMLgrad])}
def get_covariances(self,hyperparams):
"""
INPUT:
hyperparams: dictionary
OUTPUT: dictionary with the fields
Kr: kernel on rows
Kc: kernel on columns
Knoise: noise kernel
"""
if self._is_cached(hyperparams):
return self._covar_cache
if self._covar_cache==None:
self._covar_cache = {}
if not(self._is_cached(hyperparams,keys=['covar_c'])):
K_c = self.covar_c.K(hyperparams['covar_c'])
S_c,U_c = LA.eigh(K_c)
self._covar_cache['K_c'] = K_c
self._covar_cache['U_c'] = U_c
self._covar_cache['S_c'] = S_c
else:
K_c = self._covar_cache['K_c']
U_c = self._covar_cache['U_c']
S_c = self._covar_cache['S_c']
if not(self._is_cached(hyperparams,keys=['covar_r'])):
K_r = self.covar_r.K(hyperparams['covar_r'])
S_r,U_r = LA.eigh(K_r)
self._covar_cache['K_r'] = K_r
self._covar_cache['U_r'] = U_r
self._covar_cache['S_r'] = S_r
else:
K_r = self._covar_cache['K_r']
U_r = self._covar_cache['U_r']
S_r = self._covar_cache['S_r']
Binv = SP.linalg.pinv(self.basis)
S = SP.kron(S_c,S_r) + self.likelihood.Kdiag(hyperparams['lik'],self.nt)
#UYUB = SP.dot(U_r.T, SP.dot(self.Y, SP.dot(Binv.T, SP.dot(U_c, SP.dot(U_c.T, Binv)))))
UYUB = SP.dot(U_r.T, SP.dot(self.Y, SP.dot(Binv.T, U_c)))
YtildeVec = (1./S) * ravel(UYUB)
self._covar_cache['Binv'] = Binv
self._covar_cache['S'] = S
self._covar_cache['UYUB'] = UYUB
self._covar_cache['Ytilde'] = unravel(YtildeVec,self.n,self.t)
UBinv = SP.dot(U_c.T, Binv)
self._covar_cache['UBinvB'] = SP.dot(UBinv, self.basis)
self._covar_cache['UBinvBinvU'] = SP.dot(UBinv, UBinv.T)
self._covar_cache['S_c_tilde'] = SP.diag(SP.dot(self._covar_cache['UBinvB'],SP.dot(K_c, self._covar_cache['UBinvB'].T)))
self._covar_cache['hyperparams'] = copy.deepcopy(hyperparams)
return self._covar_cache
def get_covariances(self, hyperparams):
if self._is_cached(hyperparams):
return self._covar_cache
if self._covar_cache == None:
self._covar_cache = {}
if not (self._is_cached(hyperparams, keys=['covar_c'])):
K_c = self.covar_c.K(hyperparams['covar_c'])
S_c, U_c = LA.eigh(K_c)
self._covar_cache['K_c'] = K_c
self._covar_cache['U_c'] = U_c
self._covar_cache['S_c'] = S_c
else:
K_c = self._covar_cache['K_c']
U_c = self._covar_cache['U_c']
S_c = self._covar_cache['S_c']
if not (self._is_cached(hyperparams, keys=['covar_r'])):
K_r = self.covar_r.K(hyperparams['covar_r'])
S_r, U_r = LA.eigh(K_r)
self._covar_cache['K_r'] = K_r
self._covar_cache['U_r'] = U_r
self._covar_cache['S_r'] = S_r
else:
K_r = self._covar_cache['K_r']
U_r = self._covar_cache['U_r']
S_r = self._covar_cache['S_r']
Binv = SP.linalg.pinv(self.basis.T).T
S = SP.kron(S_c, S_r) + self.likelihood.Kdiag(hyperparams['lik'],
self.nt)
UYUB = reduce(SP.dot, [U_r.T, self.Y, Binv.T, U_c])
YtildeVec = (1. / S) * ravel(UYUB)
self._covar_cache['Binv'] = Binv
self._covar_cache['S'] = S
self._covar_cache['UYUB'] = UYUB
self._covar_cache['Ytilde'] = unravel(YtildeVec, self.n, self.t)
UBinv = SP.dot(U_c.T, Binv)
self._covar_cache['UBinvB'] = SP.dot(UBinv, self.basis)
self._covar_cache['UBinvBinvU'] = SP.dot(UBinv, UBinv.T)
self._covar_cache['S_c_tilde'] = SP.diag(
reduce(SP.dot, [
self._covar_cache['UBinvB'], K_c, self._covar_cache['UBinvB'].T
]))
self._covar_cache['hyperparams'] = copy.deepcopy(hyperparams)
return self._covar_cache
def _LMLgrad_s(self, hyperparams):
"""
evaluate gradients with respect to covariance matrix Sigma
"""
try:
KV = self.get_covariances(hyperparams)
except LA.LinAlgError:
LG.error('linalg exception in _LMLgrad_x_sigma')
return {'X_s': SP.zeros(hyperparams['X_s'].shape)}
Si = 1. / KV['Stilde_os']
Yhat = unravel(Si * ravel(KV['UYtildeU_os']), self.n, SP.prod(self.bn))
RV = {}
if 'covar_s' in hyperparams:
k = 0
RV['covar_s'] = []
for covar in self.covar_s:
theta = SP.zeros(len(hyperparams['covar_s'][k]))
#USU = SP.dot(KV['USi_c'][k].T,KV['Utilde_s'][k])
USU = SP.dot(
self.nbasis[k].T,
SP.dot(self.basis[k],
SP.dot(KV['USi_c'][k].T, KV['Utilde_s'][k])))
for i in range(len(theta)):
Kgrad_s = covar.Kgrad_theta(hyperparams['covar_s'][k], i)
UdKU = SP.dot(USU.T, SP.dot(Kgrad_s, USU))
temp = copy.deepcopy(KV['Stilde_s'])
for z in range(len(temp)):
temp[z] = SP.diag(temp[z])
temp[k] = UdKU
SUdKUS = reduce(SP.kron, temp[::-1])
LMLgrad_det = SP.sum(Si * SP.kron(
SP.diag(SUdKUS), reduce(SP.kron, KV['Stilde_o'])))
SYUdKU = SP.dot(SUdKUS,
reduce(SP.kron, KV['Stilde_o']) * Yhat.T)
LMLgrad_quad = -(Yhat.T * SYUdKU).sum()
LMLgrad = 0.5 * (LMLgrad_det + LMLgrad_quad)
theta[i] = LMLgrad
RV['covar_s'].append(theta)
k += 1
return RV
def _LMLgrad_lik(self, hyperparams):
"""
evaluates the gradient of the log marginal likelihood with respect to
the hyperparameters of the likelihood function
"""
try:
KV = self.get_covariances(hyperparams)
except LA.LinAlgError:
LG.error('linalg exception in _LML_covar')
return {'lik': SP.zeros(len(hyperparams['lik']))}
except ValueError:
LG.error('value error in _LML_covar')
return {'lik': SP.zeros(len(hyperparams['lik']))}
YtildeVec = ravel(KV['Ytilde'])
Kd_diag = self.likelihood.Kdiag_grad_theta(hyperparams['lik'], self.nt,
0)
LMLgrad_det = ((1. / KV['S']) * Kd_diag).sum()
LMLgrad_quad = -(YtildeVec**2 * Kd_diag).sum()
LMLgrad = 0.5 * (LMLgrad_det + LMLgrad_quad)
return {'lik': SP.array([LMLgrad])}
def _LML_covar(self, hyperparams):
"""
log marginal likelihood
"""
#self._update_inputs(hyperparams)
try:
KV = self.get_covariances(hyperparams)
except LA.LinAlgError:
pdb.set_trace()
LG.error('linalg exception in _LML_covar')
return 1E6
Si = 1. / KV['Stilde_rc']
lml_quad = 0.5 * (ravel(KV['UYtildeU_rc'])**2 * Si).sum()
lml_det = 0.5 * (
SP.log(reduce(SP.kron, KV['S_s'][::-1])).sum() * self.n +
SP.log(reduce(SP.kron, KV['S_o'])).sum() * SP.prod(self.t))
lml_det += 0.5 * SP.log(KV['Stilde_rc']).sum()
lml_const = 0.5 * self.nt * (SP.log(2 * SP.pi))
lml = lml_quad + lml_det + lml_const
return lml
def get_covariances(self,hyperparams,debugging=False):
"""
INPUT:
hyperparams: dictionary
OUTPUT: dictionary with the fields
Kr: kernel on rows
Kc: kernel on columns
Knoise: noise kernel
"""
if self._is_cached(hyperparams):
return self._covar_cache
if self._covar_cache==None:
self._covar_cache = {}
if not(self._is_cached(hyperparams,keys=['covar_c'])):
K_c = self.covar_c.K(hyperparams['covar_c'])
S_c,U_c = LA.eigh(K_c)
self._covar_cache['K_c'] = K_c
self._covar_cache['U_c'] = U_c
self._covar_cache['S_c'] = S_c
else:
K_c = self._covar_cache['K_c']
U_c = self._covar_cache['U_c']
S_c = self._covar_cache['S_c']
if not(self._is_cached(hyperparams,keys=['covar_r'])):
K_r = self.covar_r.K(hyperparams['covar_r'])
S_r,U_r = LA.eigh(K_r)
self._covar_cache['K_r'] = K_r
self._covar_cache['U_r'] = U_r
self._covar_cache['S_r'] = S_r
else:
K_r = self._covar_cache['K_r']
U_r = self._covar_cache['U_r']
S_r = self._covar_cache['S_r']
S = SP.kron(S_c,S_r) + self.likelihood.Kdiag(hyperparams['lik'],self.nt)
UYU = SP.dot(U_r.T,SP.dot(self.Y,U_c))
YtildeVec = (1./S)*ravel(UYU)
self._covar_cache['S'] = S
self._covar_cache['UYU'] = UYU
self._covar_cache['Ytilde'] = unravel(YtildeVec,self.n,self.t)
if debugging:
# test ravel operations
UYUvec = ravel(UYU)
UYU2 = unravel(UYUvec,self.n,self.t)
SP.allclose(UYU2,UYU)
# needed later
Yvec = ravel(self.Y)
K_noise = self.likelihood.K(hyperparams['lik'],self.nt) # only works for iid noise
K = SP.kron(K_c,K_r) + K_noise
#L = LA.cholesky(K).T
L = jitChol(K)[0].T # lower triangular
alpha = LA.cho_solve((L,True),Yvec)
alpha2D = SP.reshape(alpha,(self.nt,1))
Kinv = LA.cho_solve((L,True),SP.eye(self.nt))
W = Kinv - SP.dot(alpha2D,alpha2D.T)
self._covar_cache['Yvec'] = Yvec
self._covar_cache['K'] = K
self._covar_cache['Kinv'] = Kinv
self._covar_cache['L'] = L
self._covar_cache['alpha'] = alpha
self._covar_cache['W'] = W
self._covar_cache['hyperparams'] = copy.deepcopy(hyperparams)
return self._covar_cache
def predict(self, hyperparams, Xstar_r, compute_cov=False):
"""
predicting
"""
KV = self.get_covariances(hyperparams)
self.covar_r[0].Xcross = Xstar_r
Kstar_r = self.covar_r[0].Kcross(hyperparams['covar_r'][0])
USUr = SP.dot(
SP.sqrt(1. / KV['S_o'][0]) * KV['U_o'][0], KV['Utilde_r'][0])
S = KV['Stilde_rc']
RUSUrYhat = SP.dot(
Kstar_r.T,
SP.dot(
USUr,
unravel(
ravel(KV['UYtildeU_rc']) * 1. / S, self.n,
SP.prod(self.nbn))))
usuc = list()
for i in range(self.out_dims):
usuc.append(
SP.dot(
self.basis[i],
SP.dot(
KV['K_c'][i],
SP.dot(
self.basis[i].T,
SP.dot(self.nbasis[i],
SP.dot(KV['USi_s'][i].T,
KV['Utilde_c'][i]))))).T)
if SP.prod(self.t) <= 10000:
USUC = reduce(SP.kron, usuc[::-1])
Ystar = SP.dot(RUSUrYhat, USUC)
else:
Ystar = SP.zeros([Xstar_r.shape[0], SP.prod(self.t)])
if self.out_dims == 1:
for j in range(self.t[0]):
USUC = usuc[0][:, j]
Ystar[:, j] = SP.dot(RUSUrYhat, USUC)
elif self.out_dims == 2:
for j in range(self.t[0]):
for k in range(self.t[1]):
USUC = reduce(SP.kron, [usuc[1][:, k], usuc[0][:, j]])
Ystar[:, k + j * self.t[1]] = SP.dot(RUSUrYhat, USUC)
elif self.out_dims == 3:
for j in range(self.t[0]):
for k in range(self.t[1]):
for l in range(self.t[2]):
USUC = reduce(
SP.kron,
[usuc[2][:, l], usuc[1][:, k], usuc[0][:, j]])
Ystar[:, l + k * self.t[2] + j *
(self.t[1] * self.t[2])] = SP.dot(
RUSUrYhat, USUC)
Ystar_covar = []
if compute_cov:
if self.nt < 10000:
B = reduce(SP.kron, self.basis[::-1])
C = SP.dot(B, SP.dot(reduce(SP.kron, KV['K_c'][::-1]), B.T))
USUC = reduce(SP.kron, usuc[::-1])
R_star_star = self.covar_r[0].K(hyperparams['covar_r'][0],
Xstar_r)
temp = SP.kron(USUC.T, SP.dot(Kstar_r.T, USUr))
Ystar_covar = SP.kron(SP.diag(C),
SP.diag(R_star_star)) - SP.sum(
(1. / S * temp).T * temp.T, axis=0)
Ystar_covar = unravel(Ystar_covar, Xstar_r.shape[0],
SP.prod(self.t))
elif SP.prod(self.t) < 10000:
Ystar_covar = SP.zeros([Xstar_r.shape[0], SP.prod(self.t)])
B = reduce(SP.kron, self.basis[::-1])
C = SP.dot(B, SP.dot(reduce(SP.kron, KV['K_c'][::-1]), B.T))
USUC = reduce(SP.kron, usuc[::-1])
printProgressBar(0,
Xstar_r.shape[0],
prefix='Computing perdiction varaince:',
suffix='Complete',
length=20)
for i in range(Xstar_r.shape[0]):
R_star_star = self.covar_r[0].K(
hyperparams['covar_r'][0],
SP.expand_dims(Xstar_r[i, :], axis=0))
self.covar_r[0].Xcross = SP.expand_dims(Xstar_r[i, :],
axis=0)
R_tr_star = self.covar_r[0].Kcross(
hyperparams['covar_r'][0])
r = SP.dot(R_tr_star.T, USUr)
q = SP.diag(SP.kron(C, R_star_star))
t = SP.zeros([SP.prod(self.t)])
for j in range(SP.prod(self.t)):
temp = SP.kron(USUC[:, j], r)
t[j, ] = SP.sum((1. / S * temp).T * temp.T, axis=0)
Ystar_covar[i, :] = q - t
if (i + 1) % 50 == 0:
printProgressBar(
i + 1,
Xstar_r.shape[0],
prefix='Computing perdiction varaince:',
suffix='Complete',
length=20)
self.covar_r[0].Xcross = Xstar_r
elif Xstar_r.shape[0] < 2000:
Ystar_covar = SP.zeros([Xstar_r.shape[0], SP.prod(self.t)])
c_diag = list()
for j in range(self.out_dims):
temp = SP.dot(self.basis[j],
SP.dot(KV['K_c'][j], self.basis[j].T))
c_diag.append(SP.diag(temp))
C = reduce(SP.kron, c_diag[::-1])
R_star_star = self.covar_r[0].K(hyperparams['covar_r'][0],
Xstar_r)
R_tr_star = self.covar_r[0].Kcross(hyperparams['covar_r'][0])
r = SP.dot(R_tr_star.T, USUr)
#q = SP.reshape(SP.kron(SP.diag(R_star_star),C),[Ystar_covar.shape[0],Ystar_covar.shape[1]])
q = unravel(SP.kron(C, SP.diag(R_star_star)),
Ystar_covar.shape[0], Ystar_covar.shape[1])
t = SP.zeros(Ystar_covar.shape)
printProgressBar(0,
Xstar_r.shape[0],
prefix='Computing perdiction varaince:',
suffix='Complete',
length=20)
if self.out_dims == 1:
for j in range(self.t[0]):
USUC = usuc[0][:, j]
temp = SP.kron(USUC, r)
t[j, ] = SP.sum((1. / S * temp).T * temp.T, axis=0)
printProgressBar(
j + 1,
self.t[0],
prefix='Computing perdiction varaince:',
suffix='Complete',
length=20)
elif self.out_dims == 2:
for j in range(self.t[0]):
for k in range(self.t[1]):
USUC = reduce(SP.kron,
[usuc[1][:, k], usuc[0][:, j]])
temp = SP.kron(USUC, r)
t[k + (j * self.t[1]), ] = SP.sum(
(1. / S * temp).T * temp.T, axis=0)
printProgressBar(
j + 1,
self.t[0],
prefix='Computing perdiction varaince:',
suffix='Complete',
length=20)
elif self.out_dims == 3:
for j in range(self.t[0]):
for k in range(self.t[1]):
for l in range(self.t[2]):
USUC = reduce(SP.kron, [
usuc[2][:, l], usuc[1][:, k], usuc[0][:, j]
])
temp = SP.kron(USUC, r)
t[:, l + k * self.t[2] +
j * self.t[1] * self.t[2], ] = SP.sum(
(1. / S * temp).T * temp.T, axis=0)
if (j + 1) % 5 == 0:
printProgressBar(
j + 1,
self.t[0],
prefix='Computing perdiction varaince:',
suffix='Complete',
length=20)
Ystar_covar = q - t
else:
Ystar_covar = SP.zeros([Xstar_r.shape[0], SP.prod(self.t)])
c_diag = list()
for j in range(self.out_dims):
temp = SP.dot(self.basis[j],
SP.dot(KV['K_c'][j], self.basis[j].T))
c_diag.append(SP.diag(temp))
C = reduce(SP.kron, c_diag[::-1])
printProgressBar(0,
Xstar_r.shape[0],
prefix='Computing perdiction varaince:',
suffix='Complete',
length=20)
for i in range(Xstar_r.shape[0]):
R_star_star = self.covar_r[0].K(
hyperparams['covar_r'][0],
SP.expand_dims(Xstar_r[i, :], axis=0))
self.covar_r[0].Xcross = SP.expand_dims(Xstar_r[i, :],
axis=0)
R_tr_star = self.covar_r[0].Kcross(
hyperparams['covar_r'][0])
r = SP.dot(R_tr_star.T, USUr)
q = C * R_star_star
t = SP.zeros([SP.prod(self.t)])
if self.out_dims == 1:
for j in range(self.t[0]):
USUC = usuc[0][:, j]
temp = SP.kron(USUC, r)
t[j, ] = SP.sum((1. / S * temp).T * temp.T, axis=0)
elif self.out_dims == 2:
for j in range(self.t[0]):
for k in range(self.t[1]):
USUC = reduce(SP.kron,
[usuc[1][:, k], usuc[0][:, j]])
temp = SP.kron(USUC, r)
t[k + (j * self.t[1]), ] = SP.sum(
(1. / S * temp).T * temp.T, axis=0)
elif self.out_dims == 3:
for j in range(self.t[0]):
for k in range(self.t[1]):
for l in range(self.t[2]):
USUC = reduce(SP.kron, [
usuc[2][:, l], usuc[1][:, k],
usuc[0][:, j]
])
temp = SP.kron(USUC, r)
t[l + k * self.t[2] +
j * self.t[1] * self.t[2], ] = SP.sum(
(1. / S * temp).T * temp.T, axis=0)
Ystar_covar[i, :] = q - t
if (i + 1) % 10 == 0:
printProgressBar(
i + 1,
Xstar_r.shape[0],
prefix='Computing perdiction varaince:',
suffix='Complete',
length=20)
self.covar_r[0].Xcross = Xstar_r
return Ystar, Ystar_covar