def aux_var_model(f, K, sn, g=None):
"""Implementation of auxiliary variable model inside sds sampling method
Reference: http://papers.nips.cc/paper/4114-slice-sampling-covariance-hyperparameters-of-latent-gaussian-models.pdf
P(f|g,theta) = N(f; m_theta_g, R_theta_g)
Parameters
----------
f: ndarray with shape (n_samples,)
latent samples
K: ndarray with shape (n_samples, n_samples)
covariance of training samples
sn: float
noise
g: ndarray with shape (n_samples,)
auxiliary values with P(g|theta) = N(g; 0, Sigma_theta + S_theta)
"""
n = K.shape[0]
Kii = np.diagonal(K)
K_ii_inv = 1./(Kii)
v_1 = (sn**2)**(-1) + K_ii_inv # v-1 = variance of posterior P(f|L, theta) according to Laplace Approximation
Sii = 1./(v_1 - K_ii_inv)
S = np.zeros_like(K)
np.fill_diagonal(S, Sii)
S = np.maximum(S, 0.)
if g is None:
# g = np.dot(tools.jitchol(K+S), np.random.normal(size=(n,)))
g = np.random.multivariate_normal(f, S, 1).T.reshape((n,))
L = tools.jitchol(K+S)
V = np.linalg.solve(L, K) # V = L-1 * K, V.T*V = K.T * (K+S)-1 * K
R_theta = K - np.dot(V.T, V)
# R_theta = K - np.dot(np.dot(K, np.linalg.inv(K+S)), K)
# LS = np.linalg.cholesky(S)
# beta = tools.solve_chol(LS.T, g) # beta = S-1 * g
# m_theta_g = np.dot(R_theta, beta)
m_theta_g = np.dot(np.dot(R_theta, np.linalg.inv(S)), g)
chol_R_theta = tools.jitchol(R_theta+np.eye(n)*1e-11)
return g, K+S, m_theta_g, chol_R_theta, L
def inf_mcmc(f, model, ys=0):
"""Inference of fs|f
Parameters
----------
f: ndarray with shape (n_samples, n_mcmc_iters)
latent samples from MCMC
model: GP instance
Trained Gaussian process model, which has x, y, mean, cov & llk func's
ys: ndarray with shape (n_samples,)
testing target, for the purpose of computing log-likelihood
"""
x = model.x
y = model.y
xs= model.xs
my = np.mean(y)
n_samples = f.shape[1]
ns = xs.shape[0]
n, D = x.shape
m = np.tile(model.meanfunc.getMean(x), (1, n_samples))
K = model.covfunc.getCovMatrix(x=x, mode='train')
sn2 = model.likfunc.sn**2.
L = tools.jitchol(K/sn2+np.eye(n)).T
alpha = tools.solve_chol(L, f-m)/sn2 # np.dot(Sigma**(-1), f-m)
sW = np.ones((n, 1))/np.sqrt(sn2)
Ltril = np.all(np.tril(L, -1) == 0) # is L an upper triangular matrix?
kss = model.covfunc.getCovMatrix(z=xs, mode='self_test') # this only contains the diagonal terms
Ks = model.covfunc.getCovMatrix(x=x, z=xs, mode='cross')
ms = model.meanfunc.getMean(xs)
Fmu = np.tile(ms, (1, n_samples)) + np.dot(Ks.T, alpha) # conditional mean fs|f
if Ltril: # L is triangular => use Cholesky parameters (alpha, sW, L)
V = np.linalg.solve(L.T, np.tile(sW, (1, ns))*Ks) # Ks has shape (n, ns),
fs2 = kss - np.array([(V*V).sum(axis=0)]).T
else: # L is not triangular => use alternative parametrization
fs2 = kss + np.array([(Ks*np.dot(L, Ks)).sum(axis=0)]).T
# variance can only be >= 0
Fs2 = np.maximum(fs2, 0)
# Fs2 = np.tile(fs2, (1, n_samples))
Fmu = np.mean(Fmu, axis=1, keepdims=True)
Ymu, Lower, Upper = model.likfunc.evaluate(mu=Fmu, s2=Fs2)
ym = np.reshape(np.mean(Ymu, axis=1), (ns, 1)) + my
ys_lw = np.reshape(np.mean(Lower, axis=1), (ns, 1)) + my
ys_up = np.reshape(np.mean(Upper, axis=1), (ns, 1)) + my
return ym, ys_lw, ys_up, Fs2
