def plot_contours(ax, params):
samples = sample_obs(params, N_samples, inputs, layer_sizes)
y_mean, y_cov = np.mean(samples, axis=0), np.cov(samples.T)
approx_pdf = lambda x: mvn.logpdf(x, y_mean, y_cov)
real_pdf = lambda x: mvn.logpdf(x, real_mean, real_cov)
plot_isocontours(ax, approx_pdf, colors='r', label='approx')
plot_isocontours(ax, real_pdf, colors='b', label='true')
def target_lnpdf(theta):
theta = np.atleast_2d(theta)
target_lnpdf.counter += len(theta)
y = np.zeros((len(theta)))
x = np.zeros((len(theta)))
for i in range(0, num_dimensions):
y += l[i] * np.sin(np.sum(theta[:,:i + 1],1))
x += l[i] * np.cos(np.sum(theta[:,:i + 1],1))
return normal_auto.logpdf(theta, prior_mean, prior_cov) + normal_auto.logpdf(np.vstack([x, y]).transpose(),
likelihood_mean, likelihood_cov)
def plot_contours(ax, params):
samples = sample_bnn(params, N_samples, inputs, layer_sizes)
y_mean, y_cov = np.mean(samples, axis=0), np.cov(samples.T)
approx_pdf = lambda x: mvn.logpdf(x, y_mean, y_cov)
real_pdf = lambda x: mvn.logpdf(x, real_mean, real_cov)
plot_isocontours(ax, approx_pdf, colors='r', label='approx')
plot_isocontours(ax, real_pdf, colors='b', label='true')
gp_samples = sample_full_normal(real_mean, r, N_samples)
ax.scatter(gp_samples[:, 0], gp_samples[:, 1], marker='x')
def integrand(*args):
"""
If we want avoid inverting a huge matrix we need to evaluate
the log marginal through quadrature
here observations are conditionally independent
given function draw f
"""
f = np.array(args)
# UPDATE TO REFLECT DIMENSION
likelihood = np.sum(mvn.logpdf(y, f, noise_variance * np.eye(t)))
K = cov_func(cov_params, x, x)
prior = mvn.logpdf(f, np.zeros(len(f)), K)
return np.exp(likelihood + prior)
def target_lnpdf(theta):
theta = np.atleast_2d(theta)
target_lnpdf.counter += len(theta)
y = np.zeros((len(theta)))
x = np.zeros((len(theta)))
for i in range(0, num_dimensions):
y += l[i] * np.sin(np.sum(theta[:,:i+1],1))
x += l[i] * np.cos(np.sum(theta[:,:i+1],1))
likelihood = np.max((
normal_auto.logpdf(np.vstack((x,y)).transpose(),[0.7 * num_dimensions, 0], likelihood_variance * np.eye(2)),
normal_auto.logpdf(np.vstack((x,y)).transpose(),[-0.7 * num_dimensions, 0], likelihood_variance * np.eye(2)),
normal_auto.logpdf(np.vstack((x,y)).transpose(),[0, 0.7 * num_dimensions], likelihood_variance * np.eye(2)),
normal_auto.logpdf(np.vstack((x,y)).transpose(),[0, -0.7 * num_dimensions], likelihood_variance * np.eye(2))))
return np.squeeze(normal_auto.logpdf(theta, np.zeros(num_dimensions), prior_variance * np.eye(num_dimensions)) + likelihood)
def log_pdf_prior(weights, prior_params, sd, type=None):
if type is None:
return diag_gaussian_log_density(weights, 0, np.log(sd))
elif type == "diagonal":
prior_mean, prior_log_std = prior_params
return diag_gaussian_log_density(weights, prior_mean, prior_log_std)
elif type == "full":
prior_mean, prior_cov = prior_params
return mvn.logpdf(weights, prior_mean, prior_cov)
else:
prior_pis, prior_mus, prior_covs = prior_params
log_pdf = np.zeros(weights.shape[0])
for pi, mu, cov in zip(prior_pis, prior_mus, prior_covs):
log_pdf += mvn.logpdf(weights, mu, cov)
return log_pdf
def target_lnpdf(theta):
theta = np.atleast_2d(theta)
target_lnpdf.counter += len(theta)
cluster_lls = []
for i in range(0, num_true_components):
cluster_lls.append(np.log(1./num_true_components) + normal_auto.logpdf(theta, means[i], covs[i]))
return np.squeeze(logsumexp(np.vstack(cluster_lls), axis=0))
def log_likelihood(all_params): # implement mini batches later?
n_samples = 1
samples = [
sample_mean_cov_from_deep_gp(all_params, X, True)
for i in xrange(n_samples)
]
return logsumexp(np.array([mvn.logpdf(y,mean,var+1e-6*np.eye(len(var))*np.max(np.diag(var))) for mean,var in samples])) - np.log(n_samples) \
+ evaluate_prior(all_params)
def test_log_likelihood(all_params, X, y, n_samples):
rs = npr.RandomState(0)
samples = [
sample_mean_cov_from_deep_gp(all_params, X, True, rs, FITC=True)
for i in xrange(n_samples)
]
return logsumexp(
np.array([mvn.logpdf(y, mean, var) for mean, var in samples]))
def log_marginal_likelihood(params, data):
cluster_lls = []
for log_proportion, mean, chol in zip(*unpack_params(params)):
cov = np.dot(chol.T, chol) + 0.000001 * np.eye(D)
cluster_log_likelihood = log_proportion + mvn.logpdf(data, mean, cov)
cluster_lls.append(np.expand_dims(cluster_log_likelihood, axis=0))
cluster_lls = np.concatenate(cluster_lls, axis=0)
return np.sum(logsumexp(cluster_lls, axis=0))
def log_marginal_likelihood_full(params, x, y, weights=None):
if weights is None:
weights = np.ones(len(y))
mean, cov_params, noise_variance = unpack_kernel_params(params)
cov_y_y = cov_func(cov_params, x, x) + \
noise_variance * np.diag(1 / weights)
prior_mean = mean * np.ones(len(y))
return mvn.logpdf(y, prior_mean, cov_y_y)
def expected_like(params, N_samples, x, layer_sizes, mean, chol):
y = sample_obs(params, N_samples, x, layer_sizes)
mu = np.dot(rs.randn(N_samples, mean.shape[0]), chol) + mean
return np.mean(
np.array(
map(
lambda s: np.mean(
mvn.logpdf(y, s, noise_var * np.eye(mean.shape[0]))), mu)))
def log_likelihood(all_params, X, y, n_samples):
rs = npr.RandomState(0)
samples = [sample_mean_cov_from_deep_gp(all_params, X, True, rs, FITC=True) for i in xrange(n_samples)]
return (
logsumexp(np.array([mvn.logpdf(y, mean, var) for mean, var in samples]))
- np.log(n_samples)
+ evaluate_prior(all_params)
)
def log_marginal_likelihood(params, x, y):
"""
computes log p(y|X) = log N(y|mu, K + std*I)
"""
mean, cov_params, noise_scale = unpack_kernel_params(params)
cov_y_y = cov_func(cov_params, x, x) + noise_scale * np.eye(len(y))
prior_mean = mean * np.ones(len(y))
return mvn.logpdf(y, prior_mean, cov_y_y)
def log_marginal_likelihood(params, data):
cluster_lls = []
for log_proportion, mean, chol in zip(*unpack_params(params)):
cov = np.dot(chol.T, chol) + 0.000001 * np.eye(D)
cluster_log_likelihood = log_proportion + mvn.logpdf(data, mean, cov)
cluster_lls.append(np.expand_dims(cluster_log_likelihood, axis=0))
cluster_lls = np.concatenate(cluster_lls, axis=0)
return np.sum(logsumexp(cluster_lls, axis=0))
def lnq_grid(params):
assert D == 2
xg, yg = np.linspace(-4, 4, 50), np.linspace(-4, 4, 50)
xx, yy = np.meshgrid(xg, yg)
pts = np.column_stack([xx.ravel(), yy.ravel()])
zs, ldets = feed_forward(pts, params, layers)
lls = mvn.logpdf(pts, mean=np.zeros(D), cov=np.eye(D)) - ldets
return zs[:, 0].reshape(xx.shape), zs[:, 1].reshape(
yy.shape), lls.reshape(xx.shape)
def evaluate_prior(all_params): # clean up code so we don't compute matrices twice
layer_params, x0, y0 = unpack_all_params(all_params)
log_prior = 0
for layer in xrange(n_layers):
#import pdb; pdb.set_trace()
mean, cov_params, noise_scale = unpack_kernel_params(layer_params[layer])
cov_y_y = covariance_function(cov_params, x0[layer], x0[layer]) + noise_scale * np.eye(len(y0[layer]))
log_prior += mvn.logpdf(y0[layer],np.ones(len(cov_y_y))*mean,cov_y_y+np.eye(len(cov_y_y))*10)
return log_prior
def plot_posterior_contours(mean_params,logstd_params):
plt.clf()
logprob_adj = lambda two_params: logprob_given_two(mean_params, two_params)
plot_isocontours(ax, logprob_adj, cmap='Blues')
mean_2d = mean_params[contourK, [px1,px2]]
logstd_2s = logstd_params[contourK, [px1,px2]]
variational_contour = lambda x: mvn.logpdf(x, mean_2d, np.diag(np.exp(2*logstd_2s)))
plot_isocontours(ax, variational_contour, cmap='Reds')
plt.draw()
plt.pause(10)
def evaluate_prior(all_params): # clean up code so we don't compute matrices twice
all_layer_params = unpack_all_params(all_params)
log_prior = 0
deep_map = create_deep_map(all_params)
for layer,layer_map in deep_map.iteritems():
for unit,gp_map in layer_map.iteritems():
cov_y_y = covariance_function(gp_map['cov_params'],gp_map['x0'],gp_map['x0']) + gp_map['noise_scale'] * np.eye(len(gp_map['y0']))
log_prior += mvn.logpdf(gp_map['y0'],np.ones(len(cov_y_y))*gp_map['mean'],cov_y_y + np.diag(np.diag(cov_y_y))*0) # CHANGE
##log_prior += mvn.logpdf(gp_map['y0'],np.ones(len(cov_y_y))*gp_map['mean'],cov_y_y + np.eye(len(cov_y_y))*tuning_param)
###log_prior += mvn.logpdf(gp_map['y0'],np.ones(len(cov_y_y))*gp_map['mean'],np.diag(np.diag(cov_y_y))*10)
return log_prior
def evaluate_prior(all_params): # clean up code so we don't compute matrices twice
all_layer_params = unpack_all_params(all_params)
log_prior = 0
deep_map = create_deep_map(all_params)
for layer,layer_map in deep_map.iteritems():
for unit,gp_map in layer_map.iteritems():
cov_y_y = covariance_function(gp_map['cov_params'],gp_map['x0'],gp_map['x0']) + gp_map['noise_scale'] * np.eye(len(gp_map['y0']))
log_prior += mvn.logpdf(gp_map['y0'],np.ones(len(cov_y_y))*gp_map['mean'],cov_y_y + np.diag(np.diag(cov_y_y))*0) # CHANGE
##log_prior += mvn.logpdf(gp_map['y0'],np.ones(len(cov_y_y))*gp_map['mean'],cov_y_y + np.eye(len(cov_y_y))*tuning_param)
###log_prior += mvn.logpdf(gp_map['y0'],np.ones(len(cov_y_y))*gp_map['mean'],np.diag(np.diag(cov_y_y))*10)
return 0#log_prior
def evaluate_prior(all_params): # clean up code so we don't compute matrices twice
all_layer_params = unpack_all_params(all_params)
log_prior = 0
for layer in xrange(n_layers):
layer_params = all_layer_params[layer]
layer_gp_params = unpack_layer_params[layer](layer_params)
for dim in xrange(dimensions[layer+1]):
gp_params = layer_gp_params[dim]
mean, cov_params, noise_scale, x0, y0 = unpack_gp_params_all[layer][dim](gp_params)
cov_y_y = covariance_function(cov_params,x0,x0) + noise_scale * np.eye(len(y0))
log_prior += mvn.logpdf(y0,np.ones(len(cov_y_y))*mean,cov_y_y+np.eye(len(cov_y_y))*10)
return log_prior
def elbo(y, phi, lam, pi, psi, sigma2s, mus, Sigmas, kernel_params):
"""
phi [N, K] sample membership (cell line cluster)
lam [G, L] feature membership (expression cluster)
pi [K] sample mixture weight
psi [L] feature mixture weights
y[N, G, T] data
mus [K, L, T] means
"""
"""
conditional = np.array([list(map(
lambda f, s: norm.logpdf(y, f, s).sum(axis=-1), Q[:, :-1], Q[:, -1]))
for Q in np.concatenate([mus, sigma2s[:, :, np.newaxis]], 2)])
conditional = conditional + np.log(mix)[:, :, np.newaxis, np.newaxis]
assignments = np.einsum('nk, gl->klng', phi, lam)
likelihood = np.sum(conditional * assignments)
"""
likelihood = 0
# data likelihood
for l in range(L):
for k in range(K):
ll = np.sum(np.nan_to_num(norm.logpdf(
y, mus[k, l], np.sqrt(sigma2s[k, l]))), axis=-1)
ll = ll - 0.5 * (np.trace(Sigmas[k, l] / sigma2s[k, l]))
ll = ll * phi[:, k][:, np.newaxis]
ll = ll * lam[:, l]
likelihood = likelihood + np.sum(ll)
# assignment likelihood
likelihood = likelihood + np.sum(np.log(pi) * phi)
likelihood = likelihood + np.sum(np.log(psi) * lam)
# function liklihood
for k in range(K):
for l in range(L):
Ker = cov_func(kernel_params[k, l], inputs, inputs)
likelihood = likelihood \
+ mvn.logpdf(mus[k, l], np.zeros(T), Ker) \
- 0.5 * np.trace(solve(Ker, Sigmas[k, l]))
entropy = np.sum(list(map(multinomial_entropy, phi)) +
list(map(multinomial_entropy, lam)))
for k in range(K):
for l in range(L):
entropy = entropy + mvn.entropy(mus[k, l], Sigmas[k, l])
return likelihood + entropy
def evaluate_prior(
all_params): # clean up code so we don't compute matrices twice
layer_params, x0, y0 = unpack_all_params(all_params)
log_prior = 0
for layer in xrange(n_layers):
#import pdb; pdb.set_trace()
mean, cov_params, noise_scale = unpack_kernel_params(
layer_params[layer])
cov_y_y = covariance_function(
cov_params, x0[layer],
x0[layer]) + noise_scale * np.eye(len(y0[layer]))
log_prior += mvn.logpdf(y0[layer],
np.ones(len(cov_y_y)) * mean,
cov_y_y + np.eye(len(cov_y_y)) * 10)
return log_prior
def log_posterior(x,inputs ,len_sc,variance, t):
N=x.shape
sum_prob=0
params = [0, len_sc, variance, - 0.24707744]
for i in range(N[0]):
"""An example 2D intractable distribution:
a Gaussian evaluated at zero with a Gaussian prior on the log-variance."""
mu= x[i][:]
pred_mean, pred_cov = predict(params, inputs, mu, X)
prior = log_marginal_likelihood(params,inputs,mu)
posterior = mvn.logpdf(y, pred_mean, pred_cov, True)
sum_prob=posterior+sum_prob+prior
return sum_prob
def elbo(prior_params, qa_params, X, y, f_samples, arch, act, noise):
fs = sample_bnn(prior_params, X, f_samples, arch, act)
m = np.mean(fs, axis=0, keepdims=True)
qa_mean, qa_Sigma = qa_params
# m, K_ff = prior_mean_cov(prior_params, X, f_samples, arch, act)
# a_samples = sample_full_normal(qa_posterior_moments(m, K_ff, y, noise),1)
# qa_mean, qa_Sigma = qa_posterior_moments(m, K_ff, y, noise)
a_samples = sample_normal(qa_params, 1)
print(a_samples.shape, fs.shape, m.shape)
mean = a_samples * (fs - m) / unbiased(fs)
log_qy = diag_gaussian_log_density(y, mean, noise)
log_qa = mvn.logpdf(a_samples, qa_mean, qa_Sigma)
log_pa = diag_gaussian_log_density(a_samples, 0, 1)
return np.mean(log_qy - log_qa + log_pa)
def target_lnpdf(theta, without_prior=False):
theta = np.atleast_2d(theta)
target_lnpdf.counter += len(theta)
weighted_sum = np.dot(theta, X.transpose())
offset = np.maximum(weighted_sum, np.zeros(weighted_sum.shape))
denominator = offset + np.log(np.exp(weighted_sum - offset) + np.exp(-offset))
log_prediction = -denominator
swapped_y = -(y - 1)
log_prediction = log_prediction + swapped_y[np.newaxis, :] * (weighted_sum)
#log_prediction[np.where(np.isinf(log_prediction))] = 0
if (np.any(np.isnan(log_prediction)) or np.any(np.isinf(log_prediction))):
print('nan')
loglikelihood = np.sum(log_prediction,1)
if without_prior:
return np.squeeze(loglikelihood)
else:
return np.squeeze(normal_auto.logpdf(theta, prior_mean, prior_cov) + loglikelihood)
def kl_estimate(params, n_samples, arch, act):
prior_params, noise, kernel_params, x = params
x = sample_inputs('gridbox', 100, (0, 10))
y = sample_bnn(prior_params, x, n_samples, arch, act, noise) # [nf, nd]
f = sample_bnn(prior_params, x, n_samples, arch, act)
w = sample_weights(prior_params, 1)
mu, log_std = prior_params
log_prior = diag_gaussian_log_density(w, mu, log_std)
log_likelihood = diag_gaussian_log_density(y, f, noise)
jitter = 1e-7 * np.eye(y.shape[0])
cov = covariance(kernel_params, x, x) + jitter
log_pgp = mvn.logpdf(y, np.zeros(y.shape[1]), cov)
print(log_likelihood.shape, log_pgp.shape)
return np.mean(log_likelihood + log_prior - log_pgp)
def log_marginal_likelihood(params, x, y):
mean, cov_params, noise_scale = unpack_kernel_params(params)
cov_y_y = cov_func(cov_params, x, x) + noise_scale * np.eye(len(y))
prior_mean = mean * np.ones(len(y))
return mvn.logpdf(y, prior_mean, cov_y_y, True)
def log_likelihood(all_params): # implement mini batches later?
n_samples = 1
samples = [sample_mean_cov_from_deep_gp(all_params, X, True) for i in xrange(n_samples)]
return logsumexp(np.array([mvn.logpdf(y,mean,var+1e-6*np.eye(len(var))*np.max(np.diag(var))) for mean,var in samples])) - np.log(n_samples) \
+ evaluate_prior(all_params)
def logdensity(x, var_param):
mean, beta = unpack_params(var_param)
L = beta_to_L(beta[:,np.newaxis])
Sigma = [email protected]
return mvn.logpdf(x, mean, Sigma)
def gmm_log_likelihood(params, data):
cluster_lls = []
for log_proportion, mean, cov_sqrt in zip(*unpack_gmm_params(params)):
cov = np.dot(cov_sqrt.T, cov_sqrt)
cluster_lls.append(log_proportion + mvn.logpdf(data, mean, cov))
return np.sum(logsumexp(np.vstack(cluster_lls), axis=0))
def kl_estimate(params, N_samples, x, layer_sizes, mean, cov):
y = sample_obs(params, N_samples, x, layer_sizes)
return -entropy_estimate(y) - np.mean(mvn.logpdf(y, mean, cov))
def logdensity(x, var_param):
mean, log_std = unpack_params(var_param)
return mvn.logpdf(x, mean, np.diag(np.exp(2*log_std)))
def log_var_approx(self, z, params):
mu, log_sigma = self.unpack_params(params)
sigma = np.diag(np.exp(2 * log_sigma)) + 1e-6
return mvn.logpdf(z, mu, sigma)
def gmm_log_likelihood(params, data):
cluster_lls = []
for log_proportion, mean, cov_sqrt in zip(*unpack_gmm_params(params)):
cov = np.dot(cov_sqrt.T, cov_sqrt)
cluster_lls.append(log_proportion + mvn.logpdf(data, mean, cov))
return np.sum(logsumexp(np.vstack(cluster_lls), axis=0))
def log_marginal_likelihood(params, x, y):
mean, cov_params, noise_scale = unpack_kernel_params(params)
cov_y_y = cov_func(cov_params, x, x) + noise_scale * np.eye(len(y))
prior_mean = mean * np.ones(len(y))
return mvn.logpdf(y, prior_mean, cov_y_y)
def log_marginal(x, y):
cov_y_y = RBF(x, x) + noise_scale * np.eye(len(y))
return np.sum(mvn.logpdf(y, np.zeros(len(y)), cov_y_y, True))
def log_cond(x,y,xstar,ystar):
pred_mean, pred_cov= conditional(x,y,xstar)
return np.sum(mvn.logpdf(ystar,np.reshape(pred_mean,-1),pred_cov))
def _log_prior_x(self, X):
"""Return the log prior for `X`.
"""
return ag_mvn.logpdf(X, self.mu_x, self.cov_x).sum()
def make_mvn_lowrank_marginal(mean, C, s_diag, marg_idx):
mu_marg, Sigma_marg = mvn_lowrank_params(mean, C, s_diag, marg_idx)
return lambda x: mvn.logpdf(
x, mean=mu_marg, cov=Sigma_marg, allow_singular=True)