def rejection_loglike(z, ll_args):
if ll_args == 'logistic':
z = ilogit(z)
if z.min() < min_mu or z.max() > max_mu or (z[1:] -
z[:-1]).max() > 0:
return -np.inf
return log_likelihood(z, ll_args)
def _resample_R(self, data):
'''Random-walk MCMC for R'''
self.R = self.R[..., None]
logR = np.log(self.R)
# Current success probability
P = ilogit(np.einsum('nk,mtk->nmt', self.W,
self.V).clip(-10, 10))[..., None]
# Run a fixed number of random walk Metropolis-Hastings steps
for mcmc_step in range(self.nmetropolis):
# Sample a candidate R values
candidate_logR = logR + np.random.normal(
0, self.rpropstdev, size=logR.shape)
candidate_R = np.exp(candidate_logR)
# Get the prior log-likelihood ratio
accept_prior = norm.logpdf(
candidate_logR, loc=0, scale=self.rstdev) - norm.logpdf(
logR, loc=0, scale=self.rstdev)
# Get the likelihood log-likelihood ratio
accept_likelihood = (
gammaln(data + candidate_R) - gammaln(candidate_R) -
gammaln(data + self.R) + gammaln(self.R) # combinatorial bit
+ (candidate_R - self.R) * np.log(1 - P)
) # failure prob bit; success prob bit does not use R
# Multiply replicates and any other aggregated likelihoods
for dim in self.rdims:
accept_likelihood = np.nansum(accept_likelihood, axis=dim)
# Get rid of aggregated prior dims; make sure to go back to 1 for any size-1 unaggregated dims
accept_prior = np.squeeze(accept_prior).reshape(
accept_likelihood.shape)
# Get the acceptance probability for each R
accept_probs = np.exp(
np.clip(accept_prior + accept_likelihood, -10,
1)).reshape(self.R.shape)
# Accept/reject the samples
accept_indices = np.random.random(
size=accept_probs.shape) <= accept_probs
accept_indices = accept_indices & (candidate_R > 1) # TEMP
accepted_logR = candidate_logR[accept_indices]
logR[accept_indices] = accepted_logR
self.R[accept_indices] = np.exp(accepted_logR)
# Update the pseudo-counts for the PG-augmentation step
self.N = np.nansum(data + self.R,
axis=-1) # Binomial N can sum independent trials
self.R = self.R[..., 0]
def benchmarks():
'''Benchmarking GASS vs.
1) naive ESS + rejection sampling
2) logistic ESS + rejection sampling for monotonicity
3) logistic ESS + posterior projection'''
import matplotlib
import matplotlib.pyplot as plt
from functionalmf.elliptical_slice import elliptical_slice as ess
from functionalmf.utils import ilogit, pav
from scipy.stats import gamma
np.random.seed(42)
ntrials = 100
nmethods = 5
nobs = 3
sample_sizes = np.array([100, 500, 1000, 5000, 10000], dtype=int)
nsizes = len(sample_sizes)
nburn = nsamples = sample_sizes.max()
verbose = True
mu_prior = np.array([0.95, 0.8, 0.75, 0.5, 0.29, 0.2, 0.17, 0.15, 0.01, 0.0001]) # monotonic curve prior
T = len(mu_prior)
b = 3
min_mu, max_mu = 0.0, 1
sigma_prior = 0.1*np.array([np.exp(-0.5*(i - np.arange(T))**2 / b) for i in range(T)]) # Squared exponential kernel
# Get an empirical estimate of the logit-transformed covariance
print('Building empirical covariance matrix for logit transformed model')
mu_samples = np.zeros((1000, len(mu_prior)))
for i in range(mu_samples.shape[0]):
if i % 1 == 0:
print('\t', i)
mu_samples[i] = np.random.multivariate_normal(mu_prior, sigma_prior)
while mu_samples[i].min() < min_mu or mu_samples[i].max() > max_mu or (mu_samples[i][1:] - mu_samples[i][:-1]).max() > 0:
mu_samples[i] = np.random.multivariate_normal(mu_prior, sigma_prior)
mu_samples_logit = np.log(mu_samples / (1-mu_samples))
sigma_prior_logit = np.einsum('ni,nj->nij', mu_samples_logit, mu_samples_logit).mean(axis=0)
mu_prior_logit = np.log(mu_prior / (1-mu_prior))
mse = np.zeros((ntrials,nsizes,nmethods))
coverage = np.zeros((ntrials, nsizes, nmethods,T), dtype=bool)
for trial in range(ntrials):
print('Trial {}'.format(trial))
# Sample the true mean via rejection sampling
mu_truth = np.random.multivariate_normal(mu_prior, sigma_prior)
while mu_truth.min() < min_mu or mu_truth.max() > max_mu or (mu_truth[1:] - mu_truth[:-1]).max() > 0:
mu_truth = np.random.multivariate_normal(mu_prior, sigma_prior)
print(mu_truth)
# Plot some data points using the true scale
data = np.array([np.random.gamma(100, scale=mu_truth) for _ in range(nobs)]).T
samples = np.zeros((nsamples, nmethods, T))
xobs = np.tile(np.arange(T), (nobs, 1)).T
# Linear constraints requiring monotonicity and [0, 1] intervals
C_zero = np.concatenate([np.eye(T), np.zeros((T,1))+min_mu], axis=1)
C_one = np.concatenate([np.eye(T)*-1, np.ones((T,1))*-1 ], axis=1)
C_mono = np.array([np.concatenate([np.zeros(i), [1,-1], np.zeros(T-i-2), [0]]) for i in range(T-1)])
# Setup the lower bound inequalities
C_lower = np.concatenate([C_zero, C_one, C_mono], axis=0)
# Initialize to be a simple line trending downward (i.e. reasonable guess)
x = ((T - np.arange(T)) / T).clip(min_mu+0.01, max_mu-0.01)
x = np.tile(x, nmethods).reshape(nmethods, T)
# Convert to logits for the logistic models
x[2] = np.log(x[2] / (1-x[2]))
x[4] = np.log(x[4] / (1-x[4]))
# Simple iid N(y | mu, sigma^2) likelihood
def log_likelihood(z, ll_args):
if len(z.shape) == len(data.shape):
return gamma.logpdf(data[None], 100, scale=z[...,None]).sum(axis=-1).sum(axis=-1)
return gamma.logpdf(data, 100, scale=z[...,None]).sum()
# Log likelihood where we don't assume everything is valid
def rejection_loglike(z, ll_args):
if ll_args == 'logistic':
z = ilogit(z)
if z.min() < min_mu or z.max() > max_mu or (z[1:] - z[:-1]).max() > 0:
return -np.inf
return log_likelihood(z, ll_args)
# Generalized analytic slice sampling
cur_ll = [None]*nmethods
for step in range(nburn+nsamples):
if verbose and step % 1000 == 0:
print(step)
import warnings
warnings.simplefilter("ignore")
# Generalized analytic slice sampling
x[0], cur_ll[0] = gass(x[0], sigma_prior, log_likelihood, C_lower, cur_ll=cur_ll[0], mu=mu_prior)
# Naive ESS + rejection sampling
x[1], cur_ll[1] = ess(x[1], sigma_prior, rejection_loglike, cur_log_like=cur_ll[1], mu=mu_prior)
# Logistic ESS + rejection sampling
x[2], cur_ll[2] = ess(x[2], sigma_prior_logit, rejection_loglike, cur_log_like=cur_ll[2], mu=mu_prior_logit, ll_args='logistic')
# Naive ESS + posterior projection
x[3], cur_ll[3] = ess(x[3], sigma_prior, log_likelihood, cur_log_like=cur_ll[3], mu=mu_prior)
# Logistic ESS + posterior projection
x[4], cur_ll[4] = ess(x[4], sigma_prior_logit, lambda x_prop, ll_args: log_likelihood(ilogit(x_prop), ll_args),
cur_log_like=cur_ll[4], mu=mu_prior_logit)
# Save posterior samples
if step >= nburn:
samples[step-nburn] = x
# Pass the results for logistic models through the logistic function
samples[:,(2,4)] = ilogit(samples[:,(2,4)])
# Project the posteriors using PAV
print('Projecting third and fourth method posteriors')
for i in range(nsamples):
samples[i,3] = pav(samples[i,3][::-1]).clip(0,1)[::-1]
samples[i,4] = pav(samples[i,4][::-1]).clip(0,1)[::-1]
for size_idx, sample_size in enumerate(sample_sizes):
mu_hat = samples[:sample_size].mean(axis=0)
mu_lower = np.percentile(samples[:sample_size], 5, axis=0)
mu_upper = np.percentile(samples[:sample_size], 95, axis=0)
np.set_printoptions(precision=2, suppress=True)
# Calculate the mean squared error
mse[trial,size_idx] = ((mu_truth[None] - mu_hat)**2).mean(axis=-1)
# Calculate the 90th credible interval coverage
coverage[trial,size_idx] = (mu_truth[None] >= mu_lower) & (mu_truth[None] <= mu_upper)
print('Samples={} MSE={} Coverage={}'.format(sample_size,
mse[:trial+1,size_idx].mean(axis=0) * 1e3,
coverage[:trial+1,size_idx].mean(axis=0).mean(axis=1)))
print()
if trial == 0:
np.save('data/gass-benchmark-samples.npy', samples)
xobs = np.tile(np.arange(T), (nobs, 1)).T
# plt.scatter(xobs, data)
plt.plot(xobs[:,0], mu_hat[1], color='0.25', ls='--', label='ESS+Rejection')
plt.plot(xobs[:,0], mu_hat[2], color='0.4', ls=':', label='ESS+Link+Rejection')
plt.plot(xobs[:,0], mu_hat[3], color='0.65', ls='--', label='ESS+Projection')
plt.plot(xobs[:,0], mu_hat[4], color='0.8', ls=':', label='ESS+Link+Projection')
plt.plot(xobs[:,0], mu_hat[0], color='orange', label='GASS')
plt.plot(xobs[:,0], mu_truth, color='black', label='Truth')
# plt.fill_between(xobs[:,0], mu_lower, mu_upper, color='orange', alpha=0.5,)
plt.axhline(1, color='purple', ls='--', label='Upper bound')
plt.axhline(0, color='red', ls='--', label='Lower bound')
plt.legend(loc='lower left', ncol=2)
plt.savefig('plots/gass-benchmark.pdf', bbox_inches='tight')
plt.close()
import seaborn as sns
methods = ['GASS', 'RS', 'LRS', 'PP', 'LPP']
for midx, method in enumerate(methods):
with sns.axes_style('white'):
plt.rc('font', weight='bold')
plt.rc('grid', lw=3)
plt.rc('lines', lw=3)
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
# plt.scatter(xobs, data, color='gray', alpha=0.5)
plt.plot(xobs[:,0], mu_truth, color='black', label='Truth')
plt.plot(xobs[:,0], mu_hat[midx], color='orange', label=method)
plt.fill_between(xobs[:,0], mu_lower[midx], mu_upper[midx], color='orange', alpha=0.5)
plt.axhline(max_mu, color='red', ls='--', label='Upper bound')
plt.axhline(min_mu, color='red', ls='--', label='Lower bound')
plt.ylim([-0.1,1.1])
plt.ylabel('Gamma scale', fontsize=14)
plt.savefig('plots/gass-example-{}.pdf'.format(method.lower()), bbox_inches='tight')
plt.close()
np.save('data/gass-benchmark-mse.npy', mse)
np.save('data/gass-benchmark-coverage.npy', coverage)
mse = mse * 1e3
mse_mean, mse_stderr = mse.mean(axis=0), mse.std(axis=0) / np.sqrt(mse.shape[0])
coverage_mean, coverage_stderr = coverage.mean(axis=-1).mean(axis=0), coverage.mean(axis=-1).std(axis=0) / np.sqrt(coverage.shape[0])
for i in range(nmethods):
print(' & '.join(['${:.2f} \\pm {:.2f}$'.format(m,s) for m,s in zip(mse_mean[:,i], mse_stderr[:,i])]))
print()
for i in range(nmethods):
print(' & '.join(['${:.2f} \\pm {:.2f}$'.format(m,s) for m,s in zip(coverage_mean[:,i], coverage_stderr[:,i])]))
####### Run the Gibbs sampler #######
results = model.run_gibbs(Y_missing,
nburn=nburn,
nthin=nthin,
nsamples=nsamples,
print_freq=100,
verbose=True)
Ws = results['W']
Vs = results['V']
Rs = results['R']
Tau2s = results['Tau2']
lam2s = results['lam2']
sigma2s = results['sigma2']
# Get the Bayes estimate
Ps = ilogit(np.einsum('znk,zmtk->znmt', Ws, Vs).clip(-10, 10))
Mu_hat = Rs * Ps / (1 - Ps)
Mu_hat_mean = Mu_hat.mean(axis=0)
Mu_hat_upper = np.percentile(Mu_hat, 95, axis=0)
Mu_hat_lower = np.percentile(Mu_hat, 5, axis=0)
###### Plot the true curves, the noisy observations, and the fits ######
print('Plotting results')
X = np.arange(ndepth)
fig, axarr = plt.subplots(nrows,
ncols,
figsize=(5 * ncols, 5 * nrows),
sharex=True,
sharey=True)
for i in range(nrows):
for j in range(ncols):
def fun(x):
logit = np.einsum('k,nk,t->nt', x[1:], W,
concentrations) + x[0] + a[:, None]
return np.nansum((Y[:, j] - ilogit(logit))**
2) + regularizer * (x**2).mean()
def fun(x):
logit = np.einsum('k,mk,t->mt', x[1:], V,
concentrations) + x[0] + b[:, None]
return np.nansum(
(Y[i] - ilogit(logit))**2) + regularizer * (x**2).mean()
bounds = [(-10, 10) for _ in range(nembeds + 1)]
def fit_logistic_factors(Y,
nembeds,
max_steps=100,
concentrations=None,
verbose=False,
tol=1e-4,
regularizer=1e-4):
from scipy.optimize import minimize
if concentrations is None:
concentrations = np.arange(Y.shape[2])
W = np.random.normal(0, 0.1, size=(Y.shape[0], nembeds))
V = np.random.normal(0, 0.1, size=(Y.shape[1], nembeds))
a, b = np.random.normal(size=(Y.shape[0])), np.random.normal(
size=(Y.shape[1]))
# Fit the embedding model via alternating minimization
rmse = np.inf
for step in range(max_steps):
if verbose:
print('Step {}'.format(step))
# Track the previous iteration to assess convergence
prev_W, prev_V = np.copy(W), np.copy(V)
prev_rmse = rmse
# Fix V and fit W
for i in range(W.shape[0]):
def fun(x):
logit = np.einsum('k,mk,t->mt', x[1:], V,
concentrations) + x[0] + b[:, None]
return np.nansum(
(Y[i] - ilogit(logit))**2) + regularizer * (x**2).mean()
bounds = [(-10, 10) for _ in range(nembeds + 1)]
bounds = [(-10, 10) for _ in range(nembeds + 1)]
res = minimize(fun,
x0=np.concatenate([a[i:i + 1], W[i]]),
method='SLSQP',
options={
'ftol': 1e-8,
'maxiter': 1000
},
bounds=bounds)
a[i], W[i] = res.x[0], res.x[1:]
# Fix W and fit V
for j in range(V.shape[0]):
def fun(x):
logit = np.einsum('k,nk,t->nt', x[1:], W,
concentrations) + x[0] + a[:, None]
return np.nansum((Y[:, j] - ilogit(logit))**
2) + regularizer * (x**2).mean()
bounds = [(-10, 10) for _ in range(nembeds + 1)]
res = minimize(fun,
x0=np.concatenate([b[j:j + 1], V[j]]),
method='SLSQP',
options={
'ftol': 1e-8,
'maxiter': 1000
},
bounds=bounds)
b[j], V[j] = res.x[0], res.x[1:]
# delta = np.linalg.norm(np.concatenate([(prev_W - W).flatten(), (prev_V - V).flatten()]))
Mu = ilogit(
np.einsum('nk,mk,t->nmt', W, V, concentrations) +
a[:, None, None] + b[None, :, None])
rmse = np.sqrt(np.nansum((Y - Mu)**2))
delta = (prev_rmse - rmse) / rmse
if verbose:
print('delta: {}'.format(delta))
if delta <= tol:
break
Mu = ilogit(
np.einsum('nk,mk,t->nmt', W, V, concentrations) + a[:, None, None] +
b[None, :, None])
return Mu, W, V, a, b
np.random.seed(seed)
# Sample from the prior
model = init_model()
# Ground truth drawn from the model
W_true, V_true = create_wiggly_with_jumps()
# Get the true mean values
Mu = np.einsum('nk,mtk->nmt', W_true, V_true)
print('Mean ranges: [{},{}]'.format(Mu.min(), Mu.max()))
# Generate the data
N = np.full((nrows, ncols, ndepth), nreplicates)
Y = np.random.binomial(N, ilogit(Mu))
# Convert to acceptable formats for sampling
N = N.astype(float)
Y = Y.astype(float)
# Hold out some curves
Y_missing = Y.copy()
Y_missing[:3, :3] = np.nan
N_missing = N.copy()
N_missing[np.isnan(Y_missing)] = np.nan
####### Run the Gibbs sampler #######
results = model.run_gibbs((Y_missing, N_missing),
nburn=nburn,
nthin=nthin,
