class BinomialBayesianTensorFiltering(GaussianBayesianTensorFiltering):
def __init__(self, nrows, ncols, ndepth, pg_seed=42, **kwargs):
super().__init__(nrows, ncols, ndepth, **kwargs)
# Initialize the Polya-Gamma sampler
from pypolyagamma import PyPolyaGamma
self.pg = PyPolyaGamma(seed=pg_seed)
self.nu2 = np.zeros((nrows, ncols, ndepth))
self.nu2_flat = np.zeros(np.prod(self.nu2.shape))
self.sample_nu2 = True
def _resample_W(self, data):
Y, N = data
kappa = (Y - N / 2) * self.nu2
super()._resample_W(kappa)
def _resample_V(self, data):
Y, N = data
kappa = (Y - N / 2) * self.nu2
super()._resample_V(kappa)
def _resample_nu2(self, data):
'''Update the latent variables, which lead to variance terms in the
gaussian sampler steps.'''
Y, N = data
Mu = np.einsum('nk,mtk->nmt', self.W, self.V)
# missing = np.isnan(Y)
# for s in np.ndindex(Y.shape):
# if missing[s]:
# continue
# self.nu2[s] = 1/self.pg.pgdraw(N[s], Mu[s])
# print(N.flatten()[:5], Mu.flatten()[:5], self.nu2_flat[:5])
with np.errstate(divide='ignore'):
self.pg.pgdrawv(N.flatten(), Mu.flatten(), self.nu2.reshape(-1))
self.nu2 = 1 / self.nu2
def _smpl_fn(cls, rng, b, c, size):
pg = PyPolyaGamma(rng.randint(2 ** 16))
if not size and b.shape == c.shape == ():
return pg.pgdraw(b, c)
else:
b, c = np.broadcast_arrays(b, c)
out_shape = b.shape + tuple(size or ())
smpl_val = np.empty(out_shape, dtype="double")
b = np.tile(b, tuple(size or ()) + (1,))
c = np.tile(c, tuple(size or ()) + (1,))
pg.pgdrawv(
np.asarray(b.flat).astype("double", copy=True),
np.asarray(c.flat).astype("double", copy=True),
np.asarray(smpl_val.flat),
)
return smpl_val
def rng_fn(cls, rng, b, c, size):
pg = PyPolyaGamma(rng.randint(2**16))
if not size and b.shape == c.shape == ():
return pg.pgdraw(b, c)
else:
b, c = np.broadcast_arrays(b, c)
size = tuple(size or ())
if len(size) > 0:
b = np.broadcast_to(b, size)
c = np.broadcast_to(c, size)
smpl_val = np.empty(b.shape, dtype="double")
pg.pgdrawv(
np.asarray(b.flat).astype("double", copy=True),
np.asarray(c.flat).astype("double", copy=True),
np.asarray(smpl_val.flat),
)
return smpl_val
class BasicRandom():
"""
Generators of random variables from the basic distributions used in
Bayesian sparse regression.
"""
def __init__(self, seed=None):
self.np_random = np.random
self.pg = None
self.ts = None
self.set_seed(seed)
def set_seed(self, seed):
self.np_random.seed(seed)
pg_seed = np.random.randint(1, 1 + np.iinfo(np.uint32).max)
ts_seed = np.random.randint(1, 1 + np.iinfo(np.uint32).max)
self.pg = PyPolyaGamma(seed=pg_seed)
self.ts = ExpTiltedStableDist(seed=ts_seed)
def get_state(self):
rand_gen_state = {
'numpy': self.np_random.get_state(),
'tilted_stable': self.ts.get_state(),
'pypolyagamma': self.pg
# Don't know how to access the internal state, so just save
# the object itself.
}
return rand_gen_state
def set_state(self, rand_gen_state):
self.np_random.set_state(rand_gen_state['numpy'])
self.ts.set_state(rand_gen_state['tilted_stable'])
self.pg = rand_gen_state['pypolyagamma']
def polya_gamma(self, shape, tilt, size):
omega = np.zeros(size)
self.pg.pgdrawv(shape, tilt, omega)
return omega
def tilted_stable(self, char_exponent, tilt):
return self.ts.rv(char_exponent, tilt)
def nb_fit_bayes(Z):
from pypolyagamma import PyPolyaGamma
from scipy.stats import norm
results = []
pgr = PyPolyaGamma(seed=0)
model_logr = np.zeros(Z.shape[0])
model_Psi = np.zeros(Z.shape)
model_r = np.exp(model_logr)
model_P = ilogit(model_Psi)
prior_logr_sd = 100.
Omegas = np.zeros_like(Z)
for step in xrange(3000):
# Random-walk MCMC for log(r)
for mcmc_step in xrange(30):
candidate_logr = model_logr + np.random.normal(
0, 1, size=Z.shape[0])
candidate_r = np.exp(candidate_logr)
accept_prior = norm.logpdf(
candidate_logr, loc=0, scale=prior_logr_sd) - norm.logpdf(
model_logr, loc=0, scale=prior_logr_sd)
accept_likelihood = negBinomRatio(Z,
candidate_r[:, np.newaxis],
model_r[:, np.newaxis],
model_P,
model_P,
log=True).sum(axis=1)
accept_probs = np.exp(
np.clip(accept_prior + accept_likelihood, -10, 1))
accept_indices = np.random.random(size=Z.shape[0]) <= accept_probs
model_logr[accept_indices] = candidate_logr[accept_indices]
model_r = np.exp(model_logr)
# Polya-Gamma sampler -- Marginal test version only
N_ij = Z + model_r[:, np.newaxis]
[
pgr.pgdrawv(N_ij[i], np.repeat(model_Psi[i, 0], Z.shape[1]),
Omegas[i]) for i in xrange(Z.shape[0])
]
# Sample the logits using only the expressed values -- Marginal test version only
v = 1 / (Omegas.sum(axis=1) + 1 / 100.**2)
m = v * (Z.sum(axis=1) - Z.shape[1] * model_r) / 2.
model_Psi = np.random.normal(loc=m, scale=np.sqrt(v))[:, np.newaxis]
model_P = ilogit(model_Psi)
if step > 1000 and (step % 2) == 0:
results.append([model_r, model_P[:, 0]])
# print(model_r, model_P[:,0])
return np.array(results)
class _BaseLogisticRFLVM(_BaseRFLVM):
def __init__(self, rng, data, n_burn, n_iters, latent_dim, n_clusters,
n_rffs, dp_prior_obs, dp_df, disp_prior, bias_var):
"""Initialize base class for logistic RFLVMs.
"""
# `_BaseRFLVM` will call `_init_specific_params`, and these need to be
# set first.
self.disp_prior = disp_prior
self.bias_var = bias_var
super().__init__(rng=rng,
data=data,
n_burn=n_burn,
n_iters=n_iters,
latent_dim=latent_dim,
n_clusters=n_clusters,
n_rffs=n_rffs,
dp_prior_obs=dp_prior_obs,
dp_df=dp_df)
# Polya-gamma augmentation.
self.pg = PyPolyaGamma()
prior_Sigma = np.eye(self.M + 1)
prior_Sigma[-1, -1] = np.sqrt(self.bias_var)
self.inv_B = np.linalg.inv(prior_Sigma)
mu_A_b = np.zeros(self.M + 1)
self.inv_B_b = self.inv_B @ mu_A_b
self.omega = np.empty(self.Y.shape)
# Linear coefficients `beta`.
b0 = np.zeros(self.M + 1)
B0 = np.eye(self.M + 1)
self.beta = self.rng.multivariate_normal(b0, B0, size=self._j_func())
# -----------------------------------------------------------------------------
# Public API.
# -----------------------------------------------------------------------------
def log_likelihood(self, **kwargs):
"""Generalized, differentiable log likelihood function.
"""
# This function can be called for two reasons:
#
# 1. Optimize the log likelihood w.r.t. `X`.
# 2. Evaluate the log likelihood w.r.t. a MH-proposed `W`.
#
X = kwargs.get('X', self.X)
W = kwargs.get('W', self.W)
phi_X = self.phi(X, W, add_bias=True)
psi = phi_X @ self.beta.T
LL = self._log_c_func() \
+ self._a_func() * psi \
- self._b_func() * np.log(1 + np.exp(psi))
return LL.sum()
# -----------------------------------------------------------------------------
# Polya-gamma augmentation.
# -----------------------------------------------------------------------------
def _sample_beta(self):
"""Sample `β|ω ~ N(m, V)`. See (Polson 2013).
"""
phi_X = self.phi(self.X, self.W, add_bias=True)
for j in range(self.J):
# This really computes: phi_X.T @ np.diag(omega[:, j]) @ phi_X
J = (phi_X * self.omega[:, j][:, None]).T @ phi_X + \
self.inv_B
h = phi_X.T @ self._kappa_func(j) + self.inv_B_b
joint_sample = self._sample_gaussian(J=J, h=h)
self.beta[j] = joint_sample
def _sample_omega(self):
"""Sample `ω|β ~ PG(b, x*β)`. See (Polson 2013).
"""
phi_X = self.phi(self.X, self.W, add_bias=True)
psi = phi_X @ self.beta.T
b = self._b_func()
self.pg.pgdrawv(b.ravel(), psi.ravel(), self.omega.ravel())
self.omega = self.omega.reshape(self.Y.shape)
def _a_func(self, j=None):
"""This function returns `a(y)`. See the comment at the top of this
file and (Polson 2013).
"""
raise NotImplementedError()
def _b_func(self, j=None):
"""This function returns `b(y)`. See the comment at the top of this
file and (Polson 2013).
"""
raise NotImplementedError()
def _log_c_func(self):
"""This function returns `log c(y)`. This is the normalizer in logistic
models and is only used in the log likelihood calculation. See the
comment at the top of this file and (Polson 2013).
"""
raise NotImplementedError()
def _j_func(self):
"""Return number of features to iterate over. This is required because
multinomial models decompose the multinomial distribution into `J-1`
binomial distributions.
"""
raise NotImplementedError()
def _kappa_func(self, j):
"""This function returns `kappa(y)`. See the comment at the top of this
file and (Polson 2013).
"""
return self._a_func(j) - (self._b_func(j) / 2.0)