def __init__(self, data, T, alpha_beta):
mu, sigma = compute_uniform_mean_psi(T)
self.theta_prior = Gaussian(
mu=mu, sigma=sigma, mu_0=mu, sigma_0=T * sigma / 10.0, nu_0=T / 10.0, kappa_0=1.0 / 10
)
self.ppgs = initialize_polya_gamma_samplers()
self.omega = np.zeros((data.shape[0], T - 1))
super(StickbreakingCorrelatedLDA, self).__init__(data, T, alpha_beta)
def _get_lds_effective_params(self):
mu_uniform, sigma_uniform = compute_uniform_mean_psi(self.V)
mu_init = np.tile(mu_uniform, self.K)
sigma_init = np.tile(np.diag(sigma_uniform), self.K)
sigma_states = np.repeat(self.sigmasq_states, (self.V - 1) * self.K)
sigma_obs = 1.0 / self.omega
y = kappa_vec(self.time_word_topic_counts, axis=1) / self.omega
return mu_init, sigma_init, sigma_states, sigma_obs.reshape(y.shape[0], -1), y.reshape(y.shape[0], -1)
def _get_lds_effective_params(self):
mu_uniform, sigma_uniform = compute_uniform_mean_psi(self.V)
mu_init = np.tile(mu_uniform, self.K)
sigma_init = np.tile(np.diag(sigma_uniform), self.K)
sigma_states = np.repeat(self.sigmasq_states, (self.V - 1) * self.K)
sigma_obs = 1. / self.omega
y = kappa_vec(self.time_word_topic_counts, axis=1) / self.omega
return mu_init, sigma_init, sigma_states, \
sigma_obs.reshape(y.shape[0], -1), y.reshape(y.shape[0], -1)
def initialize_parameters(self):
self.sigmasq_states = 0.1 # TODO make this learned, init from hypers
mean_psi = compute_uniform_mean_psi(self.V)[0][None, :, None]
self.psi = np.tile(mean_psi, (self.T, 1, self.K))
self.omega = np.zeros_like(self.psi)
self.theta = sample_dirichlet(self.alpha_theta * np.ones((self.D, self.K)), "horiz")
self.z = np.zeros((self.data.data.shape[0], self.K), dtype="uint32")
self.resample_z()
def initialize_parameters(self):
self.sigmasq_states = 0.1 # TODO make this learned, init from hypers
mean_psi = compute_uniform_mean_psi(self.V)[0][None, :, None]
self.psi = np.tile(mean_psi, (self.T, 1, self.K))
self.omega = np.zeros_like(self.psi)
self.theta = sample_dirichlet(
self.alpha_theta * np.ones((self.D, self.K)), 'horiz')
self.z = np.zeros((self.data.data.shape[0], self.K), dtype='uint32')
self.resample_z()
def __init__(self, data, T, alpha_beta):
mu, sigma = compute_uniform_mean_psi(T)
self.theta_prior = Gaussian(mu=mu,
sigma=sigma,
mu_0=mu,
sigma_0=T * sigma / 10.,
nu_0=T / 10.,
kappa_0=1. / 10)
self.ppgs = initialize_polya_gamma_samplers()
self.omega = np.zeros((data.shape[0], T - 1))
super(StickbreakingCorrelatedLDA, self).__init__(data, T, alpha_beta)
def test_correlated_pgm_rvs(Sigma):
K = Sigma.shape[0] + 1
mu, _ = compute_uniform_mean_psi(K)
print("mu: ", mu)
# Sample a bunch of pis and look at the marginals
samples = 10000
psis = np.random.multivariate_normal(mu, Sigma, size=samples)
pis = []
for smpl in xrange(samples):
pis.append(psi_to_pi(psis[smpl]))
pis = np.array(pis)
print("E[pi]: ", pis.mean(axis=0))
print("var[pi]: ", pis.var(axis=0))
plt.figure()
plt.subplot(311)
plt.boxplot(pis)
plt.xlabel("k")
plt.ylabel("$p(\pi_k)$")
# Plot the covariance
cov = np.cov(pis.T)
plt.subplot(323)
plt.imshow(cov[:-1,:-1], interpolation="None", cmap="cool")
plt.colorbar()
plt.title("Cov($\pi$)")
plt.subplot(324)
invcov = np.linalg.inv(cov[:-1,:-1] + np.diag(1e-6 * np.ones(K-1)))
# good = np.delete(np.arange(K), np.arange(0,K,3))
# invcov = np.linalg.inv(cov[np.ix_(good,good)])
plt.imshow(invcov, interpolation="None", cmap="cool")
plt.colorbar()
plt.title("Cov$(\pi)^{-1}$")
plt.subplot(325)
plt.imshow(Sigma, interpolation="None", cmap="cool")
plt.colorbar()
plt.title("$\Sigma$")
plt.subplot(326)
plt.imshow(np.linalg.inv(Sigma), interpolation="None", cmap="cool")
plt.colorbar()
plt.title("$\Sigma^{-1}$")
plt.savefig("correlated_psi_pi.png")
plt.show()
def test_correlated_pgm_rvs(Sigma):
K = Sigma.shape[0] + 1
mu, _ = compute_uniform_mean_psi(K)
print("mu: ", mu)
# Sample a bunch of pis and look at the marginals
samples = 10000
psis = np.random.multivariate_normal(mu, Sigma, size=samples)
pis = []
for smpl in range(samples):
pis.append(psi_to_pi(psis[smpl]))
pis = np.array(pis)
print("E[pi]: ", pis.mean(axis=0))
print("var[pi]: ", pis.var(axis=0))
plt.figure()
plt.subplot(311)
plt.boxplot(pis)
plt.xlabel("k")
plt.ylabel("$p(\pi_k)$")
# Plot the covariance
cov = np.cov(pis.T)
plt.subplot(323)
plt.imshow(cov[:-1, :-1], interpolation="None", cmap="cool")
plt.colorbar()
plt.title("Cov($\pi$)")
plt.subplot(324)
invcov = np.linalg.inv(cov[:-1, :-1] + np.diag(1e-6 * np.ones(K - 1)))
# good = np.delete(np.arange(K), np.arange(0,K,3))
# invcov = np.linalg.inv(cov[np.ix_(good,good)])
plt.imshow(invcov, interpolation="None", cmap="cool")
plt.colorbar()
plt.title("Cov$(\pi)^{-1}$")
plt.subplot(325)
plt.imshow(Sigma, interpolation="None", cmap="cool")
plt.colorbar()
plt.title("$\Sigma$")
plt.subplot(326)
plt.imshow(np.linalg.inv(Sigma), interpolation="None", cmap="cool")
plt.colorbar()
plt.title("$\Sigma^{-1}$")
plt.savefig("correlated_psi_pi.png")
plt.show()
def test_pgm_rvs():
K = 10
mu, sig = compute_uniform_mean_psi(K, sigma=2)
# mu = np.zeros(K-1)
# sig = np.ones(K-1)
print("mu: ", mu)
print("sig: ", sig)
Sigma = np.diag(sig)
# Add some covariance
# Sigma[:5,:5] = 1.0 + 1e-3*np.random.randn(5,5)
# Sample a bunch of pis and look at the marginals
pgm = PGMultinomial(K, mu=mu, Sigma=Sigma)
samples = 10000
pis = []
for smpl in xrange(samples):
pgm.resample()
pis.append(pgm.pi)
pis = np.array(pis)
print("E[pi]: ", pis.mean(axis=0))
print("var[pi]: ", pis.var(axis=0))
plt.figure()
plt.subplot(121)
plt.boxplot(pis)
plt.xlabel("k")
plt.ylabel("$p(\pi_k)$")
# Plot the covariance
cov = np.cov(pis.T)
plt.subplot(122)
plt.imshow(cov, interpolation="None", cmap="cool")
plt.colorbar()
plt.title("Cov($\pi$)")
plt.show()
def test_pgm_rvs():
K = 10
mu, sig = compute_uniform_mean_psi(K, sigma=2)
# mu = np.zeros(K-1)
# sig = np.ones(K-1)
print("mu: ", mu)
print("sig: ", sig)
Sigma = np.diag(sig)
# Add some covariance
# Sigma[:5,:5] = 1.0 + 1e-3*np.random.randn(5,5)
# Sample a bunch of pis and look at the marginals
pgm = PGMultinomial(K, mu=mu, Sigma=Sigma)
samples = 10000
pis = []
for smpl in range(samples):
pgm.resample()
pis.append(pgm.pi)
pis = np.array(pis)
print("E[pi]: ", pis.mean(axis=0))
print("var[pi]: ", pis.var(axis=0))
plt.figure()
plt.subplot(121)
plt.boxplot(pis)
plt.xlabel("k")
plt.ylabel("$p(\pi_k)$")
# Plot the covariance
cov = np.cov(pis.T)
plt.subplot(122)
plt.imshow(cov, interpolation="None", cmap="cool")
plt.colorbar()
plt.title("Cov($\pi$)")
plt.show()
def __init__(self, K, kernel, D, Z=None, X=None, mu=None, mu_0=None, Sigma_0=None):
self.K = K
self.D = D
self.kernel = kernel
# Start with an empty datalist
self.data_list = []
# Initialize the GP mean prior
if not any([_ is None for _ in (mu_0, Sigma_0)]):
assert mu_0.shape == (self.K-1,)
assert Sigma_0.shape == (self.K-1, self.K-1)
self.mu_0, self.Sigma_0 = mu_0, Sigma_0
else:
self.mu_0 = compute_uniform_mean_psi(K)[0]
self.Sigma_0 = np.eye(self.K-1)
# Initialize the GP mean
if mu is None:
self.mu = self.mu_0.copy()
else:
assert isinstance(mu, np.ndarray) and mu.shape == (self.K-1,)
self.mu = mu
betas.append(model.beta.copy())
# Check that the PG-Multinomial samples are distributed like the prior
thetas = np.array(thetas)
theta_mean = thetas.mean(0)
theta_std = thetas.std(0)
betas = np.array(betas)
beta_mean = betas.mean(0)
beta_std = betas.std(0)
# Now sample from the prior for comparison
print("Sampling from prior")
from pybasicbayes.distributions import GaussianFixedMean
from pgmult.utils import compute_uniform_mean_psi, psi_to_pi
mu, sigma0 = compute_uniform_mean_psi(T)
psis_prior = np.array(
[GaussianFixedMean(mu=mu, lmbda_0=T * sigma0, nu_0=T).rvs(1)
for _ in range(N_iter)])
thetas_prior = psi_to_pi(psis_prior[:,0,:])
betas_prior = np.random.dirichlet(alpha_beta*np.ones(V), size=(N_iter,))
# print "Mean psi: ", psi_mean, " +- ", psi_std
import pybasicbayes.util.general as general
percentilecutoff = 5
def plot_1d_scaled_quantiles(p1,p2,plot_midline=True):
# scaled quantiles so that multiple calls line up
p1.sort(), p2.sort() # NOTE: destructive! but that's cool
xmin,xmax = general.scoreatpercentile(p1,percentilecutoff), \
general.scoreatpercentile(p1,100-percentilecutoff)
betas.append(model.beta.copy())
# Check that the PG-Multinomial samples are distributed like the prior
thetas = np.array(thetas)
theta_mean = thetas.mean(0)
theta_std = thetas.std(0)
betas = np.array(betas)
beta_mean = betas.mean(0)
beta_std = betas.std(0)
# Now sample from the prior for comparison
print("Sampling from prior")
from pybasicbayes.distributions import GaussianFixedMean
from pgmult.utils import compute_uniform_mean_psi, psi_to_pi
mu, sigma0 = compute_uniform_mean_psi(T)
psis_prior = np.array(
[GaussianFixedMean(mu=mu, lmbda_0=T * sigma0, nu_0=T).rvs(1)
for _ in xrange(N_iter)])
thetas_prior = psi_to_pi(psis_prior[:,0,:])
betas_prior = np.random.dirichlet(alpha_beta*np.ones(V), size=(N_iter,))
# print "Mean psi: ", psi_mean, " +- ", psi_std
import pybasicbayes.util.general as general
percentilecutoff = 5
def plot_1d_scaled_quantiles(p1,p2,plot_midline=True):
# scaled quantiles so that multiple calls line up
p1.sort(), p2.sort() # NOTE: destructive! but that's cool
xmin,xmax = general.scoreatpercentile(p1,percentilecutoff), \
general.scoreatpercentile(p1,100-percentilecutoff)
