def __init__(self, K, n, C=None, sigma_C=1, mu=None, mu_pi=None):
"""
Create a PGMultinomial distribution with mean and covariance for psi.
:param K: Dimensionality of the multinomial distribution
:param mu_C: Mean of the matrix normal distribution over C
"""
assert isinstance(K, int) and K >= 2, "K must be an integer >= 2"
self.K = K
assert isinstance(n, int) and n >= 1, "n must be an integer >= 1"
self.n = n
# Initialize emission matrix C
self.sigma_C = sigma_C
if C is None:
self.C = self.sigma_C * np.random.randn(self.K-1, self.n)
# mu, sigma = compute_psi_cmoments(np.ones(K))
# self.C = compute_psi_cmoments(np.ones(K))[0][:,None] * np.ones((self.K-1, self.n))
else:
assert C.shape == (self.K-1, self.n)
self.C = C
# Initialize the observation mean (mu)
if mu is None and mu_pi is None:
self.mu = np.zeros(self.K-1)
elif mu is not None:
assert mu.shape == (self.K-1,)
self.mu = mu
else:
assert mu_pi.shape == (self.K,)
self.mu = pi_to_psi(mu_pi)
# Initialize Polya-gamma augmentation variables
self.ppgs = initialize_polya_gamma_samplers()
def add_data(self, Z, X, optimize_hypers=True):
assert Z.ndim == 2 and Z.shape[1] == self.D
M = Z.shape[0]
assert X.shape == (M, self.K)
# Get the empirical probabilities (offset by 1 to ensure nonzero)
pi_emp_train = (self.alpha+X).astype(np.float) / \
(self.alpha + X).sum(axis=1)[:,None]
# Convert these to psi's
self.Z = Z
self.psi = np.array([pi_to_psi(pi) for pi in pi_emp_train])
# Compute the mean value of psi
self.mu = self.psi.mean(axis=0)
self.psi -= self.mu
# Create the GP Regression model
from GPy.models import GPRegression
self.model = GPRegression(Z, self.psi, self.kernel)
# Optimize the kernel parameters
if optimize_hypers:
self.model.optimize(messages=True)
def fit_gp_multinomial_model(model, test, pi_train=None, N_samples=100, run=1):
if pi_train is not None:
if isinstance(model, pgmult.gp.LogisticNormalGP):
model.data_list[0]["psi"] = ln_pi_to_psi(pi_train) - model.mu
elif isinstance(model, pgmult.gp.MultinomialGP):
model.data_list[0]["psi"] = pi_to_psi(pi_train) - model.mu
model.resample_omega()
else:
model.initialize_from_data()
### Inference
results_base = os.path.join("results", "names", "run%03d" % run, "results")
results_file = results_base + ".pkl.gz"
if os.path.exists(results_file):
with gzip.open(results_file, "r") as f:
samples, lls, pred_lls, timestamps = pickle.load(f)
else:
Z_test = get_inputs(test)
lls = [model.log_likelihood()]
samples = [model.copy_sample()]
pred_ll, pred_pi = model.predictive_log_likelihood(Z_test, test.data)
pred_lls = [pred_ll]
pred_pis = [pred_pi]
times = [0]
# Print initial values
print("Initial LL: ", lls[0])
print("Initial Pred LL: ", pred_lls[0])
for itr in range(N_samples):
print("Iteration ", itr)
tic = time.time()
model.resample_model(verbose=True)
times.append(time.time()-tic)
samples.append(model.copy_sample())
lls.append(model.log_likelihood())
pred_ll, pred_pi = model.predictive_log_likelihood(get_inputs(test), test.data)
pred_lls.append(pred_ll)
pred_pis.append(pred_pi)
print("Log likelihood: ", lls[-1])
print("Pred Log likelihood: ", pred_ll)
# Save this sample
# with gzip.open(results_file + ".itr%03d.pkl.gz" % itr, "w") as f:
# pickle.dump(model, f, protocol=-1)
lls = np.array(lls)
pred_lls = np.array(pred_lls)
timestamps = np.cumsum(times)
return samples, lls, pred_lls, pred_pis, timestamps
def fit_gp_multinomial_model(model, test, pi_train=None, N_samples=100, run=1):
if pi_train is not None:
if isinstance(model, pgmult.gp.LogisticNormalGP):
model.data_list[0]["psi"] = ln_pi_to_psi(pi_train) - model.mu
elif isinstance(model, pgmult.gp.MultinomialGP):
model.data_list[0]["psi"] = pi_to_psi(pi_train) - model.mu
model.resample_omega()
else:
model.initialize_from_data()
### Inference
results_base = os.path.join("results", "names", "run%03d" % run, "results")
results_file = results_base + ".pkl.gz"
if os.path.exists(results_file):
with gzip.open(results_file, "r") as f:
samples, lls, pred_lls, timestamps = pickle.load(f)
else:
Z_test = get_inputs(test)
lls = [model.log_likelihood()]
samples = [model.copy_sample()]
pred_ll, pred_pi = model.predictive_log_likelihood(Z_test, test.data)
pred_lls = [pred_ll]
pred_pis = [pred_pi]
times = [0]
# Print initial values
print("Initial LL: ", lls[0])
print("Initial Pred LL: ", pred_lls[0])
for itr in xrange(N_samples):
print("Iteration ", itr)
tic = time.time()
model.resample_model(verbose=True)
times.append(time.time()-tic)
samples.append(model.copy_sample())
lls.append(model.log_likelihood())
pred_ll, pred_pi = model.predictive_log_likelihood(get_inputs(test), test.data)
pred_lls.append(pred_ll)
pred_pis.append(pred_pi)
print("Log likelihood: ", lls[-1])
print("Pred Log likelihood: ", pred_ll)
# Save this sample
# with gzip.open(results_file + ".itr%03d.pkl.gz" % itr, "w") as f:
# pickle.dump(model, f, protocol=-1)
lls = np.array(lls)
pred_lls = np.array(pred_lls)
timestamps = np.cumsum(times)
return samples, lls, pred_lls, pred_pis, timestamps
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100):
"""
Fit a logistic normal LDS model with pMCMC
"""
Nx = len(Xs)
assert len(Xtest) == Nx
print("Fitting SBM-LDS with %d states using pMCMC" % D)
models = [
ParticleSBMultinomialLDS(
init_dynamics_distn=GaussianFixed(mu=np.zeros(D),
sigma=1 * np.eye(D)),
dynamics_distn=AutoRegression(nu_0=D + 1,
S_0=D * np.eye(D),
M_0=np.zeros((D, D)),
K_0=D * np.eye(D)),
emission_distn=Regression(nu_0=K + 1,
S_0=K * np.eye(K),
M_0=np.zeros((K, D)),
K_0=K * np.eye(D)),
mu=pi_to_psi(np.ones(K) / K),
sigma_C=1.0) for _ in xrange(Nx)
]
for model in models:
model.A = 0.5 * np.eye(D)
model.sigma_states = np.eye(D)
model.C = np.random.randn(K - 1, D)
model.sigma_obs = 0.1 * np.eye(K)
for X, model in zip(Xs, models):
model.add_data(X)
def compute_pred_ll():
pred_ll = 0
for Xte, model in zip(Xtest, models):
pred_ll += model.predictive_log_likelihood(Xte, Npred=100)[0]
return pred_ll
init_results = (0, None, np.nan, np.nan, compute_pred_ll())
def resample():
tic = time.time()
[model.resample_model() for model in models]
toc = time.time() - tic
return toc, None, np.nan, np.nan, compute_pred_ll()
times, samples, lls, test_lls, pred_lls = \
map(np.array, zip(*([init_results] +
[resample() for _ in progprint_xrange(N_samples, perline=5)])))
timestamps = np.cumsum(times)
return Results(lls, test_lls, pred_lls, samples, timestamps)
def test_psi_pi_conversion_3d():
K = 10
N = 10
D = 10
pi = np.random.rand(N, D, K)
pi /= np.sum(pi, axis=2, keepdims=True)
psi = np.array([pi_to_psi(p) for p in pi])
pi2 = psi_to_pi(psi, axis=2)
assert np.allclose(pi, pi2), "Mapping is not invertible."
def test_psi_pi_conversion():
K = 10
pi = np.ones(K) / float(K)
psi = pi_to_psi(pi)
pi2 = psi_to_pi(psi)
print("pi: ", pi)
print("psi: ", psi)
print("pi2: ", pi2)
assert np.allclose(pi, pi2), "Mapping is not invertible."
def test_psi_pi_conversion():
K = 10
pi = np.ones(K) / float(K)
psi = pi_to_psi(pi)
pi2 = psi_to_pi(psi)
print("pi: ", pi)
print("psi: ", psi)
print("pi2: ", pi2)
assert np.allclose(pi, pi2), "Mapping is not invertible."
def initialize_from_data(self, pi_lim=None, initialize_to_mle=True):
"""
Initialize the psi's to the empirical mean of the data
:return:
"""
for data in self.data_list:
# Compute the empirical probability
X = data["X"]
M = data["M"]
assert X.ndim == 2 and X.shape[1] == self.K
# Get the empirical probabilities (offset by 1 to ensure nonzero)
alpha = 1
pi_emp = (alpha+X).astype(np.float) / \
(alpha + X).sum(axis=1)[:,None]
pi_emp_mean = pi_emp.mean(axis=0)
# Set mu equal to the empirical mean value of pi
psi_emp_mean = pi_to_psi(pi_emp_mean)
self.mu = psi_emp_mean
# self.mu = np.zeros(self.K-1)
if initialize_to_mle:
# Convert empirical values to psi
psi_emp = np.array([pi_to_psi(p) for p in pi_emp])
psi_emp -= self.mu
assert psi_emp.shape == (M, self.K-1)
# Set the deviations from the mean to zero
data["psi"] = psi_emp
else:
data["psi"] = np.zeros((M, self.K-1))
# Resample the omegas
self.resample_omega()
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100):
"""
Fit a logistic normal LDS model with pMCMC
"""
print("Fitting SBM-LDS with %d states using pMCMC" % D)
model = ParticleSBMultinomialLDS(
init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1 * np.eye(D)),
dynamics_distn=AutoRegression(nu_0=D + 1,
S_0=D * np.eye(D),
M_0=np.zeros((D, D)),
K_0=D * np.eye(D)),
emission_distn=Regression(nu_0=K + 1,
S_0=K * np.eye(K),
M_0=np.zeros((K, D)),
K_0=K * np.eye(D)),
mu=pi_to_psi(np.ones(K) / K))
model.A = 0.5 * np.eye(D)
model.sigma_states = np.eye(D)
model.C = np.random.randn(K - 1, D)
model.sigma_obs = 0.1 * np.eye(K)
for X in Xs:
model.add_data(X)
compute_pred_ll = lambda: sum([
model.predictive_log_likelihood(Xt, data_index=i, Npred=10)[0]
for i, Xt in enumerate(Xtest)
])
init_results = (0, None, model.log_likelihood(), np.nan, compute_pred_ll())
def resample():
tic = time.time()
model.resample_model()
toc = time.time() - tic
# pred_ll = model.predictive_log_likelihood(Xtest, Npred=1000)
return toc, None, model.log_likelihood(), \
np.nan, \
compute_pred_ll()
times, samples, lls, test_lls, pred_lls = \
list(map(np.array, list(zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)])))))
timestamps = np.cumsum(times)
return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100):
"""
Fit a logistic normal LDS model with pMCMC
"""
Nx = len(Xs)
assert len(Xtest) == Nx
print("Fitting SBM-LDS with %d states using pMCMC" % D)
models = [ParticleSBMultinomialLDS(
init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)),
dynamics_distn=AutoRegression(nu_0=D+1,S_0=D*np.eye(D),M_0=np.zeros((D,D)),K_0=D*np.eye(D)),
emission_distn=Regression(nu_0=K+1,S_0=K*np.eye(K),M_0=np.zeros((K,D)),K_0=K*np.eye(D)),
mu=pi_to_psi(np.ones(K)/K),
sigma_C=1.0)
for _ in xrange(Nx)]
for model in models:
model.A = 0.5*np.eye(D)
model.sigma_states = np.eye(D)
model.C = np.random.randn(K-1,D)
model.sigma_obs = 0.1*np.eye(K)
for X, model in zip(Xs, models):
model.add_data(X)
def compute_pred_ll():
pred_ll = 0
for Xte, model in zip(Xtest, models):
pred_ll += model.predictive_log_likelihood(Xte, Npred=100)[0]
return pred_ll
init_results = (0, None, np.nan, np.nan, compute_pred_ll())
def resample():
tic = time.time()
[model.resample_model() for model in models]
toc = time.time() - tic
return toc, None, np.nan, np.nan, compute_pred_ll()
times, samples, lls, test_lls, pred_lls = \
map(np.array, zip(*([init_results] +
[resample() for _ in progprint_xrange(N_samples, perline=5)])))
timestamps = np.cumsum(times)
return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100):
"""
Fit a logistic normal LDS model with pMCMC
"""
print("Fitting SBM-LDS with %d states using pMCMC" % D)
model = ParticleSBMultinomialLDS(
init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)),
dynamics_distn=AutoRegression(nu_0=D+1,S_0=D*np.eye(D),M_0=np.zeros((D,D)),K_0=D*np.eye(D)),
emission_distn=Regression(nu_0=K+1,S_0=K*np.eye(K),M_0=np.zeros((K,D)),K_0=K*np.eye(D)),
mu=pi_to_psi(np.ones(K)/K))
model.A = 0.5*np.eye(D)
model.sigma_states = np.eye(D)
model.C = np.random.randn(K-1,D)
model.sigma_obs = 0.1*np.eye(K)
for X in Xs:
model.add_data(X)
compute_pred_ll = lambda: sum([model.predictive_log_likelihood(Xt, data_index=i, Npred=10)[0]
for i,Xt in enumerate(Xtest)])
init_results = (0, None, model.log_likelihood(),
np.nan, compute_pred_ll())
def resample():
tic = time.time()
model.resample_model()
toc = time.time() - tic
# pred_ll = model.predictive_log_likelihood(Xtest, Npred=1000)
return toc, None, model.log_likelihood(), \
np.nan, \
compute_pred_ll()
times, samples, lls, test_lls, pred_lls = \
map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)])))
timestamps = np.cumsum(times)
return Results(lls, test_lls, pred_lls, samples, timestamps)
nonempty_docs = np.asarray(model.data.sum(1) > 0).ravel()
model.theta[nonempty_docs] = ln_psi_to_pi(lmbda)
model.resample_z()
return model
fit_lda_gibbs = sampler_fitter(
'fit_lda_gibbs', StandardLDA, 'resample', lda_initializer)
fit_lda_collapsed = sampler_fitter(
'fit_lda_collapsed', StandardLDA, 'resample_collapsed', lda_initializer)
fit_lnctm_gibbs = sampler_fitter(
'fit_lnctm_gibbs', LogisticNormalCorrelatedLDA, 'resample',
make_ctm_initializer(lambda lmbda: lmbda))
fit_sbctm_gibbs = sampler_fitter(
'fit_sbctm_gibbs', StickbreakingCorrelatedLDA, 'resample',
make_ctm_initializer(lambda lmbda: pi_to_psi(ln_psi_to_pi(lmbda))))
########################
# inspecting results #
########################
def plot_sb_interpretable_results(sb_results, words):
nwords = 5
Sigma = sb_results[-1][-1]
T = Sigma.shape[0]
def get_topwords(topic):
return words[np.argsort(sb_results[-1][0][:,topic])[-nwords:]]
lim = np.abs(Sigma).max()
def pi(self, value):
self.psi = pi_to_psi(value)
nonempty_docs = np.asarray(model.data.sum(1) > 0).ravel()
model.theta[nonempty_docs] = ln_psi_to_pi(lmbda)
model.resample_z()
return model
fit_lda_gibbs = sampler_fitter(
'fit_lda_gibbs', StandardLDA, 'resample', lda_initializer)
fit_lda_collapsed = sampler_fitter(
'fit_lda_collapsed', StandardLDA, 'resample_collapsed', lda_initializer)
fit_lnctm_gibbs = sampler_fitter(
'fit_lnctm_gibbs', LogisticNormalCorrelatedLDA, 'resample',
make_ctm_initializer(lambda lmbda: lmbda))
fit_sbctm_gibbs = sampler_fitter(
'fit_sbctm_gibbs', StickbreakingCorrelatedLDA, 'resample',
make_ctm_initializer(lambda lmbda: pi_to_psi(ln_psi_to_pi(lmbda))))
########################
# inspecting results #
########################
def plot_sb_interpretable_results(sb_results, words):
nwords = 5
Sigma = sb_results[-1][-1]
T = Sigma.shape[0]
def get_topwords(topic):
return words[np.argsort(sb_results[-1][0][:,topic])[-nwords:]]
lim = np.abs(Sigma).max()
def theta(self, theta):
self.psi = pi_to_psi(theta)
def theta(self, theta):
self.psi = pi_to_psi(theta)