def __init__(self,
population,
D=2,
A=None,
C=None,
sigma_states=None,
sigma_C=1.0):
self.population = population
self.activation = population.activation_model
self.N = self.population.N
self.D = D
from pybasicbayes.distributions import Gaussian
self.init_dynamics_distn = Gaussian(mu_0=np.ones(D),
kappa_0=1.0,
sigma_0=0.000001 * np.eye(D),
nu_0=3.0)
from autoregressive.distributions import AutoRegression
self.dynamics_distn = AutoRegression(A=A,
sigma=sigma_states,
nu_0=D + 1.0,
S_0=0.5 * np.eye(D),
M_0=np.zeros((D, D)),
K_0=0.5 * np.eye(D))
# Initialize the emission matrix
if C is None:
self.C = sigma_C * np.random.randn(self.N, self.D)
else:
assert C.shape == (self.N, self.D)
self.C = C
self.sigma_C = sigma_C
def __init__(self,
N,
B=1,
dim=2,
b=0.5,
sigma=None,
Sigma_0=None,
nu_0=None,
mu_self=0.0,
eta=0.01):
super(_LatentDistanceModelGaussianMixin, self).__init__(N, B)
self.B = B
self.dim = dim
self.b = b
self.eta = eta
self.L = np.sqrt(eta) * np.random.randn(N, dim)
if Sigma_0 is None:
Sigma_0 = np.eye(B)
if nu_0 is None:
nu_0 = B + 2
self.cov = GaussianFixedMean(mu=np.zeros(B),
sigma=sigma,
lmbda_0=Sigma_0,
nu_0=nu_0)
# Special case self-weights (along the diagonal)
self._self_gaussian = Gaussian(mu_0=mu_self * np.ones(B),
sigma_0=Sigma_0,
nu_0=nu_0,
kappa_0=1.0)
def test_gaussian():
prior_data = 2*np.random.randn(5,2) + np.array([1.,3.])
a = Gaussian().empirical_bayes(prior_data)
# data = a.rvs(10)
gibbs_statistics = []
for itr in range(20000):
a.resample()
# a.resample(data)
gibbs_statistics.append(a.mu)
gibbs_statistics = np.array(gibbs_statistics)
b = AnnealedGaussianModel().empirical_bayes(prior_data)
# b.add_data(data)
pt = ParallelTempering(b,[5.])
pt_samples = pt.run(20000,1)
pt_statistics = np.array([m.mu for m in pt_samples])
fig = plt.figure()
testing.populations_eq_quantile_plot(gibbs_statistics,pt_statistics,fig=fig)
plt.savefig('gaussian_test.png')
testing.assert_populations_eq_moments(gibbs_statistics,pt_statistics), \
'Annealing MAY have failed, check FIGURES'
def __init__(self,
N,
B=1,
mu_0=None,
Sigma_0=None,
nu_0=None,
kappa_0=None):
super(NIWGaussianWeightDistribution, self).__init__(N)
self.B = B
if mu_0 is None:
mu_0 = np.zeros(B)
if Sigma_0 is None:
Sigma_0 = np.eye(B)
if nu_0 is None:
nu_0 = B + 2
if kappa_0 is None:
kappa_0 = 1.0
self._gaussian = Gaussian(mu_0=mu_0,
sigma_0=Sigma_0,
nu_0=nu_0,
kappa_0=kappa_0)
# Special case self-weights (along the diagonal)
self._self_gaussian = Gaussian(mu_0=mu_0,
sigma_0=Sigma_0,
nu_0=nu_0,
kappa_0=kappa_0)
def test_gaussian():
prior_data = 2 * np.random.randn(5, 2) + np.array([1., 3.])
a = Gaussian().empirical_bayes(prior_data)
# data = a.rvs(10)
gibbs_statistics = []
for itr in range(20000):
a.resample()
# a.resample(data)
gibbs_statistics.append(a.mu)
gibbs_statistics = np.array(gibbs_statistics)
b = AnnealedGaussianModel().empirical_bayes(prior_data)
# b.add_data(data)
pt = ParallelTempering(b, [5.])
pt_samples = pt.run(20000, 1)
pt_statistics = np.array([m.mu for m in pt_samples])
fig = plt.figure()
testing.populations_eq_quantile_plot(gibbs_statistics,
pt_statistics,
fig=fig)
plt.savefig('gaussian_test.png')
testing.assert_populations_eq_moments(gibbs_statistics,pt_statistics), \
'Annealing MAY have failed, check FIGURES'
class StickbreakingCorrelatedLDA(_LDABase):
"Correlated LDA with the stick breaking representation"
def __init__(self, data, T, alpha_beta):
mu, sigma = compute_uniform_mean_psi(T)
self.theta_prior = Gaussian(
mu=mu, sigma=sigma, mu_0=mu, sigma_0=T * sigma / 10.0, nu_0=T / 10.0, kappa_0=1.0 / 10
)
self.ppgs = initialize_polya_gamma_samplers()
self.omega = np.zeros((data.shape[0], T - 1))
super(StickbreakingCorrelatedLDA, self).__init__(data, T, alpha_beta)
@property
def theta(self):
return psi_to_pi(self.psi)
@theta.setter
def theta(self, theta):
self.psi = pi_to_psi(theta)
def initialize_theta(self):
self.psi = np.tile(self.theta_prior.mu, (self.D, 1))
def resample_theta(self):
self.resample_omega()
self.resample_psi()
def resample(self):
super(StickbreakingCorrelatedLDA, self).resample()
self.resample_theta_prior()
def resample_omega(self):
pgdrawvpar(
self.ppgs, N_vec(self.doc_topic_counts).astype("float64").ravel(), self.psi.ravel(), self.omega.ravel()
)
np.clip(self.omega, 1e-32, np.inf, out=self.omega)
def resample_psi(self):
Lmbda = np.linalg.inv(self.theta_prior.sigma)
h = Lmbda.dot(self.theta_prior.mu)
randvec = np.random.randn(self.D, self.T - 1) # pre-generate randomness
for d, c in enumerate(self.doc_topic_counts):
self.psi[d] = sample_infogaussian(Lmbda + np.diag(self.omega[d]), h + kappa_vec(c), randvec[d])
def resample_theta_prior(self):
self.theta_prior.resample(self.psi)
def copy_sample(self):
new = copy.copy(self)
new.beta = self.beta.copy()
new.psi = self.psi.copy()
new.theta_prior = self.theta_prior.copy_sample()
del new.z
del new.omega
return new
def __init__(self, data, T, alpha_beta):
mu, sigma = np.zeros(T), np.eye(T)
self.theta_prior = \
Gaussian(
mu=mu, sigma=sigma, mu_0=mu, sigma_0=T*sigma/10.,
nu_0=T/10., kappa_0=10.)
self.ppgs = initialize_polya_gamma_samplers()
self.omega = np.zeros((data.shape[0], T))
super(LogisticNormalCorrelatedLDA, self).__init__(data, T, alpha_beta)
def __init__(self,
N,
B=1,
C=2,
pi=10.0,
mu_0=None,
Sigma_0=None,
nu_0=None,
kappa_0=None,
special_case_self_conns=True):
"""
Initialize SBM with parameters defined above.
"""
super(SBMGaussianWeightDistribution, self).__init__(N)
self.B = B
assert isinstance(
C, int) and C >= 1, "C must be a positive integer number of blocks"
self.C = C
if isinstance(pi, (int, float)):
self.pi = pi * np.ones(C)
else:
assert isinstance(pi, np.ndarray) and pi.shape == (
C, ), "pi must be a sclar or a C-vector"
self.pi = pi
self.m = np.random.dirichlet(self.pi)
self.c = np.random.choice(self.C, p=self.m, size=(self.N))
if mu_0 is None:
mu_0 = np.zeros(B)
if Sigma_0 is None:
Sigma_0 = np.eye(B)
if nu_0 is None:
nu_0 = B + 2
if kappa_0 is None:
kappa_0 = 1.0
self._gaussians = [[
Gaussian(mu_0=mu_0, nu_0=nu_0, kappa_0=kappa_0, sigma_0=Sigma_0)
for _ in xrange(C)
] for _ in xrange(C)]
# Special case self-weights (along the diagonal)
self.special_case_self_conns = special_case_self_conns
if special_case_self_conns:
self._self_gaussian = Gaussian(mu_0=mu_0,
sigma_0=Sigma_0,
nu_0=nu_0,
kappa_0=kappa_0)
def test_hmmsgd_metaobs():
"""
"""
K = 2
D = 2
kappa_0 = 1
nu_0 = 4
emit1 = Gaussian(mu=np.array([0, 0]),
sigma=np.eye(2),
mu_0=np.zeros(2),
sigma_0=np.eye(2),
kappa_0=kappa_0,
nu_0=nu_0)
emit2 = Gaussian(mu=np.array([5, 5]),
sigma=np.eye(2),
mu_0=np.zeros(2),
sigma_0=np.eye(2),
kappa_0=kappa_0,
nu_0=nu_0)
emit = np.array([emit1, emit2])
N = 1000
obs = np.array([emit[int(np.round(i / N))].rvs()[0] for i in range(N)])
mu_0 = np.zeros(D)
sigma_0 = 0.75 * np.cov(obs.T)
kappa_0 = 0.01
nu_0 = 4
prior_emit = [
Gaussian(mu_0=mu_0, sigma_0=sigma_0, kappa_0=kappa_0, nu_0=nu_0)
for _ in range(K)
]
prior_emit = np.array(prior_emit)
prior_tran = np.ones(K * K).reshape((K, K))
prior_init = np.ones(K)
hmm = HMM.VBHMM(obs, prior_init, prior_tran, prior_emit)
hmm.infer()
full_var_x = hmm.full_local_update()
sts_true = np.array([int(np.round(i / N)) for i in range(N)])
# hamming distance
print('Hamming Distance = ', hmm.hamming_dist(full_var_x, sts_true)[0])
# plot learned emissions over observations
util.plot_emissions(obs, prior_emit, hmm.var_emit)
plt.show()
# plot elbo over iterations
plt.plot(hmm.elbo_vec)
plt.show()
def __init__(self, data, T, alpha_beta):
mu, sigma = compute_uniform_mean_psi(T)
self.theta_prior = Gaussian(mu=mu,
sigma=sigma,
mu_0=mu,
sigma_0=T * sigma / 10.,
nu_0=T / 10.,
kappa_0=1. / 10)
self.ppgs = initialize_polya_gamma_samplers()
self.omega = np.zeros((data.shape[0], T - 1))
super(StickbreakingCorrelatedLDA, self).__init__(data, T, alpha_beta)
def __init__(self, N, B,
mu_0=0.0, sigma_0=1.0, kappa_0=1.0, nu_0=3.0,
is_diagonal_weight_special=True,
**kwargs):
super(_IndependentGaussianMixin, self).__init__(N, B)
mu_0 = expand_scalar(mu_0, (B,))
sigma_0 = expand_cov(sigma_0, (B,B))
self._gaussian = Gaussian(mu_0=mu_0, sigma_0=sigma_0, kappa_0=kappa_0, nu_0=max(nu_0, B+2.))
self.is_diagonal_weight_special = is_diagonal_weight_special
if is_diagonal_weight_special:
self._self_gaussian = \
Gaussian(mu_0=mu_0, sigma_0=sigma_0, kappa_0=kappa_0, nu_0=nu_0)
def __init__(self, population, D=2,
A=None, C=None,
sigma_states=None,
sigma_C=1.0):
self.population = population
self.activation = population.activation_model
self.N = self.population.N
self.D = D
from pybasicbayes.distributions import Gaussian
self.init_dynamics_distn = Gaussian(mu_0=np.ones(D),
kappa_0=1.0,
sigma_0=0.000001 * np.eye(D),
nu_0=3.0)
from autoregressive.distributions import AutoRegression
self.dynamics_distn = AutoRegression(A=A, sigma=sigma_states,
nu_0=D+1.0, S_0=0.5 * np.eye(D),
M_0=np.zeros((D,D)), K_0=0.5 * np.eye(D))
# Initialize the emission matrix
if C is None:
self.C = sigma_C * np.random.randn(self.N, self.D)
else:
assert C.shape == (self.N, self.D)
self.C = C
self.sigma_C = sigma_C
def test_viterbi(T=1000, K=20, D=2):
# Create a true HMM
A = npr.rand(K, K)
A /= A.sum(axis=1, keepdims=True)
A = 0.75 * np.eye(K) + 0.25 * A
C = npr.randn(K, D)
sigma = 0.01
# Sample from the true HMM
z = np.zeros(T, dtype=int)
y = np.zeros((T, D))
for t in range(T):
if t > 0:
z[t] = np.random.choice(K, p=A[z[t-1]])
y[t] = C[z[t]] + np.sqrt(sigma) * npr.randn(D)
# Compare to pyhsmm answer
from pyhsmm.models import HMM as OldHMM
from pyhsmm.basic.distributions import Gaussian
oldhmm = OldHMM([Gaussian(mu=C[k], sigma=sigma * np.eye(D)) for k in range(K)],
trans_matrix=A,
init_state_distn="uniform")
oldhmm.add_data(y)
states = oldhmm.states_list.pop()
states.Viterbi()
z_star = states.stateseq
# Make an HMM with these parameters
hmm = ssm.HMM(K, D, observations="diagonal_gaussian")
hmm.transitions.log_Ps = np.log(A)
hmm.observations.mus = C
hmm.observations.sigmasq = sigma * np.ones((K, D))
z_star2 = hmm.most_likely_states(y)
assert np.allclose(z_star, z_star2)
def test_hmm_likelihood(T=1000, K=5, D=2):
# Create a true HMM
A = npr.rand(K, K)
A /= A.sum(axis=1, keepdims=True)
A = 0.75 * np.eye(K) + 0.25 * A
C = npr.randn(K, D)
sigma = 0.01
# Sample from the true HMM
z = np.zeros(T, dtype=int)
y = np.zeros((T, D))
for t in range(T):
if t > 0:
z[t] = np.random.choice(K, p=A[z[t - 1]])
y[t] = C[z[t]] + np.sqrt(sigma) * npr.randn(D)
# Compare to pyhsmm answer
from pyhsmm.models import HMM as OldHMM
from pybasicbayes.distributions import Gaussian
oldhmm = OldHMM(
[Gaussian(mu=C[k], sigma=sigma * np.eye(D)) for k in range(K)],
trans_matrix=A,
init_state_distn="uniform")
true_lkhd = oldhmm.log_likelihood(y)
# Make an HMM with these parameters
hmm = HMM(K, D, observations="diagonal_gaussian")
hmm.transitions.log_Ps = np.log(A)
hmm.observations.mus = C
hmm.observations.sigmasq = sigma * np.ones((K, D))
test_lkhd = hmm.log_probability(y)
assert np.allclose(true_lkhd, test_lkhd)
def make_synthetic_data():
mu_init = np.zeros(D)
# mu_init[0] = 1.0
sigma_init = 0.5 * np.eye(D)
A = np.eye(D)
# A[:2,:2] = \
# 0.99*np.array([[np.cos(np.pi/24), -np.sin(np.pi/24)],
# [np.sin(np.pi/24), np.cos(np.pi/24)]])
sigma_states = 0.1 * np.eye(D)
# C = np.hstack((np.ones((K-1, 1)), np.zeros((K-1, D-1))))
C = 0. * np.random.randn(K - 1, D)
truemodel = MultinomialLDS(K,
D,
init_dynamics_distn=Gaussian(mu=mu_init,
sigma=sigma_init),
dynamics_distn=AutoRegression(
A=A, sigma=sigma_states),
C=C)
data_list = []
Xs = []
for i in xrange(Ndata):
data = truemodel.generate(T=T, N=N)
data_list.append(data)
Xs.append(data["x"])
return data_list, Xs
def DefaultBernoulliLDS(D_obs, D_latent, D_input=0,
mu_init=None, sigma_init=None,
A=None, B=None, sigma_states=None,
C=None, D=None
):
model = LaplaceApproxBernoulliLDS(
init_dynamics_distn=
Gaussian(mu_0=np.zeros(D_latent), sigma_0=np.eye(D_latent),
kappa_0=1.0, nu_0=D_latent + 1),
dynamics_distn=Regression(
nu_0=D_latent + 1,
S_0=D_latent * np.eye(D_latent),
M_0=np.zeros((D_latent, D_latent + D_input)),
K_0=D_latent * np.eye(D_latent + D_input)),
emission_distn=
BernoulliRegression(D_obs, D_latent + D_input, verbose=False))
set_default = \
lambda prm, val, default: \
model.__setattr__(prm, val if val is not None else default)
set_default("mu_init", mu_init, np.zeros(D_latent))
set_default("sigma_init", sigma_init, np.eye(D_latent))
set_default("A", A, 0.99 * random_rotation(D_latent))
set_default("B", B, 0.1 * np.random.randn(D_latent, D_input))
set_default("sigma_states", sigma_states, 0.1 * np.eye(D_latent))
set_default("C", C, np.random.randn(D_obs, D_latent))
set_default("D", D, 0.1 * np.random.randn(D_obs, D_input))
return model
def make_rslds_parameters(C_init):
init_dynamics_distns = [
Gaussian(
mu=np.zeros(D_latent),
sigma=np.eye(D_latent),
nu_0=D_latent + 2, sigma_0=3. * np.eye(D_latent),
mu_0=np.zeros(D_latent), kappa_0=1.0,
)
for _ in range(args.K)]
dynamics_distns = [
Regression(
nu_0=D_latent + 2,
S_0=1e-4 * np.eye(D_latent),
M_0=np.hstack((np.eye(D_latent), np.zeros((D_latent, 1)))),
K_0=np.eye(D_latent + 1),
)
for _ in range(args.K)]
emission_distns = \
DiagonalRegression(args.D_obs, D_latent + 1,
A=C_init.copy(), sigmasq=np.ones(args.D_obs),
alpha_0=2.0, beta_0=2.0)
return init_dynamics_distns, dynamics_distns, emission_distns
def simulate_prior():
# Account for stick breaking asymmetry
mu_b, _ = compute_psi_cmoments(np.ones(K_true))
init_dynamics_distns = [
Gaussian(mu=np.array([-0, -0]), sigma=5 * np.eye(D_latent))
for _ in range(K)
]
dynamics_distns = make_dynamics_distns()
emission_distns = \
DiagonalRegression(D_obs, D_latent+1,
alpha_0=2.0, beta_0=2.0)
model = PGRecurrentSLDS(trans_params=dict(sigmasq_A=10., sigmasq_b=0.01),
init_state_distn='uniform',
init_dynamics_distns=init_dynamics_distns,
dynamics_distns=dynamics_distns,
emission_distns=emission_distns)
# Print the true parameters
np.set_printoptions(precision=2)
print("True W_markov:\n{}".format(model.trans_distn.A[:, :K_true]))
print("True W_input:\n{}".format(model.trans_distn.A[:, K_true:]))
return model
def __init__(self, N, B, mu, sigma, mu_self=None, sigma_self=None):
super(FixedGaussianWeightDistribution, self).__init__(N)
self.B = B
assert mu.shape == (B, )
self.mu = mu
assert sigma.shape == (B, B)
self.sigma = sigma
self._gaussian = Gaussian(mu, sigma)
if mu_self is not None and sigma_self is not None:
self._self_gaussian = Gaussian(mu_self, sigma_self)
else:
self._self_gaussian = self._gaussian
def __init__(self, data, T, alpha_beta):
mu, sigma = np.zeros(T), np.eye(T)
self.theta_prior = Gaussian(mu=mu, sigma=sigma, mu_0=mu, sigma_0=T * sigma / 10.0, nu_0=T / 10.0, kappa_0=10.0)
self.ppgs = initialize_polya_gamma_samplers()
self.omega = np.zeros((data.shape[0], T))
super(LogisticNormalCorrelatedLDA, self).__init__(data, T, alpha_beta)
def sample_slds_model():
mu_init = np.zeros(D_latent)
mu_init[0] = 2.0
sigma_init = 0.01 * np.eye(D_latent)
def random_rotation(n, theta):
rot = 0.99 * np.array([[np.cos(theta), -np.sin(theta)],
[np.sin(theta), np.cos(theta)]])
out = np.zeros((n, n))
out[:2, :2] = rot
q = np.linalg.qr(np.random.randn(n, n))[0]
return q.dot(out).dot(q.T)
def random_dynamics(n):
A = np.random.randn(n,n)
A = A.dot(A.T)
U,S,V = np.linalg.svd(A)
A_stable = U.dot(np.diag(S/(1.1*np.max(S)))).dot(V.T)
# A_stable = U.dot(0.99 * np.eye(n)).dot(V.T)
return A_stable
ths = np.linspace(0, np.pi/8., K)
As = [random_rotation(D_latent, ths[k]) for k in range(K)]
# As = [random_dynamics(D_latent) for k in range(K)]
bs = [np.zeros((D_latent, 1))] + [.25 * np.random.randn(D_latent, 1) for k in range(K-1)]
C = np.random.randn(D_obs, D_latent)
d = np.zeros((D_obs, 1))
sigma_obs = 0.5 * np.ones(D_obs)
###################
# generate data #
###################
init_dynamics_distns = [Gaussian(mu=mu_init, sigma=sigma_init) for _ in range(K)]
dynamics_distns = [Regression(A=np.hstack((A, b)), sigma=0.01 * np.eye(D_latent)) for A,b in zip(As, bs)]
emission_distns = DiagonalRegression(D_obs, D_latent+1, A=np.hstack((C, d)), sigmasq=sigma_obs)
slds = HMMSLDS(
dynamics_distns=dynamics_distns,
emission_distns=emission_distns,
init_dynamics_distns=init_dynamics_distns,
alpha=3., init_state_distn='uniform')
#### MANUALLY CREATE DATA
P = np.ones((K,K)) + 1 * np.eye(K)
P = P / np.sum(P,1,keepdims=True)
z = np.zeros(T//D, dtype=np.int32)
for t in range(1,T//D):
z[t] = np.random.choice(np.arange(K), p=P[z[t-1]])
z = np.repeat(z, D)
statesobj = slds._states_class(model=slds, T=z.size, stateseq=z, inputs=np.ones((z.size, 1)))
statesobj.generate_gaussian_states()
y = statesobj.data = statesobj.generate_obs()
x = statesobj.gaussian_states
slds.states_list.append(statesobj)
return z,x,y,slds
def __init__(self, data, T, alpha_beta):
mu, sigma = compute_uniform_mean_psi(T)
self.theta_prior = Gaussian(
mu=mu, sigma=sigma, mu_0=mu, sigma_0=T*sigma/10.,
nu_0=T/10., kappa_0=1./10)
self.ppgs = initialize_polya_gamma_samplers()
self.omega = np.zeros((data.shape[0], T-1))
super(StickbreakingCorrelatedLDA, self).__init__(data, T, alpha_beta)
def test_hmm_likelihood_perf(T=10000, K=50, D=20):
# Create a true HMM
A = npr.rand(K, K)
A /= A.sum(axis=1, keepdims=True)
A = 0.75 * np.eye(K) + 0.25 * A
C = npr.randn(K, D)
sigma = 0.01
# Sample from the true HMM
z = np.zeros(T, dtype=int)
y = np.zeros((T, D))
for t in range(T):
if t > 0:
z[t] = np.random.choice(K, p=A[z[t - 1]])
y[t] = C[z[t]] + np.sqrt(sigma) * npr.randn(D)
# Compare to pyhsmm answer
from pyhsmm.models import HMM as OldHMM
from pybasicbayes.distributions import Gaussian
oldhmm = OldHMM(
[Gaussian(mu=C[k], sigma=sigma * np.eye(D)) for k in range(K)],
trans_matrix=A,
init_state_distn="uniform")
states = oldhmm.add_data(y)
tic = time()
true_lkhd = states.log_likelihood()
pyhsmm_dt = time() - tic
print("PyHSMM: ", pyhsmm_dt, "sec. Val: ", true_lkhd)
# Make an HMM with these parameters
hmm = ssm.HMM(K, D, observations="gaussian")
hmm.transitions.log_Ps = np.log(A)
hmm.observations.mus = C
hmm.observations._sqrt_Sigmas = np.sqrt(sigma) * np.array(
[np.eye(D) for k in range(K)])
tic = time()
test_lkhd = hmm.log_probability(y)
smm_dt = time() - tic
print("SMM HMM: ", smm_dt, "sec. Val: ", test_lkhd)
# Make an ARHMM with these parameters
arhmm = ssm.HMM(K, D, observations="ar")
tic = time()
arhmm.log_probability(y)
arhmm_dt = time() - tic
print("SSM ARHMM: ", arhmm_dt, "sec.")
# Make an ARHMM with these parameters
arhmm = ssm.HMM(K, D, observations="ar")
tic = time()
arhmm.expected_states(y)
arhmm_dt = time() - tic
print("SSM ARHMM Expectations: ", arhmm_dt, "sec.")
def __init__(self, N, B=1, mu_0=None, Sigma_0=None, nu_0=None, kappa_0=None):
super(NIWGaussianWeightDistribution, self).__init__(N)
self.B = B
if mu_0 is None:
mu_0 = np.zeros(B)
if Sigma_0 is None:
Sigma_0 = np.eye(B)
if nu_0 is None:
nu_0 = B + 2
if kappa_0 is None:
kappa_0 = 1.0
self._gaussian = Gaussian(mu_0=mu_0, sigma_0=Sigma_0,
nu_0=nu_0, kappa_0=kappa_0)
# Special case self-weights (along the diagonal)
self._self_gaussian = Gaussian(mu_0=mu_0, sigma_0=Sigma_0,
nu_0=nu_0, kappa_0=kappa_0)
def fit_lds_model(Xs, Xtest, N_samples=100):
model = MultinomialLDS(K,
D,
init_dynamics_distn=Gaussian(mu_0=np.zeros(D),
sigma_0=np.eye(D),
kappa_0=1.0,
nu_0=D + 1.0),
dynamics_distn=AutoRegression(nu_0=D + 1,
S_0=np.eye(D),
M_0=np.zeros((D, D)),
K_0=np.eye(D)),
sigma_C=1)
for X in Xs:
model.add_data(X)
data = model.data_list[0]
samples = []
lls = []
test_lls = []
mc_test_lls = []
pis = []
psis = []
zs = []
timestamps = [time.time()]
for smpl in progprint_xrange(N_samples):
model.resample_model()
timestamps.append(time.time())
samples.append(model.copy_sample())
# TODO: Use log_likelihood() to marginalize over z
lls.append(model.log_likelihood())
# test_lls.append(model.heldout_log_likelihood(Xtest, M=50)[0])
mc_test_lls.append(model._mc_heldout_log_likelihood(Xtest, M=1)[0])
pis.append(model.pi(data))
psis.append(model.psi(data))
zs.append(data["states"].stateseq)
lls = np.array(lls)
test_lls = np.array(test_lls)
pis = np.array(pis)
psis = np.array(psis)
zs = np.array(zs)
timestamps = np.array(timestamps)
timestamps -= timestamps[0]
return model, lls, test_lls, mc_test_lls, pis, psis, zs, timestamps
def __init__(self, N, B=1,
C=2, pi=10.0,
mu_0=None, Sigma_0=None, nu_0=None, kappa_0=None,
special_case_self_conns=True):
"""
Initialize SBM with parameters defined above.
"""
super(SBMGaussianWeightDistribution, self).__init__(N)
self.B = B
assert isinstance(C, int) and C >= 1, "C must be a positive integer number of blocks"
self.C = C
if isinstance(pi, (int, float)):
self.pi = pi * np.ones(C)
else:
assert isinstance(pi, np.ndarray) and pi.shape == (C,), "pi must be a sclar or a C-vector"
self.pi = pi
self.m = np.random.dirichlet(self.pi)
self.c = np.random.choice(self.C, p=self.m, size=(self.N))
if mu_0 is None:
mu_0 = np.zeros(B)
if Sigma_0 is None:
Sigma_0 = np.eye(B)
if nu_0 is None:
nu_0 = B + 2
if kappa_0 is None:
kappa_0 = 1.0
self._gaussians = [[Gaussian(mu_0=mu_0, nu_0=nu_0,
kappa_0=kappa_0, sigma_0=Sigma_0)
for _ in xrange(C)]
for _ in xrange(C)]
# Special case self-weights (along the diagonal)
self.special_case_self_conns = special_case_self_conns
if special_case_self_conns:
self._self_gaussian = Gaussian(mu_0=mu_0, sigma_0=Sigma_0,
nu_0=nu_0, kappa_0=kappa_0)
def test_expectations(T=1000, K=20, D=2):
# Create a true HMM
A = npr.rand(K, K)
A /= A.sum(axis=1, keepdims=True)
A = 0.75 * np.eye(K) + 0.25 * A
C = npr.randn(K, D)
sigma = 0.01
# Sample from the true HMM
z = np.zeros(T, dtype=int)
y = np.zeros((T, D))
for t in range(T):
if t > 0:
z[t] = np.random.choice(K, p=A[z[t - 1]])
y[t] = C[z[t]] + np.sqrt(sigma) * npr.randn(D)
# Compare to pyhsmm answer
from pyhsmm.models import HMM as OldHMM
from pyhsmm.basic.distributions import Gaussian
oldhmm = OldHMM(
[Gaussian(mu=C[k], sigma=sigma * np.eye(D)) for k in range(K)],
trans_matrix=A,
init_state_distn="uniform")
oldhmm.add_data(y)
states = oldhmm.states_list.pop()
states.E_step()
true_Ez = states.expected_states
true_E_trans = states.expected_transcounts
# Make an HMM with these parameters
hmm = HMM(K, D, observations="diagonal_gaussian")
hmm.transitions.log_Ps = np.log(A)
hmm.observations.mus = C
hmm.observations.sigmasq = sigma * np.ones((K, D))
test_Ez, test_Ezzp1, _ = hmm.expected_states(y)
test_E_trans = test_Ezzp1.sum(0)
print(true_E_trans.round(3))
print(test_E_trans.round(3))
assert np.allclose(true_Ez, test_Ez)
assert np.allclose(true_E_trans, test_E_trans)
def DefaultPoissonLDS(
D_obs,
D_latent,
D_input=0,
mu_init=None,
sigma_init=None,
A=None,
B=None,
sigma_states=None,
C=None,
d=None,
):
assert D_input == 0, "Inputs are not yet supported for Poisson LDS"
model = LaplaceApproxPoissonLDS(
init_dynamics_distn=Gaussian(mu_0=np.zeros(D_latent),
sigma_0=np.eye(D_latent),
kappa_0=1.0,
nu_0=D_latent + 1),
dynamics_distn=Regression(A=0.9 * np.eye(D_latent),
sigma=np.eye(D_latent),
nu_0=D_latent + 1,
S_0=D_latent * np.eye(D_latent),
M_0=np.zeros((D_latent, D_latent)),
K_0=D_latent * np.eye(D_latent)),
emission_distn=PoissonRegression(D_obs, D_latent, verbose=False))
set_default = \
lambda prm, val, default: \
model.__setattr__(prm, val if val is not None else default)
set_default("mu_init", mu_init, np.zeros(D_latent))
set_default("sigma_init", sigma_init, np.eye(D_latent))
set_default("A", A, 0.99 * random_rotation(D_latent))
set_default("B", B, 0.1 * np.random.randn(D_latent, D_input))
set_default("sigma_states", sigma_states, 0.1 * np.eye(D_latent))
set_default("C", C, np.random.randn(D_obs, D_latent))
set_default("d", d, np.zeros((D_obs, 1)))
return model
def fit_arhmm(x, affine=True):
print("Fitting Sticky ARHMM")
dynamics_hypparams = \
dict(nu_0=D_latent + 2,
S_0=np.eye(D_latent),
M_0=np.hstack((np.eye(D_latent), np.zeros((D_latent, int(affine))))),
K_0=np.eye(D_latent + affine),
affine=affine)
dynamics_hypparams = get_empirical_ar_params([x], dynamics_hypparams)
dynamics_distns = [
AutoRegression(
A=np.column_stack((0.99 * np.eye(D_latent),
np.zeros((D_latent, int(affine))))),
sigma=np.eye(D_latent),
**dynamics_hypparams)
for _ in range(args.K)]
init_distn = Gaussian(nu_0=D_latent + 2,
sigma_0=np.eye(D_latent),
mu_0=np.zeros(D_latent),
kappa_0=1.0)
arhmm = ARWeakLimitStickyHDPHMM(
init_state_distn='uniform',
init_emission_distn=init_distn,
obs_distns=dynamics_distns,
alpha=3.0, kappa=10.0, gamma=3.0)
arhmm.add_data(x)
lps = []
for _ in tqdm(range(args.N_samples)):
arhmm.resample_model()
lps.append(arhmm.log_likelihood())
z_init = arhmm.states_list[0].stateseq
z_init = np.concatenate(([0], z_init))
return arhmm, z_init
def __init__(self,
N,
B=1,
dim=2,
b=0.5,
sigma=None,
Sigma_0=None,
nu_0=None,
mu_self=0.0,
eta=0.01):
"""
Initialize SBM with parameters defined above.
"""
super(LatentDistanceGaussianWeightDistribution, self).__init__(N)
self.B = B
self.dim = dim
self.b = b
self.eta = eta
self.L = np.sqrt(eta) * np.random.randn(N, dim)
if Sigma_0 is None:
Sigma_0 = np.eye(B)
if nu_0 is None:
nu_0 = B + 2
self.cov = GaussianFixedMean(mu=np.zeros(B),
sigma=sigma,
lmbda_0=Sigma_0,
nu_0=nu_0)
# Special case self-weights (along the diagonal)
self._self_gaussian = Gaussian(mu_0=mu_self * np.ones(B),
sigma_0=Sigma_0,
nu_0=nu_0,
kappa_0=1.0)
def make_rslds_parameters(C_init):
init_dynamics_distns = [
Gaussian(
mu=np.zeros(D_latent),
sigma=3 * np.eye(D_latent),
nu_0=D_latent + 2,
sigma_0=3. * np.eye(D_latent),
mu_0=np.zeros(D_latent),
kappa_0=1.0,
) for _ in range(K)
]
ths = np.random.uniform(np.pi / 30., 1.0, size=K)
As = [random_rotation(D_latent, th) for th in ths]
As = [np.hstack((A, np.ones((D_latent, 1)))) for A in As]
dynamics_distns = [
Regression(
A=As[k],
sigma=np.eye(D_latent),
nu_0=D_latent + 1000,
S_0=np.eye(D_latent),
M_0=np.hstack((np.eye(D_latent), np.zeros((D_latent, 1)))),
K_0=np.eye(D_latent + 1),
) for k in range(K)
]
if C_init is not None:
emission_distns = \
DiagonalRegression(D_obs, D_latent + 1,
A=C_init.copy(), sigmasq=np.ones(D_obs),
alpha_0=2.0, beta_0=2.0)
else:
emission_distns = \
DiagonalRegression(D_obs, D_latent + 1,
alpha_0=2.0, beta_0=2.0)
return init_dynamics_distns, dynamics_distns, emission_distns
class SBMGaussianWeightDistribution(GaussianWeightDistribution, GibbsSampling):
"""
A stochastic block model is a clustered network model with
C: Number of blocks
m[c]: Probability that a node belongs block c
mu[c,c']: Mean weight from node in block c to node in block c'
Sig[c,c']: Cov of weight from node in block c to node in block c'
It has hyperparameters:
pi: Parameter of Dirichlet prior over m
mu0, nu0, kappa0, Sigma0: Parameters of NIW prior over (mu,Sig)
"""
# TODO: Specify the self weight parameters in the constructor
def __init__(self, N, B=1,
C=2, pi=10.0,
mu_0=None, Sigma_0=None, nu_0=None, kappa_0=None,
special_case_self_conns=True):
"""
Initialize SBM with parameters defined above.
"""
super(SBMGaussianWeightDistribution, self).__init__(N)
self.B = B
assert isinstance(C, int) and C >= 1, "C must be a positive integer number of blocks"
self.C = C
if isinstance(pi, (int, float)):
self.pi = pi * np.ones(C)
else:
assert isinstance(pi, np.ndarray) and pi.shape == (C,), "pi must be a sclar or a C-vector"
self.pi = pi
self.m = np.random.dirichlet(self.pi)
self.c = np.random.choice(self.C, p=self.m, size=(self.N))
if mu_0 is None:
mu_0 = np.zeros(B)
if Sigma_0 is None:
Sigma_0 = np.eye(B)
if nu_0 is None:
nu_0 = B + 2
if kappa_0 is None:
kappa_0 = 1.0
self._gaussians = [[Gaussian(mu_0=mu_0, nu_0=nu_0,
kappa_0=kappa_0, sigma_0=Sigma_0)
for _ in xrange(C)]
for _ in xrange(C)]
# Special case self-weights (along the diagonal)
self.special_case_self_conns = special_case_self_conns
if special_case_self_conns:
self._self_gaussian = Gaussian(mu_0=mu_0, sigma_0=Sigma_0,
nu_0=nu_0, kappa_0=kappa_0)
@property
def _Mu(self):
return np.array([[self._gaussians[c1][c2].mu
for c2 in xrange(self.C)]
for c1 in xrange(self.C)])
@property
def _Sigma(self):
return np.array([[self._gaussians[c1][c2].sigma
for c2 in xrange(self.C)]
for c1 in xrange(self.C)])
@property
def Mu(self):
"""
Get the NxNxB matrix of weight means
:return:
"""
_Mu = self._Mu
Mu = _Mu[np.ix_(self.c, self.c)]
if self.special_case_self_conns:
for n in xrange(self.N):
Mu[n,n] = self._self_gaussian.mu
return Mu
@property
def Sigma(self):
"""
Get the NxNxBxB matrix of weight covariances
:return:
"""
_Sigma = self._Sigma
Sigma = _Sigma[np.ix_(self.c, self.c)]
if self.special_case_self_conns:
for n in xrange(self.N):
Sigma[n,n] = self._self_gaussian.sigma
return Sigma
def initialize_from_prior(self):
self.m = np.random.dirichlet(self.pi)
self.c = np.random.choice(self.C, p=self.m, size=(self.N))
for c1 in xrange(self.C):
for c2 in xrange(self.C):
self._gaussians[c1][c2].resample()
if self.special_case_self_conns:
self._self_gaussian.resample()
def initialize_hypers(self, W):
mu_0 = W.mean(axis=(0,1))
sigma_0 = np.diag(W.var(axis=(0,1)))
for c1 in xrange(self.C):
for c2 in xrange(self.C):
nu_0 = self._gaussians[c1][c2].nu_0
self._gaussians[c1][c2].mu_0 = mu_0
self._gaussians[c1][c2].sigma_0 = sigma_0 * (nu_0 - self.B - 1) / self.C
self._gaussians[c1][c2].resample()
if self.special_case_self_conns:
W_self = W[np.arange(self.N), np.arange(self.N)]
self._self_gaussian.mu_0 = W_self.mean(axis=0)
self._self_gaussian.sigma_0 = np.diag(W_self.var(axis=0))
self._self_gaussian.resample()
# Cluster the neurons based on their rows and columns
from sklearn.cluster import KMeans
features = np.hstack((W[:,:,0], W[:,:,0].T))
km = KMeans(n_clusters=self.C)
km.fit(features)
self.c = km.labels_.astype(np.int)
print "Initial c: ", self.c
def _get_mask(self, A, c1, c2):
mask = ((self.c==c1)[:,None] * (self.c==c2)[None,:])
mask &= A.astype(np.bool)
if self.special_case_self_conns:
mask &= True - np.eye(self.N, dtype=np.bool)
return mask
def log_likelihood(self, (A,W)):
N = self.N
assert A.shape == (N,N)
assert W.shape == (N,N,self.B)
ll = 0
for c1 in xrange(self.C):
for c2 in xrange(self.C):
mask = self._get_mask(A, c1, c2)
ll += self._gaussians[c1][c2].log_likelihood(W[mask]).sum()
if self.special_case_self_conns:
mask = np.eye(self.N).astype(np.bool) & A.astype(np.bool)
ll += self._self_gaussian.log_likelihood(W[mask]).sum()
return ll
sigma_init = 0.01*np.eye(2)
A = 0.99*np.array([[np.cos(np.pi/24), -np.sin(np.pi/24)],
[np.sin(np.pi/24), np.cos(np.pi/24)]])
sigma_states = 0.01*np.eye(2)
C = np.array([[2.,0.]])
D_out, D_in = C.shape
###################
# generate data #
###################
truemodel = NegativeBinomialLDS(
init_dynamics_distn=Gaussian(mu_init, sigma_init),
dynamics_distn=AutoRegression(A=A,sigma=sigma_states),
emission_distn=PGEmissions(D_out, D_in, C=C))
T = 2000
data, z_true = truemodel.generate(T)
psi_true = z_true.dot(C.T)
p_true = logistic(psi_true)
###############
# fit model #
###############
model = NegativeBinomialLDS(
init_dynamics_distn=Gaussian(mu_0=np.zeros(D_in), sigma_0=np.eye(D_in),
def main(name, datadir, datafn, K, expdir=None, nfolds=1, nrestarts=1, seed=None):
""" Run experiment on 4 state, two group synthetic data.
name : Name of experiment.
datadir : Path to directory containing data.
datafn : Prefix name to files that data and missing masks are stored
in.
K : Number of components in HMM.
expdir : Path to directory to store experiment results. If None
(default), then a directory, `name`_results, is made in the
current directory.
nfolds : Number of folds to generate if datafn is None.
nrestarts : Number of random initial parameters.
seed : Random number seed.
"""
# Set seed for reproducibility
np.random.seed(seed)
# Generate/Load data and folds (missing masks)
# These are the emission distributions for the following tests
if not os.path.exists(datadir):
raise RuntimeError("Could not find datadir: %s" % (datadir,))
else:
if not os.path.isdir(datadir):
raise RuntimeError("datadir: %s exists but is not a directory" % (datadir,))
if datafn is None:
datafn = name
dpath = os.path.join(datadir, datafn + "_data.txt")
mpath = os.path.join(datadir, datafn + "_fold*.txt")
try:
X = np.loadtxt(dpath)
except IOError:
if os.path.exists(dpath) and not os.path.isdir(dpath):
raise RuntimeError("Could not load data: %s" % (dpath,))
masks = glob.glob(mpath)
if len(masks) == 0:
masks = [None]
# Initialize parameter possibilities
obs_mean = np.mean(X, axis=0)
mu_0 = obs_mean
sigma_0 = 0.75*np.cov(X.T)
# Vague values that keeps covariance matrices p.d.
kappa_0 = 0.01
nu_0 = 4
prior_init = np.ones(K)
prior_tran = np.ones((K,K))
N, D = X.shape
rand_starts = list()
for r in xrange(nrestarts):
init_means = np.empty((K,D))
init_cov = list()
for k in xrange(K):
init_means[k,:] = mvnrand(mu_0, cov=sigma_0)
init_cov.append(sample_invwishart(np.linalg.inv(sigma_0), nu_0))
# We use prior b/c mu and sigma are sampled here
prior_emit = np.array([Gaussian(mu=init_means[k,:], sigma=sigma_0,
mu_0=mu_0, sigma_0=sigma_0,
kappa_0=kappa_0, nu_0=nu_0)
for k in xrange(K)])
init_init = np.random.rand(K)
init_init /= np.sum(init_init)
init_tran = np.random.rand(K,K)
init_tran /= np.sum(init_tran, axis=1)[:,np.newaxis]
# Make dict with initial parameters to pass to experiment.
pd = {'init_init': init_init, 'init_tran': init_tran,
'prior_init': prior_init, 'prior_tran': prior_tran,
'prior_emit': prior_emit, 'maxit': maxit, 'verbose': verbose}
rand_starts.append(pd)
# Compute Cartesian product of random starts with other possible parameter
# values, make a generator to fill in entries in the par dicts created
# above, and then construct the par_list by calling the generator with the
# Cartesian product iterator.
par_prod_iter = itertools.product(rand_starts, taus, kappas, reuse_msg,
grow_buffer, Ls, correct_trans)
def gen_par(par_tuple):
d = copy.copy(par_tuple[0])
d['tau'] = par_tuple[1]
d['kappa'] = par_tuple[2]
d['reuseMsg'] = par_tuple[3]
d['growBuffer'] = par_tuple[4]
d['metaobs_half'] = par_tuple[5]
d['correctTrans'] = par_tuple[6]
d['mb_sz'] = 100//(2*par_tuple[5]+1)
return d
# Call gen_par on each par product to pack into dictionary to pass to
# experiment.
par_list = itertools.imap(gen_par, par_prod_iter)
# Create ExperimentSequential and call run_exper
dname = os.path.join(datadir, datafn + "_data.txt")
exp = ExpSeq(datafn, dname, run_exper, par_list,
masks=masks, exper_dir=expdir)
exp.run()
class _IndependentGaussianMixin(_NetworkModel):
"""
Each weight is an independent Gaussian with a shared NIW prior.
Special case the self-connections.
"""
def __init__(self, N, B,
mu_0=0.0, sigma_0=1.0, kappa_0=1.0, nu_0=3.0,
is_diagonal_weight_special=True,
**kwargs):
super(_IndependentGaussianMixin, self).__init__(N, B)
mu_0 = expand_scalar(mu_0, (B,))
sigma_0 = expand_cov(sigma_0, (B,B))
self._gaussian = Gaussian(mu_0=mu_0, sigma_0=sigma_0, kappa_0=kappa_0, nu_0=max(nu_0, B+2.))
self.is_diagonal_weight_special = is_diagonal_weight_special
if is_diagonal_weight_special:
self._self_gaussian = \
Gaussian(mu_0=mu_0, sigma_0=sigma_0, kappa_0=kappa_0, nu_0=nu_0)
@property
def mu_W(self):
N, B = self.N, self.B
mu = np.zeros((N, N, B))
if self.is_diagonal_weight_special:
# Set off-diagonal weights
mask = np.ones((N, N), dtype=bool)
mask[np.diag_indices(N)] = False
mu[mask] = self._gaussian.mu
# set diagonal weights
mask = np.eye(N).astype(bool)
mu[mask] = self._self_gaussian.mu
else:
mu = np.tile(self._gaussian.mu[None,None,:], (N, N, 1))
return mu
@property
def sigma_W(self):
N, B = self.N, self.B
if self.is_diagonal_weight_special:
sigma = np.zeros((N, N, B, B))
# Set off-diagonal weights
mask = np.ones((N, N), dtype=bool)
mask[np.diag_indices(N)] = False
sigma[mask] = self._gaussian.sigma
# set diagonal weights
mask = np.eye(N).astype(bool)
sigma[mask] = self._self_gaussian.sigma
else:
sigma = np.tile(self._gaussian.mu[None, None, :, :], (N, N, 1, 1))
return sigma
def resample(self, data=[]):
super(_IndependentGaussianMixin, self).resample(data)
A, W = data
N, B = self.N, self.B
if self.is_diagonal_weight_special:
# Resample prior for off-diagonal weights
mask = np.ones((N, N), dtype=bool)
mask[np.diag_indices(N)] = False
mask = mask & A
self._gaussian.resample(W[mask])
# Resample prior for diagonal weights
mask = np.eye(N).astype(bool) & A
self._self_gaussian.resample(W[mask])
else:
# Resample prior for all weights
self._gaussian.resample(W[A])
A[:2,:2] = \
0.99*np.array([[np.cos(np.pi/24), -np.sin(np.pi/24)],
[np.sin(np.pi/24), np.cos(np.pi/24)]])
sigma_states = 0.1 * np.eye(D)
K = 3
# C = np.hstack((np.ones((K-1, 1)), np.zeros((K-1, D-1))))
C = np.random.randn(K - 1, D)
###################
# generate data #
###################
model = MultinomialLDS(K,
D,
init_dynamics_distn=Gaussian(mu=mu_init,
sigma=sigma_init),
dynamics_distn=AutoRegression(A=A, sigma=sigma_states),
C=C)
data = model.generate(T=T, N=N, keep=False)
# data["x"] = np.hstack([np.zeros((T,K-1)), np.ones((T,1))])
# Estimate the held out likelihood using Monte Carlo
M = 10000
hll_mc, std_mc = model._mc_heldout_log_likelihood(data["x"], M=M)
# Estimate the held out log likelihood
# hll_info, std_info = model._info_form_heldout_log_likelihood(data["x"], M=M)
hll_dist, std_dist = model._distn_form_heldout_log_likelihood(data["x"], M=M)
print("MC. Model: ", hll_mc, " +- ", std_mc)
# print "AN. Model (info): ", hll_info, " +- ", std_info
import numpy as np
import sys
from pybasicbayes.distributions import Gaussian
from gen_synthetic import generate_data
if __name__ == '__main__':
prefix = 'cy'
K=5
A = np.array([[.00, .99,0.00, .00, .01],
[.01, .00, .99, .00, .00],
[.00, .01, .00, .99, .00],
[.00, .00, .01, .00, .99],
[.99, .00, .00, .01, .00]])
means= [(0, 0), (10, 10), (10, 10), (0, 0), (10, 10)]
means = [Gaussian(mu=np.array(m), sigma=np.eye(2)) for m in means]
means = np.array(means)
obs, sts, _ = generate_data(A, means, 10000)
np.savetxt('data/%s_data.txt' % prefix, obs)
np.savetxt('data/%s_sts.txt' % prefix, sts)
D_in = 2
D_obs = 2
Nmax = 2
tgrid = np.linspace(-5 * np.pi, 5 * np.pi, T)
covariate_seq = np.column_stack((np.sin(tgrid), np.cos(tgrid)))
covariate_seq += 0.001 * np.random.randn(T, D_in)
obs_hypparams = {
'mu_0': np.zeros(D_obs),
'sigma_0': np.eye(D_obs),
'kappa_0': 1.0,
'nu_0': D_obs + 2
}
true_model = \
PGInputHMM(obs_distns=[Gaussian(**obs_hypparams) for state in range(Nmax)],
init_state_concentration=1.0,
D_in=D_in,
trans_params=dict(sigmasq_A=4.0, sigmasq_b=0.001))
# Set the weights by hand such that they primarily
# depend on the input
true_model.trans_distn.A[0][Nmax:] = [5., 5.]
# true_model.trans_distn.A[1][Nmax:] = [-2., 2.]
# true_model.trans_distn.A[2][Nmax:] = [-2., -2.]
# generate fake data and plot
dataset = [
true_model.generate(T, covariates=covariate_seq[:, :D_in])
for _ in range(5)
]
class LinearDynamicalSystemBackground(Component):
"""
Linear Dynamical System model for the background activation.
Since the potentials for the activation are of a Gaussian form,
we can perform conjugate Gibbs sampling or variational inference
for a Gaussian LDS model.
"""
def __init__(self, population, D=2,
A=None, C=None,
sigma_states=None,
sigma_C=1.0):
self.population = population
self.activation = population.activation_model
self.N = self.population.N
self.D = D
from pybasicbayes.distributions import Gaussian
self.init_dynamics_distn = Gaussian(mu_0=np.ones(D),
kappa_0=1.0,
sigma_0=0.000001 * np.eye(D),
nu_0=3.0)
from autoregressive.distributions import AutoRegression
self.dynamics_distn = AutoRegression(A=A, sigma=sigma_states,
nu_0=D+1.0, S_0=0.5 * np.eye(D),
M_0=np.zeros((D,D)), K_0=0.5 * np.eye(D))
# Initialize the emission matrix
if C is None:
self.C = sigma_C * np.random.randn(self.N, self.D)
else:
assert C.shape == (self.N, self.D)
self.C = C
self.sigma_C = sigma_C
def augment_data(self, augmented_data):
# Add a latent state sequence
augmented_data["states"] = self.generate_states(augmented_data["T"])
def log_likelihood(self, augmented_data):
raise NotImplementedError
def generate(self,T):
states = self.generate_states(T)
return states.dot(self.C.T)
def generate_states(self, T):
stateseq = np.empty((T,self.D))
stateseq[0] = self.init_dynamics_distn.rvs()
chol = np.linalg.cholesky(self.dynamics_distn.sigma)
randseq = np.random.randn(T-1,self.D)
for t in xrange(1,T):
stateseq[t] = \
self.dynamics_distn.A.dot(stateseq[t-1]) \
+ chol.dot(randseq[t-1])
return stateseq
def mean_background_activation(self, augmented_data):
return augmented_data["states"].dot(self.C.T)
def resample(self, augmented_data_list):
self.resample_states(augmented_data_list)
self.resample_parameters(augmented_data_list)
def resample_states(self, augmented_data_list):
from pylds.lds_messages import filter_and_sample
for data in augmented_data_list:
# Compute the residual activation from other components
psi = self.activation.compute_psi(data)
psi_residual = psi - self.mean_background_activation(data)
# Get the observed mean and variance
mu_obs = self.activation.new_mean(data)
prec_obs = self.activation.new_precision(data)
# Subtract off the activation from other components
mu_obs -= psi_residual
# Convert prec_obs into an array of diagonal covariance matrices
sigma_obs = np.empty((data["T"], self.N, self.N), order="C")
for t in xrange(data["T"]):
sigma_obs[t,:,:] = np.diag(1./prec_obs[t,:])
data["states"] = filter_and_sample(
self.init_dynamics_distn.mu,
self.init_dynamics_distn.sigma,
self.dynamics_distn.A,
self.dynamics_distn.sigma,
self.C,
sigma_obs,
mu_obs)
def resample_parameters(self, augmented_data_list):
self.resample_init_dynamics_distn(augmented_data_list)
self.resample_dynamics_distn(augmented_data_list)
self.resample_emission_distn(augmented_data_list)
def resample_init_dynamics_distn(self, augmented_data_list):
states_list = [ad["states"][0] for ad in augmented_data_list]
self.init_dynamics_distn.resample(states_list)
def resample_dynamics_distn(self, augmented_data_list):
from pyhsmm.util.general import AR_striding
states_list = [ad["states"] for ad in augmented_data_list]
strided_states_list = [AR_striding(s,1) for s in states_list]
self.dynamics_distn.resample(strided_states_list)
def resample_emission_distn(self, augmented_data_list):
"""
Resample the observation vectors. Since the emission noise is diagonal,
we can resample the columns of C independently
:return:
"""
# Get the prior
prior_precision = 1./self.sigma_C * np.eye(self.D)
prior_mean = np.zeros(self.D)
prior_mean_dot_precision = prior_mean.dot(prior_precision)
# Get the sufficient statistics from the likelihood
lkhd_precision = np.zeros((self.N, self.D, self.D))
lkhd_mean_dot_precision = np.zeros((self.N, self.D))
for data in augmented_data_list:
# Compute the residual activation from other components
psi = self.activation.compute_psi(data)
psi_residual = psi - self.mean_background_activation(data)
# Get the observed mean and variance
mu_obs = self.activation.new_mean(data)
prec_obs = self.activation.new_precision(data)
# Subtract off the residual
mu_obs -= psi_residual
# Update the sufficient statistics for each neuron
for n in xrange(self.N):
lkhd_precision[n,:,:] += (data["states"] * prec_obs[:,n][:,None]).T.dot(data["states"])
lkhd_mean_dot_precision[n,:] += \
(mu_obs[:,n] * prec_obs[:,n]).T.dot(data["states"])
# Sample each column of C
for n in xrange(self.N):
post_prec = prior_precision + lkhd_precision[n,:,:]
post_cov = np.linalg.inv(post_prec)
post_mu = (prior_mean_dot_precision +
lkhd_mean_dot_precision[n,:]).dot(post_cov)
post_mu = post_mu.ravel()
self.C[n,:] = np.random.multivariate_normal(post_mu, post_cov)
### Variational inference
def meanfieldupdate(self, augmented_data): raise NotImplementedError
def get_vlb(self, augmented_data): raise NotImplementedError
def resample_from_mf(self, augmented_data): raise NotImplementedError
def svi_step(self, augmented_data, minibatchfrac, stepsize): raise NotImplementedError
class NIWGaussianWeightDistribution(GaussianWeightDistribution, GibbsSampling):
"""
Gaussian weight distribution with a normal inverse-Wishart prior.
"""
# TODO: Specify the self weight parameters in the constructor
def __init__(self, N, B=1, mu_0=None, Sigma_0=None, nu_0=None, kappa_0=None):
super(NIWGaussianWeightDistribution, self).__init__(N)
self.B = B
if mu_0 is None:
mu_0 = np.zeros(B)
if Sigma_0 is None:
Sigma_0 = np.eye(B)
if nu_0 is None:
nu_0 = B + 2
if kappa_0 is None:
kappa_0 = 1.0
self._gaussian = Gaussian(mu_0=mu_0, sigma_0=Sigma_0,
nu_0=nu_0, kappa_0=kappa_0)
# Special case self-weights (along the diagonal)
self._self_gaussian = Gaussian(mu_0=mu_0, sigma_0=Sigma_0,
nu_0=nu_0, kappa_0=kappa_0)
@property
def Mu(self):
mu = self._gaussian.mu
Mu = np.tile(mu[None,None,:], (self.N, self.N,1))
for n in xrange(self.N):
Mu[n,n,:] = self._self_gaussian.mu
return Mu
@property
def Sigma(self):
sig = self._gaussian.sigma
Sig = np.tile(sig[None,None,:,:], (self.N, self.N,1,1))
for n in xrange(self.N):
Sig[n,n,:,:] = self._self_gaussian.sigma
return Sig
def initialize_from_prior(self):
self._gaussian.resample()
self._self_gaussian.resample()
def initialize_hypers(self, W):
# self.B = W.shape[2]
mu_0 = W.mean(axis=(0,1))
sigma_0 = np.diag(W.var(axis=(0,1)))
self._gaussian.mu_0 = mu_0
self._gaussian.sigma_0 = sigma_0
self._gaussian.resample()
# self._gaussian.nu_0 = self.B + 2
W_self = W[np.arange(self.N), np.arange(self.N)]
self._self_gaussian.mu_0 = W_self.mean(axis=0)
self._self_gaussian.sigma_0 = np.diag(W_self.var(axis=0))
self._self_gaussian.resample()
# self._self_gaussian.nu_0 = self.B + 2
def log_prior(self):
from graphistician.internals.utils import normal_inverse_wishart_log_prob
lp = 0
lp += normal_inverse_wishart_log_prob(self._gaussian)
lp += normal_inverse_wishart_log_prob(self._self_gaussian)
return lp
def sample_predictive_parameters(self):
Murow = Mucol = np.tile(self._gaussian.mu[None,:], (self.N+1,1))
Lrow = Lcol = np.tile(self._gaussian.sigma_chol[None,:,:], (self.N+1,1,1))
Murow[-1,:] = self._self_gaussian.mu
Mucol[-1,:] = self._self_gaussian.mu
Lrow[-1,:,:] = self._self_gaussian.sigma_chol
Lcol[-1,:,:] = self._self_gaussian.sigma_chol
return Murow, Mucol, Lrow, Lcol
def resample(self, (A,W)):
# Resample the Normal-inverse Wishart prior over mu and W
# given W for which A=1
A_offdiag = A.copy()
np.fill_diagonal(A_offdiag, 0)
A_ondiag = A * np.eye(self.N)
self._gaussian.resample(W[A_offdiag==1])
self._self_gaussian.resample(W[A_ondiag==1])
class NIWGaussianWeightDistribution(GaussianWeightDistribution, GibbsSampling):
"""
Gaussian weight distribution with a normal inverse-Wishart prior.
"""
# TODO: Specify the self weight parameters in the constructor
def __init__(self,
N,
B=1,
mu_0=None,
Sigma_0=None,
nu_0=None,
kappa_0=None):
super(NIWGaussianWeightDistribution, self).__init__(N)
self.B = B
if mu_0 is None:
mu_0 = np.zeros(B)
if Sigma_0 is None:
Sigma_0 = np.eye(B)
if nu_0 is None:
nu_0 = B + 2
if kappa_0 is None:
kappa_0 = 1.0
self._gaussian = Gaussian(mu_0=mu_0,
sigma_0=Sigma_0,
nu_0=nu_0,
kappa_0=kappa_0)
# Special case self-weights (along the diagonal)
self._self_gaussian = Gaussian(mu_0=mu_0,
sigma_0=Sigma_0,
nu_0=nu_0,
kappa_0=kappa_0)
@property
def Mu(self):
mu = self._gaussian.mu
Mu = np.tile(mu[None, None, :], (self.N, self.N, 1))
for n in xrange(self.N):
Mu[n, n, :] = self._self_gaussian.mu
return Mu
@property
def Sigma(self):
sig = self._gaussian.sigma
Sig = np.tile(sig[None, None, :, :], (self.N, self.N, 1, 1))
for n in xrange(self.N):
Sig[n, n, :, :] = self._self_gaussian.sigma
return Sig
def initialize_from_prior(self):
self._gaussian.resample()
self._self_gaussian.resample()
def initialize_hypers(self, W):
# self.B = W.shape[2]
mu_0 = W.mean(axis=(0, 1))
sigma_0 = np.diag(W.var(axis=(0, 1)))
self._gaussian.mu_0 = mu_0
self._gaussian.sigma_0 = sigma_0
self._gaussian.resample()
# self._gaussian.nu_0 = self.B + 2
W_self = W[np.arange(self.N), np.arange(self.N)]
self._self_gaussian.mu_0 = W_self.mean(axis=0)
self._self_gaussian.sigma_0 = np.diag(W_self.var(axis=0))
self._self_gaussian.resample()
# self._self_gaussian.nu_0 = self.B + 2
def log_prior(self):
from graphistician.internals.utils import normal_inverse_wishart_log_prob
lp = 0
lp += normal_inverse_wishart_log_prob(self._gaussian)
lp += normal_inverse_wishart_log_prob(self._self_gaussian)
return lp
def sample_predictive_parameters(self):
Murow = Mucol = np.tile(self._gaussian.mu[None, :], (self.N + 1, 1))
Lrow = Lcol = np.tile(self._gaussian.sigma_chol[None, :, :],
(self.N + 1, 1, 1))
Murow[-1, :] = self._self_gaussian.mu
Mucol[-1, :] = self._self_gaussian.mu
Lrow[-1, :, :] = self._self_gaussian.sigma_chol
Lcol[-1, :, :] = self._self_gaussian.sigma_chol
return Murow, Mucol, Lrow, Lcol
def resample(self, (A, W)):
# Resample the Normal-inverse Wishart prior over mu and W
# given W for which A=1
A_offdiag = A.copy()
np.fill_diagonal(A_offdiag, 0)
A_ondiag = A * np.eye(self.N)
self._gaussian.resample(W[A_offdiag == 1])
self._self_gaussian.resample(W[A_ondiag == 1])
class LogisticNormalCorrelatedLDA(_LDABase):
"Correlated LDA with the stick breaking representation"
def __init__(self, data, T, alpha_beta):
mu, sigma = np.zeros(T), np.eye(T)
self.theta_prior = \
Gaussian(
mu=mu, sigma=sigma, mu_0=mu, sigma_0=T*sigma/10.,
nu_0=T/10., kappa_0=10.)
self.ppgs = initialize_polya_gamma_samplers()
self.omega = np.zeros((data.shape[0], T))
super(LogisticNormalCorrelatedLDA, self).__init__(data, T, alpha_beta)
@property
def theta(self):
return ln_psi_to_pi(self.psi)
@theta.setter
def theta(self, theta):
self.psi = ln_pi_to_psi(theta)
def initialize_theta(self):
self.psi = np.tile(self.theta_prior.mu, (self.D, 1))
def resample_theta(self):
self.resample_psi_and_omega()
def resample(self):
super(LogisticNormalCorrelatedLDA, self).resample()
self.resample_theta_prior()
def resample_psi_and_omega(self):
Lmbda = np.linalg.inv(self.theta_prior.sigma)
for d in xrange(self.D):
N = self.data[d].sum()
c = self.doc_topic_counts[d]
for t in xrange(self.T):
self.omega[d,t] = self.ppgs[0].pgdraw(
N, self._conditional_omega(d,t))
mu_cond, sigma_cond = self._conditional_psi(d, t, Lmbda, N, c)
self.psi[d,t] = np.random.normal(mu_cond, np.sqrt(sigma_cond))
def _conditional_psi(self, d, t, Lmbda, N, c):
nott = np.arange(self.T) != t
psi = self.psi[d]
omega = self.omega[d]
mu = self.theta_prior.mu
zetat = logsumexp(psi[nott])
mut_marg = mu[t] - 1./Lmbda[t,t] * Lmbda[t,nott].dot(psi[nott] - mu[nott])
sigmat_marg = 1./Lmbda[t,t]
sigmat_cond = 1./(omega[t] + 1./sigmat_marg)
# kappa is the mean dot precision, i.e. the sufficient statistic of a Gaussian
# therefore we can sum over datapoints
kappa = (c[t] - N/2.0).sum()
mut_cond = sigmat_cond * (kappa + mut_marg / sigmat_marg + omega[t]*zetat)
return mut_cond, sigmat_cond
def _conditional_omega(self, d, t):
nott = np.arange(self.T) != t
psi = self.psi[d]
zetat = logsumexp(psi[nott])
return psi[t] - zetat
def resample_theta_prior(self):
self.theta_prior.resample(self.psi)
def copy_sample(self):
new = copy.copy(self)
new.beta = self.beta.copy()
new.psi = self.psi.copy()
new.theta_prior = self.theta_prior.copy_sample()
del new.z
del new.omega
return new
