def __init__(self,
population,
D=2,
A=None,
C=None,
sigma_states=None,
sigma_C=1.0):
self.population = population
self.activation = population.activation_model
self.N = self.population.N
self.D = D
from pybasicbayes.distributions import Gaussian
self.init_dynamics_distn = Gaussian(mu_0=np.ones(D),
kappa_0=1.0,
sigma_0=0.000001 * np.eye(D),
nu_0=3.0)
from autoregressive.distributions import AutoRegression
self.dynamics_distn = AutoRegression(A=A,
sigma=sigma_states,
nu_0=D + 1.0,
S_0=0.5 * np.eye(D),
M_0=np.zeros((D, D)),
K_0=0.5 * np.eye(D))
# Initialize the emission matrix
if C is None:
self.C = sigma_C * np.random.randn(self.N, self.D)
else:
assert C.shape == (self.N, self.D)
self.C = C
self.sigma_C = sigma_C
def __init__(self, population, D=2,
A=None, C=None,
sigma_states=None,
sigma_C=1.0):
self.population = population
self.activation = population.activation_model
self.N = self.population.N
self.D = D
from pybasicbayes.distributions import Gaussian
self.init_dynamics_distn = Gaussian(mu_0=np.ones(D),
kappa_0=1.0,
sigma_0=0.000001 * np.eye(D),
nu_0=3.0)
from autoregressive.distributions import AutoRegression
self.dynamics_distn = AutoRegression(A=A, sigma=sigma_states,
nu_0=D+1.0, S_0=0.5 * np.eye(D),
M_0=np.zeros((D,D)), K_0=0.5 * np.eye(D))
# Initialize the emission matrix
if C is None:
self.C = sigma_C * np.random.randn(self.N, self.D)
else:
assert C.shape == (self.N, self.D)
self.C = C
self.sigma_C = sigma_C
def fit_lds_model(Xs, Xtest, D, N_samples=100):
model = MultinomialLDS(K,
D,
init_dynamics_distn=GaussianFixed(mu=np.zeros(D),
sigma=1 *
np.eye(D)),
dynamics_distn=AutoRegression(nu_0=D + 1,
S_0=1 * np.eye(D),
M_0=np.zeros((D, D)),
K_0=1 * np.eye(D)),
sigma_C=0.01)
for X in Xs:
model.add_data(X)
model.resample_parameters()
init_results = (0, model, model.log_likelihood(),
model.heldout_log_likelihood(Xtest, M=1),
model.predictive_log_likelihood(Xtest, M=1000))
def resample():
tic = time.time()
model.resample_model()
toc = time.time() - tic
return toc, None, model.log_likelihood(), \
model.heldout_log_likelihood(Xtest, M=1), \
model.predictive_log_likelihood(Xtest, M=1000)
times, samples, lls, test_lls, pred_lls = \
map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)])))
timestamps = np.cumsum(times)
return Results(lls, test_lls, pred_lls, samples, timestamps)
def make_synthetic_data():
mu_init = np.zeros(D)
# mu_init[0] = 1.0
sigma_init = 0.5 * np.eye(D)
A = np.eye(D)
# A[:2,:2] = \
# 0.99*np.array([[np.cos(np.pi/24), -np.sin(np.pi/24)],
# [np.sin(np.pi/24), np.cos(np.pi/24)]])
sigma_states = 0.1 * np.eye(D)
# C = np.hstack((np.ones((K-1, 1)), np.zeros((K-1, D-1))))
C = 0. * np.random.randn(K - 1, D)
truemodel = MultinomialLDS(K,
D,
init_dynamics_distn=Gaussian(mu=mu_init,
sigma=sigma_init),
dynamics_distn=AutoRegression(
A=A, sigma=sigma_states),
C=C)
data_list = []
Xs = []
for i in xrange(Ndata):
data = truemodel.generate(T=T, N=N)
data_list.append(data)
Xs.append(data["x"])
return data_list, Xs
def fit_gaussian_lds_model(Xs, Xtest, D_gauss_lds, N_samples=100):
Nx = len(Xs)
assert len(Xtest) == Nx
print("Fitting Gaussian (Raw) LDS with %d states" % D_gauss_lds)
models = [NonstationaryLDS(
init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)),
dynamics_distn=AutoRegression(nu_0=D+1,S_0=1*np.eye(D),M_0=np.zeros((D,D)),K_0=1*np.eye(D)),
emission_distn=Regression(nu_0=K+1,S_0=K*np.eye(K),M_0=np.zeros((K,D)),K_0=K*np.eye(D)))
for _ in xrange(Nx)]
Xs_centered = [X - np.mean(X, axis=0)[None,:] + 1e-3*np.random.randn(*X.shape) for X in Xs]
for X, model in zip(Xs_centered, models):
model.add_data(X)
def compute_pred_ll():
pred_ll = 0
for Xtr, Xte, model in zip(Xs_centered, Xtest, models):
# Monte Carlo sample to get pi density implied by Gaussian LDS
Npred = 10
Tpred = Xte.shape[0]
preds = model.sample_predictions(Xtr, Tpred, Npred=Npred)
# Convert predictions to a distribution by finding the
# largest dimension for each predicted Gaussian.
# Preds is T x K x Npred, inds is TxNpred
inds = np.argmax(preds, axis=1)
pi = np.array([np.bincount(inds[t], minlength=K) for t in xrange(Tpred)]) / float(Npred)
assert np.allclose(pi.sum(axis=1), 1.0)
pi = np.clip(pi, 1e-8, 1.0)
pi /= pi.sum(axis=1)[:,None]
# Compute the log likelihood under pi
pred_ll += np.sum([Multinomial(weights=pi[t], K=K).log_likelihood(Xte[t][None,:])
for t in xrange(Tpred)])
return pred_ll
# TODO: Get initial pred ll
init_results = (0, None, np.nan, np.nan, compute_pred_ll())
def resample():
tic = time.time()
[model.resample_model() for model in models]
toc = time.time() - tic
return toc, None, np.nan, np.nan, compute_pred_ll()
times, samples, lls, test_lls, pred_lls = \
map(np.array, zip(*([init_results] +
[resample() for _ in progprint_xrange(N_samples, perline=5)])))
timestamps = np.cumsum(times)
return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100):
"""
Fit a logistic normal LDS model with pMCMC
"""
Nx = len(Xs)
assert len(Xtest) == Nx
print("Fitting SBM-LDS with %d states using pMCMC" % D)
models = [
ParticleSBMultinomialLDS(
init_dynamics_distn=GaussianFixed(mu=np.zeros(D),
sigma=1 * np.eye(D)),
dynamics_distn=AutoRegression(nu_0=D + 1,
S_0=D * np.eye(D),
M_0=np.zeros((D, D)),
K_0=D * np.eye(D)),
emission_distn=Regression(nu_0=K + 1,
S_0=K * np.eye(K),
M_0=np.zeros((K, D)),
K_0=K * np.eye(D)),
mu=pi_to_psi(np.ones(K) / K),
sigma_C=1.0) for _ in xrange(Nx)
]
for model in models:
model.A = 0.5 * np.eye(D)
model.sigma_states = np.eye(D)
model.C = np.random.randn(K - 1, D)
model.sigma_obs = 0.1 * np.eye(K)
for X, model in zip(Xs, models):
model.add_data(X)
def compute_pred_ll():
pred_ll = 0
for Xte, model in zip(Xtest, models):
pred_ll += model.predictive_log_likelihood(Xte, Npred=100)[0]
return pred_ll
init_results = (0, None, np.nan, np.nan, compute_pred_ll())
def resample():
tic = time.time()
[model.resample_model() for model in models]
toc = time.time() - tic
return toc, None, np.nan, np.nan, compute_pred_ll()
times, samples, lls, test_lls, pred_lls = \
map(np.array, zip(*([init_results] +
[resample() for _ in progprint_xrange(N_samples, perline=5)])))
timestamps = np.cumsum(times)
return Results(lls, test_lls, pred_lls, samples, timestamps)
def DefaultLDS(n, p):
model = LDS(
dynamics_distn=AutoRegression(
nu_0=n+1, S_0=n*np.eye(n), M_0=np.zeros((n, n)), K_0=n*np.eye(n)),
emission_distn=Regression(
nu_0=p+1, S_0=p*np.eye(p), M_0=np.zeros((p, n)), K_0=p*np.eye(n)))
model.A = 0.99*np.eye(n)
model.sigma_states = np.eye(n)
model.C = np.random.randn(p, n)
model.sigma_obs = 0.1*np.eye(p)
return model
def fit_lds_model(Xs, Xtest, D, N_samples=100):
Nx = len(Xs)
assert len(Xtest) == Nx
mus = [X.sum(0) + 0.1 for X in Xs]
mus = [mu / mu.sum() for mu in mus]
# mus = [np.ones(K)/float(K) for _ in Xs]
models = [
MultinomialLDS(K,
D,
init_dynamics_distn=GaussianFixed(mu=np.zeros(D),
sigma=1 * np.eye(D)),
dynamics_distn=AutoRegression(nu_0=D + 1,
S_0=1 * np.eye(D),
M_0=np.zeros((D, D)),
K_0=1 * np.eye(D)),
sigma_C=1.,
mu_pi=mus[i]) for i in xrange(Nx)
]
for X, model in zip(Xs, models):
model.add_data(X)
[model.resample_parameters() for model in models]
def compute_pred_ll():
pred_ll = 0
for Xt, model in zip(Xtest, models):
pred_ll += model.predictive_log_likelihood(Xt, M=1)[0]
return pred_ll
init_results = (0, models, np.nan, np.nan, compute_pred_ll())
def resample():
tic = time.time()
[model.resample_model() for model in models]
toc = time.time() - tic
return toc, None, np.nan, np.nan, compute_pred_ll()
times, samples, lls, test_lls, pred_lls = \
map(np.array, zip(*([init_results] +
[resample() for _ in progprint_xrange(N_samples, perline=5)])))
timestamps = np.cumsum(times)
return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_lds_model_with_pmcmc(Xs, Xtest, D, N_samples=100):
"""
Fit a logistic normal LDS model with pMCMC
"""
print("Fitting SBM-LDS with %d states using pMCMC" % D)
model = ParticleSBMultinomialLDS(
init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1 * np.eye(D)),
dynamics_distn=AutoRegression(nu_0=D + 1,
S_0=D * np.eye(D),
M_0=np.zeros((D, D)),
K_0=D * np.eye(D)),
emission_distn=Regression(nu_0=K + 1,
S_0=K * np.eye(K),
M_0=np.zeros((K, D)),
K_0=K * np.eye(D)),
mu=pi_to_psi(np.ones(K) / K))
model.A = 0.5 * np.eye(D)
model.sigma_states = np.eye(D)
model.C = np.random.randn(K - 1, D)
model.sigma_obs = 0.1 * np.eye(K)
for X in Xs:
model.add_data(X)
compute_pred_ll = lambda: sum([
model.predictive_log_likelihood(Xt, data_index=i, Npred=10)[0]
for i, Xt in enumerate(Xtest)
])
init_results = (0, None, model.log_likelihood(), np.nan, compute_pred_ll())
def resample():
tic = time.time()
model.resample_model()
toc = time.time() - tic
# pred_ll = model.predictive_log_likelihood(Xtest, Npred=1000)
return toc, None, model.log_likelihood(), \
np.nan, \
compute_pred_ll()
times, samples, lls, test_lls, pred_lls = \
list(map(np.array, list(zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)])))))
timestamps = np.cumsum(times)
return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_lds_model(Xs, Xtest, D, N_samples=100):
Nx = len(Xs)
assert len(Xtest) == Nx
model = MultinomialLDS(K,
D,
init_dynamics_distn=GaussianFixed(mu=np.zeros(D),
sigma=1 *
np.eye(D)),
dynamics_distn=AutoRegression(nu_0=D + 1,
S_0=1 * np.eye(D),
M_0=np.zeros((D, D)),
K_0=1 * np.eye(D)),
sigma_C=1.)
for X in Xs:
model.add_data(X)
model.resample_parameters()
compute_pred_ll = lambda: sum([
model.predictive_log_likelihood(Xt, data_index=i, M=10)[0]
for i, Xt in enumerate(Xtest)
])
init_results = (
0,
None,
model.log_likelihood(),
# model.heldout_log_likelihood(Xtest, M=1),
np.nan,
compute_pred_ll())
def resample():
tic = time.time()
model.resample_model()
toc = time.time() - tic
return toc, None, model.log_likelihood(), \
np.nan,\
compute_pred_ll()
times, samples, lls, test_lls, pred_lls = \
list(map(np.array, list(zip(*([init_results] +
[resample() for _ in progprint_xrange(N_samples, perline=5)])))))
timestamps = np.cumsum(times)
return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_ln_lds_model(Xs, Xtest, D, N_samples=100):
"""
Fit a logistic normal LDS model with pMCMC
"""
Nx = len(Xs)
assert len(Xtest) == Nx
print("Fitting Logistic Normal LDS with %d states" % D)
mus = [X.sum(0) + 0.1 for X in Xs]
mus = [np.log(mu / mu.sum()) for mu in mus]
models = [LogisticNormalMultinomialLDS(
init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1*np.eye(D)),
dynamics_distn=AutoRegression(nu_0=D+1,S_0=D*np.eye(D),M_0=np.zeros((D,D)),K_0=D*np.eye(D)),
emission_distn=Regression(nu_0=K+1,S_0=K*np.eye(K),M_0=np.zeros((K,D)),K_0=K*np.eye(D)),
sigma_C=1.0, mu=mu) \
for mu in mus]
for model in models:
model.A = 0.5 * np.eye(D)
model.sigma_states = np.eye(D)
model.C = 1.0 * np.random.randn(K, D)
model.sigma_obs = 0.1 * np.eye(K)
for X, model in zip(Xs, models):
model.add_data(X)
def compute_pred_ll():
pred_ll = 0
for Xte, model in zip(Xtest, models):
pred_ll += model.predictive_log_likelihood(Xte, Npred=1)[0]
return pred_ll
init_results = (0, None, np.nan, np.nan, compute_pred_ll())
def resample():
tic = time.time()
[model.resample_model() for model in models]
toc = time.time() - tic
return toc, None, np.nan, np.nan, compute_pred_ll()
times, samples, lls, test_lls, pred_lls = \
list(map(np.array, list(zip(*([init_results] +
[resample() for _ in progprint_xrange(N_samples, perline=5)])))))
timestamps = np.cumsum(times)
return Results(lls, test_lls, pred_lls, samples, timestamps)
def fit_lds_model(Xs, Xtest, N_samples=100):
model = MultinomialLDS(K,
D,
init_dynamics_distn=Gaussian(mu_0=np.zeros(D),
sigma_0=np.eye(D),
kappa_0=1.0,
nu_0=D + 1.0),
dynamics_distn=AutoRegression(nu_0=D + 1,
S_0=np.eye(D),
M_0=np.zeros((D, D)),
K_0=np.eye(D)),
sigma_C=1)
for X in Xs:
model.add_data(X)
data = model.data_list[0]
samples = []
lls = []
test_lls = []
mc_test_lls = []
pis = []
psis = []
zs = []
timestamps = [time.time()]
for smpl in progprint_xrange(N_samples):
model.resample_model()
timestamps.append(time.time())
samples.append(model.copy_sample())
# TODO: Use log_likelihood() to marginalize over z
lls.append(model.log_likelihood())
# test_lls.append(model.heldout_log_likelihood(Xtest, M=50)[0])
mc_test_lls.append(model._mc_heldout_log_likelihood(Xtest, M=1)[0])
pis.append(model.pi(data))
psis.append(model.psi(data))
zs.append(data["states"].stateseq)
lls = np.array(lls)
test_lls = np.array(test_lls)
pis = np.array(pis)
psis = np.array(psis)
zs = np.array(zs)
timestamps = np.array(timestamps)
timestamps -= timestamps[0]
return model, lls, test_lls, mc_test_lls, pis, psis, zs, timestamps
def fit_ln_lds_model(Xs, Xtest, D, N_samples=100):
"""
Fit a logistic normal LDS model with pMCMC
"""
print("Fitting Logistic Normal LDS with %d states" % D)
model = LogisticNormalMultinomialLDS(
init_dynamics_distn=GaussianFixed(mu=np.zeros(D), sigma=1 * np.eye(D)),
dynamics_distn=AutoRegression(nu_0=D + 1,
S_0=D * np.eye(D),
M_0=np.zeros((D, D)),
K_0=D * np.eye(D)),
emission_distn=Regression(nu_0=K + 1,
S_0=K * np.eye(K),
M_0=np.zeros((K, D)),
K_0=K * np.eye(D)),
sigma_C=0.1)
model.A = 0.5 * np.eye(D)
model.sigma_states = np.eye(D)
model.C = 0.33 * np.random.randn(K, D)
model.sigma_obs = 0.1 * np.eye(K)
for X in Xs:
model.add_data(X)
init_results = (0, None, model.log_likelihood(), np.nan,
model.predictive_log_likelihood(Xtest, Npred=1000))
def resample():
tic = time.time()
model.resample_model()
toc = time.time() - tic
pred_ll = model.predictive_log_likelihood(Xtest, Npred=1000)
return toc, None, model.log_likelihood(), \
np.nan, \
pred_ll
times, samples, lls, test_lls, pred_lls = \
map(np.array, zip(*([init_results] + [resample() for _ in progprint_xrange(N_samples)])))
timestamps = np.cumsum(times)
return Results(lls, test_lls, pred_lls, samples, timestamps)
[np.sin(np.pi/24), np.cos(np.pi/24)]])
sigma_states = 0.1 * np.eye(D)
K = 3
# C = np.hstack((np.ones((K-1, 1)), np.zeros((K-1, D-1))))
C = np.random.randn(K - 1, D)
###################
# generate data #
###################
model = MultinomialLDS(K,
D,
init_dynamics_distn=Gaussian(mu=mu_init,
sigma=sigma_init),
dynamics_distn=AutoRegression(A=A, sigma=sigma_states),
C=C)
data = model.generate(T=T, N=N, keep=False)
# data["x"] = np.hstack([np.zeros((T,K-1)), np.ones((T,1))])
# Estimate the held out likelihood using Monte Carlo
M = 10000
hll_mc, std_mc = model._mc_heldout_log_likelihood(data["x"], M=M)
# Estimate the held out log likelihood
# hll_info, std_info = model._info_form_heldout_log_likelihood(data["x"], M=M)
hll_dist, std_dist = model._distn_form_heldout_log_likelihood(data["x"], M=M)
print("MC. Model: ", hll_mc, " +- ", std_mc)
# print "AN. Model (info): ", hll_info, " +- ", std_info
print("AN. Model (dist): ", hll_dist, " +- ", std_dist)
class LinearDynamicalSystemBackground(Component):
"""
Linear Dynamical System model for the background activation.
Since the potentials for the activation are of a Gaussian form,
we can perform conjugate Gibbs sampling or variational inference
for a Gaussian LDS model.
"""
def __init__(self, population, D=2,
A=None, C=None,
sigma_states=None,
sigma_C=1.0):
self.population = population
self.activation = population.activation_model
self.N = self.population.N
self.D = D
from pybasicbayes.distributions import Gaussian
self.init_dynamics_distn = Gaussian(mu_0=np.ones(D),
kappa_0=1.0,
sigma_0=0.000001 * np.eye(D),
nu_0=3.0)
from autoregressive.distributions import AutoRegression
self.dynamics_distn = AutoRegression(A=A, sigma=sigma_states,
nu_0=D+1.0, S_0=0.5 * np.eye(D),
M_0=np.zeros((D,D)), K_0=0.5 * np.eye(D))
# Initialize the emission matrix
if C is None:
self.C = sigma_C * np.random.randn(self.N, self.D)
else:
assert C.shape == (self.N, self.D)
self.C = C
self.sigma_C = sigma_C
def augment_data(self, augmented_data):
# Add a latent state sequence
augmented_data["states"] = self.generate_states(augmented_data["T"])
def log_likelihood(self, augmented_data):
raise NotImplementedError
def generate(self,T):
states = self.generate_states(T)
return states.dot(self.C.T)
def generate_states(self, T):
stateseq = np.empty((T,self.D))
stateseq[0] = self.init_dynamics_distn.rvs()
chol = np.linalg.cholesky(self.dynamics_distn.sigma)
randseq = np.random.randn(T-1,self.D)
for t in xrange(1,T):
stateseq[t] = \
self.dynamics_distn.A.dot(stateseq[t-1]) \
+ chol.dot(randseq[t-1])
return stateseq
def mean_background_activation(self, augmented_data):
return augmented_data["states"].dot(self.C.T)
def resample(self, augmented_data_list):
self.resample_states(augmented_data_list)
self.resample_parameters(augmented_data_list)
def resample_states(self, augmented_data_list):
from pylds.lds_messages import filter_and_sample
for data in augmented_data_list:
# Compute the residual activation from other components
psi = self.activation.compute_psi(data)
psi_residual = psi - self.mean_background_activation(data)
# Get the observed mean and variance
mu_obs = self.activation.new_mean(data)
prec_obs = self.activation.new_precision(data)
# Subtract off the activation from other components
mu_obs -= psi_residual
# Convert prec_obs into an array of diagonal covariance matrices
sigma_obs = np.empty((data["T"], self.N, self.N), order="C")
for t in xrange(data["T"]):
sigma_obs[t,:,:] = np.diag(1./prec_obs[t,:])
data["states"] = filter_and_sample(
self.init_dynamics_distn.mu,
self.init_dynamics_distn.sigma,
self.dynamics_distn.A,
self.dynamics_distn.sigma,
self.C,
sigma_obs,
mu_obs)
def resample_parameters(self, augmented_data_list):
self.resample_init_dynamics_distn(augmented_data_list)
self.resample_dynamics_distn(augmented_data_list)
self.resample_emission_distn(augmented_data_list)
def resample_init_dynamics_distn(self, augmented_data_list):
states_list = [ad["states"][0] for ad in augmented_data_list]
self.init_dynamics_distn.resample(states_list)
def resample_dynamics_distn(self, augmented_data_list):
from pyhsmm.util.general import AR_striding
states_list = [ad["states"] for ad in augmented_data_list]
strided_states_list = [AR_striding(s,1) for s in states_list]
self.dynamics_distn.resample(strided_states_list)
def resample_emission_distn(self, augmented_data_list):
"""
Resample the observation vectors. Since the emission noise is diagonal,
we can resample the columns of C independently
:return:
"""
# Get the prior
prior_precision = 1./self.sigma_C * np.eye(self.D)
prior_mean = np.zeros(self.D)
prior_mean_dot_precision = prior_mean.dot(prior_precision)
# Get the sufficient statistics from the likelihood
lkhd_precision = np.zeros((self.N, self.D, self.D))
lkhd_mean_dot_precision = np.zeros((self.N, self.D))
for data in augmented_data_list:
# Compute the residual activation from other components
psi = self.activation.compute_psi(data)
psi_residual = psi - self.mean_background_activation(data)
# Get the observed mean and variance
mu_obs = self.activation.new_mean(data)
prec_obs = self.activation.new_precision(data)
# Subtract off the residual
mu_obs -= psi_residual
# Update the sufficient statistics for each neuron
for n in xrange(self.N):
lkhd_precision[n,:,:] += (data["states"] * prec_obs[:,n][:,None]).T.dot(data["states"])
lkhd_mean_dot_precision[n,:] += \
(mu_obs[:,n] * prec_obs[:,n]).T.dot(data["states"])
# Sample each column of C
for n in xrange(self.N):
post_prec = prior_precision + lkhd_precision[n,:,:]
post_cov = np.linalg.inv(post_prec)
post_mu = (prior_mean_dot_precision +
lkhd_mean_dot_precision[n,:]).dot(post_cov)
post_mu = post_mu.ravel()
self.C[n,:] = np.random.multivariate_normal(post_mu, post_cov)
### Variational inference
def meanfieldupdate(self, augmented_data): raise NotImplementedError
def get_vlb(self, augmented_data): raise NotImplementedError
def resample_from_mf(self, augmented_data): raise NotImplementedError
def svi_step(self, augmented_data, minibatchfrac, stepsize): raise NotImplementedError
class LinearDynamicalSystemBackground(Component):
"""
Linear Dynamical System model for the background activation.
Since the potentials for the activation are of a Gaussian form,
we can perform conjugate Gibbs sampling or variational inference
for a Gaussian LDS model.
"""
def __init__(self,
population,
D=2,
A=None,
C=None,
sigma_states=None,
sigma_C=1.0):
self.population = population
self.activation = population.activation_model
self.N = self.population.N
self.D = D
from pybasicbayes.distributions import Gaussian
self.init_dynamics_distn = Gaussian(mu_0=np.ones(D),
kappa_0=1.0,
sigma_0=0.000001 * np.eye(D),
nu_0=3.0)
from autoregressive.distributions import AutoRegression
self.dynamics_distn = AutoRegression(A=A,
sigma=sigma_states,
nu_0=D + 1.0,
S_0=0.5 * np.eye(D),
M_0=np.zeros((D, D)),
K_0=0.5 * np.eye(D))
# Initialize the emission matrix
if C is None:
self.C = sigma_C * np.random.randn(self.N, self.D)
else:
assert C.shape == (self.N, self.D)
self.C = C
self.sigma_C = sigma_C
def augment_data(self, augmented_data):
# Add a latent state sequence
augmented_data["states"] = self.generate_states(augmented_data["T"])
def log_likelihood(self, augmented_data):
raise NotImplementedError
def generate(self, T):
states = self.generate_states(T)
return states.dot(self.C.T)
def generate_states(self, T):
stateseq = np.empty((T, self.D))
stateseq[0] = self.init_dynamics_distn.rvs()
chol = np.linalg.cholesky(self.dynamics_distn.sigma)
randseq = np.random.randn(T - 1, self.D)
for t in xrange(1, T):
stateseq[t] = \
self.dynamics_distn.A.dot(stateseq[t-1]) \
+ chol.dot(randseq[t-1])
return stateseq
def mean_background_activation(self, augmented_data):
return augmented_data["states"].dot(self.C.T)
def resample(self, augmented_data_list):
self.resample_states(augmented_data_list)
self.resample_parameters(augmented_data_list)
def resample_states(self, augmented_data_list):
from pylds.lds_messages import filter_and_sample
for data in augmented_data_list:
# Compute the residual activation from other components
psi = self.activation.compute_psi(data)
psi_residual = psi - self.mean_background_activation(data)
# Get the observed mean and variance
mu_obs = self.activation.new_mean(data)
prec_obs = self.activation.new_precision(data)
# Subtract off the activation from other components
mu_obs -= psi_residual
# Convert prec_obs into an array of diagonal covariance matrices
sigma_obs = np.empty((data["T"], self.N, self.N), order="C")
for t in xrange(data["T"]):
sigma_obs[t, :, :] = np.diag(1. / prec_obs[t, :])
data["states"] = filter_and_sample(self.init_dynamics_distn.mu,
self.init_dynamics_distn.sigma,
self.dynamics_distn.A,
self.dynamics_distn.sigma,
self.C, sigma_obs, mu_obs)
def resample_parameters(self, augmented_data_list):
self.resample_init_dynamics_distn(augmented_data_list)
self.resample_dynamics_distn(augmented_data_list)
self.resample_emission_distn(augmented_data_list)
def resample_init_dynamics_distn(self, augmented_data_list):
states_list = [ad["states"][0] for ad in augmented_data_list]
self.init_dynamics_distn.resample(states_list)
def resample_dynamics_distn(self, augmented_data_list):
from pyhsmm.util.general import AR_striding
states_list = [ad["states"] for ad in augmented_data_list]
strided_states_list = [AR_striding(s, 1) for s in states_list]
self.dynamics_distn.resample(strided_states_list)
def resample_emission_distn(self, augmented_data_list):
"""
Resample the observation vectors. Since the emission noise is diagonal,
we can resample the columns of C independently
:return:
"""
# Get the prior
prior_precision = 1. / self.sigma_C * np.eye(self.D)
prior_mean = np.zeros(self.D)
prior_mean_dot_precision = prior_mean.dot(prior_precision)
# Get the sufficient statistics from the likelihood
lkhd_precision = np.zeros((self.N, self.D, self.D))
lkhd_mean_dot_precision = np.zeros((self.N, self.D))
for data in augmented_data_list:
# Compute the residual activation from other components
psi = self.activation.compute_psi(data)
psi_residual = psi - self.mean_background_activation(data)
# Get the observed mean and variance
mu_obs = self.activation.new_mean(data)
prec_obs = self.activation.new_precision(data)
# Subtract off the residual
mu_obs -= psi_residual
# Update the sufficient statistics for each neuron
for n in xrange(self.N):
lkhd_precision[n, :, :] += (data["states"] *
prec_obs[:, n][:, None]).T.dot(
data["states"])
lkhd_mean_dot_precision[n,:] += \
(mu_obs[:,n] * prec_obs[:,n]).T.dot(data["states"])
# Sample each column of C
for n in xrange(self.N):
post_prec = prior_precision + lkhd_precision[n, :, :]
post_cov = np.linalg.inv(post_prec)
post_mu = (prior_mean_dot_precision +
lkhd_mean_dot_precision[n, :]).dot(post_cov)
post_mu = post_mu.ravel()
self.C[n, :] = np.random.multivariate_normal(post_mu, post_cov)
### Variational inference
def meanfieldupdate(self, augmented_data):
raise NotImplementedError
def get_vlb(self, augmented_data):
raise NotImplementedError
def resample_from_mf(self, augmented_data):
raise NotImplementedError
def svi_step(self, augmented_data, minibatchfrac, stepsize):
raise NotImplementedError
Kmax = 10 # number of latent discrete states
D_latent = 2 # latent linear dynamics' dimension
D_input = 1 # latent linear dynamics' dimension
D_obs = 2 # data dimension
N_iter = 200 # number of VBEM iterations
As = [
np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta),
np.cos(theta)]])
for alpha, theta in ((0.95, 0.1), (0.95, -0.1), (1., 0.))
]
truemodel = ARHSMM(
alpha=4.,
init_state_concentration=4.,
obs_distns=[AutoRegression(A=A, sigma=0.05 * np.eye(2)) for A in As],
dur_distns=[PoissonDuration(alpha_0=3 * 50, beta_0=3) for _ in As])
truemodel.prefix = np.array([[0., 3.]])
data, labels = truemodel.generate(T)
data = data[truemodel.nlags:]
plt.figure()
plt.plot(data[:, 0], data[:, 1], 'x-')
#################
# build model #
#################
Cs = [np.eye(D_obs) for _ in range(Kmax)] # Shared emission matrices
sigma_obss = [0.05 * np.eye(D_obs)
for _ in range(Kmax)] # Emission noise covariances
A = 0.999*np.array([[np.cos(np.pi/24), -np.sin(np.pi/24)],
[np.sin(np.pi/24), np.cos(np.pi/24)]])
sigma_states = 0.0001*np.eye(D)
K = 4
# C = np.hstack((np.ones((K-1, 1)), np.zeros((K-1, D-1))))
C = np.random.randn(K-1, D)
sigma_obs = 0.01 * np.eye(K)
###################
# generate data #
###################
truemodel = MultinomialLDS(K, D,
init_dynamics_distn=Gaussian(mu=mu_init,sigma=sigma_init),
dynamics_distn=AutoRegression(A=A,sigma=sigma_states),
C=C
)
data = truemodel.generate(T=T)
###################
# inference #
###################
testmodel = MultinomialLDS(K, D,
init_dynamics_distn=Gaussian(mu_0=mu_init, sigma_0=sigma_init, kappa_0=1.0, nu_0=3.0),
dynamics_distn=AutoRegression(nu_0=D+1,S_0=np.eye(D),M_0=np.zeros((D,D)),K_0=np.eye(D)),
sigma_C=1
)
A = 0.99*np.array([[np.cos(np.pi/24), -np.sin(np.pi/24)],
[np.sin(np.pi/24), np.cos(np.pi/24)]])
sigma_states = 0.01*np.eye(2)
C = np.array([[2.,0.]])
D_out, D_in = C.shape
###################
# generate data #
###################
truemodel = NegativeBinomialLDS(
init_dynamics_distn=Gaussian(mu_init, sigma_init),
dynamics_distn=AutoRegression(A=A,sigma=sigma_states),
emission_distn=PGEmissions(D_out, D_in, C=C))
T = 2000
data, z_true = truemodel.generate(T)
psi_true = z_true.dot(C.T)
p_true = logistic(psi_true)
###############
# fit model #
###############
model = NegativeBinomialLDS(
init_dynamics_distn=Gaussian(mu_0=np.zeros(D_in), sigma_0=np.eye(D_in),
kappa_0=1.0, nu_0=D_in+1),
mu_init = np.array([0., 1.])
sigma_init = 0.01 * np.eye(2)
A = 0.99 * np.array([[np.cos(np.pi / 24), -np.sin(np.pi / 24)],
[np.sin(np.pi / 24),
np.cos(np.pi / 24)]])
sigma_states = 0.01 * np.eye(2)
C = np.array([[10., 0.]])
sigma_obs = 0.01 * np.eye(1)
###################
# generate data #
###################
truemodel = LDS(dynamics_distn=AutoRegression(A=A, sigma=sigma_states),
emission_distn=Regression(A=C, sigma=sigma_obs))
data, stateseq = truemodel.generate(2000)
###############
# fit model #
###############
def update(model):
model.resample_model()
return model.log_likelihood()
model = DefaultLDS(n=2, p=data.shape[1]).add_data(data)