def linear_state_space_model(D=3, N=100, M=10):
# Dynamics matrix with ARD
alpha = Gamma(1e-5, 1e-5, plates=(D, ), name='alpha')
A = Gaussian(np.zeros(D), diagonal(alpha), plates=(D, ), name='A')
# Latent states with dynamics
X = GaussianMarkovChain(
np.zeros(D), # mean of x0
1e-3 * np.identity(D), # prec of x0
A, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X')
# Mixing matrix from latent space to observation space using ARD
gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma')
C = Gaussian(np.zeros(D), diagonal(gamma), plates=(M, 1), name='C')
# Observation noise
tau = Gamma(1e-5, 1e-5, name='tau')
# Observations
CX = Dot(C, X.as_gaussian())
Y = Normal(CX, tau, name='Y')
return (Y, CX, X, tau, C, gamma, A, alpha)
def pca_model(M, N, D):
# Construct the PCA model with ARD
# ARD
alpha = nodes.Gamma(1e-2,
1e-2,
plates=(D,),
name='alpha')
# Loadings
W = nodes.Gaussian(np.zeros(D),
diagonal(alpha),
name="W",
plates=(M,1))
# States
X = nodes.Gaussian(np.zeros(D),
np.identity(D),
name="X",
plates=(1,N))
# PCA
WX = nodes.Dot(W, X, name="WX")
# Noise
tau = nodes.Gamma(1e-2, 1e-2, name="tau", plates=())
# Noisy observations
Y = nodes.Normal(WX, tau, name="Y", plates=(M,N))
return (Y, WX, W, X, tau, alpha)
def linear_state_space_model(D=3, N=100, M=10):
# Dynamics matrix with ARD
alpha = Gamma(1e-5,
1e-5,
plates=(D,),
name='alpha')
A = Gaussian(np.zeros(D),
diagonal(alpha),
plates=(D,),
name='A')
# Latent states with dynamics
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
A, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X')
# Mixing matrix from latent space to observation space using ARD
gamma = Gamma(1e-5,
1e-5,
plates=(D,),
name='gamma')
C = Gaussian(np.zeros(D),
diagonal(gamma),
plates=(M,1),
name='C')
# Observation noise
tau = Gamma(1e-5,
1e-5,
name='tau')
# Observations
CX = Dot(C, X.as_gaussian())
Y = Normal(CX,
tau,
name='Y')
return (Y, CX, X, tau, C, gamma, A, alpha)
def run_dlssm(y, f, mask, D, K, maxiter):
"""
Run VB inference for linear state space model with drifting dynamics.
"""
(M, N) = np.shape(y)
# Dynamics matrix with ARD
# alpha : (D) x ()
alpha = Gamma(1e-5, 1e-5, plates=(K, ), name='alpha')
# A : (K) x (K)
A = Gaussian(
np.zeros(K),
#np.identity(K),
diagonal(alpha),
plates=(K, ),
name='A_S')
A.initialize_from_value(np.identity(K))
# rho
## rho = Gamma(1e-5,
## 1e-5,
## plates=(K,),
## name="rho")
# S : () x (N-1,K)
S = GaussianMarkovChain(np.ones(K),
1e-6 * np.identity(K),
A,
np.ones(K),
n=N - 1,
name='S')
S.initialize_from_value(1 * np.ones((N - 1, K)))
# Projection matrix of the dynamics matrix
# beta : (K) x ()
beta = Gamma(1e-5, 1e-5, plates=(K, ), name='beta')
# B : (D) x (D*K)
B = Gaussian(np.zeros(D * K),
diagonal(tile(beta, D)),
plates=(D, ),
name='B')
b = np.zeros((D, D, K))
b[np.arange(D), np.arange(D), np.zeros(D, dtype=int)] = 1
B.initialize_from_value(np.reshape(1 * b, (D, D * K)))
# A : (N-1,D) x (D)
BS = MatrixDot(B, S.as_gaussian().add_plate_axis(-1), name='BS')
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(
np.zeros(D), # mean of x0
1e-3 * np.identity(D), # prec of x0
BS, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N, D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma')
# C : (M,1) x (D)
C = Gaussian(np.zeros(D), diagonal(gamma), plates=(M, 1), name='C')
C.initialize_from_value(np.random.randn(M, 1, D))
# Observation noise
# tau : () x ()
tau = Gamma(1e-5, 1e-5, name='tau')
# Observations
# Y : (M,N) x ()
CX = Dot(C, X.as_gaussian())
Y = Normal(CX, tau, name='Y')
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, S, A, alpha, B, beta, C, gamma, tau)
#
# Run inference with rotations.
#
# Rotate the D-dimensional state space (C, X)
rotB = transformations.RotateGaussianMatrixARD(B, beta, axis='rows')
rotX = transformations.RotateDriftingMarkovChain(X, B, S, rotB)
rotC = transformations.RotateGaussianARD(C, gamma)
R_X = transformations.RotationOptimizer(rotX, rotC, D)
# Rotate the K-dimensional latent dynamics space (B, S)
rotA = transformations.RotateGaussianARD(A, alpha)
rotS = transformations.RotateGaussianMarkovChain(S, A, rotA)
rotB = transformations.RotateGaussianMatrixARD(B, beta, axis='cols')
R_S = transformations.RotationOptimizer(rotS, rotB, K)
# Iterate
for ind in range(int(maxiter / 5)):
Q.update(repeat=5)
#Q.update(X, S, A, alpha, rho, B, beta, C, gamma, tau, repeat=maxiter)
R_X.rotate()
R_S.rotate()
## R_X.rotate(
## check_bound=Q.compute_lowerbound,
## check_bound_terms=Q.compute_lowerbound_terms,
## check_gradient=True
## )
## R_S.rotate(
## check_bound=Q.compute_lowerbound,
## check_bound_terms=Q.compute_lowerbound_terms,
## check_gradient=True
## )
#
# SHOW RESULTS
#
# Mean and standard deviation of the posterior
(f_mean, f_squared) = CX.get_moments()
f_std = np.sqrt(f_squared - f_mean**2)
# Plot observations space
for m in range(M):
plt.subplot(M, 1, m + 1)
plt.plot(y[m, :], 'r.')
plt.plot(f[m, :], 'b-')
bpplt.errorplot(y=f_mean[m, :], error=2 * f_std[m, :])
def run_lssm(y, f, mask, D, maxiter):
"""
Run VB inference for linear state space model.
"""
(M, N) = np.shape(y)
#
# CONSTRUCT THE MODEL
#
# Dynamic matrix
# alpha: (D) x ()
alpha = Gamma(1e-5, 1e-5, plates=(D, ), name='alpha')
# A : (D) x (D)
A = Gaussian(np.zeros(D), diagonal(alpha), plates=(D, ), name='A')
A.initialize_from_value(np.identity(D))
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(
np.zeros(D), # mean of x0
1e-3 * np.identity(D), # prec of x0
A, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N, D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma')
# C : (M,1) x (D)
C = Gaussian(np.zeros(D), diagonal(gamma), plates=(M, 1), name='C')
C.initialize_from_value(np.random.randn(M, 1, D))
# Observation noise
# tau : () x ()
tau = Gamma(1e-5, 1e-5, name='tau')
# Observations
# Y : (M,N) x ()
CX = Dot(C, X.as_gaussian())
Y = Normal(CX, tau, name='Y')
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, A, alpha, C, gamma, tau)
# Iterate
Q.update(X, A, alpha, C, gamma, tau, repeat=maxiter)
#
# SHOW RESULTS
#
# Mean and standard deviation of the posterior
(f_mean, f_squared) = CX.get_moments()
f_std = np.sqrt(f_squared - f_mean**2)
# Plot observations space
#plt.figure()
for m in range(M):
plt.subplot(M, 1, m + 1)
plt.plot(y[m, :], 'r.')
plt.plot(f[m, :], 'b-')
bpplt.errorplot(y=f_mean[m, :], error=2 * f_std[m, :])
def run_dlssm(y, f, mask, D, K, maxiter):
"""
Run VB inference for linear state space model with drifting dynamics.
"""
(M, N) = np.shape(y)
# Dynamics matrix with ARD
# alpha : (D) x ()
alpha = Gamma(1e-5,
1e-5,
plates=(K,),
name='alpha')
# A : (K) x (K)
A = Gaussian(np.zeros(K),
#np.identity(K),
diagonal(alpha),
plates=(K,),
name='A_S')
A.initialize_from_value(np.identity(K))
# rho
## rho = Gamma(1e-5,
## 1e-5,
## plates=(K,),
## name="rho")
# S : () x (N-1,K)
S = GaussianMarkovChain(np.ones(K),
1e-6*np.identity(K),
A,
np.ones(K),
n=N-1,
name='S')
S.initialize_from_value(1*np.ones((N-1,K)))
# Projection matrix of the dynamics matrix
# beta : (K) x ()
beta = Gamma(1e-5,
1e-5,
plates=(K,),
name='beta')
# B : (D) x (D*K)
B = Gaussian(np.zeros(D*K),
diagonal(tile(beta, D)),
plates=(D,),
name='B')
b = np.zeros((D,D,K))
b[np.arange(D),np.arange(D),np.zeros(D,dtype=int)] = 1
B.initialize_from_value(np.reshape(1*b, (D,D*K)))
# A : (N-1,D) x (D)
BS = MatrixDot(B,
S.as_gaussian().add_plate_axis(-1),
name='BS')
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
BS, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N,D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5,
1e-5,
plates=(D,),
name='gamma')
# C : (M,1) x (D)
C = Gaussian(np.zeros(D),
diagonal(gamma),
plates=(M,1),
name='C')
C.initialize_from_value(np.random.randn(M,1,D))
# Observation noise
# tau : () x ()
tau = Gamma(1e-5,
1e-5,
name='tau')
# Observations
# Y : (M,N) x ()
CX = Dot(C, X.as_gaussian())
Y = Normal(CX,
tau,
name='Y')
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, S, A, alpha, B, beta, C, gamma, tau)
#
# Run inference with rotations.
#
# Rotate the D-dimensional state space (C, X)
rotB = transformations.RotateGaussianMatrixARD(B, beta, axis='rows')
rotX = transformations.RotateDriftingMarkovChain(X, B, S, rotB)
rotC = transformations.RotateGaussianARD(C, gamma)
R_X = transformations.RotationOptimizer(rotX, rotC, D)
# Rotate the K-dimensional latent dynamics space (B, S)
rotA = transformations.RotateGaussianARD(A, alpha)
rotS = transformations.RotateGaussianMarkovChain(S, A, rotA)
rotB = transformations.RotateGaussianMatrixARD(B, beta, axis='cols')
R_S = transformations.RotationOptimizer(rotS, rotB, K)
# Iterate
for ind in range(int(maxiter/5)):
Q.update(repeat=5)
#Q.update(X, S, A, alpha, rho, B, beta, C, gamma, tau, repeat=maxiter)
R_X.rotate()
R_S.rotate()
## R_X.rotate(
## check_bound=Q.compute_lowerbound,
## check_bound_terms=Q.compute_lowerbound_terms,
## check_gradient=True
## )
## R_S.rotate(
## check_bound=Q.compute_lowerbound,
## check_bound_terms=Q.compute_lowerbound_terms,
## check_gradient=True
## )
#
# SHOW RESULTS
#
# Mean and standard deviation of the posterior
(f_mean, f_squared) = CX.get_moments()
f_std = np.sqrt(f_squared - f_mean**2)
# Plot observations space
for m in range(M):
plt.subplot(M,1,m+1)
plt.plot(y[m,:], 'r.')
plt.plot(f[m,:], 'b-')
bpplt.errorplot(y=f_mean[m,:], error=2*f_std[m,:])
def run_lssm(y, f, mask, D, maxiter):
"""
Run VB inference for linear state space model.
"""
(M, N) = np.shape(y)
#
# CONSTRUCT THE MODEL
#
# Dynamic matrix
# alpha: (D) x ()
alpha = Gamma(1e-5,
1e-5,
plates=(D,),
name='alpha')
# A : (D) x (D)
A = Gaussian(np.zeros(D),
diagonal(alpha),
plates=(D,),
name='A')
A.initialize_from_value(np.identity(D))
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
A, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N,D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5,
1e-5,
plates=(D,),
name='gamma')
# C : (M,1) x (D)
C = Gaussian(np.zeros(D),
diagonal(gamma),
plates=(M,1),
name='C')
C.initialize_from_value(np.random.randn(M,1,D))
# Observation noise
# tau : () x ()
tau = Gamma(1e-5,
1e-5,
name='tau')
# Observations
# Y : (M,N) x ()
CX = Dot(C, X.as_gaussian())
Y = Normal(CX,
tau,
name='Y')
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, A, alpha, C, gamma, tau)
# Iterate
Q.update(X, A, alpha, C, gamma, tau, repeat=maxiter)
#
# SHOW RESULTS
#
# Mean and standard deviation of the posterior
(f_mean, f_squared) = CX.get_moments()
f_std = np.sqrt(f_squared - f_mean**2)
# Plot observations space
#plt.figure()
for m in range(M):
plt.subplot(M,1,m+1)
plt.plot(y[m,:], 'r.')
plt.plot(f[m,:], 'b-')
bpplt.errorplot(y=f_mean[m,:], error=2*f_std[m,:])
