def linear_state_space_model(D=3, N=100, M=10):
# Dynamics matrix with ARD
alpha = Gamma(1e-5,
1e-5,
plates=(D,),
name='alpha')
A = GaussianArrayARD(0,
alpha,
shape=(D,),
plates=(D,),
name='A')
# Latent states with dynamics
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
A, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X')
# Mixing matrix from latent space to observation space using ARD
gamma = Gamma(1e-5,
1e-5,
plates=(D,),
name='gamma')
C = GaussianArrayARD(0,
gamma,
shape=(D,),
plates=(M,1),
name='C')
# Observation noise
tau = Gamma(1e-5,
1e-5,
name='tau')
# Underlying noiseless function
F = SumMultiply('i,i',
C,
X.as_gaussian())
# Noisy observations
Y = GaussianArrayARD(F,
tau,
name='Y')
return (Y, F, X, tau, C, gamma, A, alpha)
def model(M=10, N=100, D=3):
"""
Construct linear state-space model.
See, for instance, the following publication:
"Fast variational Bayesian linear state-space model"
Luttinen (ECML 2013)
"""
# Dynamics matrix with ARD
alpha = Gamma(1e-5,
1e-5,
plates=(D,),
name='alpha')
A = GaussianARD(0,
alpha,
shape=(D,),
plates=(D,),
plotter=bpplt.GaussianHintonPlotter(rows=0,
cols=1,
scale=0),
name='A')
A.initialize_from_value(np.identity(D))
# Latent states with dynamics
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
A, # dynamics
np.ones(D), # innovation
n=N, # time instances
plotter=bpplt.GaussianMarkovChainPlotter(scale=2),
name='X')
X.initialize_from_value(np.random.randn(N,D))
# Mixing matrix from latent space to observation space using ARD
gamma = Gamma(1e-5,
1e-5,
plates=(D,),
name='gamma')
gamma.initialize_from_value(1e-2*np.ones(D))
C = GaussianARD(0,
gamma,
shape=(D,),
plates=(M,1),
plotter=bpplt.GaussianHintonPlotter(rows=0,
cols=2,
scale=0),
name='C')
C.initialize_from_value(np.random.randn(M,1,D))
# Observation noise
tau = Gamma(1e-5,
1e-5,
name='tau')
tau.initialize_from_value(1e2)
# Underlying noiseless function
F = SumMultiply('i,i',
C,
X,
name='F')
# Noisy observations
Y = GaussianARD(F,
tau,
name='Y')
Q = VB(Y, F, C, gamma, X, A, alpha, tau, C)
return Q
def run_dlssm(y, f, mask, D, K, maxiter):
"""
Run VB inference for linear state space model with drifting dynamics.
"""
(M, N) = np.shape(y)
# Dynamics matrix with ARD
# alpha : (K) x ()
alpha = Gamma(1e-5,
1e-5,
plates=(K,),
name='alpha')
# A : (K) x (K)
A = GaussianArrayARD(np.identity(K),
alpha,
shape=(K,),
plates=(K,),
name='A_S',
initialize=False)
A.initialize_from_value(np.identity(K))
# State of the drift
# S : () x (N,K)
S = GaussianMarkovChain(np.ones(K),
1e-6*np.identity(K),
A,
np.ones(K),
n=N,
name='S',
initialize=False)
S.initialize_from_value(np.ones((N,K)))
# Projection matrix of the dynamics matrix
# Initialize S and B such that BS is identity matrix
# beta : (K) x ()
beta = Gamma(1e-5,
1e-5,
plates=(D,K),
name='beta')
# B : (D) x (D,K)
b = np.zeros((D,D,K))
b[np.arange(D),np.arange(D),np.zeros(D,dtype=int)] = 1
B = GaussianArrayARD(0,
beta,
plates=(D,),
name='B',
initialize=False)
B.initialize_from_value(np.reshape(1*b, (D,D,K)))
# BS : (N-1,D) x (D)
# TODO/FIXME: Implement __getitem__ method
BS = SumMultiply('dk,k->d',
B,
S.as_gaussian()[...,np.newaxis],
# iterator_axis=0,
name='BS')
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
BS, # dynamics
np.ones(D), # innovation
n=N+1, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N+1,D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5,
1e-5,
plates=(D,K),
name='gamma')
# C : (M,1) x (D,K)
C = GaussianArrayARD(0,
gamma,
plates=(M,1),
name='C',
initialize=False)
C.initialize_from_random()
# Observation noise
# tau : () x ()
tau = Gamma(1e-5,
1e-5,
name='tau')
# Observations
# Y : (M,N) x ()
F = SumMultiply('dk,d,k',
C,
X.as_gaussian()[1:],
S.as_gaussian(),
name='F')
Y = GaussianArrayARD(F,
tau,
name='Y')
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, S, A, alpha, B, beta, C, gamma, tau)
#
# Run inference with rotations.
#
rotate = False
if rotate:
# Rotate the D-dimensional state space (C, X)
rotB = transformations.RotateGaussianMatrixARD(B, beta, D, K, axis='rows')
rotX = transformations.RotateDriftingMarkovChain(X, B, S, rotB)
rotC = transformations.RotateGaussianARD(C, gamma)
R_X = transformations.RotationOptimizer(rotX, rotC, D)
# Rotate the K-dimensional latent dynamics space (B, S)
rotA = transformations.RotateGaussianARD(A, alpha)
rotS = transformations.RotateGaussianMarkovChain(S, A, rotA)
rotB = transformations.RotateGaussianMatrixARD(B, beta, D, K, axis='cols')
R_S = transformations.RotationOptimizer(rotS, rotB, K)
# Iterate
for ind in range(maxiter):
print("X update")
Q.update(X)
print("S update")
Q.update(S)
print("A update")
Q.update(A)
print("alpha update")
Q.update(alpha)
print("B update")
Q.update(B)
print("beta update")
Q.update(beta)
print("C update")
Q.update(C)
print("gamma update")
Q.update(gamma)
print("tau update")
Q.update(tau)
if rotate:
if ind >= 0:
R_X.rotate()
if ind >= 0:
R_S.rotate()
Q.plot_iteration_by_nodes()
#
# SHOW RESULTS
#
# Plot observations space
plt.figure()
bpplt.timeseries_normal(F)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
# Plot latent space
plt.figure()
bpplt.timeseries_gaussian_mc(X, scale=2)
# Plot drift space
plt.figure()
bpplt.timeseries_gaussian_mc(S, scale=2)
def run_lssm(y, f, mask, D, maxiter):
"""
Run VB inference for linear state space model.
"""
(M, N) = np.shape(y)
#
# CONSTRUCT THE MODEL
#
# Dynamic matrix
# alpha: (D) x ()
alpha = Gamma(1e-5,
1e-5,
plates=(D,),
name='alpha')
# A : (D) x (D)
A = Gaussian(np.zeros(D),
diagonal(alpha),
plates=(D,),
name='A')
A.initialize_from_value(np.identity(D))
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
A, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N,D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5,
1e-5,
plates=(D,),
name='gamma')
# C : (M,1) x (D)
C = Gaussian(np.zeros(D),
diagonal(gamma),
plates=(M,1),
name='C')
C.initialize_from_value(np.random.randn(M,1,D))
# Observation noise
# tau : () x ()
tau = Gamma(1e-5,
1e-5,
name='tau')
# Observations
# Y : (M,N) x ()
CX = Dot(C, X.as_gaussian())
Y = Normal(CX,
tau,
name='Y')
# Rotate the D-dimensional latent space
rotA = transformations.RotateGaussianARD(A, alpha)
rotX = transformations.RotateGaussianMarkovChain(X, A, rotA)
rotC = transformations.RotateGaussianARD(C, gamma)
R = transformations.RotationOptimizer(rotX, rotC, D)
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, A, alpha, C, gamma, tau)
# Iterate
for ind in range(maxiter):
Q.update(X, A, alpha, C, gamma, tau)
R.rotate()
#
# SHOW RESULTS
#
plt.figure()
bpplt.timeseries_normal(CX)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
def model(M, N, D, K):
"""
Construct the linear state-space model with time-varying dynamics
For reference, see the following publication:
(TODO)
"""
#
# The model block for the latent mixing weight process
#
# Dynamics matrix with ARD
# beta : (K) x ()
beta = Gamma(1e-5,
1e-5,
plates=(K,),
name='beta')
# B : (K) x (K)
B = GaussianARD(np.identity(K),
beta,
shape=(K,),
plates=(K,),
name='B',
plotter=bpplt.GaussianHintonPlotter(rows=0,
cols=1,
scale=0),
initialize=False)
B.initialize_from_value(np.identity(K))
# Mixing weight process, that is, the weights in the linear combination of
# state dynamics matrices
# S : () x (N,K)
S = GaussianMarkovChain(np.ones(K),
1e-6*np.identity(K),
B,
np.ones(K),
n=N,
name='S',
plotter=bpplt.GaussianMarkovChainPlotter(scale=2),
initialize=False)
s = 10*np.random.randn(N,K)
s[:,0] = 10
S.initialize_from_value(s)
#
# The model block for the latent states
#
# Projection matrix of the dynamics matrix
# alpha : (K) x ()
alpha = Gamma(1e-5,
1e-5,
plates=(D,K),
name='alpha')
alpha.initialize_from_value(1*np.ones((D,K)))
# A : (D) x (D,K)
A = GaussianARD(0,
alpha,
shape=(D,K),
plates=(D,),
name='A',
plotter=bpplt.GaussianHintonPlotter(rows=0,
cols=1,
scale=0),
initialize=False)
# Initialize S and A such that A*S is almost an identity matrix
a = np.zeros((D,D,K))
a[np.arange(D),np.arange(D),np.zeros(D,dtype=int)] = 1
a[:,:,0] = np.identity(D) / s[0,0]
a[:,:,1:] = 0.1/s[0,0]*np.random.randn(D,D,K-1)
A.initialize_from_value(a)
# Latent states with dynamics
# X : () x (N,D)
X = VaryingGaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
A, # dynamics matrices
S._convert(GaussianMoments)[1:], # temporal weights
np.ones(D), # innovation
n=N, # time instances
name='X',
plotter=bpplt.GaussianMarkovChainPlotter(scale=2),
initialize=False)
X.initialize_from_value(np.random.randn(N,D))
#
# The model block for observations
#
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5,
1e-5,
plates=(D,),
name='gamma')
gamma.initialize_from_value(1e-2*np.ones(D))
# C : (M,1) x (D)
C = GaussianARD(0,
gamma,
shape=(D,),
plates=(M,1),
name='C',
plotter=bpplt.GaussianHintonPlotter(rows=0,
cols=2,
scale=0))
C.initialize_from_value(np.random.randn(M,1,D))
# Noiseless process
# F : (M,N) x ()
F = SumMultiply('d,d',
C,
X,
name='F')
# Observation noise
# tau : () x ()
tau = Gamma(1e-5,
1e-5,
name='tau')
tau.initialize_from_value(1e2)
# Observations
# Y: (M,N) x ()
Y = GaussianARD(F,
tau,
name='Y')
# Construct inference machine
Q = VB(Y, F, C, gamma, X, A, alpha, tau, S, B, beta)
return Q
def lssm(M, N, D, K=1, drift_C=False, drift_A=False):
if (drift_C or drift_A) and not K > 0:
raise ValueError("K must be positive integer when using drift")
# Drift weights
if drift_A or drift_C:
# Dynamics matrix with ARD
# beta : (K) x ()
beta = Gamma(1e-5,
1e-5,
plates=(K,),
name='beta')
# B : (K) x (K)
B = GaussianArrayARD(np.identity(K),
beta,
shape=(K,),
plates=(K,),
name='B',
plotter=bpplt.GaussianHintonPlotter(rows=0,
cols=1,
scale=0),
initialize=False)
B.initialize_from_value(np.identity(K))
#B.initialize_from_mean_and_covariance(np.identity(K),
# 0.1*np.identity(K))
# State of the drift, that is, temporal weights for dynamics matrices
# S : () x (N,K)
S = GaussianMarkovChain(np.ones(K),
1e-6*np.identity(K),
B,
np.ones(K),
n=N,
name='S',
plotter=bpplt.GaussianMarkovChainPlotter(scale=2),
initialize=False)
#s = np.cumsum(np.random.randn(N,K), axis=0)
s = np.random.randn(N,K)
s[:,0] = 10
S.initialize_from_value(s)
#S.initialize_from_value(np.ones((N,K))+0.01*np.random.randn(N,K))
if not drift_A:
# Dynamic matrix
# alpha: (D) x ()
alpha = Gamma(1e-5,
1e-5,
plates=(D,),
name='alpha')
# A : (D) x (D)
A = GaussianArrayARD(0,
alpha,
shape=(D,),
plates=(D,),
name='A',
plotter=bpplt.GaussianHintonPlotter(rows=0,
cols=1,
scale=0),
initialize=False)
A.initialize_from_value(np.identity(D))
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
A, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X',
plotter=bpplt.GaussianMarkovChainPlotter(),
initialize=False)
X.initialize_from_value(np.random.randn(N,D))
else:
# Projection matrix of the dynamics matrix
# alpha : (K) x ()
alpha = Gamma(1e-5,
1e-5,
plates=(D,K),
name='alpha')
# A : (D) x (D,K)
A = GaussianArrayARD(0,
alpha,
shape=(D,K),
plates=(D,),
name='A',
plotter=bpplt.GaussianHintonPlotter(rows=0,
cols=1,
scale=0),
initialize=False)
# Initialize S and A such that A*S is almost an identity matrix
a = np.zeros((D,D,K))
a[np.arange(D),np.arange(D),np.zeros(D,dtype=int)] = 1
a[:,:,0] = np.identity(D) / s[0,0]
a[:,:,1:] = 0.1/s[0,0]*np.random.randn(D,D,K-1)
A.initialize_from_value(a)
#A.initialize_from_mean_and_covariance(a,
# 0.1/s[0,0]**2*utils.identity(D,K))
#A.initialize_from_value(a + 0.01*np.random.randn(D,D,K))
# Latent states with dynamics
# X : () x (N,D)
X = DriftingGaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
A, # dynamics matrices
S.as_gaussian()[1:], # temporal weights
np.ones(D), # innovation
n=N, # time instances
name='X',
plotter=bpplt.GaussianMarkovChainPlotter(scale=2),
initialize=False)
X.initialize_from_value(np.random.randn(N,D))
if not drift_C:
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5,
1e-5,
plates=(D,),
name='gamma')
# C : (M,1) x (D)
C = GaussianArrayARD(0,
gamma,
shape=(D,),
plates=(M,1),
name='C',
plotter=bpplt.GaussianHintonPlotter(rows=0,
cols=2,
scale=0))
C.initialize_from_value(np.random.randn(M,1,D))
#C.initialize_from_random()
#C.initialize_from_mean_and_covariance(C.random(),
# 0.1*utils.identity(D))
# Noiseless process
# F : (M,N) x ()
F = SumMultiply('d,d',
C,
X.as_gaussian(),
name='F')
else:
# Mixing matrix from latent space to observation space using ARD
# gamma : (D,K) x ()
gamma = Gamma(1e-5,
1e-5,
plates=(D,K),
name='gamma')
# C : (M,1) x (D,K)
C = GaussianArrayARD(0,
gamma,
shape=(D,K),
plates=(M,1),
name='C',
plotter=bpplt.GaussianHintonPlotter(rows=0,
cols=2,
scale=0))
C.initialize_from_random()
#C.initialize_from_mean_and_covariance(C.random(),
# 0.1*utils.identity(D, K))
# Noiseless process
# F : (M,N) x ()
F = SumMultiply('dk,d,k',
C,
X.as_gaussian(),
S.as_gaussian(),
name='F')
# Observation noise
# tau : () x ()
tau = Gamma(1e-5,
1e-5,
name='tau')
tau.initialize_from_value(1e2)
# Observations
# Y: (M,N) x ()
Y = GaussianArrayARD(F,
tau,
name='Y')
# Construct inference machine
if drift_C or drift_A:
Q = VB(Y, F, C, gamma, X, A, alpha, tau, S, B, beta)
else:
Q = VB(Y, F, C, gamma, X, A, alpha, tau)
return Q
import numpy as np
np.random.seed(1)
from bayespy.nodes import GaussianARD, GaussianMarkovChain, Gamma, Dot
M = 30
N = 400
D = 10
alpha = Gamma(1e-5, 1e-5, plates=(D, ), name='alpha')
A = GaussianARD(0, alpha, shape=(D, ), plates=(D, ), name='A')
X = GaussianMarkovChain(np.zeros(D),
1e-3 * np.identity(D),
A,
np.ones(D),
n=N,
name='X')
gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma')
C = GaussianARD(0, gamma, shape=(D, ), plates=(M, 1), name='C')
F = Dot(C, X, name='F')
C.initialize_from_random()
tau = Gamma(1e-5, 1e-5, name='tau')
Y = GaussianARD(F, tau, name='Y')
from bayespy.inference import VB
Q = VB(X, C, gamma, A, alpha, tau, Y)
w = 0.3
a = np.array([[np.cos(w), -np.sin(w), 0, 0], [np.sin(w),
np.cos(w), 0, 0], [0, 0, 1, 0],
[0, 0, 0, 0]])
c = np.random.randn(M, 4)
x = np.empty((N, 4))
f = np.empty((M, N))
y = np.empty((M, N))
x[0] = 10 * np.random.randn(4)
import numpy as np
np.random.seed(1)
from bayespy.nodes import GaussianARD, GaussianMarkovChain, Gamma, Dot
M = 30
N = 400
D = 10
alpha = Gamma(1e-5, 1e-5, plates=(D, ), name='alpha')
A = GaussianARD(0, alpha, shape=(D, ), plates=(D, ), name='A')
X = GaussianMarkovChain(np.zeros(D),
1e-3 * np.identity(D),
A,
np.ones(D),
n=N,
name='X')
gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma')
C = GaussianARD(0, gamma, shape=(D, ), plates=(M, 1), name='C')
F = Dot(C, X, name='F')
C.initialize_from_random()
tau = Gamma(1e-5, 1e-5, name='tau')
Y = GaussianARD(F, tau, name='Y')
from bayespy.inference import VB
Q = VB(X, C, gamma, A, alpha, tau, Y)
w = 0.3
a = np.array([[np.cos(w), -np.sin(w), 0, 0], [np.sin(w),
np.cos(w), 0, 0], [0, 0, 1, 0],
[0, 0, 0, 0]])
c = np.random.randn(M, 4)
x = np.empty((N, 4))