def demo(M=6, N=200, D=3, maxiter=100, debug=False, seed=42, rotate=True,
precompute=False, plot=True):
"""
Run the demo for linear state-space model.
"""
# Use deterministic random numbers
if seed is not None:
np.random.seed(seed)
# Get data
(y, f) = simulate_data(M, N)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:,30:80] = False
y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting.
# Run inference
Q = infer(y, D,
mask=mask,
rotate=rotate,
debug=debug,
maxiter=maxiter)
if plot:
# Show results
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
plt.show()
def run(M=10, N=100, D_y=3, D=5, seed=42, rotate=False, maxiter=100, debug=False):
if seed is not None:
np.random.seed(seed)
# Generate data
w = np.random.normal(0, 1, size=(M,1,D_y))
x = np.random.normal(0, 1, size=(1,N,D_y))
f = utils.utils.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.2, size=(M,N))
# Construct model
(Y, F, W, X, tau, alpha) = model(M, N, D)
# Data with missing values
mask = utils.random.mask(M, N, p=0.5) # randomly missing
y[~mask] = np.nan
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, W, X, tau, alpha)
# Initialize some nodes randomly
X.initialize_from_random()
W.initialize_from_random()
# Run inference algorithm
if rotate:
# Use rotations to speed up learning
rotW = transformations.RotateGaussianArrayARD(W, alpha)
rotX = transformations.RotateGaussianArrayARD(X)
R = transformations.RotationOptimizer(rotW, rotX, D)
for ind in range(maxiter):
Q.update()
if debug:
R.rotate(check_bound=True,
check_gradient=True)
else:
R.rotate()
else:
# Use standard VB-EM alone
Q.update(repeat=maxiter)
# Plot results
plt.figure()
bpplt.timeseries_normal(F, scale=2)
bpplt.timeseries(f, 'g-')
bpplt.timeseries(y, 'r+')
plt.show()
def run(M=6, N=200, D=3, maxiter=100, debug=False, seed=42, rotate=False, precompute=False):
# Use deterministic random numbers
if seed is not None:
np.random.seed(seed)
# Simulate some data
K = 3
c = np.random.randn(M,K)
w = 0.3
a = np.array([[np.cos(w), -np.sin(w), 0],
[np.sin(w), np.cos(w), 0],
[0, 0, 1]])
x = np.empty((N,K))
f = np.empty((M,N))
y = np.empty((M,N))
x[0] = 10*np.random.randn(K)
f[:,0] = np.dot(c,x[0])
y[:,0] = f[:,0] + 3*np.random.randn(M)
for n in range(N-1):
x[n+1] = np.dot(a,x[n]) + np.random.randn(K)
f[:,n+1] = np.dot(c,x[n+1])
y[:,n+1] = f[:,n+1] + 3*np.random.randn(M)
# Create the model
(Y, CX, X, tau, C, gamma, A, alpha) = linear_state_space_model(D=D,
N=N,
M=M)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:,30:80] = False
y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting.
# Observe the data
Y.observe(y, mask=mask)
# Initialize nodes (must use some randomness for C)
C.initialize_from_random()
# Run inference
Q = VB(Y, X, C, gamma, A, alpha, tau)
#
# Run inference with rotations.
#
if rotate:
rotA = transformations.RotateGaussianArrayARD(A, alpha, precompute=precompute)
rotX = transformations.RotateGaussianMarkovChain(X, A, rotA)
rotC = transformations.RotateGaussianArrayARD(C, gamma)
R = transformations.RotationOptimizer(rotX, rotC, D)
for ind in range(maxiter):
Q.update()
if debug:
R.rotate(maxiter=10,
check_gradient=True,
check_bound=True)
else:
R.rotate()
else:
Q.update(repeat=maxiter)
# Show results
plt.figure()
bpplt.timeseries_normal(CX, scale=2)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
plt.show()
def run_dlssm(y, f, mask, D, K, maxiter):
"""
Run VB inference for linear state space model with drifting dynamics.
"""
(M, N) = np.shape(y)
# Dynamics matrix with ARD
# alpha : (K) x ()
alpha = Gamma(1e-5,
1e-5,
plates=(K,),
name='alpha')
# A : (K) x (K)
A = GaussianArrayARD(np.identity(K),
alpha,
shape=(K,),
plates=(K,),
name='A_S',
initialize=False)
A.initialize_from_value(np.identity(K))
# State of the drift
# S : () x (N,K)
S = GaussianMarkovChain(np.ones(K),
1e-6*np.identity(K),
A,
np.ones(K),
n=N,
name='S',
initialize=False)
S.initialize_from_value(np.ones((N,K)))
# Projection matrix of the dynamics matrix
# Initialize S and B such that BS is identity matrix
# beta : (K) x ()
beta = Gamma(1e-5,
1e-5,
plates=(D,K),
name='beta')
# B : (D) x (D,K)
b = np.zeros((D,D,K))
b[np.arange(D),np.arange(D),np.zeros(D,dtype=int)] = 1
B = GaussianArrayARD(0,
beta,
plates=(D,),
name='B',
initialize=False)
B.initialize_from_value(np.reshape(1*b, (D,D,K)))
# BS : (N-1,D) x (D)
# TODO/FIXME: Implement __getitem__ method
BS = SumMultiply('dk,k->d',
B,
S.as_gaussian()[...,np.newaxis],
# iterator_axis=0,
name='BS')
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
BS, # dynamics
np.ones(D), # innovation
n=N+1, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N+1,D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5,
1e-5,
plates=(D,K),
name='gamma')
# C : (M,1) x (D,K)
C = GaussianArrayARD(0,
gamma,
plates=(M,1),
name='C',
initialize=False)
C.initialize_from_random()
# Observation noise
# tau : () x ()
tau = Gamma(1e-5,
1e-5,
name='tau')
# Observations
# Y : (M,N) x ()
F = SumMultiply('dk,d,k',
C,
X.as_gaussian()[1:],
S.as_gaussian(),
name='F')
Y = GaussianArrayARD(F,
tau,
name='Y')
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, S, A, alpha, B, beta, C, gamma, tau)
#
# Run inference with rotations.
#
rotate = False
if rotate:
# Rotate the D-dimensional state space (C, X)
rotB = transformations.RotateGaussianMatrixARD(B, beta, D, K, axis='rows')
rotX = transformations.RotateDriftingMarkovChain(X, B, S, rotB)
rotC = transformations.RotateGaussianARD(C, gamma)
R_X = transformations.RotationOptimizer(rotX, rotC, D)
# Rotate the K-dimensional latent dynamics space (B, S)
rotA = transformations.RotateGaussianARD(A, alpha)
rotS = transformations.RotateGaussianMarkovChain(S, A, rotA)
rotB = transformations.RotateGaussianMatrixARD(B, beta, D, K, axis='cols')
R_S = transformations.RotationOptimizer(rotS, rotB, K)
# Iterate
for ind in range(maxiter):
print("X update")
Q.update(X)
print("S update")
Q.update(S)
print("A update")
Q.update(A)
print("alpha update")
Q.update(alpha)
print("B update")
Q.update(B)
print("beta update")
Q.update(beta)
print("C update")
Q.update(C)
print("gamma update")
Q.update(gamma)
print("tau update")
Q.update(tau)
if rotate:
if ind >= 0:
R_X.rotate()
if ind >= 0:
R_S.rotate()
Q.plot_iteration_by_nodes()
#
# SHOW RESULTS
#
# Plot observations space
plt.figure()
bpplt.timeseries_normal(F)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
# Plot latent space
plt.figure()
bpplt.timeseries_gaussian_mc(X, scale=2)
# Plot drift space
plt.figure()
bpplt.timeseries_gaussian_mc(S, scale=2)
def run_lssm(y, f, mask, D, maxiter):
"""
Run VB inference for linear state space model.
"""
(M, N) = np.shape(y)
#
# CONSTRUCT THE MODEL
#
# Dynamic matrix
# alpha: (D) x ()
alpha = Gamma(1e-5,
1e-5,
plates=(D,),
name='alpha')
# A : (D) x (D)
A = Gaussian(np.zeros(D),
diagonal(alpha),
plates=(D,),
name='A')
A.initialize_from_value(np.identity(D))
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
A, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N,D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5,
1e-5,
plates=(D,),
name='gamma')
# C : (M,1) x (D)
C = Gaussian(np.zeros(D),
diagonal(gamma),
plates=(M,1),
name='C')
C.initialize_from_value(np.random.randn(M,1,D))
# Observation noise
# tau : () x ()
tau = Gamma(1e-5,
1e-5,
name='tau')
# Observations
# Y : (M,N) x ()
CX = Dot(C, X.as_gaussian())
Y = Normal(CX,
tau,
name='Y')
# Rotate the D-dimensional latent space
rotA = transformations.RotateGaussianARD(A, alpha)
rotX = transformations.RotateGaussianMarkovChain(X, A, rotA)
rotC = transformations.RotateGaussianARD(C, gamma)
R = transformations.RotationOptimizer(rotX, rotC, D)
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, A, alpha, C, gamma, tau)
# Iterate
for ind in range(maxiter):
Q.update(X, A, alpha, C, gamma, tau)
R.rotate()
#
# SHOW RESULTS
#
plt.figure()
bpplt.timeseries_normal(CX)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
def demo(N=1000, D=5, K=4, seed=42, maxiter=200, rotate=True, debug=False,
precompute=False, plot=True):
# Seed for random number generator
if seed is not None:
np.random.seed(seed)
# Create data
(y, f) = simulate_data(N)
# Create some gaps
mask_gaps = utils.trues(N)
for m in range(100, N, 140):
start = m
end = min(m+15, N-1)
mask_gaps[start:end] = False
# Randomly missing values
mask_random = np.logical_or(random.mask(N, p=0.8),
np.logical_not(mask_gaps))
# Remove the observations
mask = np.logical_and(mask_gaps, mask_random)
y[~mask] = np.nan # BayesPy doesn't require NaNs, they're just for plotting.
# Add row axes
y = y[None,...]
f = f[None,...]
mask = mask[None,...]
mask_gaps = mask_gaps[None,...]
mask_random = mask_random[None,...]
# Run the method
Q = infer(y, D, K,
mask=mask,
maxiter=maxiter,
rotate=rotate,
debug=debug,
precompute=precompute,
update_hyper=10,
start_rotating_weights=20,
monitor=True)
if plot:
# Plot observations
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
plt.ylim([-2, 2])
# Plot latent space
Q.plot('X')
# Plot mixing weight space
Q.plot('S')
# Compute RMSE
rmse_random = utils.rmse(Q['Y'].get_moments()[0][~mask_random],
f[~mask_random])
rmse_gaps = utils.rmse(Q['Y'].get_moments()[0][~mask_gaps],
f[~mask_gaps])
print("RMSE for randomly missing values: %f" % rmse_random)
print("RMSE for gap values: %f" % rmse_gaps)
plt.show()
def run(M=1, N=1000, D=5, K=4, seed=42, maxiter=200,
rotate=True, debug=False, precompute=False,
drift=False,
plot_X=False, plot_Y=True, plot_S=False):
# Seed for random number generator
if seed is not None:
np.random.seed(seed)
# Create data
(y, f) = simulate_drifting_lssm(M, N)
# Create some gaps
mask_gaps = utils.trues((M,N))
for m in range(100, N, 140):
start = m
end = min(m+15, N-1)
mask_gaps[:,start:end] = False
# Missing values
mask_random = np.logical_or(random.mask(M, N, p=0.8),
np.logical_not(mask_gaps))
# Remove the observations
mask = np.logical_and(mask_gaps, mask_random)
# Remove the observations
y[~mask] = np.nan # BayesPy doesn't require NaNs, they're just for plotting.
# Run the method
Q = model_lssm.run_lssm(y, D,
mask=mask,
K=K,
maxiter=maxiter,
rotate=rotate,
debug=debug,
precompute=precompute,
drift_A=drift,
update_drift=10,
start_rotating_drift=20)
## plt.figure()
## bpplt.timeseries(f, 'b-')
## plt.ylim([-2, 2])
# Plot observations
if plot_Y:
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
plt.ylim([-2, 2])
# Plot latent space
if plot_X:
Q.plot('X')
# Plot drift space
if plot_S and drift:
Q.plot('S')
# Compute RMSE
rmse_random = utils.rmse(Q['F'].get_moments()[0][~mask_random],
f[~mask_random])
rmse_gaps = utils.rmse(Q['F'].get_moments()[0][~mask_gaps],
f[~mask_gaps])
print("RMSE for randomly missing values: %f" % rmse_random)
print("RMSE for gap values: %f" % rmse_gaps)
plt.show()
def demo(N=1000,
D=5,
K=4,
seed=42,
maxiter=200,
rotate=True,
debug=False,
precompute=False,
plot=True):
# Seed for random number generator
if seed is not None:
np.random.seed(seed)
# Create data
(y, f) = simulate_data(N)
# Create some gaps
mask_gaps = utils.trues(N)
for m in range(100, N, 140):
start = m
end = min(m + 15, N - 1)
mask_gaps[start:end] = False
# Randomly missing values
mask_random = np.logical_or(random.mask(N, p=0.8),
np.logical_not(mask_gaps))
# Remove the observations
mask = np.logical_and(mask_gaps, mask_random)
y[~mask] = np.nan # BayesPy doesn't require NaNs, they're just for plotting.
# Add row axes
y = y[None, ...]
f = f[None, ...]
mask = mask[None, ...]
mask_gaps = mask_gaps[None, ...]
mask_random = mask_random[None, ...]
# Run the method
Q = infer(y,
D,
K,
mask=mask,
maxiter=maxiter,
rotate=rotate,
debug=debug,
precompute=precompute,
update_hyper=10,
start_rotating_weights=20,
monitor=True)
if plot:
# Plot observations
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
plt.ylim([-2, 2])
# Plot latent space
Q.plot('X')
# Plot mixing weight space
Q.plot('S')
# Compute RMSE
rmse_random = utils.rmse(Q['Y'].get_moments()[0][~mask_random],
f[~mask_random])
rmse_gaps = utils.rmse(Q['Y'].get_moments()[0][~mask_gaps], f[~mask_gaps])
print("RMSE for randomly missing values: %f" % rmse_random)
print("RMSE for gap values: %f" % rmse_gaps)
plt.show()
def run(M=100, N=2000, D=20, K=5, rotate=True, maxiter=200, seed=42,
debug=False, precompute=False, resolution=30, dynamic=True,
drift_A=False, drift_C=False,
plot_Y=True, plot_X=True, plot_S=True):
# Seed for random number generator
if seed is not None:
np.random.seed(seed)
# Create data
if dynamic:
decay = 1.0 - 5e-3
else:
decay = 1.0
(U, y, f, X) = simulate_data(M=M,
N=N,
resolution=resolution,
burnin=2000,
thin=20,
velocity=6e-2,
diffusion=1e-4,
decay=decay,
innovation_noise=1e-4,
innovation_lengthscale=2.0,
noise_ratio=5e-1)
plt.ion()
plt.plot(X[:,0], X[:,1], 'kx')
animation = bpplt.matrix_animation(U)
#bpplt.save_animation(animation, 'demo_drift_lssm_02_spde.mp4')
plt.show()
plt.ioff()
# Create some gaps
mask_gaps = utils.trues((M,N))
gap = 15
interval = 100
for m in range(100, N, interval):
start = m
end = min(m+gap, N-1)
mask_gaps[:,start:end] = False
# Missing values
mask_random = np.logical_or(random.mask(M, N, p=0.8),
np.logical_not(mask_gaps))
# Remove the observations
mask = np.logical_and(mask_gaps, mask_random)
y[~mask] = np.nan # BayesPy doesn't require NaNs, they're just for plotting.
# Run the method
Q = model_lssm.run_lssm(y, D,
mask=mask,
K=K,
maxiter=maxiter,
rotate=rotate,
debug=debug,
precompute=precompute,
drift_A=drift_A,
drift_C=drift_C,
update_drift=20,
start_rotating_drift=30)
if plot_Y:
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
# Plot latent space
if plot_X:
Q.plot('X')
# Plot drift space
if plot_S and (drift_A or drift_C):
Q.plot('S')
# Compute RMSE
rmse_random = utils.rmse(Q['F'].get_moments()[0][~mask_random],
f[~mask_random])
rmse_gaps = utils.rmse(Q['F'].get_moments()[0][~mask_gaps],
f[~mask_gaps])
print("RMSE for randomly missing values: %f" % rmse_random)
print("RMSE for gap values: %f" % rmse_gaps)
plt.show()
return