def run_dlssm(y, f, mask, D, K, maxiter):
"""
Run VB inference for linear state space model with drifting dynamics.
"""
(M, N) = np.shape(y)
# Dynamics matrix with ARD
# alpha : (K) x ()
alpha = Gamma(1e-5,
1e-5,
plates=(K,),
name='alpha')
# A : (K) x (K)
A = GaussianArrayARD(np.identity(K),
alpha,
shape=(K,),
plates=(K,),
name='A_S',
initialize=False)
A.initialize_from_value(np.identity(K))
# State of the drift
# S : () x (N,K)
S = GaussianMarkovChain(np.ones(K),
1e-6*np.identity(K),
A,
np.ones(K),
n=N,
name='S',
initialize=False)
S.initialize_from_value(np.ones((N,K)))
# Projection matrix of the dynamics matrix
# Initialize S and B such that BS is identity matrix
# beta : (K) x ()
beta = Gamma(1e-5,
1e-5,
plates=(D,K),
name='beta')
# B : (D) x (D,K)
b = np.zeros((D,D,K))
b[np.arange(D),np.arange(D),np.zeros(D,dtype=int)] = 1
B = GaussianArrayARD(0,
beta,
plates=(D,),
name='B',
initialize=False)
B.initialize_from_value(np.reshape(1*b, (D,D,K)))
# BS : (N-1,D) x (D)
# TODO/FIXME: Implement __getitem__ method
BS = SumMultiply('dk,k->d',
B,
S.as_gaussian()[...,np.newaxis],
# iterator_axis=0,
name='BS')
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
BS, # dynamics
np.ones(D), # innovation
n=N+1, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N+1,D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5,
1e-5,
plates=(D,K),
name='gamma')
# C : (M,1) x (D,K)
C = GaussianArrayARD(0,
gamma,
plates=(M,1),
name='C',
initialize=False)
C.initialize_from_random()
# Observation noise
# tau : () x ()
tau = Gamma(1e-5,
1e-5,
name='tau')
# Observations
# Y : (M,N) x ()
F = SumMultiply('dk,d,k',
C,
X.as_gaussian()[1:],
S.as_gaussian(),
name='F')
Y = GaussianArrayARD(F,
tau,
name='Y')
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, S, A, alpha, B, beta, C, gamma, tau)
#
# Run inference with rotations.
#
rotate = False
if rotate:
# Rotate the D-dimensional state space (C, X)
rotB = transformations.RotateGaussianMatrixARD(B, beta, D, K, axis='rows')
rotX = transformations.RotateDriftingMarkovChain(X, B, S, rotB)
rotC = transformations.RotateGaussianARD(C, gamma)
R_X = transformations.RotationOptimizer(rotX, rotC, D)
# Rotate the K-dimensional latent dynamics space (B, S)
rotA = transformations.RotateGaussianARD(A, alpha)
rotS = transformations.RotateGaussianMarkovChain(S, A, rotA)
rotB = transformations.RotateGaussianMatrixARD(B, beta, D, K, axis='cols')
R_S = transformations.RotationOptimizer(rotS, rotB, K)
# Iterate
for ind in range(maxiter):
print("X update")
Q.update(X)
print("S update")
Q.update(S)
print("A update")
Q.update(A)
print("alpha update")
Q.update(alpha)
print("B update")
Q.update(B)
print("beta update")
Q.update(beta)
print("C update")
Q.update(C)
print("gamma update")
Q.update(gamma)
print("tau update")
Q.update(tau)
if rotate:
if ind >= 0:
R_X.rotate()
if ind >= 0:
R_S.rotate()
Q.plot_iteration_by_nodes()
#
# SHOW RESULTS
#
# Plot observations space
plt.figure()
bpplt.timeseries_normal(F)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
# Plot latent space
plt.figure()
bpplt.timeseries_gaussian_mc(X, scale=2)
# Plot drift space
plt.figure()
bpplt.timeseries_gaussian_mc(S, scale=2)
def run(M=10, N=100, D_y=3, D=5):
seed = 45
print('seed =', seed)
np.random.seed(seed)
# Check HDF5 version.
if h5py.version.hdf5_version_tuple < (1, 8, 7):
print(
"WARNING! Your HDF5 version is %s. HDF5 versions <1.8.7 are not "
"able to save empty arrays, thus you may experience problems if "
"you for instance try to save before running any iteration steps."
% str(h5py.version.hdf5_version_tuple))
# Generate data
w = np.random.normal(0, 1, size=(M, 1, D_y))
x = np.random.normal(0, 1, size=(1, N, D_y))
f = misc.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.5, size=(M, N))
# Construct model
(Y, WX, W, X, tau, alpha) = pca_model(M, N, D)
# Data with missing values
mask = random.mask(M, N, p=0.9) # randomly missing
mask[:, 20:40] = False # gap missing
y[~mask] = np.nan
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, W, X, tau, alpha, autosave_iterations=5)
# Initialize some nodes randomly
X.initialize_from_value(X.random())
W.initialize_from_value(W.random())
# Save the state into a HDF5 file
filename = tempfile.NamedTemporaryFile(suffix='hdf5').name
Q.update(X, W, alpha, tau, repeat=1)
Q.save(filename=filename)
# Inference loop.
Q.update(X, W, alpha, tau, repeat=10)
# Reload the state from the HDF5 file
Q.load(filename=filename)
# Inference loop again.
Q.update(X, W, alpha, tau, repeat=10)
# NOTE: Saving and loading requires that you have the model
# constructed. "Save" does not store the model structure nor does "load"
# read it. They are just used for reading and writing the contents of the
# nodes. Thus, if you want to load, you first need to construct the same
# model that was used for saving and then use load to set the states of the
# nodes.
plt.clf()
WX_params = WX.get_parameters()
fh = WX_params[0] * np.ones(y.shape)
err_fh = 2 * np.sqrt(WX_params[1] + 1 / tau.get_moments()[0]) * np.ones(
y.shape)
for m in range(M):
plt.subplot(M, 1, m + 1)
#errorplot(y, error=None, x=None, lower=None, upper=None):
bpplt.errorplot(fh[m], x=np.arange(N), error=err_fh[m])
plt.plot(np.arange(N), f[m], 'g')
plt.plot(np.arange(N), y[m], 'r+')
plt.figure()
Q.plot_iteration_by_nodes()
plt.figure()
plt.subplot(2, 2, 1)
bpplt.binary_matrix(W.mask)
plt.subplot(2, 2, 2)
bpplt.binary_matrix(X.mask)
plt.subplot(2, 2, 3)
#bpplt.binary_matrix(WX.get_mask())
plt.subplot(2, 2, 4)
bpplt.binary_matrix(Y.mask)
def run(maxiter=100):
seed = 496 #np.random.randint(1000)
print("seed = ", seed)
np.random.seed(seed)
# Simulate some data
D = 3
M = 6
N = 200
c = np.random.randn(M, D)
w = 0.3
a = np.array([[np.cos(w), -np.sin(w), 0], [np.sin(w),
np.cos(w), 0], [0, 0, 1]])
x = np.empty((N, D))
f = np.empty((M, N))
y = np.empty((M, N))
x[0] = 10 * np.random.randn(D)
f[:, 0] = np.dot(c, x[0])
y[:, 0] = f[:, 0] + 3 * np.random.randn(M)
for n in range(N - 1):
x[n + 1] = np.dot(a, x[n]) + np.random.randn(D)
f[:, n + 1] = np.dot(c, x[n + 1])
y[:, n + 1] = f[:, n + 1] + 3 * np.random.randn(M)
# Create the model
(Y, CX, X, tau, C, gamma, A, alpha) = linear_state_space_model(D, N, M)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:, 30:80] = False
y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting.
# Observe the data
Y.observe(y, mask=mask)
# Initialize nodes (must use some randomness for C)
C.initialize_from_random()
# Run inference
Q = VB(Y, X, C, gamma, A, alpha, tau)
#
# Run inference with rotations.
#
rotA = transformations.RotateGaussianARD(A, alpha)
rotX = transformations.RotateGaussianMarkovChain(X, A, rotA)
rotC = transformations.RotateGaussianARD(C, gamma)
R = transformations.RotationOptimizer(rotX, rotC, D)
#maxiter = 84
for ind in range(maxiter):
Q.update()
#print('C term', C.lower_bound_contribution())
R.rotate(
maxiter=10,
check_gradient=True,
verbose=False,
check_bound=Q.compute_lowerbound,
#check_bound=None,
check_bound_terms=Q.compute_lowerbound_terms)
#check_bound_terms=None)
X_vb = X.u[0]
varX_vb = utils.diagonal(X.u[1] - X_vb[..., np.newaxis, :] *
X_vb[..., :, np.newaxis])
u_CX = CX.get_moments()
CX_vb = u_CX[0]
varCX_vb = u_CX[1] - CX_vb**2
# Show results
plt.figure(3)
plt.clf()
for m in range(M):
plt.subplot(M, 1, m + 1)
plt.plot(y[m, :], 'r.')
plt.plot(f[m, :], 'b-')
bpplt.errorplot(y=CX_vb[m, :], error=2 * np.sqrt(varCX_vb[m, :]))
plt.figure()
Q.plot_iteration_by_nodes()
def run(maxiter=100):
seed = 496#np.random.randint(1000)
print("seed = ", seed)
np.random.seed(seed)
# Simulate some data
D = 3
M = 6
N = 200
c = np.random.randn(M,D)
w = 0.3
a = np.array([[np.cos(w), -np.sin(w), 0],
[np.sin(w), np.cos(w), 0],
[0, 0, 1]])
x = np.empty((N,D))
f = np.empty((M,N))
y = np.empty((M,N))
x[0] = 10*np.random.randn(D)
f[:,0] = np.dot(c,x[0])
y[:,0] = f[:,0] + 3*np.random.randn(M)
for n in range(N-1):
x[n+1] = np.dot(a,x[n]) + np.random.randn(D)
f[:,n+1] = np.dot(c,x[n+1])
y[:,n+1] = f[:,n+1] + 3*np.random.randn(M)
# Create the model
(Y, CX, X, tau, C, gamma, A, alpha) = linear_state_space_model(D, N, M)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:,30:80] = False
y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting.
# Observe the data
Y.observe(y, mask=mask)
# Initialize nodes (must use some randomness for C)
C.initialize_from_random()
# Run inference
Q = VB(Y, X, C, gamma, A, alpha, tau)
#
# Run inference with rotations.
#
rotA = transformations.RotateGaussianARD(A, alpha)
rotX = transformations.RotateGaussianMarkovChain(X, A, rotA)
rotC = transformations.RotateGaussianARD(C, gamma)
R = transformations.RotationOptimizer(rotX, rotC, D)
#maxiter = 84
for ind in range(maxiter):
Q.update()
#print('C term', C.lower_bound_contribution())
R.rotate(maxiter=10,
check_gradient=True,
verbose=False,
check_bound=Q.compute_lowerbound,
#check_bound=None,
check_bound_terms=Q.compute_lowerbound_terms)
#check_bound_terms=None)
X_vb = X.u[0]
varX_vb = utils.diagonal(X.u[1] - X_vb[...,np.newaxis,:] * X_vb[...,:,np.newaxis])
u_CX = CX.get_moments()
CX_vb = u_CX[0]
varCX_vb = u_CX[1] - CX_vb**2
# Show results
plt.figure(3)
plt.clf()
for m in range(M):
plt.subplot(M,1,m+1)
plt.plot(y[m,:], 'r.')
plt.plot(f[m,:], 'b-')
bpplt.errorplot(y=CX_vb[m,:],
error=2*np.sqrt(varCX_vb[m,:]))
plt.figure()
Q.plot_iteration_by_nodes()
def run(M=10, N=100, D_y=3, D=5):
seed = 45
print('seed =', seed)
np.random.seed(seed)
# Check HDF5 version.
if h5py.version.hdf5_version_tuple < (1,8,7):
print("WARNING! Your HDF5 version is %s. HDF5 versions <1.8.7 are not "
"able to save empty arrays, thus you may experience problems if "
"you for instance try to save before running any iteration steps."
% str(h5py.version.hdf5_version_tuple))
# Generate data
w = np.random.normal(0, 1, size=(M,1,D_y))
x = np.random.normal(0, 1, size=(1,N,D_y))
f = misc.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.5, size=(M,N))
# Construct model
(Y, WX, W, X, tau, alpha) = pca_model(M, N, D)
# Data with missing values
mask = random.mask(M, N, p=0.9) # randomly missing
mask[:,20:40] = False # gap missing
y[~mask] = np.nan
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, W, X, tau, alpha, autosave_iterations=5)
# Initialize some nodes randomly
X.initialize_from_value(X.random())
W.initialize_from_value(W.random())
# Save the state into a HDF5 file
filename = tempfile.NamedTemporaryFile(suffix='hdf5').name
Q.update(X, W, alpha, tau, repeat=1)
Q.save(filename=filename)
# Inference loop.
Q.update(X, W, alpha, tau, repeat=10)
# Reload the state from the HDF5 file
Q.load(filename=filename)
# Inference loop again.
Q.update(X, W, alpha, tau, repeat=10)
# NOTE: Saving and loading requires that you have the model
# constructed. "Save" does not store the model structure nor does "load"
# read it. They are just used for reading and writing the contents of the
# nodes. Thus, if you want to load, you first need to construct the same
# model that was used for saving and then use load to set the states of the
# nodes.
plt.clf()
WX_params = WX.get_parameters()
fh = WX_params[0] * np.ones(y.shape)
err_fh = 2*np.sqrt(WX_params[1] + 1/tau.get_moments()[0]) * np.ones(y.shape)
for m in range(M):
plt.subplot(M,1,m+1)
#errorplot(y, error=None, x=None, lower=None, upper=None):
bpplt.errorplot(fh[m], x=np.arange(N), error=err_fh[m])
plt.plot(np.arange(N), f[m], 'g')
plt.plot(np.arange(N), y[m], 'r+')
plt.figure()
Q.plot_iteration_by_nodes()
plt.figure()
plt.subplot(2,2,1)
bpplt.binary_matrix(W.mask)
plt.subplot(2,2,2)
bpplt.binary_matrix(X.mask)
plt.subplot(2,2,3)
#bpplt.binary_matrix(WX.get_mask())
plt.subplot(2,2,4)
bpplt.binary_matrix(Y.mask)
