def plot(self):
x = self.u[0]
xx = self.u[1]
var = np.einsum('...ii->...i', xx) - x**2
plt.figure()
D = np.shape(x)[-1]
for d in range(D):
plt.subplot(D, 1, d + 1)
bpplt.errorplot(y=x[..., d], error=2 * np.sqrt(var[..., d]))
def run():
# Create some data
N = 500
D = 2
# Initial state
x0 = np.array([0.5, -0.5])
# Dynamics (time varying)
A0 = np.array([[.9, -.4], [.4, .9]])
A1 = np.array([[.98, -.1], [.1, .98]])
l = np.linspace(0, 1, N - 1).reshape((-1, 1, 1))
A = (1 - l) * A0 + l * A1
# Innovation covariance matrix
V = np.identity(D)
# Observation noise covariance matrix
C = np.tile(np.identity(D), (N, 1, 1))
## C0 = 10*np.array([[1, 0], [0, 1]])
## C1 = 0.01*np.array([[1, 0], [0, 1]])
## C = (1-l)**2*C0 + l**2*C1
X = np.empty((N, D))
Y = np.empty((N, D))
# Simulate data
x = x0
X[0, :] = x
Y[0, :] = x + np.random.multivariate_normal(np.zeros(D), C[0, :, :])
for n in range(N - 1):
x = np.dot(A[n, :, :], x) + np.random.multivariate_normal(
np.zeros(D), V)
X[n + 1, :] = x
Y[n + 1, :] = x + np.random.multivariate_normal(
np.zeros(D), C[n + 1, :, :])
# Invert observation noise covariance to observation precision matrices
U = np.empty((N, D, D))
UY = np.empty((N, D))
for n in range(N):
U[n, :, :] = np.linalg.inv(C[n, :, :])
UY[n, :] = np.linalg.solve(C[n, :, :], Y[n, :])
# Construct VB model
Xh = GaussianMarkovChain(np.zeros(D), np.identity(D), A, np.ones(D), n=N)
Yh = Gaussian(Xh.as_gaussian(), np.identity(D), plates=(N, ))
Yh.observe(Y)
Xh.update()
xh = Xh.u[0]
varxh = utils.diagonal(Xh.u[1]) - xh**2
#err = 2 * np.sqrt(varxh)
plt.figure(1)
plt.clf()
for d in range(D):
plt.subplot(D, 1, d)
bpplt.errorplot(xh[:, d], error=2 * np.sqrt(varxh[:, d]))
plt.plot(X[:, d], 'r-')
plt.plot(Y[:, d], '.')
def plot(self):
x = self.u[0]
xx = self.u[1]
var = np.einsum('...ii->...i', xx) - x**2
plt.figure()
D = np.shape(x)[-1]
for d in range(D):
plt.subplot(D,1,d+1)
bpplt.errorplot(y=x[...,d],
error=2*np.sqrt(var[...,d]))
def run():
# Create some data
N = 500
D = 2
# Initial state
x0 = np.array([0.5, -0.5])
# Dynamics (time varying)
A0 = np.array([[.9, -.4], [.4, .9]])
A1 = np.array([[.98, -.1], [.1, .98]])
l = np.linspace(0, 1, N-1).reshape((-1,1,1))
A = (1-l)*A0 + l*A1
# Innovation covariance matrix
V = np.identity(D)
# Observation noise covariance matrix
C = np.tile(np.identity(D), (N, 1, 1))
## C0 = 10*np.array([[1, 0], [0, 1]])
## C1 = 0.01*np.array([[1, 0], [0, 1]])
## C = (1-l)**2*C0 + l**2*C1
X = np.empty((N,D))
Y = np.empty((N,D))
# Simulate data
x = x0
X[0,:] = x
Y[0,:] = x + np.random.multivariate_normal(np.zeros(D), C[0,:,:])
for n in range(N-1):
x = np.dot(A[n,:,:],x) + np.random.multivariate_normal(np.zeros(D), V)
X[n+1,:] = x
Y[n+1,:] = x + np.random.multivariate_normal(np.zeros(D), C[n+1,:,:])
# Invert observation noise covariance to observation precision matrices
U = np.empty((N,D,D))
UY = np.empty((N,D))
for n in range(N):
U[n,:,:] = np.linalg.inv(C[n,:,:])
UY[n,:] = np.linalg.solve(C[n,:,:], Y[n,:])
# Construct VB model
Xh = GaussianMarkovChain(np.zeros(D), np.identity(D), A, np.ones(D), n=N)
Yh = Gaussian(Xh.as_gaussian(), np.identity(D), plates=(N,))
Yh.observe(Y)
Xh.update()
xh = Xh.u[0]
varxh = utils.diagonal(Xh.u[1]) - xh**2
#err = 2 * np.sqrt(varxh)
plt.figure(1)
plt.clf()
for d in range(D):
plt.subplot(D,1,d)
bpplt.errorplot(xh[:,d], error=2*np.sqrt(varxh[:,d]))
plt.plot(X[:,d], 'r-')
plt.plot(Y[:,d], '.')
def run():
# Create some data
N = 500
D = 2
# Initial state
x0 = np.array([0.5, -0.5])
# Dynamics (time varying)
A0 = np.array([[.9, -.4], [.4, .9]])
A1 = np.array([[.98, -.1], [.1, .98]])
l = np.linspace(0, 1, N).reshape((-1, 1, 1))
A = (1 - l) * A0 + l * A1
# Innovation covariance matrix
V = np.array([[1, 0], [0, 1]])
# Observation noise covariance matrix
C0 = 10 * np.array([[1, 0], [0, 1]])
C1 = 0.01 * np.array([[1, 0], [0, 1]])
C = (1 - l)**2 * C0 + l**2 * C1
X = np.empty((N, D))
Y = np.empty((N, D))
# Simulate data
x = x0
for n in range(N):
x = np.dot(A[n, :, :], x) + np.random.multivariate_normal(
np.zeros(D), V)
X[n, :] = x
Y[n, :] = x + np.random.multivariate_normal(np.zeros(D), C[n, :, :])
U = np.empty((N, D, D))
UY = np.empty((N, D))
for n in range(N):
U[n, :, :] = np.linalg.inv(C[n, :, :])
UY[n, :] = np.linalg.solve(C[n, :, :], Y[n, :])
# Create iterators for the static matrices
#U = N*(U,)
#A = N*(A,)
V = N * (V, )
(Xh, CovXh) = utils.kalman_filter(UY, U, A, V, np.zeros(2),
10 * np.identity(2))
(Xh, CovXh) = utils.rts_smoother(Xh, CovXh, A, V)
plt.clf()
for d in range(D):
plt.subplot(D, 1, d)
#plt.plot(Xh[:,d], 'r--')
bpplt.errorplot(Xh[:, d], error=2 * np.sqrt(CovXh[:, d, d]))
plt.plot(X[:, d], 'r-')
plt.plot(Y[:, d], '.')
plt.show()
def run():
# Create some data
N = 500
D = 2
# Initial state
x0 = np.array([0.5, -0.5])
# Dynamics (time varying)
A0 = np.array([[.9, -.4], [.4, .9]])
A1 = np.array([[.98, -.1], [.1, .98]])
l = np.linspace(0, 1, N).reshape((-1,1,1))
A = (1-l)*A0 + l*A1
# Innovation covariance matrix
V = np.array([[1, 0], [0, 1]])
# Observation noise covariance matrix
C0 = 10*np.array([[1, 0], [0, 1]])
C1 = 0.01*np.array([[1, 0], [0, 1]])
C = (1-l)**2*C0 + l**2*C1
X = np.empty((N,D))
Y = np.empty((N,D))
# Simulate data
x = x0
for n in range(N):
x = np.dot(A[n,:,:],x) + np.random.multivariate_normal(np.zeros(D), V)
X[n,:] = x
Y[n,:] = x + np.random.multivariate_normal(np.zeros(D), C[n,:,:])
U = np.empty((N,D,D))
UY = np.empty((N,D))
for n in range(N):
U[n,:,:] = np.linalg.inv(C[n,:,:])
UY[n,:] = np.linalg.solve(C[n,:,:], Y[n,:])
# Create iterators for the static matrices
#U = N*(U,)
#A = N*(A,)
V = N*(V,)
(Xh, CovXh) = utils.kalman_filter(UY, U, A, V, np.zeros(2), 10*np.identity(2))
(Xh, CovXh) = utils.rts_smoother(Xh, CovXh, A, V)
plt.clf()
for d in range(D):
plt.subplot(D,1,d)
#plt.plot(Xh[:,d], 'r--')
bpplt.errorplot(Xh[:,d], error=2*np.sqrt(CovXh[:,d,d]))
plt.plot(X[:,d], 'r-')
plt.plot(Y[:,d], '.')
plt.show()
def run(M=10, N=100, D_y=3, D=5):
seed = 45
print('seed =', seed)
np.random.seed(seed)
# Generate data
w = np.random.normal(0, 1, size=(M,1,D_y))
x = np.random.normal(0, 1, size=(1,N,D_y))
f = utils.utils.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.5, size=(M,N))
# Construct model
(Y, WX, W, X, tau, alpha) = pca_model(M, N, D)
# Data with missing values
mask = utils.random.mask(M, N, p=0.9) # randomly missing
mask[:,20:40] = False # gap missing
y[~mask] = np.nan
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, W, X, tau, alpha)
# Initialize some nodes randomly
X.initialize_from_value(X.random())
W.initialize_from_value(W.random())
# Inference loop.
Q.update(repeat=100)
# Plot results
plt.clf()
WX_params = WX.get_parameters()
fh = WX_params[0] * np.ones(y.shape)
err_fh = 2*np.sqrt(WX_params[1] + 1/tau.get_moments()[0]) * np.ones(y.shape)
for m in range(M):
plt.subplot(M,1,m+1)
myplt.errorplot(fh[m], x=np.arange(N), error=err_fh[m])
plt.plot(np.arange(N), f[m], 'g')
plt.plot(np.arange(N), y[m], 'r+')
plt.show()
def run_lssm(y, f, mask, D, maxiter):
"""
Run VB inference for linear state space model.
"""
(M, N) = np.shape(y)
#
# CONSTRUCT THE MODEL
#
# Dynamic matrix
# alpha: (D) x ()
alpha = Gamma(1e-5,
1e-5,
plates=(D,),
name='alpha')
# A : (D) x (D)
A = Gaussian(np.zeros(D),
diagonal(alpha),
plates=(D,),
name='A')
A.initialize_from_value(np.identity(D))
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
A, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N,D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5,
1e-5,
plates=(D,),
name='gamma')
# C : (M,1) x (D)
C = Gaussian(np.zeros(D),
diagonal(gamma),
plates=(M,1),
name='C')
C.initialize_from_value(np.random.randn(M,1,D))
# Observation noise
# tau : () x ()
tau = Gamma(1e-5,
1e-5,
name='tau')
# Observations
# Y : (M,N) x ()
CX = Dot(C, X.as_gaussian())
Y = Normal(CX,
tau,
name='Y')
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, A, alpha, C, gamma, tau)
# Iterate
Q.update(X, A, alpha, C, gamma, tau, repeat=maxiter)
#
# SHOW RESULTS
#
# Mean and standard deviation of the posterior
(f_mean, f_squared) = CX.get_moments()
f_std = np.sqrt(f_squared - f_mean**2)
# Plot observations space
#plt.figure()
for m in range(M):
plt.subplot(M,1,m+1)
plt.plot(y[m,:], 'r.')
plt.plot(f[m,:], 'b-')
bpplt.errorplot(y=f_mean[m,:], error=2*f_std[m,:])
def run(maxiter=100):
seed = 496#np.random.randint(1000)
print("seed = ", seed)
np.random.seed(seed)
# Simulate some data
D = 3
M = 6
N = 200
c = np.random.randn(M,D)
w = 0.3
a = np.array([[np.cos(w), -np.sin(w), 0],
[np.sin(w), np.cos(w), 0],
[0, 0, 1]])
x = np.empty((N,D))
f = np.empty((M,N))
y = np.empty((M,N))
x[0] = 10*np.random.randn(D)
f[:,0] = np.dot(c,x[0])
y[:,0] = f[:,0] + 3*np.random.randn(M)
for n in range(N-1):
x[n+1] = np.dot(a,x[n]) + np.random.randn(D)
f[:,n+1] = np.dot(c,x[n+1])
y[:,n+1] = f[:,n+1] + 3*np.random.randn(M)
# Create the model
(Y, CX, X, tau, C, gamma, A, alpha) = linear_state_space_model(D, N, M)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:,30:80] = False
y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting.
# Observe the data
Y.observe(y, mask=mask)
# Initialize nodes (must use some randomness for C)
C.initialize_from_random()
# Run inference
Q = VB(Y, X, C, gamma, A, alpha, tau)
#
# Run inference with rotations.
#
rotA = transformations.RotateGaussianARD(A, alpha)
rotX = transformations.RotateGaussianMarkovChain(X, A, rotA)
rotC = transformations.RotateGaussianARD(C, gamma)
R = transformations.RotationOptimizer(rotX, rotC, D)
#maxiter = 84
for ind in range(maxiter):
Q.update()
#print('C term', C.lower_bound_contribution())
R.rotate(maxiter=10,
check_gradient=True,
verbose=False,
check_bound=Q.compute_lowerbound,
#check_bound=None,
check_bound_terms=Q.compute_lowerbound_terms)
#check_bound_terms=None)
X_vb = X.u[0]
varX_vb = utils.diagonal(X.u[1] - X_vb[...,np.newaxis,:] * X_vb[...,:,np.newaxis])
u_CX = CX.get_moments()
CX_vb = u_CX[0]
varCX_vb = u_CX[1] - CX_vb**2
# Show results
plt.figure(3)
plt.clf()
for m in range(M):
plt.subplot(M,1,m+1)
plt.plot(y[m,:], 'r.')
plt.plot(f[m,:], 'b-')
bpplt.errorplot(y=CX_vb[m,:],
error=2*np.sqrt(varCX_vb[m,:]))
plt.figure()
Q.plot_iteration_by_nodes()
def run(M=10, N=100, D_y=3, D=5):
seed = 45
print('seed =', seed)
np.random.seed(seed)
# Check HDF5 version.
if h5py.version.hdf5_version_tuple < (1, 8, 7):
print(
"WARNING! Your HDF5 version is %s. HDF5 versions <1.8.7 are not "
"able to save empty arrays, thus you may experience problems if "
"you for instance try to save before running any iteration steps."
% str(h5py.version.hdf5_version_tuple))
# Generate data
w = np.random.normal(0, 1, size=(M, 1, D_y))
x = np.random.normal(0, 1, size=(1, N, D_y))
f = utils.utils.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.5, size=(M, N))
# Construct model
(Y, WX, W, X, tau, alpha) = pca_model(M, N, D)
# Data with missing values
mask = utils.random.mask(M, N, p=0.9) # randomly missing
mask[:, 20:40] = False # gap missing
y[~mask] = np.nan
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, W, X, tau, alpha, autosave_iterations=5)
# Initialize some nodes randomly
X.initialize_from_value(X.random())
W.initialize_from_value(W.random())
# Save the state into a HDF5 file
filename = tempfile.NamedTemporaryFile(suffix='hdf5').name
Q.update(X, W, alpha, tau, repeat=1)
Q.save(filename)
# Inference loop.
Q.update(X, W, alpha, tau, repeat=10)
# Reload the state from the HDF5 file
Q.load(filename=filename)
# Inference loop again.
Q.update(X, W, alpha, tau, repeat=10)
# NOTE: Saving and loading requires that you have the model
# constructed. "Save" does not store the model structure nor does "load"
# read it. They are just used for reading and writing the contents of the
# nodes. Thus, if you want to load, you first need to construct the same
# model that was used for saving and then use load to set the states of the
# nodes.
plt.clf()
WX_params = WX.get_parameters()
fh = WX_params[0] * np.ones(y.shape)
err_fh = 2 * np.sqrt(WX_params[1] + 1 / tau.get_moments()[0]) * np.ones(
y.shape)
for m in range(M):
plt.subplot(M, 1, m + 1)
#errorplot(y, error=None, x=None, lower=None, upper=None):
myplt.errorplot(fh[m], x=np.arange(N), error=err_fh[m])
plt.plot(np.arange(N), f[m], 'g')
plt.plot(np.arange(N), y[m], 'r+')
plt.figure()
Q.plot_iteration_by_nodes()
plt.figure()
plt.subplot(2, 2, 1)
myplt.binary_matrix(W.mask)
plt.subplot(2, 2, 2)
myplt.binary_matrix(X.mask)
plt.subplot(2, 2, 3)
#myplt.binary_matrix(WX.get_mask())
plt.subplot(2, 2, 4)
myplt.binary_matrix(Y.mask)
plt.show()
def run():
## Generate data
# Noisy observations from a sinusoid
N = 100
func = lambda x: np.sin(x*2*np.pi/50)
x = np.random.uniform(low=0, high=N, size=(N,))
f = func(x)
y = f + np.random.normal(0, 0.2, np.shape(f))
# Plot data
plt.clf()
plt.plot(x,y,'r+')
## Construct model
# Covariance function stuff
ls = EF.NodeConstantScalar(10, name='lengthscale')
amp = EF.NodeConstantScalar(1.0, name='amplitude')
noise = EF.NodeConstantScalar(0.2, name='noise')
# Latent process covariance
#K_f = CF.SquaredExponential(amp, ls)
K_f = CF.SquaredExponential(amp, ls)
# Noise process covariance
K_noise = CF.Delta(noise)
# Observation process covariance
K_y = CF.Sum(K_f, K_noise)
# Joint covariance
#K_joint = CF.Multiple([[K_f, K_f],[K_f,K_y]], sparse=True)
# Mean function stuff
M = GP.Constant(lambda x: np.zeros(np.shape(x)[0]))
# Means for latent and observation processes
#M_multi = GP.Multiple([M, M])
# Gaussian process
F = GP.GaussianProcess(M, [[K_f, K_f], [K_f, K_y]])
#F = GP.GaussianProcess(M, [[K_f, K_f], [K_f, K_y]])
#F = GP.GaussianProcess(M_multi, K_joint)
## Inference
F.observe([[],x], y)
utils.vb_optimize_nodes(ls, amp, noise)
F.update()
u = F.get_parameters()
## Show results
# Print hyperparameters
#print('parameters')
#print(ls.name, ls.u[0])
#print(amp.name, amp.u[0])
#print(noise.name, noise.u[0])
# Posterior predictions
xh = np.arange(np.min(x)-5, np.max(x)+100, 0.1)
(fh, varfh) = u([[],xh], covariance=1)
#(fh, varfh) = u([xh,[]], covariance=1)
# Plot predictive distribution
varfh[varfh<0] = 0
errfh = np.sqrt(varfh)
myplt.errorplot(fh, x=xh, error=errfh)
return
def run(M=10, N=100, D_y=3, D=5):
seed = 45
print("seed =", seed)
np.random.seed(seed)
# Check HDF5 version.
if h5py.version.hdf5_version_tuple < (1, 8, 7):
print(
"WARNING! Your HDF5 version is %s. HDF5 versions <1.8.7 are not "
"able to save empty arrays, thus you may experience problems if "
"you for instance try to save before running any iteration steps." % str(h5py.version.hdf5_version_tuple)
)
# Generate data
w = np.random.normal(0, 1, size=(M, 1, D_y))
x = np.random.normal(0, 1, size=(1, N, D_y))
f = utils.utils.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.5, size=(M, N))
# Construct model
(Y, WX, W, X, tau, alpha) = pca_model(M, N, D)
# Data with missing values
mask = utils.random.mask(M, N, p=0.9) # randomly missing
mask[:, 20:40] = False # gap missing
y[~mask] = np.nan
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, W, X, tau, alpha, autosave_iterations=5)
# Initialize some nodes randomly
X.initialize_from_value(X.random())
W.initialize_from_value(W.random())
# Save the state into a HDF5 file
filename = tempfile.NamedTemporaryFile(suffix="hdf5").name
Q.update(X, W, alpha, tau, repeat=1)
Q.save(filename)
# Inference loop.
Q.update(X, W, alpha, tau, repeat=10)
# Reload the state from the HDF5 file
Q.load(filename=filename)
# Inference loop again.
Q.update(X, W, alpha, tau, repeat=10)
# NOTE: Saving and loading requires that you have the model
# constructed. "Save" does not store the model structure nor does "load"
# read it. They are just used for reading and writing the contents of the
# nodes. Thus, if you want to load, you first need to construct the same
# model that was used for saving and then use load to set the states of the
# nodes.
plt.clf()
WX_params = WX.get_parameters()
fh = WX_params[0] * np.ones(y.shape)
err_fh = 2 * np.sqrt(WX_params[1] + 1 / tau.get_moments()[0]) * np.ones(y.shape)
for m in range(M):
plt.subplot(M, 1, m + 1)
# errorplot(y, error=None, x=None, lower=None, upper=None):
myplt.errorplot(fh[m], x=np.arange(N), error=err_fh[m])
plt.plot(np.arange(N), f[m], "g")
plt.plot(np.arange(N), y[m], "r+")
plt.figure()
Q.plot_iteration_by_nodes()
plt.figure()
plt.subplot(2, 2, 1)
myplt.binary_matrix(W.mask)
plt.subplot(2, 2, 2)
myplt.binary_matrix(X.mask)
plt.subplot(2, 2, 3)
# myplt.binary_matrix(WX.get_mask())
plt.subplot(2, 2, 4)
myplt.binary_matrix(Y.mask)
plt.show()
def run(maxiter=100):
seed = 496 #np.random.randint(1000)
print("seed = ", seed)
np.random.seed(seed)
# Simulate some data
D = 3
M = 6
N = 200
c = np.random.randn(M, D)
w = 0.3
a = np.array([[np.cos(w), -np.sin(w), 0], [np.sin(w),
np.cos(w), 0], [0, 0, 1]])
x = np.empty((N, D))
f = np.empty((M, N))
y = np.empty((M, N))
x[0] = 10 * np.random.randn(D)
f[:, 0] = np.dot(c, x[0])
y[:, 0] = f[:, 0] + 3 * np.random.randn(M)
for n in range(N - 1):
x[n + 1] = np.dot(a, x[n]) + np.random.randn(D)
f[:, n + 1] = np.dot(c, x[n + 1])
y[:, n + 1] = f[:, n + 1] + 3 * np.random.randn(M)
# Create the model
(Y, CX, X, tau, C, gamma, A, alpha) = linear_state_space_model(D, N, M)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:, 30:80] = False
y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting.
# Observe the data
Y.observe(y, mask=mask)
# Initialize nodes (must use some randomness for C)
C.initialize_from_random()
# Run inference
Q = VB(Y, X, C, gamma, A, alpha, tau)
#
# Run inference with rotations.
#
rotA = transformations.RotateGaussianARD(A, alpha)
rotX = transformations.RotateGaussianMarkovChain(X, A, rotA)
rotC = transformations.RotateGaussianARD(C, gamma)
R = transformations.RotationOptimizer(rotX, rotC, D)
#maxiter = 84
for ind in range(maxiter):
Q.update()
#print('C term', C.lower_bound_contribution())
R.rotate(
maxiter=10,
check_gradient=True,
verbose=False,
check_bound=Q.compute_lowerbound,
#check_bound=None,
check_bound_terms=Q.compute_lowerbound_terms)
#check_bound_terms=None)
X_vb = X.u[0]
varX_vb = utils.diagonal(X.u[1] - X_vb[..., np.newaxis, :] *
X_vb[..., :, np.newaxis])
u_CX = CX.get_moments()
CX_vb = u_CX[0]
varCX_vb = u_CX[1] - CX_vb**2
# Show results
plt.figure(3)
plt.clf()
for m in range(M):
plt.subplot(M, 1, m + 1)
plt.plot(y[m, :], 'r.')
plt.plot(f[m, :], 'b-')
bpplt.errorplot(y=CX_vb[m, :], error=2 * np.sqrt(varCX_vb[m, :]))
plt.figure()
Q.plot_iteration_by_nodes()
def run():
## Generate data
# Noisy observations from a sinusoid
N = 100
func = lambda x: np.sin(x * 2 * np.pi / 50)
x = np.random.uniform(low=0, high=N, size=(N, ))
f = func(x)
y = f + np.random.normal(0, 0.2, np.shape(f))
# Plot data
plt.clf()
plt.plot(x, y, 'r+')
## Construct model
# Covariance function stuff
ls = EF.NodeConstantScalar(10, name='lengthscale')
amp = EF.NodeConstantScalar(1.0, name='amplitude')
noise = EF.NodeConstantScalar(0.2, name='noise')
# Latent process covariance
#K_f = CF.SquaredExponential(amp, ls)
K_f = CF.SquaredExponential(amp, ls)
# Noise process covariance
K_noise = CF.Delta(noise)
# Observation process covariance
K_y = CF.Sum(K_f, K_noise)
# Joint covariance
#K_joint = CF.Multiple([[K_f, K_f],[K_f,K_y]], sparse=True)
# Mean function stuff
M = GP.Constant(lambda x: np.zeros(np.shape(x)[0]))
# Means for latent and observation processes
#M_multi = GP.Multiple([M, M])
# Gaussian process
F = GP.GaussianProcess(M, [[K_f, K_f], [K_f, K_y]])
#F = GP.GaussianProcess(M, [[K_f, K_f], [K_f, K_y]])
#F = GP.GaussianProcess(M_multi, K_joint)
## Inference
F.observe([[], x], y)
utils.vb_optimize_nodes(ls, amp, noise)
F.update()
u = F.get_parameters()
## Show results
# Print hyperparameters
#print('parameters')
#print(ls.name, ls.u[0])
#print(amp.name, amp.u[0])
#print(noise.name, noise.u[0])
# Posterior predictions
xh = np.arange(np.min(x) - 5, np.max(x) + 100, 0.1)
(fh, varfh) = u([[], xh], covariance=1)
#(fh, varfh) = u([xh,[]], covariance=1)
# Plot predictive distribution
varfh[varfh < 0] = 0
errfh = np.sqrt(varfh)
myplt.errorplot(fh, x=xh, error=errfh)
return
def run_lssm(y, f, mask, D, maxiter):
"""
Run VB inference for linear state space model.
"""
(M, N) = np.shape(y)
#
# CONSTRUCT THE MODEL
#
# Dynamic matrix
# alpha: (D) x ()
alpha = Gamma(1e-5, 1e-5, plates=(D, ), name='alpha')
# A : (D) x (D)
A = Gaussian(np.zeros(D), diagonal(alpha), plates=(D, ), name='A')
A.initialize_from_value(np.identity(D))
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(
np.zeros(D), # mean of x0
1e-3 * np.identity(D), # prec of x0
A, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N, D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma')
# C : (M,1) x (D)
C = Gaussian(np.zeros(D), diagonal(gamma), plates=(M, 1), name='C')
C.initialize_from_value(np.random.randn(M, 1, D))
# Observation noise
# tau : () x ()
tau = Gamma(1e-5, 1e-5, name='tau')
# Observations
# Y : (M,N) x ()
CX = Dot(C, X.as_gaussian())
Y = Normal(CX, tau, name='Y')
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, A, alpha, C, gamma, tau)
# Iterate
Q.update(X, A, alpha, C, gamma, tau, repeat=maxiter)
#
# SHOW RESULTS
#
# Mean and standard deviation of the posterior
(f_mean, f_squared) = CX.get_moments()
f_std = np.sqrt(f_squared - f_mean**2)
# Plot observations space
#plt.figure()
for m in range(M):
plt.subplot(M, 1, m + 1)
plt.plot(y[m, :], 'r.')
plt.plot(f[m, :], 'b-')
bpplt.errorplot(y=f_mean[m, :], error=2 * f_std[m, :])
def run_dlssm(y, f, mask, D, K, maxiter):
"""
Run VB inference for linear state space model with drifting dynamics.
"""
(M, N) = np.shape(y)
# Dynamics matrix with ARD
# alpha : (D) x ()
alpha = Gamma(1e-5,
1e-5,
plates=(K,),
name='alpha')
# A : (K) x (K)
A = Gaussian(np.zeros(K),
#np.identity(K),
diagonal(alpha),
plates=(K,),
name='A_S')
A.initialize_from_value(np.identity(K))
# rho
## rho = Gamma(1e-5,
## 1e-5,
## plates=(K,),
## name="rho")
# S : () x (N-1,K)
S = GaussianMarkovChain(np.ones(K),
1e-6*np.identity(K),
A,
np.ones(K),
n=N-1,
name='S')
S.initialize_from_value(1*np.ones((N-1,K)))
# Projection matrix of the dynamics matrix
# beta : (K) x ()
beta = Gamma(1e-5,
1e-5,
plates=(K,),
name='beta')
# B : (D) x (D*K)
B = Gaussian(np.zeros(D*K),
diagonal(tile(beta, D)),
plates=(D,),
name='B')
b = np.zeros((D,D,K))
b[np.arange(D),np.arange(D),np.zeros(D,dtype=int)] = 1
B.initialize_from_value(np.reshape(1*b, (D,D*K)))
# A : (N-1,D) x (D)
BS = MatrixDot(B,
S.as_gaussian().add_plate_axis(-1),
name='BS')
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(np.zeros(D), # mean of x0
1e-3*np.identity(D), # prec of x0
BS, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N,D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5,
1e-5,
plates=(D,),
name='gamma')
# C : (M,1) x (D)
C = Gaussian(np.zeros(D),
diagonal(gamma),
plates=(M,1),
name='C')
C.initialize_from_value(np.random.randn(M,1,D))
# Observation noise
# tau : () x ()
tau = Gamma(1e-5,
1e-5,
name='tau')
# Observations
# Y : (M,N) x ()
CX = Dot(C, X.as_gaussian())
Y = Normal(CX,
tau,
name='Y')
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, S, A, alpha, B, beta, C, gamma, tau)
#
# Run inference with rotations.
#
# Rotate the D-dimensional state space (C, X)
rotB = transformations.RotateGaussianMatrixARD(B, beta, axis='rows')
rotX = transformations.RotateDriftingMarkovChain(X, B, S, rotB)
rotC = transformations.RotateGaussianARD(C, gamma)
R_X = transformations.RotationOptimizer(rotX, rotC, D)
# Rotate the K-dimensional latent dynamics space (B, S)
rotA = transformations.RotateGaussianARD(A, alpha)
rotS = transformations.RotateGaussianMarkovChain(S, A, rotA)
rotB = transformations.RotateGaussianMatrixARD(B, beta, axis='cols')
R_S = transformations.RotationOptimizer(rotS, rotB, K)
# Iterate
for ind in range(int(maxiter/5)):
Q.update(repeat=5)
#Q.update(X, S, A, alpha, rho, B, beta, C, gamma, tau, repeat=maxiter)
R_X.rotate()
R_S.rotate()
## R_X.rotate(
## check_bound=Q.compute_lowerbound,
## check_bound_terms=Q.compute_lowerbound_terms,
## check_gradient=True
## )
## R_S.rotate(
## check_bound=Q.compute_lowerbound,
## check_bound_terms=Q.compute_lowerbound_terms,
## check_gradient=True
## )
#
# SHOW RESULTS
#
# Mean and standard deviation of the posterior
(f_mean, f_squared) = CX.get_moments()
f_std = np.sqrt(f_squared - f_mean**2)
# Plot observations space
for m in range(M):
plt.subplot(M,1,m+1)
plt.plot(y[m,:], 'r.')
plt.plot(f[m,:], 'b-')
bpplt.errorplot(y=f_mean[m,:], error=2*f_std[m,:])
def run_dlssm(y, f, mask, D, K, maxiter):
"""
Run VB inference for linear state space model with drifting dynamics.
"""
(M, N) = np.shape(y)
# Dynamics matrix with ARD
# alpha : (D) x ()
alpha = Gamma(1e-5, 1e-5, plates=(K, ), name='alpha')
# A : (K) x (K)
A = Gaussian(
np.zeros(K),
#np.identity(K),
diagonal(alpha),
plates=(K, ),
name='A_S')
A.initialize_from_value(np.identity(K))
# rho
## rho = Gamma(1e-5,
## 1e-5,
## plates=(K,),
## name="rho")
# S : () x (N-1,K)
S = GaussianMarkovChain(np.ones(K),
1e-6 * np.identity(K),
A,
np.ones(K),
n=N - 1,
name='S')
S.initialize_from_value(1 * np.ones((N - 1, K)))
# Projection matrix of the dynamics matrix
# beta : (K) x ()
beta = Gamma(1e-5, 1e-5, plates=(K, ), name='beta')
# B : (D) x (D*K)
B = Gaussian(np.zeros(D * K),
diagonal(tile(beta, D)),
plates=(D, ),
name='B')
b = np.zeros((D, D, K))
b[np.arange(D), np.arange(D), np.zeros(D, dtype=int)] = 1
B.initialize_from_value(np.reshape(1 * b, (D, D * K)))
# A : (N-1,D) x (D)
BS = MatrixDot(B, S.as_gaussian().add_plate_axis(-1), name='BS')
# Latent states with dynamics
# X : () x (N,D)
X = GaussianMarkovChain(
np.zeros(D), # mean of x0
1e-3 * np.identity(D), # prec of x0
BS, # dynamics
np.ones(D), # innovation
n=N, # time instances
name='X',
initialize=False)
X.initialize_from_value(np.random.randn(N, D))
# Mixing matrix from latent space to observation space using ARD
# gamma : (D) x ()
gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma')
# C : (M,1) x (D)
C = Gaussian(np.zeros(D), diagonal(gamma), plates=(M, 1), name='C')
C.initialize_from_value(np.random.randn(M, 1, D))
# Observation noise
# tau : () x ()
tau = Gamma(1e-5, 1e-5, name='tau')
# Observations
# Y : (M,N) x ()
CX = Dot(C, X.as_gaussian())
Y = Normal(CX, tau, name='Y')
#
# RUN INFERENCE
#
# Observe data
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, X, S, A, alpha, B, beta, C, gamma, tau)
#
# Run inference with rotations.
#
# Rotate the D-dimensional state space (C, X)
rotB = transformations.RotateGaussianMatrixARD(B, beta, axis='rows')
rotX = transformations.RotateDriftingMarkovChain(X, B, S, rotB)
rotC = transformations.RotateGaussianARD(C, gamma)
R_X = transformations.RotationOptimizer(rotX, rotC, D)
# Rotate the K-dimensional latent dynamics space (B, S)
rotA = transformations.RotateGaussianARD(A, alpha)
rotS = transformations.RotateGaussianMarkovChain(S, A, rotA)
rotB = transformations.RotateGaussianMatrixARD(B, beta, axis='cols')
R_S = transformations.RotationOptimizer(rotS, rotB, K)
# Iterate
for ind in range(int(maxiter / 5)):
Q.update(repeat=5)
#Q.update(X, S, A, alpha, rho, B, beta, C, gamma, tau, repeat=maxiter)
R_X.rotate()
R_S.rotate()
## R_X.rotate(
## check_bound=Q.compute_lowerbound,
## check_bound_terms=Q.compute_lowerbound_terms,
## check_gradient=True
## )
## R_S.rotate(
## check_bound=Q.compute_lowerbound,
## check_bound_terms=Q.compute_lowerbound_terms,
## check_gradient=True
## )
#
# SHOW RESULTS
#
# Mean and standard deviation of the posterior
(f_mean, f_squared) = CX.get_moments()
f_std = np.sqrt(f_squared - f_mean**2)
# Plot observations space
for m in range(M):
plt.subplot(M, 1, m + 1)
plt.plot(y[m, :], 'r.')
plt.plot(f[m, :], 'b-')
bpplt.errorplot(y=f_mean[m, :], error=2 * f_std[m, :])
