def run(method):
# Seed for random number generator
seed = 495
np.random.seed(seed)
print("seed = ", seed)
# Create data
M = 10
N = 200
(y, f) = simulate_drifting_lssm(M, N)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:, 70:120] = False
#mask[:] = True
y[~mask] = np.nan # BayesPy doesn't require NaNs, they're just for plotting.
# Run the method
if method == 'lssm':
run_lssm(y, f, mask, 3, 200)
elif method == 'dlssm':
run_dlssm(y, f, mask, 3, 2, 200)
else:
raise Exception("Unknown method requested")
def demo(M=6, N=200, D=3, maxiter=100, debug=False, seed=42, rotate=True,
precompute=False, plot=True, monitor=True):
"""
Run the demo for linear state-space model.
"""
# Use deterministic random numbers
if seed is not None:
np.random.seed(seed)
# Get data
(y, f) = simulate_data(M, N)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:,30:80] = False
y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting.
# Run inference
Q = infer(y, D,
mask=mask,
rotate=rotate,
debug=debug,
monitor=monitor,
maxiter=maxiter)
if plot:
# Show results
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, linestyle='-', color='b')
bpplt.timeseries(y, linestyle='None', color='r', marker='.')
plt.show()
def run(method):
# Seed for random number generator
seed = 495
np.random.seed(seed)
print("seed = ", seed)
# Create data
M = 10
N = 200
(y, f) = simulate_drifting_lssm(M, N)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:,70:120] = False
#mask[:] = True
y[~mask] = np.nan # BayesPy doesn't require NaNs, they're just for plotting.
# Run the method
if method == 'lssm':
run_lssm(y, f, mask, 3, 200)
elif method == 'dlssm':
run_dlssm(y, f, mask, 3, 2, 200)
else:
raise Exception("Unknown method requested")
def demo(M=6, N=200, D=3, maxiter=100, debug=False, seed=42, rotate=True,
precompute=False, plot=True):
"""
Run the demo for linear state-space model.
"""
# Use deterministic random numbers
if seed is not None:
np.random.seed(seed)
# Get data
(y, f) = simulate_data(M, N)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:,30:80] = False
y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting.
# Run inference
Q = infer(y, D,
mask=mask,
rotate=rotate,
debug=debug,
maxiter=maxiter)
if plot:
# Show results
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
plt.show()
def test_parent_X_mv(self):
"""
Test messages from MatrixDot to X with missing values.
Compare the results of X to the analytic VB solution.
"""
self.check_X(random.mask(self.D4, self.D3, self.D2, self.D1, p=0.5))
def test_parent_X_mv(self):
"""
Test messages from MatrixDot to X with missing values.
Compare the results of X to the analytic VB solution.
"""
self.check_X(random.mask(self.D4,self.D3,self.D2,self.D1,p=0.5))
def run(M=10,
N=100,
D_y=3,
D=5,
seed=42,
rotate=False,
maxiter=100,
debug=False,
plot=True):
if seed is not None:
np.random.seed(seed)
# Generate data
w = np.random.normal(0, 1, size=(M, 1, D_y))
x = np.random.normal(0, 1, size=(1, N, D_y))
f = misc.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.2, size=(M, N))
# Construct model
(Y, F, W, X, tau, alpha) = model(M, N, D)
# Data with missing values
mask = random.mask(M, N, p=0.5) # randomly missing
y[~mask] = np.nan
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, W, X, tau, alpha)
# Initialize some nodes randomly
X.initialize_from_random()
W.initialize_from_random()
# Run inference algorithm
if rotate:
# Use rotations to speed up learning
rotW = transformations.RotateGaussianARD(W, alpha)
rotX = transformations.RotateGaussianARD(X)
R = transformations.RotationOptimizer(rotW, rotX, D)
for ind in range(maxiter):
Q.update()
if debug:
R.rotate(check_bound=True, check_gradient=True)
else:
R.rotate()
else:
# Use standard VB-EM alone
Q.update(repeat=maxiter)
# Plot results
if plot:
plt.figure()
bpplt.timeseries_normal(F, scale=2)
bpplt.timeseries(f, color='g', linestyle='-')
bpplt.timeseries(y, color='r', linestyle='None', marker='+')
def run(M=40, N=100, D_y=6, D=8, seed=42, rotate=False, maxiter=1000, debug=False, plot=True):
"""
Run pattern search demo for PCA.
"""
if seed is not None:
np.random.seed(seed)
# Generate data
w = np.random.normal(0, 1, size=(M,1,D_y))
x = np.random.normal(0, 1, size=(1,N,D_y))
f = misc.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.2, size=(M,N))
# Construct model
Q = VB(*(pca.model(M, N, D)))
# Data with missing values
mask = random.mask(M, N, p=0.5) # randomly missing
y[~mask] = np.nan
Q['Y'].observe(y, mask=mask)
# Initialize some nodes randomly
Q['X'].initialize_from_random()
Q['W'].initialize_from_random()
# Use a few VB-EM updates at the beginning
Q.update(repeat=10)
Q.save()
# Standard VB-EM as a baseline
Q.update(repeat=maxiter)
if plot:
bpplt.pyplot.plot(np.cumsum(Q.cputime), Q.L, 'k-')
# Restore initial state
Q.load()
# Pattern search method for comparison
for n in range(maxiter):
Q.pattern_search('W', 'tau', maxiter=3, collapsed=['X', 'alpha'])
Q.update(repeat=20)
if Q.has_converged():
break
if plot:
bpplt.pyplot.plot(np.cumsum(Q.cputime), Q.L, 'r:')
bpplt.pyplot.xlabel('CPU time (in seconds)')
bpplt.pyplot.ylabel('VB lower bound')
bpplt.pyplot.legend(['VB-EM', 'Pattern search'], loc='lower right')
def run(M=10, N=100, D_y=3, D=5, seed=42, rotate=False, maxiter=100, debug=False):
if seed is not None:
np.random.seed(seed)
# Generate data
w = np.random.normal(0, 1, size=(M,1,D_y))
x = np.random.normal(0, 1, size=(1,N,D_y))
f = misc.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.2, size=(M,N))
# Construct model
(Y, F, W, X, tau, alpha) = model(M, N, D)
# Data with missing values
mask = random.mask(M, N, p=0.5) # randomly missing
y[~mask] = np.nan
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, W, X, tau, alpha)
# Initialize some nodes randomly
X.initialize_from_random()
W.initialize_from_random()
# Run inference algorithm
if rotate:
# Use rotations to speed up learning
rotW = transformations.RotateGaussianARD(W, alpha)
rotX = transformations.RotateGaussianARD(X)
R = transformations.RotationOptimizer(rotW, rotX, D)
for ind in range(maxiter):
Q.update()
if debug:
R.rotate(check_bound=True,
check_gradient=True)
else:
R.rotate()
else:
# Use standard VB-EM alone
Q.update(repeat=maxiter)
# Plot results
plt.figure()
bpplt.timeseries_normal(F, scale=2)
bpplt.timeseries(f, color='g', linestyle='-')
bpplt.timeseries(y, color='r', linestyle='None', marker='+')
plt.show()
def run(M=10,
N=100,
D_y=3,
D=5,
seed=42,
rotate=False,
maxiter=1000,
debug=False,
plot=True):
if seed is not None:
np.random.seed(seed)
# Generate data
w = np.random.normal(0, 1, size=(M, 1, D_y))
x = np.random.normal(0, 1, size=(1, N, D_y))
f = misc.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.1, size=(M, N))
# Construct model
Q = model(M, N, D)
# Data with missing values
mask = random.mask(M, N, p=0.5) # randomly missing
y[~mask] = np.nan
Q['Y'].observe(y, mask=mask)
# Run inference algorithm
if rotate:
# Use rotations to speed up learning
rotW = transformations.RotateGaussianARD(Q['W'], Q['alpha'])
rotX = transformations.RotateGaussianARD(Q['X'])
R = transformations.RotationOptimizer(rotW, rotX, D)
if debug:
Q.callback = lambda: R.rotate(check_bound=True,
check_gradient=True)
else:
Q.callback = R.rotate
# Use standard VB-EM alone
Q.update(repeat=maxiter)
# Plot results
if plot:
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, color='g', linestyle='-')
bpplt.timeseries(y, color='r', linestyle='None', marker='+')
def run(M=30, D=5):
# Generate data
y = np.random.randint(D, size=(M, ))
# Construct model
p = nodes.Dirichlet(1 * np.ones(D), name='p')
z = nodes.Categorical(p, plates=(M, ), name='z')
# Observe the data with randomly missing values
mask = random.mask(M, p=0.5)
z.observe(y, mask=mask)
# Run VB-EM
Q = VB(p, z)
Q.update()
# Show results
z.show()
p.show()
def run(M=10, N=100, D_y=3, D=5, seed=42, rotate=False, maxiter=1000, debug=False, plot=True):
if seed is not None:
np.random.seed(seed)
# Generate data
w = np.random.normal(0, 1, size=(M,1,D_y))
x = np.random.normal(0, 1, size=(1,N,D_y))
f = misc.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.1, size=(M,N))
# Construct model
Q = model(M, N, D)
# Data with missing values
mask = random.mask(M, N, p=0.5) # randomly missing
y[~mask] = np.nan
Q['Y'].observe(y, mask=mask)
# Run inference algorithm
if rotate:
# Use rotations to speed up learning
rotW = transformations.RotateGaussianARD(Q['W'], Q['alpha'])
rotX = transformations.RotateGaussianARD(Q['X'])
R = transformations.RotationOptimizer(rotW, rotX, D)
if debug:
Q.callback = lambda : R.rotate(check_bound=True,
check_gradient=True)
else:
Q.callback = R.rotate
# Use standard VB-EM alone
Q.update(repeat=maxiter)
# Plot results
if plot:
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, color='g', linestyle='-')
bpplt.timeseries(y, color='r', linestyle='None', marker='+')
def run(M=30, D=5):
# Generate data
y = np.random.randint(D, size=(M,))
# Construct model
p = nodes.Dirichlet(1*np.ones(D),
name='p')
z = nodes.Categorical(p,
plates=(M,),
name='z')
# Observe the data with randomly missing values
mask = random.mask(M, p=0.5)
z.observe(y, mask=mask)
# Run VB-EM
Q = VB(p, z)
Q.update()
# Show results
z.show()
p.show()
def run(drift=True, seed=42, maxiter=50):
# Seed for random number generator
if seed is not None:
np.random.seed(seed)
# Create data
M = 10
N = 200
(y, f) = simulate_drifting_lssm(M, N)
# Add missing values randomly
mask = random.mask(M, N, p=0.8)
# Add missing values to a period of time
#mask[:,100:110] = False
mask[:,70:120] = False
#mask[:] = True
y[~mask] = np.nan # BayesPy doesn't require NaNs, they're just for plotting.
# Run the method
if drift:
run_dlssm(y, f, mask, 4, 3, maxiter)
else:
run_lssm(y, f, mask, 4, maxiter)
def run(M=40,
N=100,
D_y=6,
D=8,
seed=42,
rotate=False,
maxiter=1000,
debug=False,
plot=True):
"""
Run pattern search demo for PCA.
"""
if seed is not None:
np.random.seed(seed)
# Generate data
w = np.random.normal(0, 1, size=(M, 1, D_y))
x = np.random.normal(0, 1, size=(1, N, D_y))
f = misc.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.2, size=(M, N))
# Construct model
Q = VB(*(pca.model(M, N, D)))
# Data with missing values
mask = random.mask(M, N, p=0.5) # randomly missing
y[~mask] = np.nan
Q['Y'].observe(y, mask=mask)
# Initialize some nodes randomly
Q['X'].initialize_from_random()
Q['W'].initialize_from_random()
# Use a few VB-EM updates at the beginning
Q.update(repeat=10)
Q.save()
# Standard VB-EM as a baseline
Q.update(repeat=maxiter)
if plot:
bpplt.pyplot.plot(np.cumsum(Q.cputime), Q.L, 'k-')
# Restore initial state
Q.load()
# Pattern search method for comparison
for n in range(maxiter):
Q.pattern_search('W', 'tau', maxiter=3, collapsed=['X', 'alpha'])
Q.update(repeat=20)
if Q.has_converged():
break
if plot:
bpplt.pyplot.plot(np.cumsum(Q.cputime), Q.L, 'r:')
bpplt.pyplot.xlabel('CPU time (in seconds)')
bpplt.pyplot.ylabel('VB lower bound')
bpplt.pyplot.legend(['VB-EM', 'Pattern search'], loc='lower right')
def test_parent_A_mv(self):
"""
Test messages from MatrixDot to A with missing values.
"""
self.check_A(random.mask(self.D4,self.D3,self.D2,self.D1,p=0.5))
def run(M=100, N=2000, D=20, K=5, rotate=True, maxiter=200, seed=42,
debug=False, precompute=False, resolution=30, dynamic=True,
drift_A=False, drift_C=False,
plot_Y=True, plot_X=True, plot_S=True):
# Seed for random number generator
if seed is not None:
np.random.seed(seed)
# Create data
if dynamic:
decay = 1.0 - 5e-3
else:
decay = 1.0
(U, y, f, X) = simulate_data(M=M,
N=N,
resolution=resolution,
burnin=2000,
thin=20,
velocity=6e-2,
diffusion=1e-4,
decay=decay,
innovation_noise=1e-4,
innovation_lengthscale=2.0,
noise_ratio=5e-1)
plt.ion()
plt.plot(X[:,0], X[:,1], 'kx')
animation = bpplt.matrix_animation(U)
#bpplt.save_animation(animation, 'demo_drift_lssm_02_spde.mp4')
plt.show()
plt.ioff()
# Create some gaps
mask_gaps = utils.trues((M,N))
gap = 15
interval = 100
for m in range(100, N, interval):
start = m
end = min(m+gap, N-1)
mask_gaps[:,start:end] = False
# Missing values
mask_random = np.logical_or(random.mask(M, N, p=0.8),
np.logical_not(mask_gaps))
# Remove the observations
mask = np.logical_and(mask_gaps, mask_random)
y[~mask] = np.nan # BayesPy doesn't require NaNs, they're just for plotting.
# Run the method
Q = model_lssm.run_lssm(y, D,
mask=mask,
K=K,
maxiter=maxiter,
rotate=rotate,
debug=debug,
precompute=precompute,
drift_A=drift_A,
drift_C=drift_C,
update_drift=20,
start_rotating_drift=30)
if plot_Y:
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
# Plot latent space
if plot_X:
Q.plot('X')
# Plot drift space
if plot_S and (drift_A or drift_C):
Q.plot('S')
# Compute RMSE
rmse_random = utils.rmse(Q['F'].get_moments()[0][~mask_random],
f[~mask_random])
rmse_gaps = utils.rmse(Q['F'].get_moments()[0][~mask_gaps],
f[~mask_gaps])
print("RMSE for randomly missing values: %f" % rmse_random)
print("RMSE for gap values: %f" % rmse_gaps)
plt.show()
return
def test_parent_A_mv(self):
"""
Test messages from MatrixDot to A with missing values.
"""
self.check_A(random.mask(self.D4, self.D3, self.D2, self.D1, p=0.5))
def run(M=10, N=100, D_y=3, D=5):
seed = 45
print('seed =', seed)
np.random.seed(seed)
# Check HDF5 version.
if h5py.version.hdf5_version_tuple < (1, 8, 7):
print(
"WARNING! Your HDF5 version is %s. HDF5 versions <1.8.7 are not "
"able to save empty arrays, thus you may experience problems if "
"you for instance try to save before running any iteration steps."
% str(h5py.version.hdf5_version_tuple))
# Generate data
w = np.random.normal(0, 1, size=(M, 1, D_y))
x = np.random.normal(0, 1, size=(1, N, D_y))
f = misc.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.5, size=(M, N))
# Construct model
(Y, WX, W, X, tau, alpha) = pca_model(M, N, D)
# Data with missing values
mask = random.mask(M, N, p=0.9) # randomly missing
mask[:, 20:40] = False # gap missing
y[~mask] = np.nan
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, W, X, tau, alpha, autosave_iterations=5)
# Initialize some nodes randomly
X.initialize_from_value(X.random())
W.initialize_from_value(W.random())
# Save the state into a HDF5 file
filename = tempfile.NamedTemporaryFile(suffix='hdf5').name
Q.update(X, W, alpha, tau, repeat=1)
Q.save(filename=filename)
# Inference loop.
Q.update(X, W, alpha, tau, repeat=10)
# Reload the state from the HDF5 file
Q.load(filename=filename)
# Inference loop again.
Q.update(X, W, alpha, tau, repeat=10)
# NOTE: Saving and loading requires that you have the model
# constructed. "Save" does not store the model structure nor does "load"
# read it. They are just used for reading and writing the contents of the
# nodes. Thus, if you want to load, you first need to construct the same
# model that was used for saving and then use load to set the states of the
# nodes.
plt.clf()
WX_params = WX.get_parameters()
fh = WX_params[0] * np.ones(y.shape)
err_fh = 2 * np.sqrt(WX_params[1] + 1 / tau.get_moments()[0]) * np.ones(
y.shape)
for m in range(M):
plt.subplot(M, 1, m + 1)
#errorplot(y, error=None, x=None, lower=None, upper=None):
bpplt.errorplot(fh[m], x=np.arange(N), error=err_fh[m])
plt.plot(np.arange(N), f[m], 'g')
plt.plot(np.arange(N), y[m], 'r+')
plt.figure()
Q.plot_iteration_by_nodes()
plt.figure()
plt.subplot(2, 2, 1)
bpplt.binary_matrix(W.mask)
plt.subplot(2, 2, 2)
bpplt.binary_matrix(X.mask)
plt.subplot(2, 2, 3)
#bpplt.binary_matrix(WX.get_mask())
plt.subplot(2, 2, 4)
bpplt.binary_matrix(Y.mask)
from bayespy.inference import VB
Q = VB(X, C, gamma, A, alpha, tau, Y)
w = 0.3
a = np.array([[np.cos(w), -np.sin(w), 0, 0], [np.sin(w),
np.cos(w), 0, 0], [0, 0, 1, 0],
[0, 0, 0, 0]])
c = np.random.randn(M, 4)
x = np.empty((N, 4))
f = np.empty((M, N))
y = np.empty((M, N))
x[0] = 10 * np.random.randn(4)
f[:, 0] = np.dot(c, x[0])
y[:, 0] = f[:, 0] + 3 * np.random.randn(M)
for n in range(N - 1):
x[n + 1] = np.dot(a, x[n]) + [1, 1, 10, 10] * np.random.randn(4)
f[:, n + 1] = np.dot(c, x[n + 1])
y[:, n + 1] = f[:, n + 1] + 3 * np.random.randn(M)
from bayespy.utils import random
mask = random.mask(M, N, p=0.2)
Y.observe(y, mask=mask)
import bayespy.plot as bpplt
Q.update(repeat=10)
from bayespy.inference.vmp import transformations
rotC = transformations.RotateGaussianARD(C, gamma)
rotA = transformations.RotateGaussianARD(A, alpha)
rotX = transformations.RotateGaussianMarkovChain(X, rotA)
R = transformations.RotationOptimizer(rotX, rotC, D)
Q.callback = R.rotate
Q.update(repeat=1000)
bpplt.plot(F, center=True)
bpplt.pyplot.show()
def run(M=1, N=1000, D=5, K=4, seed=42, maxiter=200,
rotate=True, debug=False, precompute=False,
drift=False,
plot_X=False, plot_Y=True, plot_S=False):
# Seed for random number generator
if seed is not None:
np.random.seed(seed)
# Create data
(y, f) = simulate_drifting_lssm(M, N)
# Create some gaps
mask_gaps = utils.trues((M,N))
for m in range(100, N, 140):
start = m
end = min(m+15, N-1)
mask_gaps[:,start:end] = False
# Missing values
mask_random = np.logical_or(random.mask(M, N, p=0.8),
np.logical_not(mask_gaps))
# Remove the observations
mask = np.logical_and(mask_gaps, mask_random)
# Remove the observations
y[~mask] = np.nan # BayesPy doesn't require NaNs, they're just for plotting.
# Run the method
Q = model_lssm.run_lssm(y, D,
mask=mask,
K=K,
maxiter=maxiter,
rotate=rotate,
debug=debug,
precompute=precompute,
drift_A=drift,
update_drift=10,
start_rotating_drift=20)
## plt.figure()
## bpplt.timeseries(f, 'b-')
## plt.ylim([-2, 2])
# Plot observations
if plot_Y:
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
plt.ylim([-2, 2])
# Plot latent space
if plot_X:
Q.plot('X')
# Plot drift space
if plot_S and drift:
Q.plot('S')
# Compute RMSE
rmse_random = utils.rmse(Q['F'].get_moments()[0][~mask_random],
f[~mask_random])
rmse_gaps = utils.rmse(Q['F'].get_moments()[0][~mask_gaps],
f[~mask_gaps])
print("RMSE for randomly missing values: %f" % rmse_random)
print("RMSE for gap values: %f" % rmse_gaps)
plt.show()
def run(M=6, N=200, D=3, maxiter=100, debug=False, seed=42, rotate=False, precompute=False):
# Use deterministic random numbers
if seed is not None:
np.random.seed(seed)
# Simulate some data
K = 3
c = np.random.randn(M,K)
w = 0.3
a = np.array([[np.cos(w), -np.sin(w), 0],
[np.sin(w), np.cos(w), 0],
[0, 0, 1]])
x = np.empty((N,K))
f = np.empty((M,N))
y = np.empty((M,N))
x[0] = 10*np.random.randn(K)
f[:,0] = np.dot(c,x[0])
y[:,0] = f[:,0] + 3*np.random.randn(M)
for n in range(N-1):
x[n+1] = np.dot(a,x[n]) + np.random.randn(K)
f[:,n+1] = np.dot(c,x[n+1])
y[:,n+1] = f[:,n+1] + 3*np.random.randn(M)
# Create the model
(Y, CX, X, tau, C, gamma, A, alpha) = linear_state_space_model(D=D,
N=N,
M=M)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:,30:80] = False
y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting.
# Observe the data
Y.observe(y, mask=mask)
# Initialize nodes (must use some randomness for C)
C.initialize_from_random()
# Run inference
Q = VB(Y, X, C, gamma, A, alpha, tau)
#
# Run inference with rotations.
#
if rotate:
rotA = transformations.RotateGaussianArrayARD(A, alpha, precompute=precompute)
rotX = transformations.RotateGaussianMarkovChain(X, A, rotA)
rotC = transformations.RotateGaussianArrayARD(C, gamma)
R = transformations.RotationOptimizer(rotX, rotC, D)
for ind in range(maxiter):
Q.update()
if debug:
R.rotate(maxiter=10,
check_gradient=True,
check_bound=True)
else:
R.rotate()
else:
Q.update(repeat=maxiter)
# Show results
plt.figure()
bpplt.timeseries_normal(CX, scale=2)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
plt.show()
# In[3]:
from bayespy.inference.vmp.vmp import VB
Q = VB(X, C, gamma, A, alpha, tau, Y)
#
# Observe the data partially (80% is marked missing):
#
# In[4]:
from bayespy.utils import random
# Add missing values randomly (keep only 20%)
mask = random.mask(M, N, p=0.2)
Y.observe(y, mask=mask)
# Then inference (100 iterations) can be run simply as
# In[5]:
Q.update(repeat=10)
### Speeding up with parameter expansion
# VB inference can converge extremely slowly if the variables are strongly coupled. Because VMP updates one variable at a time, it may lead to slow zigzagging. This can be solved by using parameter expansion which reduces the coupling. In state-space models, the states $\mathbf{x}_n$ and the loadings $\mathbf{C}$ are coupled through a dot product $\mathbf{Cx}_n$, which is unaltered if the latent space is rotated arbitrarily:
#
# $$
def run(maxiter=100):
seed = 496 #np.random.randint(1000)
print("seed = ", seed)
np.random.seed(seed)
# Simulate some data
D = 3
M = 6
N = 200
c = np.random.randn(M, D)
w = 0.3
a = np.array([[np.cos(w), -np.sin(w), 0], [np.sin(w),
np.cos(w), 0], [0, 0, 1]])
x = np.empty((N, D))
f = np.empty((M, N))
y = np.empty((M, N))
x[0] = 10 * np.random.randn(D)
f[:, 0] = np.dot(c, x[0])
y[:, 0] = f[:, 0] + 3 * np.random.randn(M)
for n in range(N - 1):
x[n + 1] = np.dot(a, x[n]) + np.random.randn(D)
f[:, n + 1] = np.dot(c, x[n + 1])
y[:, n + 1] = f[:, n + 1] + 3 * np.random.randn(M)
# Create the model
(Y, CX, X, tau, C, gamma, A, alpha) = linear_state_space_model(D, N, M)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:, 30:80] = False
y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting.
# Observe the data
Y.observe(y, mask=mask)
# Initialize nodes (must use some randomness for C)
C.initialize_from_random()
# Run inference
Q = VB(Y, X, C, gamma, A, alpha, tau)
#
# Run inference with rotations.
#
rotA = transformations.RotateGaussianARD(A, alpha)
rotX = transformations.RotateGaussianMarkovChain(X, A, rotA)
rotC = transformations.RotateGaussianARD(C, gamma)
R = transformations.RotationOptimizer(rotX, rotC, D)
#maxiter = 84
for ind in range(maxiter):
Q.update()
#print('C term', C.lower_bound_contribution())
R.rotate(
maxiter=10,
check_gradient=True,
verbose=False,
check_bound=Q.compute_lowerbound,
#check_bound=None,
check_bound_terms=Q.compute_lowerbound_terms)
#check_bound_terms=None)
X_vb = X.u[0]
varX_vb = utils.diagonal(X.u[1] - X_vb[..., np.newaxis, :] *
X_vb[..., :, np.newaxis])
u_CX = CX.get_moments()
CX_vb = u_CX[0]
varCX_vb = u_CX[1] - CX_vb**2
# Show results
plt.figure(3)
plt.clf()
for m in range(M):
plt.subplot(M, 1, m + 1)
plt.plot(y[m, :], 'r.')
plt.plot(f[m, :], 'b-')
bpplt.errorplot(y=CX_vb[m, :], error=2 * np.sqrt(varCX_vb[m, :]))
plt.figure()
Q.plot_iteration_by_nodes()
def demo(N=1000, D=5, K=4, seed=42, maxiter=200, rotate=True, debug=False,
precompute=False, plot=True):
# Seed for random number generator
if seed is not None:
np.random.seed(seed)
# Create data
(y, f) = simulate_data(N)
# Create some gaps
mask_gaps = utils.trues(N)
for m in range(100, N, 140):
start = m
end = min(m+15, N-1)
mask_gaps[start:end] = False
# Randomly missing values
mask_random = np.logical_or(random.mask(N, p=0.8),
np.logical_not(mask_gaps))
# Remove the observations
mask = np.logical_and(mask_gaps, mask_random)
y[~mask] = np.nan # BayesPy doesn't require NaNs, they're just for plotting.
# Add row axes
y = y[None,...]
f = f[None,...]
mask = mask[None,...]
mask_gaps = mask_gaps[None,...]
mask_random = mask_random[None,...]
# Run the method
Q = infer(y, D, K,
mask=mask,
maxiter=maxiter,
rotate=rotate,
debug=debug,
precompute=precompute,
update_hyper=10,
start_rotating_weights=20,
monitor=True)
if plot:
# Plot observations
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, 'b-')
bpplt.timeseries(y, 'r.')
plt.ylim([-2, 2])
# Plot latent space
Q.plot('X')
# Plot mixing weight space
Q.plot('S')
# Compute RMSE
rmse_random = utils.rmse(Q['Y'].get_moments()[0][~mask_random],
f[~mask_random])
rmse_gaps = utils.rmse(Q['Y'].get_moments()[0][~mask_gaps],
f[~mask_gaps])
print("RMSE for randomly missing values: %f" % rmse_random)
print("RMSE for gap values: %f" % rmse_gaps)
plt.show()
def demo(N=1000,
D=5,
K=4,
seed=42,
maxiter=200,
rotate=True,
debug=False,
precompute=False,
plot=True):
# Seed for random number generator
if seed is not None:
np.random.seed(seed)
# Create data
(y, f) = simulate_data(N)
# Create some gaps
mask_gaps = misc.trues(N)
for m in range(100, N, 140):
start = m
end = min(m + 15, N - 1)
mask_gaps[start:end] = False
# Randomly missing values
mask_random = np.logical_or(random.mask(N, p=0.8),
np.logical_not(mask_gaps))
# Remove the observations
mask = np.logical_and(mask_gaps, mask_random)
y[~mask] = np.nan # BayesPy doesn't require NaNs, they're just for plotting.
# Add row axes
y = y[None, ...]
f = f[None, ...]
mask = mask[None, ...]
mask_gaps = mask_gaps[None, ...]
mask_random = mask_random[None, ...]
# Run the method
Q = infer(y,
D,
K,
mask=mask,
maxiter=maxiter,
rotate=rotate,
debug=debug,
precompute=precompute,
update_hyper=10,
start_rotating_weights=20,
monitor=True)
if plot:
# Plot observations
plt.figure()
bpplt.timeseries_normal(Q['F'], scale=2)
bpplt.timeseries(f, linestyle='-', color='b')
bpplt.timeseries(y, linestyle='None', color='r', marker='.')
plt.ylim([-2, 2])
# Plot latent space
Q.plot('X')
# Plot mixing weight space
Q.plot('S')
# Compute RMSE
rmse_random = misc.rmse(Q['Y'].get_moments()[0][~mask_random],
f[~mask_random])
rmse_gaps = misc.rmse(Q['Y'].get_moments()[0][~mask_gaps], f[~mask_gaps])
print("RMSE for randomly missing values: %f" % rmse_random)
print("RMSE for gap values: %f" % rmse_gaps)
def run(maxiter=100):
seed = 496#np.random.randint(1000)
print("seed = ", seed)
np.random.seed(seed)
# Simulate some data
D = 3
M = 6
N = 200
c = np.random.randn(M,D)
w = 0.3
a = np.array([[np.cos(w), -np.sin(w), 0],
[np.sin(w), np.cos(w), 0],
[0, 0, 1]])
x = np.empty((N,D))
f = np.empty((M,N))
y = np.empty((M,N))
x[0] = 10*np.random.randn(D)
f[:,0] = np.dot(c,x[0])
y[:,0] = f[:,0] + 3*np.random.randn(M)
for n in range(N-1):
x[n+1] = np.dot(a,x[n]) + np.random.randn(D)
f[:,n+1] = np.dot(c,x[n+1])
y[:,n+1] = f[:,n+1] + 3*np.random.randn(M)
# Create the model
(Y, CX, X, tau, C, gamma, A, alpha) = linear_state_space_model(D, N, M)
# Add missing values randomly
mask = random.mask(M, N, p=0.3)
# Add missing values to a period of time
mask[:,30:80] = False
y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting.
# Observe the data
Y.observe(y, mask=mask)
# Initialize nodes (must use some randomness for C)
C.initialize_from_random()
# Run inference
Q = VB(Y, X, C, gamma, A, alpha, tau)
#
# Run inference with rotations.
#
rotA = transformations.RotateGaussianARD(A, alpha)
rotX = transformations.RotateGaussianMarkovChain(X, A, rotA)
rotC = transformations.RotateGaussianARD(C, gamma)
R = transformations.RotationOptimizer(rotX, rotC, D)
#maxiter = 84
for ind in range(maxiter):
Q.update()
#print('C term', C.lower_bound_contribution())
R.rotate(maxiter=10,
check_gradient=True,
verbose=False,
check_bound=Q.compute_lowerbound,
#check_bound=None,
check_bound_terms=Q.compute_lowerbound_terms)
#check_bound_terms=None)
X_vb = X.u[0]
varX_vb = utils.diagonal(X.u[1] - X_vb[...,np.newaxis,:] * X_vb[...,:,np.newaxis])
u_CX = CX.get_moments()
CX_vb = u_CX[0]
varCX_vb = u_CX[1] - CX_vb**2
# Show results
plt.figure(3)
plt.clf()
for m in range(M):
plt.subplot(M,1,m+1)
plt.plot(y[m,:], 'r.')
plt.plot(f[m,:], 'b-')
bpplt.errorplot(y=CX_vb[m,:],
error=2*np.sqrt(varCX_vb[m,:]))
plt.figure()
Q.plot_iteration_by_nodes()
def run(M=10, N=100, D_y=3, D=5):
seed = 45
print('seed =', seed)
np.random.seed(seed)
# Check HDF5 version.
if h5py.version.hdf5_version_tuple < (1,8,7):
print("WARNING! Your HDF5 version is %s. HDF5 versions <1.8.7 are not "
"able to save empty arrays, thus you may experience problems if "
"you for instance try to save before running any iteration steps."
% str(h5py.version.hdf5_version_tuple))
# Generate data
w = np.random.normal(0, 1, size=(M,1,D_y))
x = np.random.normal(0, 1, size=(1,N,D_y))
f = misc.sum_product(w, x, axes_to_sum=[-1])
y = f + np.random.normal(0, 0.5, size=(M,N))
# Construct model
(Y, WX, W, X, tau, alpha) = pca_model(M, N, D)
# Data with missing values
mask = random.mask(M, N, p=0.9) # randomly missing
mask[:,20:40] = False # gap missing
y[~mask] = np.nan
Y.observe(y, mask=mask)
# Construct inference machine
Q = VB(Y, W, X, tau, alpha, autosave_iterations=5)
# Initialize some nodes randomly
X.initialize_from_value(X.random())
W.initialize_from_value(W.random())
# Save the state into a HDF5 file
filename = tempfile.NamedTemporaryFile(suffix='hdf5').name
Q.update(X, W, alpha, tau, repeat=1)
Q.save(filename=filename)
# Inference loop.
Q.update(X, W, alpha, tau, repeat=10)
# Reload the state from the HDF5 file
Q.load(filename=filename)
# Inference loop again.
Q.update(X, W, alpha, tau, repeat=10)
# NOTE: Saving and loading requires that you have the model
# constructed. "Save" does not store the model structure nor does "load"
# read it. They are just used for reading and writing the contents of the
# nodes. Thus, if you want to load, you first need to construct the same
# model that was used for saving and then use load to set the states of the
# nodes.
plt.clf()
WX_params = WX.get_parameters()
fh = WX_params[0] * np.ones(y.shape)
err_fh = 2*np.sqrt(WX_params[1] + 1/tau.get_moments()[0]) * np.ones(y.shape)
for m in range(M):
plt.subplot(M,1,m+1)
#errorplot(y, error=None, x=None, lower=None, upper=None):
bpplt.errorplot(fh[m], x=np.arange(N), error=err_fh[m])
plt.plot(np.arange(N), f[m], 'g')
plt.plot(np.arange(N), y[m], 'r+')
plt.figure()
Q.plot_iteration_by_nodes()
plt.figure()
plt.subplot(2,2,1)
bpplt.binary_matrix(W.mask)
plt.subplot(2,2,2)
bpplt.binary_matrix(X.mask)
plt.subplot(2,2,3)
#bpplt.binary_matrix(WX.get_mask())
plt.subplot(2,2,4)
bpplt.binary_matrix(Y.mask)