def plot(self): x = self.u[0] xx = self.u[1] var = np.einsum('...ii->...i', xx) - x**2 plt.figure() D = np.shape(x)[-1] for d in range(D): plt.subplot(D, 1, d + 1) bpplt.errorplot(y=x[..., d], error=2 * np.sqrt(var[..., d]))
def run(): # Create some data N = 500 D = 2 # Initial state x0 = np.array([0.5, -0.5]) # Dynamics (time varying) A0 = np.array([[.9, -.4], [.4, .9]]) A1 = np.array([[.98, -.1], [.1, .98]]) l = np.linspace(0, 1, N - 1).reshape((-1, 1, 1)) A = (1 - l) * A0 + l * A1 # Innovation covariance matrix V = np.identity(D) # Observation noise covariance matrix C = np.tile(np.identity(D), (N, 1, 1)) ## C0 = 10*np.array([[1, 0], [0, 1]]) ## C1 = 0.01*np.array([[1, 0], [0, 1]]) ## C = (1-l)**2*C0 + l**2*C1 X = np.empty((N, D)) Y = np.empty((N, D)) # Simulate data x = x0 X[0, :] = x Y[0, :] = x + np.random.multivariate_normal(np.zeros(D), C[0, :, :]) for n in range(N - 1): x = np.dot(A[n, :, :], x) + np.random.multivariate_normal( np.zeros(D), V) X[n + 1, :] = x Y[n + 1, :] = x + np.random.multivariate_normal( np.zeros(D), C[n + 1, :, :]) # Invert observation noise covariance to observation precision matrices U = np.empty((N, D, D)) UY = np.empty((N, D)) for n in range(N): U[n, :, :] = np.linalg.inv(C[n, :, :]) UY[n, :] = np.linalg.solve(C[n, :, :], Y[n, :]) # Construct VB model Xh = GaussianMarkovChain(np.zeros(D), np.identity(D), A, np.ones(D), n=N) Yh = Gaussian(Xh.as_gaussian(), np.identity(D), plates=(N, )) Yh.observe(Y) Xh.update() xh = Xh.u[0] varxh = utils.diagonal(Xh.u[1]) - xh**2 #err = 2 * np.sqrt(varxh) plt.figure(1) plt.clf() for d in range(D): plt.subplot(D, 1, d) bpplt.errorplot(xh[:, d], error=2 * np.sqrt(varxh[:, d])) plt.plot(X[:, d], 'r-') plt.plot(Y[:, d], '.')
def plot(self): x = self.u[0] xx = self.u[1] var = np.einsum('...ii->...i', xx) - x**2 plt.figure() D = np.shape(x)[-1] for d in range(D): plt.subplot(D,1,d+1) bpplt.errorplot(y=x[...,d], error=2*np.sqrt(var[...,d]))
def run(): # Create some data N = 500 D = 2 # Initial state x0 = np.array([0.5, -0.5]) # Dynamics (time varying) A0 = np.array([[.9, -.4], [.4, .9]]) A1 = np.array([[.98, -.1], [.1, .98]]) l = np.linspace(0, 1, N-1).reshape((-1,1,1)) A = (1-l)*A0 + l*A1 # Innovation covariance matrix V = np.identity(D) # Observation noise covariance matrix C = np.tile(np.identity(D), (N, 1, 1)) ## C0 = 10*np.array([[1, 0], [0, 1]]) ## C1 = 0.01*np.array([[1, 0], [0, 1]]) ## C = (1-l)**2*C0 + l**2*C1 X = np.empty((N,D)) Y = np.empty((N,D)) # Simulate data x = x0 X[0,:] = x Y[0,:] = x + np.random.multivariate_normal(np.zeros(D), C[0,:,:]) for n in range(N-1): x = np.dot(A[n,:,:],x) + np.random.multivariate_normal(np.zeros(D), V) X[n+1,:] = x Y[n+1,:] = x + np.random.multivariate_normal(np.zeros(D), C[n+1,:,:]) # Invert observation noise covariance to observation precision matrices U = np.empty((N,D,D)) UY = np.empty((N,D)) for n in range(N): U[n,:,:] = np.linalg.inv(C[n,:,:]) UY[n,:] = np.linalg.solve(C[n,:,:], Y[n,:]) # Construct VB model Xh = GaussianMarkovChain(np.zeros(D), np.identity(D), A, np.ones(D), n=N) Yh = Gaussian(Xh.as_gaussian(), np.identity(D), plates=(N,)) Yh.observe(Y) Xh.update() xh = Xh.u[0] varxh = utils.diagonal(Xh.u[1]) - xh**2 #err = 2 * np.sqrt(varxh) plt.figure(1) plt.clf() for d in range(D): plt.subplot(D,1,d) bpplt.errorplot(xh[:,d], error=2*np.sqrt(varxh[:,d])) plt.plot(X[:,d], 'r-') plt.plot(Y[:,d], '.')
def run(): # Create some data N = 500 D = 2 # Initial state x0 = np.array([0.5, -0.5]) # Dynamics (time varying) A0 = np.array([[.9, -.4], [.4, .9]]) A1 = np.array([[.98, -.1], [.1, .98]]) l = np.linspace(0, 1, N).reshape((-1, 1, 1)) A = (1 - l) * A0 + l * A1 # Innovation covariance matrix V = np.array([[1, 0], [0, 1]]) # Observation noise covariance matrix C0 = 10 * np.array([[1, 0], [0, 1]]) C1 = 0.01 * np.array([[1, 0], [0, 1]]) C = (1 - l)**2 * C0 + l**2 * C1 X = np.empty((N, D)) Y = np.empty((N, D)) # Simulate data x = x0 for n in range(N): x = np.dot(A[n, :, :], x) + np.random.multivariate_normal( np.zeros(D), V) X[n, :] = x Y[n, :] = x + np.random.multivariate_normal(np.zeros(D), C[n, :, :]) U = np.empty((N, D, D)) UY = np.empty((N, D)) for n in range(N): U[n, :, :] = np.linalg.inv(C[n, :, :]) UY[n, :] = np.linalg.solve(C[n, :, :], Y[n, :]) # Create iterators for the static matrices #U = N*(U,) #A = N*(A,) V = N * (V, ) (Xh, CovXh) = utils.kalman_filter(UY, U, A, V, np.zeros(2), 10 * np.identity(2)) (Xh, CovXh) = utils.rts_smoother(Xh, CovXh, A, V) plt.clf() for d in range(D): plt.subplot(D, 1, d) #plt.plot(Xh[:,d], 'r--') bpplt.errorplot(Xh[:, d], error=2 * np.sqrt(CovXh[:, d, d])) plt.plot(X[:, d], 'r-') plt.plot(Y[:, d], '.') plt.show()
def run(): # Create some data N = 500 D = 2 # Initial state x0 = np.array([0.5, -0.5]) # Dynamics (time varying) A0 = np.array([[.9, -.4], [.4, .9]]) A1 = np.array([[.98, -.1], [.1, .98]]) l = np.linspace(0, 1, N).reshape((-1,1,1)) A = (1-l)*A0 + l*A1 # Innovation covariance matrix V = np.array([[1, 0], [0, 1]]) # Observation noise covariance matrix C0 = 10*np.array([[1, 0], [0, 1]]) C1 = 0.01*np.array([[1, 0], [0, 1]]) C = (1-l)**2*C0 + l**2*C1 X = np.empty((N,D)) Y = np.empty((N,D)) # Simulate data x = x0 for n in range(N): x = np.dot(A[n,:,:],x) + np.random.multivariate_normal(np.zeros(D), V) X[n,:] = x Y[n,:] = x + np.random.multivariate_normal(np.zeros(D), C[n,:,:]) U = np.empty((N,D,D)) UY = np.empty((N,D)) for n in range(N): U[n,:,:] = np.linalg.inv(C[n,:,:]) UY[n,:] = np.linalg.solve(C[n,:,:], Y[n,:]) # Create iterators for the static matrices #U = N*(U,) #A = N*(A,) V = N*(V,) (Xh, CovXh) = utils.kalman_filter(UY, U, A, V, np.zeros(2), 10*np.identity(2)) (Xh, CovXh) = utils.rts_smoother(Xh, CovXh, A, V) plt.clf() for d in range(D): plt.subplot(D,1,d) #plt.plot(Xh[:,d], 'r--') bpplt.errorplot(Xh[:,d], error=2*np.sqrt(CovXh[:,d,d])) plt.plot(X[:,d], 'r-') plt.plot(Y[:,d], '.') plt.show()
def run(M=10, N=100, D_y=3, D=5): seed = 45 print('seed =', seed) np.random.seed(seed) # Generate data w = np.random.normal(0, 1, size=(M,1,D_y)) x = np.random.normal(0, 1, size=(1,N,D_y)) f = utils.utils.sum_product(w, x, axes_to_sum=[-1]) y = f + np.random.normal(0, 0.5, size=(M,N)) # Construct model (Y, WX, W, X, tau, alpha) = pca_model(M, N, D) # Data with missing values mask = utils.random.mask(M, N, p=0.9) # randomly missing mask[:,20:40] = False # gap missing y[~mask] = np.nan Y.observe(y, mask=mask) # Construct inference machine Q = VB(Y, W, X, tau, alpha) # Initialize some nodes randomly X.initialize_from_value(X.random()) W.initialize_from_value(W.random()) # Inference loop. Q.update(repeat=100) # Plot results plt.clf() WX_params = WX.get_parameters() fh = WX_params[0] * np.ones(y.shape) err_fh = 2*np.sqrt(WX_params[1] + 1/tau.get_moments()[0]) * np.ones(y.shape) for m in range(M): plt.subplot(M,1,m+1) myplt.errorplot(fh[m], x=np.arange(N), error=err_fh[m]) plt.plot(np.arange(N), f[m], 'g') plt.plot(np.arange(N), y[m], 'r+') plt.show()
def run_lssm(y, f, mask, D, maxiter): """ Run VB inference for linear state space model. """ (M, N) = np.shape(y) # # CONSTRUCT THE MODEL # # Dynamic matrix # alpha: (D) x () alpha = Gamma(1e-5, 1e-5, plates=(D,), name='alpha') # A : (D) x (D) A = Gaussian(np.zeros(D), diagonal(alpha), plates=(D,), name='A') A.initialize_from_value(np.identity(D)) # Latent states with dynamics # X : () x (N,D) X = GaussianMarkovChain(np.zeros(D), # mean of x0 1e-3*np.identity(D), # prec of x0 A, # dynamics np.ones(D), # innovation n=N, # time instances name='X', initialize=False) X.initialize_from_value(np.random.randn(N,D)) # Mixing matrix from latent space to observation space using ARD # gamma : (D) x () gamma = Gamma(1e-5, 1e-5, plates=(D,), name='gamma') # C : (M,1) x (D) C = Gaussian(np.zeros(D), diagonal(gamma), plates=(M,1), name='C') C.initialize_from_value(np.random.randn(M,1,D)) # Observation noise # tau : () x () tau = Gamma(1e-5, 1e-5, name='tau') # Observations # Y : (M,N) x () CX = Dot(C, X.as_gaussian()) Y = Normal(CX, tau, name='Y') # # RUN INFERENCE # # Observe data Y.observe(y, mask=mask) # Construct inference machine Q = VB(Y, X, A, alpha, C, gamma, tau) # Iterate Q.update(X, A, alpha, C, gamma, tau, repeat=maxiter) # # SHOW RESULTS # # Mean and standard deviation of the posterior (f_mean, f_squared) = CX.get_moments() f_std = np.sqrt(f_squared - f_mean**2) # Plot observations space #plt.figure() for m in range(M): plt.subplot(M,1,m+1) plt.plot(y[m,:], 'r.') plt.plot(f[m,:], 'b-') bpplt.errorplot(y=f_mean[m,:], error=2*f_std[m,:])
def run(maxiter=100): seed = 496#np.random.randint(1000) print("seed = ", seed) np.random.seed(seed) # Simulate some data D = 3 M = 6 N = 200 c = np.random.randn(M,D) w = 0.3 a = np.array([[np.cos(w), -np.sin(w), 0], [np.sin(w), np.cos(w), 0], [0, 0, 1]]) x = np.empty((N,D)) f = np.empty((M,N)) y = np.empty((M,N)) x[0] = 10*np.random.randn(D) f[:,0] = np.dot(c,x[0]) y[:,0] = f[:,0] + 3*np.random.randn(M) for n in range(N-1): x[n+1] = np.dot(a,x[n]) + np.random.randn(D) f[:,n+1] = np.dot(c,x[n+1]) y[:,n+1] = f[:,n+1] + 3*np.random.randn(M) # Create the model (Y, CX, X, tau, C, gamma, A, alpha) = linear_state_space_model(D, N, M) # Add missing values randomly mask = random.mask(M, N, p=0.3) # Add missing values to a period of time mask[:,30:80] = False y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting. # Observe the data Y.observe(y, mask=mask) # Initialize nodes (must use some randomness for C) C.initialize_from_random() # Run inference Q = VB(Y, X, C, gamma, A, alpha, tau) # # Run inference with rotations. # rotA = transformations.RotateGaussianARD(A, alpha) rotX = transformations.RotateGaussianMarkovChain(X, A, rotA) rotC = transformations.RotateGaussianARD(C, gamma) R = transformations.RotationOptimizer(rotX, rotC, D) #maxiter = 84 for ind in range(maxiter): Q.update() #print('C term', C.lower_bound_contribution()) R.rotate(maxiter=10, check_gradient=True, verbose=False, check_bound=Q.compute_lowerbound, #check_bound=None, check_bound_terms=Q.compute_lowerbound_terms) #check_bound_terms=None) X_vb = X.u[0] varX_vb = utils.diagonal(X.u[1] - X_vb[...,np.newaxis,:] * X_vb[...,:,np.newaxis]) u_CX = CX.get_moments() CX_vb = u_CX[0] varCX_vb = u_CX[1] - CX_vb**2 # Show results plt.figure(3) plt.clf() for m in range(M): plt.subplot(M,1,m+1) plt.plot(y[m,:], 'r.') plt.plot(f[m,:], 'b-') bpplt.errorplot(y=CX_vb[m,:], error=2*np.sqrt(varCX_vb[m,:])) plt.figure() Q.plot_iteration_by_nodes()
def run(M=10, N=100, D_y=3, D=5): seed = 45 print('seed =', seed) np.random.seed(seed) # Check HDF5 version. if h5py.version.hdf5_version_tuple < (1, 8, 7): print( "WARNING! Your HDF5 version is %s. HDF5 versions <1.8.7 are not " "able to save empty arrays, thus you may experience problems if " "you for instance try to save before running any iteration steps." % str(h5py.version.hdf5_version_tuple)) # Generate data w = np.random.normal(0, 1, size=(M, 1, D_y)) x = np.random.normal(0, 1, size=(1, N, D_y)) f = utils.utils.sum_product(w, x, axes_to_sum=[-1]) y = f + np.random.normal(0, 0.5, size=(M, N)) # Construct model (Y, WX, W, X, tau, alpha) = pca_model(M, N, D) # Data with missing values mask = utils.random.mask(M, N, p=0.9) # randomly missing mask[:, 20:40] = False # gap missing y[~mask] = np.nan Y.observe(y, mask=mask) # Construct inference machine Q = VB(Y, W, X, tau, alpha, autosave_iterations=5) # Initialize some nodes randomly X.initialize_from_value(X.random()) W.initialize_from_value(W.random()) # Save the state into a HDF5 file filename = tempfile.NamedTemporaryFile(suffix='hdf5').name Q.update(X, W, alpha, tau, repeat=1) Q.save(filename) # Inference loop. Q.update(X, W, alpha, tau, repeat=10) # Reload the state from the HDF5 file Q.load(filename=filename) # Inference loop again. Q.update(X, W, alpha, tau, repeat=10) # NOTE: Saving and loading requires that you have the model # constructed. "Save" does not store the model structure nor does "load" # read it. They are just used for reading and writing the contents of the # nodes. Thus, if you want to load, you first need to construct the same # model that was used for saving and then use load to set the states of the # nodes. plt.clf() WX_params = WX.get_parameters() fh = WX_params[0] * np.ones(y.shape) err_fh = 2 * np.sqrt(WX_params[1] + 1 / tau.get_moments()[0]) * np.ones( y.shape) for m in range(M): plt.subplot(M, 1, m + 1) #errorplot(y, error=None, x=None, lower=None, upper=None): myplt.errorplot(fh[m], x=np.arange(N), error=err_fh[m]) plt.plot(np.arange(N), f[m], 'g') plt.plot(np.arange(N), y[m], 'r+') plt.figure() Q.plot_iteration_by_nodes() plt.figure() plt.subplot(2, 2, 1) myplt.binary_matrix(W.mask) plt.subplot(2, 2, 2) myplt.binary_matrix(X.mask) plt.subplot(2, 2, 3) #myplt.binary_matrix(WX.get_mask()) plt.subplot(2, 2, 4) myplt.binary_matrix(Y.mask) plt.show()
def run(): ## Generate data # Noisy observations from a sinusoid N = 100 func = lambda x: np.sin(x*2*np.pi/50) x = np.random.uniform(low=0, high=N, size=(N,)) f = func(x) y = f + np.random.normal(0, 0.2, np.shape(f)) # Plot data plt.clf() plt.plot(x,y,'r+') ## Construct model # Covariance function stuff ls = EF.NodeConstantScalar(10, name='lengthscale') amp = EF.NodeConstantScalar(1.0, name='amplitude') noise = EF.NodeConstantScalar(0.2, name='noise') # Latent process covariance #K_f = CF.SquaredExponential(amp, ls) K_f = CF.SquaredExponential(amp, ls) # Noise process covariance K_noise = CF.Delta(noise) # Observation process covariance K_y = CF.Sum(K_f, K_noise) # Joint covariance #K_joint = CF.Multiple([[K_f, K_f],[K_f,K_y]], sparse=True) # Mean function stuff M = GP.Constant(lambda x: np.zeros(np.shape(x)[0])) # Means for latent and observation processes #M_multi = GP.Multiple([M, M]) # Gaussian process F = GP.GaussianProcess(M, [[K_f, K_f], [K_f, K_y]]) #F = GP.GaussianProcess(M, [[K_f, K_f], [K_f, K_y]]) #F = GP.GaussianProcess(M_multi, K_joint) ## Inference F.observe([[],x], y) utils.vb_optimize_nodes(ls, amp, noise) F.update() u = F.get_parameters() ## Show results # Print hyperparameters #print('parameters') #print(ls.name, ls.u[0]) #print(amp.name, amp.u[0]) #print(noise.name, noise.u[0]) # Posterior predictions xh = np.arange(np.min(x)-5, np.max(x)+100, 0.1) (fh, varfh) = u([[],xh], covariance=1) #(fh, varfh) = u([xh,[]], covariance=1) # Plot predictive distribution varfh[varfh<0] = 0 errfh = np.sqrt(varfh) myplt.errorplot(fh, x=xh, error=errfh) return
def run(M=10, N=100, D_y=3, D=5): seed = 45 print("seed =", seed) np.random.seed(seed) # Check HDF5 version. if h5py.version.hdf5_version_tuple < (1, 8, 7): print( "WARNING! Your HDF5 version is %s. HDF5 versions <1.8.7 are not " "able to save empty arrays, thus you may experience problems if " "you for instance try to save before running any iteration steps." % str(h5py.version.hdf5_version_tuple) ) # Generate data w = np.random.normal(0, 1, size=(M, 1, D_y)) x = np.random.normal(0, 1, size=(1, N, D_y)) f = utils.utils.sum_product(w, x, axes_to_sum=[-1]) y = f + np.random.normal(0, 0.5, size=(M, N)) # Construct model (Y, WX, W, X, tau, alpha) = pca_model(M, N, D) # Data with missing values mask = utils.random.mask(M, N, p=0.9) # randomly missing mask[:, 20:40] = False # gap missing y[~mask] = np.nan Y.observe(y, mask=mask) # Construct inference machine Q = VB(Y, W, X, tau, alpha, autosave_iterations=5) # Initialize some nodes randomly X.initialize_from_value(X.random()) W.initialize_from_value(W.random()) # Save the state into a HDF5 file filename = tempfile.NamedTemporaryFile(suffix="hdf5").name Q.update(X, W, alpha, tau, repeat=1) Q.save(filename) # Inference loop. Q.update(X, W, alpha, tau, repeat=10) # Reload the state from the HDF5 file Q.load(filename=filename) # Inference loop again. Q.update(X, W, alpha, tau, repeat=10) # NOTE: Saving and loading requires that you have the model # constructed. "Save" does not store the model structure nor does "load" # read it. They are just used for reading and writing the contents of the # nodes. Thus, if you want to load, you first need to construct the same # model that was used for saving and then use load to set the states of the # nodes. plt.clf() WX_params = WX.get_parameters() fh = WX_params[0] * np.ones(y.shape) err_fh = 2 * np.sqrt(WX_params[1] + 1 / tau.get_moments()[0]) * np.ones(y.shape) for m in range(M): plt.subplot(M, 1, m + 1) # errorplot(y, error=None, x=None, lower=None, upper=None): myplt.errorplot(fh[m], x=np.arange(N), error=err_fh[m]) plt.plot(np.arange(N), f[m], "g") plt.plot(np.arange(N), y[m], "r+") plt.figure() Q.plot_iteration_by_nodes() plt.figure() plt.subplot(2, 2, 1) myplt.binary_matrix(W.mask) plt.subplot(2, 2, 2) myplt.binary_matrix(X.mask) plt.subplot(2, 2, 3) # myplt.binary_matrix(WX.get_mask()) plt.subplot(2, 2, 4) myplt.binary_matrix(Y.mask) plt.show()
def run(maxiter=100): seed = 496 #np.random.randint(1000) print("seed = ", seed) np.random.seed(seed) # Simulate some data D = 3 M = 6 N = 200 c = np.random.randn(M, D) w = 0.3 a = np.array([[np.cos(w), -np.sin(w), 0], [np.sin(w), np.cos(w), 0], [0, 0, 1]]) x = np.empty((N, D)) f = np.empty((M, N)) y = np.empty((M, N)) x[0] = 10 * np.random.randn(D) f[:, 0] = np.dot(c, x[0]) y[:, 0] = f[:, 0] + 3 * np.random.randn(M) for n in range(N - 1): x[n + 1] = np.dot(a, x[n]) + np.random.randn(D) f[:, n + 1] = np.dot(c, x[n + 1]) y[:, n + 1] = f[:, n + 1] + 3 * np.random.randn(M) # Create the model (Y, CX, X, tau, C, gamma, A, alpha) = linear_state_space_model(D, N, M) # Add missing values randomly mask = random.mask(M, N, p=0.3) # Add missing values to a period of time mask[:, 30:80] = False y[~mask] = np.nan # BayesPy doesn't require this. Just for plotting. # Observe the data Y.observe(y, mask=mask) # Initialize nodes (must use some randomness for C) C.initialize_from_random() # Run inference Q = VB(Y, X, C, gamma, A, alpha, tau) # # Run inference with rotations. # rotA = transformations.RotateGaussianARD(A, alpha) rotX = transformations.RotateGaussianMarkovChain(X, A, rotA) rotC = transformations.RotateGaussianARD(C, gamma) R = transformations.RotationOptimizer(rotX, rotC, D) #maxiter = 84 for ind in range(maxiter): Q.update() #print('C term', C.lower_bound_contribution()) R.rotate( maxiter=10, check_gradient=True, verbose=False, check_bound=Q.compute_lowerbound, #check_bound=None, check_bound_terms=Q.compute_lowerbound_terms) #check_bound_terms=None) X_vb = X.u[0] varX_vb = utils.diagonal(X.u[1] - X_vb[..., np.newaxis, :] * X_vb[..., :, np.newaxis]) u_CX = CX.get_moments() CX_vb = u_CX[0] varCX_vb = u_CX[1] - CX_vb**2 # Show results plt.figure(3) plt.clf() for m in range(M): plt.subplot(M, 1, m + 1) plt.plot(y[m, :], 'r.') plt.plot(f[m, :], 'b-') bpplt.errorplot(y=CX_vb[m, :], error=2 * np.sqrt(varCX_vb[m, :])) plt.figure() Q.plot_iteration_by_nodes()
def run(): ## Generate data # Noisy observations from a sinusoid N = 100 func = lambda x: np.sin(x * 2 * np.pi / 50) x = np.random.uniform(low=0, high=N, size=(N, )) f = func(x) y = f + np.random.normal(0, 0.2, np.shape(f)) # Plot data plt.clf() plt.plot(x, y, 'r+') ## Construct model # Covariance function stuff ls = EF.NodeConstantScalar(10, name='lengthscale') amp = EF.NodeConstantScalar(1.0, name='amplitude') noise = EF.NodeConstantScalar(0.2, name='noise') # Latent process covariance #K_f = CF.SquaredExponential(amp, ls) K_f = CF.SquaredExponential(amp, ls) # Noise process covariance K_noise = CF.Delta(noise) # Observation process covariance K_y = CF.Sum(K_f, K_noise) # Joint covariance #K_joint = CF.Multiple([[K_f, K_f],[K_f,K_y]], sparse=True) # Mean function stuff M = GP.Constant(lambda x: np.zeros(np.shape(x)[0])) # Means for latent and observation processes #M_multi = GP.Multiple([M, M]) # Gaussian process F = GP.GaussianProcess(M, [[K_f, K_f], [K_f, K_y]]) #F = GP.GaussianProcess(M, [[K_f, K_f], [K_f, K_y]]) #F = GP.GaussianProcess(M_multi, K_joint) ## Inference F.observe([[], x], y) utils.vb_optimize_nodes(ls, amp, noise) F.update() u = F.get_parameters() ## Show results # Print hyperparameters #print('parameters') #print(ls.name, ls.u[0]) #print(amp.name, amp.u[0]) #print(noise.name, noise.u[0]) # Posterior predictions xh = np.arange(np.min(x) - 5, np.max(x) + 100, 0.1) (fh, varfh) = u([[], xh], covariance=1) #(fh, varfh) = u([xh,[]], covariance=1) # Plot predictive distribution varfh[varfh < 0] = 0 errfh = np.sqrt(varfh) myplt.errorplot(fh, x=xh, error=errfh) return
def run_lssm(y, f, mask, D, maxiter): """ Run VB inference for linear state space model. """ (M, N) = np.shape(y) # # CONSTRUCT THE MODEL # # Dynamic matrix # alpha: (D) x () alpha = Gamma(1e-5, 1e-5, plates=(D, ), name='alpha') # A : (D) x (D) A = Gaussian(np.zeros(D), diagonal(alpha), plates=(D, ), name='A') A.initialize_from_value(np.identity(D)) # Latent states with dynamics # X : () x (N,D) X = GaussianMarkovChain( np.zeros(D), # mean of x0 1e-3 * np.identity(D), # prec of x0 A, # dynamics np.ones(D), # innovation n=N, # time instances name='X', initialize=False) X.initialize_from_value(np.random.randn(N, D)) # Mixing matrix from latent space to observation space using ARD # gamma : (D) x () gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma') # C : (M,1) x (D) C = Gaussian(np.zeros(D), diagonal(gamma), plates=(M, 1), name='C') C.initialize_from_value(np.random.randn(M, 1, D)) # Observation noise # tau : () x () tau = Gamma(1e-5, 1e-5, name='tau') # Observations # Y : (M,N) x () CX = Dot(C, X.as_gaussian()) Y = Normal(CX, tau, name='Y') # # RUN INFERENCE # # Observe data Y.observe(y, mask=mask) # Construct inference machine Q = VB(Y, X, A, alpha, C, gamma, tau) # Iterate Q.update(X, A, alpha, C, gamma, tau, repeat=maxiter) # # SHOW RESULTS # # Mean and standard deviation of the posterior (f_mean, f_squared) = CX.get_moments() f_std = np.sqrt(f_squared - f_mean**2) # Plot observations space #plt.figure() for m in range(M): plt.subplot(M, 1, m + 1) plt.plot(y[m, :], 'r.') plt.plot(f[m, :], 'b-') bpplt.errorplot(y=f_mean[m, :], error=2 * f_std[m, :])
def run_dlssm(y, f, mask, D, K, maxiter): """ Run VB inference for linear state space model with drifting dynamics. """ (M, N) = np.shape(y) # Dynamics matrix with ARD # alpha : (D) x () alpha = Gamma(1e-5, 1e-5, plates=(K,), name='alpha') # A : (K) x (K) A = Gaussian(np.zeros(K), #np.identity(K), diagonal(alpha), plates=(K,), name='A_S') A.initialize_from_value(np.identity(K)) # rho ## rho = Gamma(1e-5, ## 1e-5, ## plates=(K,), ## name="rho") # S : () x (N-1,K) S = GaussianMarkovChain(np.ones(K), 1e-6*np.identity(K), A, np.ones(K), n=N-1, name='S') S.initialize_from_value(1*np.ones((N-1,K))) # Projection matrix of the dynamics matrix # beta : (K) x () beta = Gamma(1e-5, 1e-5, plates=(K,), name='beta') # B : (D) x (D*K) B = Gaussian(np.zeros(D*K), diagonal(tile(beta, D)), plates=(D,), name='B') b = np.zeros((D,D,K)) b[np.arange(D),np.arange(D),np.zeros(D,dtype=int)] = 1 B.initialize_from_value(np.reshape(1*b, (D,D*K))) # A : (N-1,D) x (D) BS = MatrixDot(B, S.as_gaussian().add_plate_axis(-1), name='BS') # Latent states with dynamics # X : () x (N,D) X = GaussianMarkovChain(np.zeros(D), # mean of x0 1e-3*np.identity(D), # prec of x0 BS, # dynamics np.ones(D), # innovation n=N, # time instances name='X', initialize=False) X.initialize_from_value(np.random.randn(N,D)) # Mixing matrix from latent space to observation space using ARD # gamma : (D) x () gamma = Gamma(1e-5, 1e-5, plates=(D,), name='gamma') # C : (M,1) x (D) C = Gaussian(np.zeros(D), diagonal(gamma), plates=(M,1), name='C') C.initialize_from_value(np.random.randn(M,1,D)) # Observation noise # tau : () x () tau = Gamma(1e-5, 1e-5, name='tau') # Observations # Y : (M,N) x () CX = Dot(C, X.as_gaussian()) Y = Normal(CX, tau, name='Y') # # RUN INFERENCE # # Observe data Y.observe(y, mask=mask) # Construct inference machine Q = VB(Y, X, S, A, alpha, B, beta, C, gamma, tau) # # Run inference with rotations. # # Rotate the D-dimensional state space (C, X) rotB = transformations.RotateGaussianMatrixARD(B, beta, axis='rows') rotX = transformations.RotateDriftingMarkovChain(X, B, S, rotB) rotC = transformations.RotateGaussianARD(C, gamma) R_X = transformations.RotationOptimizer(rotX, rotC, D) # Rotate the K-dimensional latent dynamics space (B, S) rotA = transformations.RotateGaussianARD(A, alpha) rotS = transformations.RotateGaussianMarkovChain(S, A, rotA) rotB = transformations.RotateGaussianMatrixARD(B, beta, axis='cols') R_S = transformations.RotationOptimizer(rotS, rotB, K) # Iterate for ind in range(int(maxiter/5)): Q.update(repeat=5) #Q.update(X, S, A, alpha, rho, B, beta, C, gamma, tau, repeat=maxiter) R_X.rotate() R_S.rotate() ## R_X.rotate( ## check_bound=Q.compute_lowerbound, ## check_bound_terms=Q.compute_lowerbound_terms, ## check_gradient=True ## ) ## R_S.rotate( ## check_bound=Q.compute_lowerbound, ## check_bound_terms=Q.compute_lowerbound_terms, ## check_gradient=True ## ) # # SHOW RESULTS # # Mean and standard deviation of the posterior (f_mean, f_squared) = CX.get_moments() f_std = np.sqrt(f_squared - f_mean**2) # Plot observations space for m in range(M): plt.subplot(M,1,m+1) plt.plot(y[m,:], 'r.') plt.plot(f[m,:], 'b-') bpplt.errorplot(y=f_mean[m,:], error=2*f_std[m,:])
def run_dlssm(y, f, mask, D, K, maxiter): """ Run VB inference for linear state space model with drifting dynamics. """ (M, N) = np.shape(y) # Dynamics matrix with ARD # alpha : (D) x () alpha = Gamma(1e-5, 1e-5, plates=(K, ), name='alpha') # A : (K) x (K) A = Gaussian( np.zeros(K), #np.identity(K), diagonal(alpha), plates=(K, ), name='A_S') A.initialize_from_value(np.identity(K)) # rho ## rho = Gamma(1e-5, ## 1e-5, ## plates=(K,), ## name="rho") # S : () x (N-1,K) S = GaussianMarkovChain(np.ones(K), 1e-6 * np.identity(K), A, np.ones(K), n=N - 1, name='S') S.initialize_from_value(1 * np.ones((N - 1, K))) # Projection matrix of the dynamics matrix # beta : (K) x () beta = Gamma(1e-5, 1e-5, plates=(K, ), name='beta') # B : (D) x (D*K) B = Gaussian(np.zeros(D * K), diagonal(tile(beta, D)), plates=(D, ), name='B') b = np.zeros((D, D, K)) b[np.arange(D), np.arange(D), np.zeros(D, dtype=int)] = 1 B.initialize_from_value(np.reshape(1 * b, (D, D * K))) # A : (N-1,D) x (D) BS = MatrixDot(B, S.as_gaussian().add_plate_axis(-1), name='BS') # Latent states with dynamics # X : () x (N,D) X = GaussianMarkovChain( np.zeros(D), # mean of x0 1e-3 * np.identity(D), # prec of x0 BS, # dynamics np.ones(D), # innovation n=N, # time instances name='X', initialize=False) X.initialize_from_value(np.random.randn(N, D)) # Mixing matrix from latent space to observation space using ARD # gamma : (D) x () gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma') # C : (M,1) x (D) C = Gaussian(np.zeros(D), diagonal(gamma), plates=(M, 1), name='C') C.initialize_from_value(np.random.randn(M, 1, D)) # Observation noise # tau : () x () tau = Gamma(1e-5, 1e-5, name='tau') # Observations # Y : (M,N) x () CX = Dot(C, X.as_gaussian()) Y = Normal(CX, tau, name='Y') # # RUN INFERENCE # # Observe data Y.observe(y, mask=mask) # Construct inference machine Q = VB(Y, X, S, A, alpha, B, beta, C, gamma, tau) # # Run inference with rotations. # # Rotate the D-dimensional state space (C, X) rotB = transformations.RotateGaussianMatrixARD(B, beta, axis='rows') rotX = transformations.RotateDriftingMarkovChain(X, B, S, rotB) rotC = transformations.RotateGaussianARD(C, gamma) R_X = transformations.RotationOptimizer(rotX, rotC, D) # Rotate the K-dimensional latent dynamics space (B, S) rotA = transformations.RotateGaussianARD(A, alpha) rotS = transformations.RotateGaussianMarkovChain(S, A, rotA) rotB = transformations.RotateGaussianMatrixARD(B, beta, axis='cols') R_S = transformations.RotationOptimizer(rotS, rotB, K) # Iterate for ind in range(int(maxiter / 5)): Q.update(repeat=5) #Q.update(X, S, A, alpha, rho, B, beta, C, gamma, tau, repeat=maxiter) R_X.rotate() R_S.rotate() ## R_X.rotate( ## check_bound=Q.compute_lowerbound, ## check_bound_terms=Q.compute_lowerbound_terms, ## check_gradient=True ## ) ## R_S.rotate( ## check_bound=Q.compute_lowerbound, ## check_bound_terms=Q.compute_lowerbound_terms, ## check_gradient=True ## ) # # SHOW RESULTS # # Mean and standard deviation of the posterior (f_mean, f_squared) = CX.get_moments() f_std = np.sqrt(f_squared - f_mean**2) # Plot observations space for m in range(M): plt.subplot(M, 1, m + 1) plt.plot(y[m, :], 'r.') plt.plot(f[m, :], 'b-') bpplt.errorplot(y=f_mean[m, :], error=2 * f_std[m, :])