def linear_state_space_model(D=3, N=100, M=10): # Dynamics matrix with ARD alpha = Gamma(1e-5, 1e-5, plates=(D,), name='alpha') A = GaussianArrayARD(0, alpha, shape=(D,), plates=(D,), name='A') # Latent states with dynamics X = GaussianMarkovChain(np.zeros(D), # mean of x0 1e-3*np.identity(D), # prec of x0 A, # dynamics np.ones(D), # innovation n=N, # time instances name='X') # Mixing matrix from latent space to observation space using ARD gamma = Gamma(1e-5, 1e-5, plates=(D,), name='gamma') C = GaussianArrayARD(0, gamma, shape=(D,), plates=(M,1), name='C') # Observation noise tau = Gamma(1e-5, 1e-5, name='tau') # Underlying noiseless function F = SumMultiply('i,i', C, X.as_gaussian()) # Noisy observations Y = GaussianArrayARD(F, tau, name='Y') return (Y, F, X, tau, C, gamma, A, alpha)
def model(M=10, N=100, D=3): """ Construct linear state-space model. See, for instance, the following publication: "Fast variational Bayesian linear state-space model" Luttinen (ECML 2013) """ # Dynamics matrix with ARD alpha = Gamma(1e-5, 1e-5, plates=(D,), name='alpha') A = GaussianARD(0, alpha, shape=(D,), plates=(D,), plotter=bpplt.GaussianHintonPlotter(rows=0, cols=1, scale=0), name='A') A.initialize_from_value(np.identity(D)) # Latent states with dynamics X = GaussianMarkovChain(np.zeros(D), # mean of x0 1e-3*np.identity(D), # prec of x0 A, # dynamics np.ones(D), # innovation n=N, # time instances plotter=bpplt.GaussianMarkovChainPlotter(scale=2), name='X') X.initialize_from_value(np.random.randn(N,D)) # Mixing matrix from latent space to observation space using ARD gamma = Gamma(1e-5, 1e-5, plates=(D,), name='gamma') gamma.initialize_from_value(1e-2*np.ones(D)) C = GaussianARD(0, gamma, shape=(D,), plates=(M,1), plotter=bpplt.GaussianHintonPlotter(rows=0, cols=2, scale=0), name='C') C.initialize_from_value(np.random.randn(M,1,D)) # Observation noise tau = Gamma(1e-5, 1e-5, name='tau') tau.initialize_from_value(1e2) # Underlying noiseless function F = SumMultiply('i,i', C, X, name='F') # Noisy observations Y = GaussianARD(F, tau, name='Y') Q = VB(Y, F, C, gamma, X, A, alpha, tau, C) return Q
def run_dlssm(y, f, mask, D, K, maxiter): """ Run VB inference for linear state space model with drifting dynamics. """ (M, N) = np.shape(y) # Dynamics matrix with ARD # alpha : (K) x () alpha = Gamma(1e-5, 1e-5, plates=(K,), name='alpha') # A : (K) x (K) A = GaussianArrayARD(np.identity(K), alpha, shape=(K,), plates=(K,), name='A_S', initialize=False) A.initialize_from_value(np.identity(K)) # State of the drift # S : () x (N,K) S = GaussianMarkovChain(np.ones(K), 1e-6*np.identity(K), A, np.ones(K), n=N, name='S', initialize=False) S.initialize_from_value(np.ones((N,K))) # Projection matrix of the dynamics matrix # Initialize S and B such that BS is identity matrix # beta : (K) x () beta = Gamma(1e-5, 1e-5, plates=(D,K), name='beta') # B : (D) x (D,K) b = np.zeros((D,D,K)) b[np.arange(D),np.arange(D),np.zeros(D,dtype=int)] = 1 B = GaussianArrayARD(0, beta, plates=(D,), name='B', initialize=False) B.initialize_from_value(np.reshape(1*b, (D,D,K))) # BS : (N-1,D) x (D) # TODO/FIXME: Implement __getitem__ method BS = SumMultiply('dk,k->d', B, S.as_gaussian()[...,np.newaxis], # iterator_axis=0, name='BS') # Latent states with dynamics # X : () x (N,D) X = GaussianMarkovChain(np.zeros(D), # mean of x0 1e-3*np.identity(D), # prec of x0 BS, # dynamics np.ones(D), # innovation n=N+1, # time instances name='X', initialize=False) X.initialize_from_value(np.random.randn(N+1,D)) # Mixing matrix from latent space to observation space using ARD # gamma : (D) x () gamma = Gamma(1e-5, 1e-5, plates=(D,K), name='gamma') # C : (M,1) x (D,K) C = GaussianArrayARD(0, gamma, plates=(M,1), name='C', initialize=False) C.initialize_from_random() # Observation noise # tau : () x () tau = Gamma(1e-5, 1e-5, name='tau') # Observations # Y : (M,N) x () F = SumMultiply('dk,d,k', C, X.as_gaussian()[1:], S.as_gaussian(), name='F') Y = GaussianArrayARD(F, tau, name='Y') # # RUN INFERENCE # # Observe data Y.observe(y, mask=mask) # Construct inference machine Q = VB(Y, X, S, A, alpha, B, beta, C, gamma, tau) # # Run inference with rotations. # rotate = False if rotate: # Rotate the D-dimensional state space (C, X) rotB = transformations.RotateGaussianMatrixARD(B, beta, D, K, axis='rows') rotX = transformations.RotateDriftingMarkovChain(X, B, S, rotB) rotC = transformations.RotateGaussianARD(C, gamma) R_X = transformations.RotationOptimizer(rotX, rotC, D) # Rotate the K-dimensional latent dynamics space (B, S) rotA = transformations.RotateGaussianARD(A, alpha) rotS = transformations.RotateGaussianMarkovChain(S, A, rotA) rotB = transformations.RotateGaussianMatrixARD(B, beta, D, K, axis='cols') R_S = transformations.RotationOptimizer(rotS, rotB, K) # Iterate for ind in range(maxiter): print("X update") Q.update(X) print("S update") Q.update(S) print("A update") Q.update(A) print("alpha update") Q.update(alpha) print("B update") Q.update(B) print("beta update") Q.update(beta) print("C update") Q.update(C) print("gamma update") Q.update(gamma) print("tau update") Q.update(tau) if rotate: if ind >= 0: R_X.rotate() if ind >= 0: R_S.rotate() Q.plot_iteration_by_nodes() # # SHOW RESULTS # # Plot observations space plt.figure() bpplt.timeseries_normal(F) bpplt.timeseries(f, 'b-') bpplt.timeseries(y, 'r.') # Plot latent space plt.figure() bpplt.timeseries_gaussian_mc(X, scale=2) # Plot drift space plt.figure() bpplt.timeseries_gaussian_mc(S, scale=2)
def run_lssm(y, f, mask, D, maxiter): """ Run VB inference for linear state space model. """ (M, N) = np.shape(y) # # CONSTRUCT THE MODEL # # Dynamic matrix # alpha: (D) x () alpha = Gamma(1e-5, 1e-5, plates=(D,), name='alpha') # A : (D) x (D) A = Gaussian(np.zeros(D), diagonal(alpha), plates=(D,), name='A') A.initialize_from_value(np.identity(D)) # Latent states with dynamics # X : () x (N,D) X = GaussianMarkovChain(np.zeros(D), # mean of x0 1e-3*np.identity(D), # prec of x0 A, # dynamics np.ones(D), # innovation n=N, # time instances name='X', initialize=False) X.initialize_from_value(np.random.randn(N,D)) # Mixing matrix from latent space to observation space using ARD # gamma : (D) x () gamma = Gamma(1e-5, 1e-5, plates=(D,), name='gamma') # C : (M,1) x (D) C = Gaussian(np.zeros(D), diagonal(gamma), plates=(M,1), name='C') C.initialize_from_value(np.random.randn(M,1,D)) # Observation noise # tau : () x () tau = Gamma(1e-5, 1e-5, name='tau') # Observations # Y : (M,N) x () CX = Dot(C, X.as_gaussian()) Y = Normal(CX, tau, name='Y') # Rotate the D-dimensional latent space rotA = transformations.RotateGaussianARD(A, alpha) rotX = transformations.RotateGaussianMarkovChain(X, A, rotA) rotC = transformations.RotateGaussianARD(C, gamma) R = transformations.RotationOptimizer(rotX, rotC, D) # # RUN INFERENCE # # Observe data Y.observe(y, mask=mask) # Construct inference machine Q = VB(Y, X, A, alpha, C, gamma, tau) # Iterate for ind in range(maxiter): Q.update(X, A, alpha, C, gamma, tau) R.rotate() # # SHOW RESULTS # plt.figure() bpplt.timeseries_normal(CX) bpplt.timeseries(f, 'b-') bpplt.timeseries(y, 'r.')
def model(M, N, D, K): """ Construct the linear state-space model with time-varying dynamics For reference, see the following publication: (TODO) """ # # The model block for the latent mixing weight process # # Dynamics matrix with ARD # beta : (K) x () beta = Gamma(1e-5, 1e-5, plates=(K,), name='beta') # B : (K) x (K) B = GaussianARD(np.identity(K), beta, shape=(K,), plates=(K,), name='B', plotter=bpplt.GaussianHintonPlotter(rows=0, cols=1, scale=0), initialize=False) B.initialize_from_value(np.identity(K)) # Mixing weight process, that is, the weights in the linear combination of # state dynamics matrices # S : () x (N,K) S = GaussianMarkovChain(np.ones(K), 1e-6*np.identity(K), B, np.ones(K), n=N, name='S', plotter=bpplt.GaussianMarkovChainPlotter(scale=2), initialize=False) s = 10*np.random.randn(N,K) s[:,0] = 10 S.initialize_from_value(s) # # The model block for the latent states # # Projection matrix of the dynamics matrix # alpha : (K) x () alpha = Gamma(1e-5, 1e-5, plates=(D,K), name='alpha') alpha.initialize_from_value(1*np.ones((D,K))) # A : (D) x (D,K) A = GaussianARD(0, alpha, shape=(D,K), plates=(D,), name='A', plotter=bpplt.GaussianHintonPlotter(rows=0, cols=1, scale=0), initialize=False) # Initialize S and A such that A*S is almost an identity matrix a = np.zeros((D,D,K)) a[np.arange(D),np.arange(D),np.zeros(D,dtype=int)] = 1 a[:,:,0] = np.identity(D) / s[0,0] a[:,:,1:] = 0.1/s[0,0]*np.random.randn(D,D,K-1) A.initialize_from_value(a) # Latent states with dynamics # X : () x (N,D) X = VaryingGaussianMarkovChain(np.zeros(D), # mean of x0 1e-3*np.identity(D), # prec of x0 A, # dynamics matrices S._convert(GaussianMoments)[1:], # temporal weights np.ones(D), # innovation n=N, # time instances name='X', plotter=bpplt.GaussianMarkovChainPlotter(scale=2), initialize=False) X.initialize_from_value(np.random.randn(N,D)) # # The model block for observations # # Mixing matrix from latent space to observation space using ARD # gamma : (D) x () gamma = Gamma(1e-5, 1e-5, plates=(D,), name='gamma') gamma.initialize_from_value(1e-2*np.ones(D)) # C : (M,1) x (D) C = GaussianARD(0, gamma, shape=(D,), plates=(M,1), name='C', plotter=bpplt.GaussianHintonPlotter(rows=0, cols=2, scale=0)) C.initialize_from_value(np.random.randn(M,1,D)) # Noiseless process # F : (M,N) x () F = SumMultiply('d,d', C, X, name='F') # Observation noise # tau : () x () tau = Gamma(1e-5, 1e-5, name='tau') tau.initialize_from_value(1e2) # Observations # Y: (M,N) x () Y = GaussianARD(F, tau, name='Y') # Construct inference machine Q = VB(Y, F, C, gamma, X, A, alpha, tau, S, B, beta) return Q
def lssm(M, N, D, K=1, drift_C=False, drift_A=False): if (drift_C or drift_A) and not K > 0: raise ValueError("K must be positive integer when using drift") # Drift weights if drift_A or drift_C: # Dynamics matrix with ARD # beta : (K) x () beta = Gamma(1e-5, 1e-5, plates=(K,), name='beta') # B : (K) x (K) B = GaussianArrayARD(np.identity(K), beta, shape=(K,), plates=(K,), name='B', plotter=bpplt.GaussianHintonPlotter(rows=0, cols=1, scale=0), initialize=False) B.initialize_from_value(np.identity(K)) #B.initialize_from_mean_and_covariance(np.identity(K), # 0.1*np.identity(K)) # State of the drift, that is, temporal weights for dynamics matrices # S : () x (N,K) S = GaussianMarkovChain(np.ones(K), 1e-6*np.identity(K), B, np.ones(K), n=N, name='S', plotter=bpplt.GaussianMarkovChainPlotter(scale=2), initialize=False) #s = np.cumsum(np.random.randn(N,K), axis=0) s = np.random.randn(N,K) s[:,0] = 10 S.initialize_from_value(s) #S.initialize_from_value(np.ones((N,K))+0.01*np.random.randn(N,K)) if not drift_A: # Dynamic matrix # alpha: (D) x () alpha = Gamma(1e-5, 1e-5, plates=(D,), name='alpha') # A : (D) x (D) A = GaussianArrayARD(0, alpha, shape=(D,), plates=(D,), name='A', plotter=bpplt.GaussianHintonPlotter(rows=0, cols=1, scale=0), initialize=False) A.initialize_from_value(np.identity(D)) # Latent states with dynamics # X : () x (N,D) X = GaussianMarkovChain(np.zeros(D), # mean of x0 1e-3*np.identity(D), # prec of x0 A, # dynamics np.ones(D), # innovation n=N, # time instances name='X', plotter=bpplt.GaussianMarkovChainPlotter(), initialize=False) X.initialize_from_value(np.random.randn(N,D)) else: # Projection matrix of the dynamics matrix # alpha : (K) x () alpha = Gamma(1e-5, 1e-5, plates=(D,K), name='alpha') # A : (D) x (D,K) A = GaussianArrayARD(0, alpha, shape=(D,K), plates=(D,), name='A', plotter=bpplt.GaussianHintonPlotter(rows=0, cols=1, scale=0), initialize=False) # Initialize S and A such that A*S is almost an identity matrix a = np.zeros((D,D,K)) a[np.arange(D),np.arange(D),np.zeros(D,dtype=int)] = 1 a[:,:,0] = np.identity(D) / s[0,0] a[:,:,1:] = 0.1/s[0,0]*np.random.randn(D,D,K-1) A.initialize_from_value(a) #A.initialize_from_mean_and_covariance(a, # 0.1/s[0,0]**2*utils.identity(D,K)) #A.initialize_from_value(a + 0.01*np.random.randn(D,D,K)) # Latent states with dynamics # X : () x (N,D) X = DriftingGaussianMarkovChain(np.zeros(D), # mean of x0 1e-3*np.identity(D), # prec of x0 A, # dynamics matrices S.as_gaussian()[1:], # temporal weights np.ones(D), # innovation n=N, # time instances name='X', plotter=bpplt.GaussianMarkovChainPlotter(scale=2), initialize=False) X.initialize_from_value(np.random.randn(N,D)) if not drift_C: # Mixing matrix from latent space to observation space using ARD # gamma : (D) x () gamma = Gamma(1e-5, 1e-5, plates=(D,), name='gamma') # C : (M,1) x (D) C = GaussianArrayARD(0, gamma, shape=(D,), plates=(M,1), name='C', plotter=bpplt.GaussianHintonPlotter(rows=0, cols=2, scale=0)) C.initialize_from_value(np.random.randn(M,1,D)) #C.initialize_from_random() #C.initialize_from_mean_and_covariance(C.random(), # 0.1*utils.identity(D)) # Noiseless process # F : (M,N) x () F = SumMultiply('d,d', C, X.as_gaussian(), name='F') else: # Mixing matrix from latent space to observation space using ARD # gamma : (D,K) x () gamma = Gamma(1e-5, 1e-5, plates=(D,K), name='gamma') # C : (M,1) x (D,K) C = GaussianArrayARD(0, gamma, shape=(D,K), plates=(M,1), name='C', plotter=bpplt.GaussianHintonPlotter(rows=0, cols=2, scale=0)) C.initialize_from_random() #C.initialize_from_mean_and_covariance(C.random(), # 0.1*utils.identity(D, K)) # Noiseless process # F : (M,N) x () F = SumMultiply('dk,d,k', C, X.as_gaussian(), S.as_gaussian(), name='F') # Observation noise # tau : () x () tau = Gamma(1e-5, 1e-5, name='tau') tau.initialize_from_value(1e2) # Observations # Y: (M,N) x () Y = GaussianArrayARD(F, tau, name='Y') # Construct inference machine if drift_C or drift_A: Q = VB(Y, F, C, gamma, X, A, alpha, tau, S, B, beta) else: Q = VB(Y, F, C, gamma, X, A, alpha, tau) return Q
import numpy as np np.random.seed(1) from bayespy.nodes import GaussianARD, GaussianMarkovChain, Gamma, Dot M = 30 N = 400 D = 10 alpha = Gamma(1e-5, 1e-5, plates=(D, ), name='alpha') A = GaussianARD(0, alpha, shape=(D, ), plates=(D, ), name='A') X = GaussianMarkovChain(np.zeros(D), 1e-3 * np.identity(D), A, np.ones(D), n=N, name='X') gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma') C = GaussianARD(0, gamma, shape=(D, ), plates=(M, 1), name='C') F = Dot(C, X, name='F') C.initialize_from_random() tau = Gamma(1e-5, 1e-5, name='tau') Y = GaussianARD(F, tau, name='Y') from bayespy.inference import VB Q = VB(X, C, gamma, A, alpha, tau, Y) w = 0.3 a = np.array([[np.cos(w), -np.sin(w), 0, 0], [np.sin(w), np.cos(w), 0, 0], [0, 0, 1, 0], [0, 0, 0, 0]]) c = np.random.randn(M, 4) x = np.empty((N, 4)) f = np.empty((M, N)) y = np.empty((M, N)) x[0] = 10 * np.random.randn(4)
import numpy as np np.random.seed(1) from bayespy.nodes import GaussianARD, GaussianMarkovChain, Gamma, Dot M = 30 N = 400 D = 10 alpha = Gamma(1e-5, 1e-5, plates=(D, ), name='alpha') A = GaussianARD(0, alpha, shape=(D, ), plates=(D, ), name='A') X = GaussianMarkovChain(np.zeros(D), 1e-3 * np.identity(D), A, np.ones(D), n=N, name='X') gamma = Gamma(1e-5, 1e-5, plates=(D, ), name='gamma') C = GaussianARD(0, gamma, shape=(D, ), plates=(M, 1), name='C') F = Dot(C, X, name='F') C.initialize_from_random() tau = Gamma(1e-5, 1e-5, name='tau') Y = GaussianARD(F, tau, name='Y') from bayespy.inference import VB Q = VB(X, C, gamma, A, alpha, tau, Y) w = 0.3 a = np.array([[np.cos(w), -np.sin(w), 0, 0], [np.sin(w), np.cos(w), 0, 0], [0, 0, 1, 0], [0, 0, 0, 0]]) c = np.random.randn(M, 4) x = np.empty((N, 4))