def test_is_transition_matrix(self): self.assertTrue(is_transition_matrix(self.T)) """Larger test-case to prevent too restrictive tolerance settings""" X = np.random.random((2000, 2000)) Tlarge = X / X.sum(axis=1)[:, np.newaxis] self.assertTrue(is_transition_matrix(Tlarge))
def test_discrete_4_2(self): # 4x4 transition matrix nstates = 2 P = np.array([[0.90, 0.10, 0.00, 0.00], [0.10, 0.89, 0.01, 0.00], [0.00, 0.01, 0.89, 0.10], [0.00, 0.00, 0.10, 0.90]]) # generate realization import msmtools.generation as msmgen T = 10000 dtrajs = [msmgen.generate_traj(P, T)] C = msmest.count_matrix(dtrajs, 1).toarray() # estimate initial HMM with 2 states - should be identical to P hmm = init_discrete_hmm(dtrajs, nstates) # Test if model fit is close to reference. Note that we do not have an exact reference, so we cannot set the # tolerance in a rigorous way to test statistical significance. These are just sanity checks. Tij = hmm.transition_matrix B = hmm.output_model.output_probabilities # Test stochasticity import msmtools.analysis as msmana msmana.is_transition_matrix(Tij) np.allclose(B.sum(axis=1), np.ones(B.shape[0])) # if (B[0,0]<B[1,0]): # B = B[np.array([1,0]),:] Tij_ref = np.array([[0.99, 0.01], [0.01, 0.99]]) Bref = np.array([[0.5, 0.5, 0.0, 0.0], [0.0, 0.0, 0.5, 0.5]]) assert(np.max(Tij-Tij_ref) < 0.01) assert(np.max(B-Bref) < 0.05 or np.max(B[[1, 0]]-Bref) < 0.05)
def test_sample_nonrev_1(self): P = sample_tmatrix(self.C, reversible=False) assert np.all(P.shape == self.C.shape) assert is_transition_matrix(P) # same with boject sampler = tmatrix_sampler(self.C, reversible=False) P = sampler.sample() assert np.all(P.shape == self.C.shape) assert is_transition_matrix(P)
def test_sample_nonrev_10(self): sampler = tmatrix_sampler(self.C, reversible=False) Ps = sampler.sample(nsamples=10) assert len(Ps) == 10 for i in range(10): assert np.all(Ps[i].shape == self.C.shape) assert is_transition_matrix(Ps[i])
def update(self, Tij, Pi=None): r""" Updates the transition matrix and recomputes all derived quantities """ # EMMA imports from msmtools import analysis as msmana # save a copy of the transition matrix self._Tij = np.array(Tij) assert msmana.is_transition_matrix( self._Tij), 'Given transition matrix is not a stochastic matrix' assert self._Tij.shape[ 0] == self._nstates, 'Given transition matrix has unexpected number of states ' # initial / stationary distribution if (Pi is not None): assert np.all( Pi >= 0 ), 'Given initial distribution contains negative elements.' Pi = np.array(Pi) / np.sum( Pi) # ensure normalization and make a copy if (self._stationary): pT = msmana.stationary_distribution(self._Tij) if Pi is None: # stationary and no stationary distribution fixed, so computing it from trans. mat. self._Pi = pT else: # stationary but stationary distribution is fixed, so the transition matrix must be consistent assert np.allclose(Pi, pT), 'Stationary HMM requested, but given distribution is not the ' \ 'stationary distribution of the given transition matrix.' self._Pi = Pi else: if Pi is None: # no initial distribution given, so use stationary distribution anyway self._Pi = msmana.stationary_distribution(self._Tij) else: self._Pi = Pi # reversible if self._reversible: assert msmana.is_reversible( Tij ), 'Reversible HMM requested, but given transition matrix is not reversible.' # try to do eigendecomposition by default, because it's very cheap for hidden transition matrices from scipy.linalg import LinAlgError try: if self._reversible: self._R, self._D, self._L = msmana.rdl_decomposition( self._Tij, norm='reversible') # everything must be real-valued self._R = self._R.real self._D = self._D.real self._L = self._L.real else: self._R, self._D, self._L = msmana.rdl_decomposition( self._Tij, norm='standard') self._eigenvalues = np.diag(self._D) self._spectral_decomp_available = True except LinAlgError: logger().warn('Eigendecomposition failed for transition matrix\n' + str(self._Tij) + '\nspectral properties will not be available') self._spectral_decomp_available = False
def test_discrete_4_2(self): # 4x4 transition matrix n_states = 2 P = np.array([[0.90, 0.10, 0.00, 0.00], [0.10, 0.89, 0.01, 0.00], [0.00, 0.01, 0.89, 0.10], [0.00, 0.00, 0.10, 0.90]]) # generate realization T = 50000 dtrajs = [MarkovStateModel(P).simulate(T)] # estimate initial HMM with 2 states - should be identical to P hmm = initial_guess_discrete_from_data(dtrajs, n_states, lagtime=1, regularize=False) # Test if model fit is close to reference. Note that we do not have an exact reference, so we cannot set the # tolerance in a rigorous way to test statistical significance. These are just sanity checks. Tij = hmm.transition_model.transition_matrix B = hmm.output_model.output_probabilities # Test stochasticity np.testing.assert_(msmana.is_transition_matrix(Tij)) np.testing.assert_allclose(B.sum(axis=1), np.ones(B.shape[0])) Tij_ref = np.array([[0.99, 0.01], [0.01, 0.99]]) Bref = np.array([[0.5, 0.5, 0.0, 0.0], [0.0, 0.0, 0.5, 0.5]]) np.testing.assert_array_almost_equal(Tij, Tij_ref, decimal=2) if np.max(B - Bref) < .05: np.testing.assert_allclose(B, Bref, atol=0.06) else: np.testing.assert_allclose(B[[1, 0]], Bref, atol=0.06)
def test_transition_matrix(self): import msmtools.analysis as msmana for P in [ self.hmm_lag1.transition_model.transition_matrix, self.hmm_lag10.transition_model.transition_matrix ]: np.testing.assert_(msmana.is_transition_matrix(P)) np.testing.assert_(msmana.is_reversible(P))
def test_transition_matrix(self): import msmtools.analysis as msmana for P in [ self.hmsm_lag1.transition_matrix, self.hmsm_lag1.transition_matrix ]: assert msmana.is_transition_matrix(P) assert msmana.is_reversible(P)
def test_transition_matrix(self): """Test example transition matrices. """ Tij = testsystems.generate_transition_matrix(n_states=3, reversible=False) assert not is_reversible(Tij) Tij = testsystems.generate_transition_matrix(n_states=3, reversible=True) assert Tij.shape == (3, 3) assert is_transition_matrix(Tij)
def test_3state_prev(self): dtraj = np.array([0, 1, 2, 0, 3, 4]) import msmtools.estimation as msmest for rev in [True, False]: hmm = initial_guess_discrete_from_data(dtraj, n_hidden_states=3, lagtime=1, reversible=rev) assert msmana.is_transition_matrix(hmm.transition_model.transition_matrix) if rev: assert msmana.is_reversible(hmm.transition_model.transition_matrix) assert np.allclose(hmm.output_probabilities.sum(axis=1), 1) for rev in [True, False]: C = TransitionCountEstimator(lagtime=1, count_mode="sliding").fit(dtraj).fetch_model().count_matrix C += msmest.prior_neighbor(C, 0.001) hmm = initial_guess_discrete_from_data(dtraj, n_hidden_states=3, lagtime=1, reversible=rev) np.testing.assert_(msmana.is_transition_matrix(hmm.transition_model.transition_matrix)) if rev: np.testing.assert_(msmana.is_reversible(hmm.transition_model.transition_matrix)) np.testing.assert_allclose(hmm.output_model.output_probabilities.sum(axis=1), 1.)
def test_transition_matrix_obs(self): assert np.array_equal(self.hmsm_lag1.transition_matrix_obs().shape, (self.hmsm_lag1.nstates_obs,self.hmsm_lag1.nstates_obs)) assert np.array_equal(self.hmsm_lag10.transition_matrix_obs().shape, (self.hmsm_lag10.nstates_obs,self.hmsm_lag10.nstates_obs)) for T in [self.hmsm_lag1.transition_matrix_obs(), self.hmsm_lag1.transition_matrix_obs(k=2), self.hmsm_lag10.transition_matrix_obs(), self.hmsm_lag10.transition_matrix_obs(k=4)]: assert msmana.is_transition_matrix(T) assert msmana.is_reversible(T)
def test_transition_matrix_samples(self): Psamples = self.sampled_hmm_lag10.transition_matrix_samples # shape assert np.array_equal(Psamples.shape, (self.nsamples, self.nstates, self.nstates)) # consistency import msmtools.analysis as msmana for P in Psamples: assert msmana.is_transition_matrix(P) assert msmana.is_reversible(P)
def test_transition_matrix_samples(self): Psamples = np.array([m.transition_matrix for m in self.bhmm]) # shape assert np.array_equal(np.shape(Psamples), (self.n_samples, self.n_states, self.n_states)) # consistency import msmtools.analysis as msmana for P in Psamples: assert msmana.is_transition_matrix(P) assert msmana.is_reversible(P)
def _transition_matrix_samples(self, msm): Psamples = msm.sample_f('transition_matrix') # shape assert np.array_equal(np.shape(Psamples), (self.nsamples, self.nstates, self.nstates)) # consistency import msmtools.analysis as msmana for P in Psamples: assert msmana.is_transition_matrix(P) assert msmana.is_reversible(P)
def test_2state_2obs_Pgiven(self): obs = np.array([0, 0, 1, 1, 0]) Aref = np.array([[1.0]]) for rev in [True, False]: # reversibiliy doesn't matter in this example hmm = initial_guess_discrete_from_data(obs, n_hidden_states=1, lagtime=1, reversible=rev) np.testing.assert_(msmana.is_transition_matrix(hmm.transition_model.transition_matrix)) np.testing.assert_allclose(hmm.transition_model.transition_matrix, Aref) # output must be 1 x 2, and no zeros np.testing.assert_equal(hmm.output_model.output_probabilities.shape, (1, 2)) np.testing.assert_(np.all(hmm.output_model.output_probabilities > 0))
def test_3state_prev(self): import msmtools.analysis as msmana dtraj = np.array([0, 1, 2, 0, 3, 4]) C = msmest.count_matrix(dtraj, 1).toarray() for rev in [True, False]: hmm = init_discrete_hmm(dtraj, 3, reversible=rev) assert msmana.is_transition_matrix(hmm.transition_matrix) if rev: assert msmana.is_reversible(hmm.transition_matrix) assert np.allclose(hmm.output_model.output_probabilities.sum(axis=1), 1)
def test_discrete_2_2(self): # 2x2 transition matrix P = np.array([[0.99, 0.01], [0.01, 0.99]]) # generate realization import msmtools.generation as msmgen T = 10000 dtrajs = [msmgen.generate_traj(P, T)] # estimate initial HMM with 2 states - should be identical to P hmm = initdisc.initial_model_discrete(dtrajs, 2) # test A = hmm.transition_matrix B = hmm.output_model.output_probabilities # Test stochasticity import msmtools.analysis as msmana msmana.is_transition_matrix(A) np.allclose(B.sum(axis=1), np.ones(B.shape[0])) # A should be close to P if (B[0, 0] < B[1, 0]): B = B[np.array([1, 0]), :] assert (np.max(A - P) < 0.01) assert (np.max(B - np.eye(2)) < 0.01)
def test_discrete_6_3(self): # 4x4 transition matrix nstates = 3 P = np.array([[0.90, 0.10, 0.00, 0.00, 0.00, 0.00], [0.20, 0.79, 0.01, 0.00, 0.00, 0.00], [0.00, 0.01, 0.84, 0.15, 0.00, 0.00], [0.00, 0.00, 0.05, 0.94, 0.01, 0.00], [0.00, 0.00, 0.00, 0.02, 0.78, 0.20], [0.00, 0.00, 0.00, 0.00, 0.10, 0.90]]) # generate realization import msmtools.generation as msmgen T = 10000 dtrajs = [msmgen.generate_traj(P, T)] # estimate initial HMM with 2 states - should be identical to P hmm = initdisc.initial_model_discrete(dtrajs, nstates) # Test stochasticity and reversibility Tij = hmm.transition_matrix B = hmm.output_model.output_probabilities import msmtools.analysis as msmana msmana.is_transition_matrix(Tij) msmana.is_reversible(Tij) np.allclose(B.sum(axis=1), np.ones(B.shape[0]))
def test_discrete_2_2(self): # 2x2 transition matrix P = np.array([[0.99, 0.01], [0.01, 0.99]]) # generate realization import msmtools.generation as msmgen T = 10000 dtrajs = [msmgen.generate_traj(P, T)] C = msmest.count_matrix(dtrajs, 1).toarray() # estimate initial HMM with 2 states - should be identical to P hmm = init_discrete_hmm(dtrajs, 2) # test A = hmm.transition_matrix B = hmm.output_model.output_probabilities # Test stochasticity import msmtools.analysis as msmana msmana.is_transition_matrix(A) np.allclose(B.sum(axis=1), np.ones(B.shape[0])) # A should be close to P if B[0, 0] < B[1, 0]: B = B[np.array([1, 0]), :] assert(np.max(A-P) < 0.01) assert(np.max(B-np.eye(2)) < 0.01)
def test_discrete_6_3(self): # 4x4 transition matrix nstates = 3 P = np.array([[0.90, 0.10, 0.00, 0.00, 0.00, 0.00], [0.20, 0.79, 0.01, 0.00, 0.00, 0.00], [0.00, 0.01, 0.84, 0.15, 0.00, 0.00], [0.00, 0.00, 0.05, 0.94, 0.01, 0.00], [0.00, 0.00, 0.00, 0.02, 0.78, 0.20], [0.00, 0.00, 0.00, 0.00, 0.10, 0.90]]) # generate realization import msmtools.generation as msmgen T = 10000 dtrajs = [msmgen.generate_traj(P, T)] C = msmest.count_matrix(dtrajs, 1).toarray() # estimate initial HMM with 2 states - should be identical to P hmm = init_discrete_hmm(dtrajs, nstates) # Test stochasticity and reversibility Tij = hmm.transition_matrix B = hmm.output_model.output_probabilities import msmtools.analysis as msmana msmana.is_transition_matrix(Tij) msmana.is_reversible(Tij) np.allclose(B.sum(axis=1), np.ones(B.shape[0]))
def P(self, value): self._P = value import msmtools.analysis as msmana # check input if self._P is not None: if not msmana.is_transition_matrix(self._P, tol=1e-8): raise ValueError('T is not a transition matrix.') # set states self.nstates = _np.shape(self._P)[0] if self.reversible is None: self.reversible = msmana.is_reversible(self._P) from scipy.sparse import issparse self.sparse = issparse(self._P)
def transition_matrix(self, T): # Check transition matrix. if mana.is_transition_matrix(T): if not mana.is_reversible(T): warnings.warn("The transition matrix is not reversible.") if not mana.is_connected(T): warnings.warn("The transition matrix is not connected.") self._transition_matrix = T else: warnings.warn( "Not a transition matrix. Has to be square and rows must sum to one." )
def update(self, Tij, Pi=None): r""" Updates the transition matrix and recomputes all derived quantities """ # EMMA imports from msmtools import analysis as msmana # save a copy of the transition matrix self._Tij = np.array(Tij) assert msmana.is_transition_matrix(self._Tij), 'Given transition matrix is not a stochastic matrix' assert self._Tij.shape[0] == self._nstates, 'Given transition matrix has unexpected number of states ' # initial / stationary distribution if (Pi is not None): assert np.all(Pi >= 0), 'Given initial distribution contains negative elements.' Pi = np.array(Pi) / np.sum(Pi) # ensure normalization and make a copy if (self._stationary): pT = msmana.stationary_distribution(self._Tij) if Pi is None: # stationary and no stationary distribution fixed, so computing it from trans. mat. self._Pi = pT else: # stationary but stationary distribution is fixed, so the transition matrix must be consistent assert np.allclose(Pi, pT), 'Stationary HMM requested, but given distribution is not the ' \ 'stationary distribution of the given transition matrix.' self._Pi = Pi else: if Pi is None: # no initial distribution given, so use stationary distribution anyway self._Pi = msmana.stationary_distribution(self._Tij) else: self._Pi = Pi # reversible if self._reversible: assert msmana.is_reversible(Tij), 'Reversible HMM requested, but given transition matrix is not reversible.' # try to do eigendecomposition by default, because it's very cheap for hidden transition matrices from scipy.linalg import LinAlgError try: if self._reversible: self._R, self._D, self._L = msmana.rdl_decomposition(self._Tij, norm='reversible') # everything must be real-valued self._R = self._R.real self._D = self._D.real self._L = self._L.real else: self._R, self._D, self._L = msmana.rdl_decomposition(self._Tij, norm='standard') self._eigenvalues = np.diag(self._D) self._spectral_decomp_available = True except LinAlgError: logger().warn('Eigendecomposition failed for transition matrix\n'+str(self._Tij)+ '\nspectral properties will not be available') self._spectral_decomp_available = False
def is_reversible(P): """ Returns if P is reversible on its weakly connected sets """ import msmtools.analysis as msmana # treat each weakly connected set separately sets = connected_sets(P, strong=False) for s in sets: Ps = P[s, :][:, s] if not msmana.is_transition_matrix(Ps): return False # isn't even a transition matrix! pi = msmana.stationary_distribution(Ps) X = pi[:, None] * Ps if not np.allclose(X, X.T): return False # survived. return True
def update(self, Pi, Tij): r""" Updates the transition matrix and recomputes all derived quantities """ from msmtools import analysis as msmana # update transition matrix by copy self._Tij = np.array(Tij) assert msmana.is_transition_matrix(self._Tij), 'Given transition matrix is not a stochastic matrix' assert self._Tij.shape[0] == self._nstates, 'Given transition matrix has unexpected number of states ' # reset spectral decomposition self._spectral_decomp_available = False # check initial distribution assert np.all(Pi >= 0), 'Given initial distribution contains negative elements.' assert np.any(Pi > 0), 'Given initial distribution is zero' self._Pi = np.array(Pi) / np.sum(Pi) # ensure normalization and make a copy
def _transition_matrix(self, msm): P = msm.transition_matrix # should be ndarray by default # assert (isinstance(P, np.ndarray)) assert (isinstance(P, np.ndarray) or isinstance(P, scipy.sparse.csr_matrix)) # shape assert (np.all(P.shape == (msm.n_states, msm.n_states))) # test transition matrix properties import msmtools.analysis as msmana assert (msmana.is_transition_matrix(P)) assert (msmana.is_connected(P)) # REVERSIBLE if msm.reversible: assert (msmana.is_reversible(P))
def _transition_matrix_samples(self, msm, given_pi): Psamples = [s.transition_matrix for s in msm.samples] # shape assert np.array_equal(np.shape(Psamples), (self.nsamples, self.n_states, self.n_states)) # consistency import msmtools.analysis as msmana for P in Psamples: assert msmana.is_transition_matrix(P) try: assert msmana.is_reversible(P) except AssertionError: # re-do calculation msmtools just performed to get details from msmtools.analysis import stationary_distribution mu = stationary_distribution(P) X = mu[:, np.newaxis] * P np.testing.assert_allclose(X, np.transpose(X), atol=1e-12, err_msg="P not reversible, given_pi={}".format(given_pi))
def __init__(self, T, K0, pi, dt=1.0, sparsity=None, t_agg=None, tol=1.0E7, maxiter=100000, on_error='raise'): from msmtools.analysis import is_transition_matrix super(CrommelinVandenEijndenEstimator, self).__init__(T, pi, dt=dt, sparsity=sparsity, t_agg=t_agg, tol=tol, maxiter=maxiter, on_error=on_error) if not K0.shape[0] == K0.shape[1] == self.N: raise ValueError( 'Shapes of K0 matrix (initial guess) and count matrix do not match.' ) if not is_transition_matrix(T): raise_or_warn('T is not a valid transition matrix.', self.on_error) evals, self.U, self.Uinv = eigen_decomposition(T, self.pi) if not np.all( np.abs(evals) > 0.0): # don't allow eigenvalue==exactly zero raise ValueError( 'T has eigenvalues that are exactly zero, can\'t proceed with rate matrix estimation. ' 'If the CVE method is only used to intitialize the KL method, you might try to call the KL ' 'method with an initial guess of the rate matrix (K0) instead of intializing with CVE.' ) assert np.allclose(self.Uinv.dot(T).dot(self.U), np.diag(evals)) # self-consistency test self.c = np.abs(evals) self.L = np.diag(np.log(np.abs(evals)) / self.dt) theta = self.pi[self.I] * K0[self.I, self.J] self.initial = np.maximum(theta, self.lower_bounds)
def _transition_matrix_stats(self, msm): import msmtools.analysis as msmana # mean Pmean = msm.sample_mean('transition_matrix') # test shape and consistency assert np.array_equal(Pmean.shape, (self.nstates, self.nstates)) assert msmana.is_transition_matrix(Pmean) # std Pstd = msm.sample_std('transition_matrix') # test shape assert np.array_equal(Pstd.shape, (self.nstates, self.nstates)) # conf L, R = msm.sample_conf('transition_matrix') # test shape assert np.array_equal(L.shape, (self.nstates, self.nstates)) assert np.array_equal(R.shape, (self.nstates, self.nstates)) # test consistency assert np.all(L <= Pmean) assert np.all(R >= Pmean)
def test_transition_matrix_stats(self): import msmtools.analysis as msmana # mean Pmean = self.sampled_hmm_lag10.transition_matrix_mean # test shape and consistency assert np.array_equal(Pmean.shape, (self.nstates, self.nstates)) assert msmana.is_transition_matrix(Pmean) # std Pstd = self.sampled_hmm_lag10.transition_matrix_std # test shape assert np.array_equal(Pstd.shape, (self.nstates, self.nstates)) # conf L, R = self.sampled_hmm_lag10.transition_matrix_conf # test shape assert np.array_equal(L.shape, (self.nstates, self.nstates)) assert np.array_equal(R.shape, (self.nstates, self.nstates)) # test consistency assert np.all(L <= Pmean) assert np.all(R >= Pmean)
def test_discrete_2_2(self): # 2x2 transition matrix P = np.array([[0.99, 0.01], [0.01, 0.99]]) # generate realization T = 10000 dtrajs = [MarkovStateModel(P).simulate(T)] # estimate initial HMM with 2 states - should be identical to P init_hmm = initial_guess_discrete_from_data(dtrajs, n_hidden_states=2, lagtime=1) # test A = init_hmm.transition_model.transition_matrix B = init_hmm.output_model.output_probabilities # Test stochasticity np.testing.assert_(msmana.is_transition_matrix(A)) np.testing.assert_allclose(B.sum(axis=1), np.ones(B.shape[0])) # A should be close to P if B[0, 0] < B[1, 0]: B = B[np.array([1, 0]), :] np.testing.assert_array_almost_equal(A, P, decimal=2) np.testing.assert_array_almost_equal(B, np.eye(2), decimal=2)
def transition_matrix(self, T): """Set transition matrix Parameters ---------- T : 2D numpy.ndarray. Transition matrix. Should be row stochastic and ergodic. """ # Check transition matrix. if mana.is_transition_matrix(T): if not mana.is_reversible(T): warnings.warn("The transition matrix is not reversible.") if not mana.is_connected(T): warnings.warn("The transition matrix is not connected.") else: warnings.warn( "Not a transition matrix. Has to be square and rows must sum to one." ) self._transition_matrix = T
def _transition_matrix_stats(self, msm): import msmtools.analysis as msmana # mean Ps = np.array([s.transition_matrix for s in msm.samples]) Pmean = Ps.mean(axis=0) # test shape and consistency assert np.array_equal(Pmean.shape, (self.n_states, self.n_states)) assert msmana.is_transition_matrix(Pmean) # std Pstd = Ps.std(axis=0) # test shape assert np.array_equal(Pstd.shape, (self.n_states, self.n_states)) # conf L, R = confidence_interval(Ps) # test shape assert np.array_equal(L.shape, (self.n_states, self.n_states)) assert np.array_equal(R.shape, (self.n_states, self.n_states)) # test consistency assert np.all(L <= Pmean) assert np.all(R >= Pmean)
def test_init(self): initial_model = init_model_gaussian1d(self._observations, self._nstates) assert initial_model.nstates == self._nstates assert msmana.is_transition_matrix(initial_model.transition_matrix)
def test_transition_matrix(self): import msmtools.analysis as msmana for P in [self.hmm_lag1.transition_matrix, self.hmm_lag1.transition_matrix]: assert msmana.is_transition_matrix(P) assert msmana.is_reversible(P)