def stationary_distribution_sensitivity(T, j):
r"""Sensitivity matrix of a stationary distribution element.
Parameters
----------
T : (M, M) ndarray
Transition matrix (stochastic matrix).
j : int
Index of stationary distribution element
for which sensitivity matrix is computed.
Returns
-------
S : (M, M) ndarray
Sensitivity matrix for the specified element
of the stationary distribution.
"""
if issparse(T):
_showSparseConversionWarning()
stationary_distribution_sensitivity(T.todense(), j)
elif isdense(T):
return dense.sensitivity.stationary_distribution_sensitivity(T, j)
else:
raise _type_not_supported
def eigenvector_sensitivity(T, k, j, right=True):
r"""Sensitivity matrix of a selected eigenvector element.
Parameters
----------
T : (M, M) ndarray
Transition matrix (stochastic matrix).
k : int
Eigenvector index
j : int
Element index
right : bool
If True compute for right eigenvector, otherwise compute for left eigenvector.
Returns
-------
S : (M, M) ndarray
Sensitivity matrix for the j-th element of the k-th eigenvector.
"""
if issparse(T):
_showSparseConversionWarning()
eigenvector_sensitivity(T.todense(), k, j, right=right)
elif isdense(T):
return dense.sensitivity.eigenvector_sensitivity(T, k, j, right=right)
else:
raise _type_not_supported
def coarsegrain(F, sets):
r"""Coarse-grains the flux to the given sets.
Parameters
----------
F : (n, n) ndarray or scipy.sparse matrix
Matrix of flux values between pairs of states.
sets : list of array-like of ints
The sets of states onto which the flux is coarse-grained.
Notes
-----
The coarse grained flux is defined as
.. math:: fc_{I,J} = \sum_{i \in I,j \in J} f_{i,j}
Note that if you coarse-grain a net flux, it does n ot necessarily
have a net flux property anymore. If want to make sure you get a
netflux, use to_netflux(coarsegrain(F,sets)).
References
----------
.. [1] F. Noe, Ch. Schuette, E. Vanden-Eijnden, L. Reich and
T. Weikl: Constructing the Full Ensemble of Folding Pathways
from Short Off-Equilibrium Simulations.
Proc. Natl. Acad. Sci. USA, 106, 19011-19016 (2009)
"""
if issparse(F):
return sparse.tpt.coarsegrain(F, sets)
elif isdense(F):
return dense.tpt.coarsegrain(F, sets)
else:
raise _type_not_supported
def check_positive(X):
"""Check if all values are positives.
Parameters
----------
X: numpy array or scipy sparse matrix
Matrix to be analyzed
Raises
------
ValueError
If the matrix contains negative values.
Returns
-------
numpy array or scipy sparse matrix
X
"""
if isinstance(X, dok_matrix):
values = np.array(list(X.values()))
elif isinstance(X, lil_matrix):
values = np.array([v for e in X.data for v in e])
elif isdense(X):
values = X
else:
values = X.data
if (values < 0).any():
raise ValueError("The matrix contains negative values.")
return X
def __eq__(self, other):
# Scalar other.
if isscalarlike(other):
if np.isnan(other):
return csr_matrix(self.shape, dtype=np.bool_)
if other == 0:
warn("Comparing a sparse matrix with 0 using == is inefficient"
", try using != instead.", SparseEfficiencyWarning)
all_true = _all_true(self.shape)
inv = self._scalar_binopt(other, operator.ne)
return all_true - inv
else:
return self._scalar_binopt(other, operator.eq)
# Dense other.
elif isdense(other):
return self.todense() == other
# Sparse other.
elif isspmatrix(other):
warn("Comparing sparse matrices using == is inefficient, try using"
" != instead.", SparseEfficiencyWarning)
#TODO sparse broadcasting
if self.shape != other.shape:
return False
elif self.format != other.format:
other = other.asformat(self.format)
res = self._binopt(other,'_ne_')
all_true = _all_true(self.shape)
return all_true - res
else:
return False
def _inequality(self, other, op, op_name, bad_scalar_msg):
# Scalar other.
if isscalarlike(other):
if 0 == other and op_name in ('_le_', '_ge_'):
raise NotImplementedError(" >= and <= don't work with 0.")
elif op(0, other):
warn(bad_scalar_msg, SparseEfficiencyWarning)
other_arr = np.empty(self.shape, dtype=np.result_type(other))
other_arr.fill(other)
other_arr = csr_matrix(other_arr)
return self._binopt(other_arr, op_name)
else:
return self._scalar_binopt(other, op)
# Dense other.
elif isdense(other):
return op(self.todense(), other)
# Sparse other.
elif isspmatrix(other):
#TODO sparse broadcasting
if self.shape != other.shape:
raise ValueError("inconsistent shapes")
elif self.format != other.format:
other = other.asformat(self.format)
if op_name not in ('_ge_', '_le_'):
return self._binopt(other, op_name)
warn("Comparing sparse matrices using >= and <= is inefficient, "
"using <, >, or !=, instead.", SparseEfficiencyWarning)
all_true = _all_true(self.shape)
res = self._binopt(other, '_gt_' if op_name == '_le_' else '_lt_')
return all_true - res
else:
raise ValueError("Operands could not be compared.")
def __ne__(self, other):
# Scalar other.
if isscalarlike(other):
if np.isnan(other):
warn("Comparing a sparse matrix with nan using != is inefficient",
SparseEfficiencyWarning)
all_true = _all_true(self.shape)
return all_true
elif other != 0:
warn("Comparing a sparse matrix with a nonzero scalar using !="
" is inefficient, try using == instead.", SparseEfficiencyWarning)
all_true = _all_true(self.shape)
inv = self._scalar_binopt(other, operator.eq)
return all_true - inv
else:
return self._scalar_binopt(other, operator.ne)
# Dense other.
elif isdense(other):
return self.todense() != other
# Sparse other.
elif isspmatrix(other):
#TODO sparse broadcasting
if self.shape != other.shape:
return True
elif self.format != other.format:
other = other.asformat(self.format)
return self._binopt(other,'_ne_')
else:
return True
def to_netflux(flux):
r"""Compute the netflux from the gross flux.
Parameters
----------
flux : (M, M) ndarray
Matrix of flux values between pairs of states.
Returns
-------
netflux : (M, M) ndarray
Matrix of netflux values between pairs of states.
Notes
-----
The netflux or effective current is defined as
.. math:: f_{ij}^{+}=\max \{ f_{ij}-f_{ji}, 0 \}
:math:`f_{ij}` is the flux for the transition from :math:`A` to
:math:`B`.
References
----------
.. [1] P. Metzner, C. Schuette and E. Vanden-Eijnden.
Transition Path Theory for Markov Jump Processes.
Multiscale Model Simul 7: 1192-1219 (2009)
"""
if issparse(flux):
return sparse.tpt.to_netflux(flux)
elif isdense(flux):
return dense.tpt.to_netflux(flux)
else:
raise _type_not_supported
def is_connected(C, directed=True):
r"""Return true, if the input count matrix is completely connected.
Effectively checking if the number of connected components equals one.
(EMMA function)
Parameters
----------
C : scipy.sparse matrix or numpy ndarray
Count matrix specifying edge weights.
directed : bool, optional
Whether to compute connected components for a directed or
undirected graph. Default is True.
Returns
-------
connected : boolean, returning true only if C is connected.
"""
from scipy.sparse import csr_matrix
from scipy.sparse.sputils import isdense
import scipy.sparse.csgraph as csgraph
if isdense(C):
C = csr_matrix(C)
nc=csgraph.connected_components(C, directed=directed, connection='strong', return_labels=False)
return nc == 1
def total_flux(F, A = None):
r"""Compute the total flux, or turnover flux, that is produced by
the flux sources and consumed by the flux sinks.
Parameters
----------
F : (M, M) ndarray
Matrix of flux values between pairs of states.
A : array_like (optional)
List of integer state labels for set A (reactant)
Returns
-------
F : float
The total flux, or turnover flux, that is produced by the flux
sources and consumed by the flux sinks
References
----------
.. [1] P. Metzner, C. Schuette and E. Vanden-Eijnden.
Transition Path Theory for Markov Jump Processes.
Multiscale Model Simul 7: 1192-1219 (2009)
"""
if issparse(F):
return sparse.tpt.total_flux(F, A = A)
elif isdense(F):
return dense.tpt.total_flux(F, A = A)
else:
raise _type_not_supported
def is_ergodic(T, tol):
"""
checks if T is 'ergodic'
Parameters
----------
T : scipy.sparse matrix
Transition matrix
tol : float
tolerance
Returns
-------
Truth value : bool
True, if # strongly connected components = 1
False, otherwise
"""
if isdense(T):
T = T.tocsr()
if not is_transition_matrix(T, tol):
raise ValueError("given matrix is not a valid transition matrix.")
num_components = connected_components(T, directed=True, \
connection='strong', \
return_labels=False)
return num_components == 1
def committor_sensitivity(T, A, B, i, forward=True):
r"""Sensitivity matrix of a specified committor entry.
Parameters
----------
T : (M, M) ndarray
Transition matrix
A : array_like
List of integer state labels for set A
B : array_like
List of integer state labels for set B
i : int
Compute the sensitivity for committor entry `i`
forward : bool (optional)
Compute the forward committor. If forward
is False compute the backward committor.
Returns
-------
S : (M, M) ndarray
Sensitivity matrix of the specified committor entry.
"""
if issparse(T):
_showSparseConversionWarning()
committor_sensitivity(T.todense(), A, B, i, forward)
elif isdense(T):
if forward:
return dense.sensitivity.forward_committor_sensitivity(T, A, B, i)
else:
return dense.sensitivity.backward_committor_sensitivity(T, A, B, i)
else:
raise _type_not_supported
def mfpt_sensitivity(T, target, i):
r"""Sensitivity matrix of the mean first-passage time from specified state.
Parameters
----------
T : (M, M) ndarray
Transition matrix
target : int or list
Target state or set for mfpt computation
i : int
Compute the sensitivity for state `i`
Returns
-------
S : (M, M) ndarray
Sensitivity matrix for specified state
"""
if issparse(T):
_showSparseConversionWarning()
mfpt_sensitivity(T.todense(), target, i)
elif isdense(T):
return dense.sensitivity.mfpt_sensitivity(T, target, i)
else:
raise _type_not_supported
def f1_per_sample(y_true, y_pred):
if isdense(y_true) or isdense(y_pred):
y_true = sp.csr_matrix(y_true)
y_pred = sp.csr_matrix(y_pred)
sum_axis = 1
true_and_pred = y_true.multiply(y_pred)
tp_sum = count_nonzero(true_and_pred, axis=sum_axis)
pred_sum = count_nonzero(y_pred, axis=sum_axis)
true_sum = count_nonzero(y_true, axis=sum_axis)
with np.errstate(divide='ignore', invalid='ignore'):
precision = _prf_divide(tp_sum, pred_sum)
recall = _prf_divide(tp_sum, true_sum)
f_score = (2 * precision * recall / (1 * precision + recall))
f_score[tp_sum == 0] = 0.0
return f_score
def largest_connected_submatrix(C, directed=True, lcc=None):
r"""Compute the count matrix on the largest connected set.
Parameters
----------
C : scipy.sparse matrix
Count matrix specifying edge weights.
directed : bool, optional
Whether to compute connected components for a directed or
undirected graph. Default is True
lcc : (M,) ndarray, optional
The largest connected set
Returns
-------
C_cc : scipy.sparse matrix
Count matrix of largest completely
connected set of vertices (states)
See also
--------
largest_connected_set
Notes
-----
Viewing the count matrix as the adjacency matrix of a (directed)
graph the larest connected submatrix is the adjacency matrix of
the largest connected set of the corresponding graph. The largest
connected submatrix can be efficiently computed using Tarjan's algorithm.
References
----------
.. [1] Tarjan, R E. 1972. Depth-first search and linear graph
algorithms. SIAM Journal on Computing 1 (2): 146-160.
Examples
--------
>>> from pyemma.msm.estimation import largest_connected_submatrix
>>> C=np.array([10, 1, 0], [2, 0, 3], [0, 0, 4]])
>>> C_cc_directed=largest_connected_submatrix(C)
>>> C_cc_directed
array([[10, 1],
[ 2, 0]])
>>> C_cc_undirected=largest_connected_submatrix(C, directed=False)
>>> C_cc_undirected
array([[10, 1, 0],
[ 2, 0, 3],
[ 0, 0, 4]])
"""
if isdense(C):
return sparse.connectivity.largest_connected_submatrix(csr_matrix(C), directed=directed, lcc=lcc).toarray()
else:
return sparse.connectivity.largest_connected_submatrix(C, directed=directed, lcc=lcc)
def multiply(self, other):
"""Point-wise multiplication by another matrix, vector, or
scalar.
"""
# Scalar multiplication.
if isscalarlike(other):
return self._mul_scalar(other)
# Sparse matrix or vector.
if isspmatrix(other):
if self.shape == other.shape:
if not isinstance(other, fast_csr_matrix):
other = csr_matrix(other)
return self._binopt(other, '_elmul_')
# Single element.
elif other.shape == (1,1):
return self._mul_scalar(other.toarray()[0, 0])
elif self.shape == (1,1):
return other._mul_scalar(self.toarray()[0, 0])
# A row times a column.
elif self.shape[1] == other.shape[0] and self.shape[1] == 1:
return self._mul_sparse_matrix(other.tocsc())
elif self.shape[0] == other.shape[1] and self.shape[0] == 1:
return other._mul_sparse_matrix(self.tocsc())
# Row vector times matrix. other is a row.
elif other.shape[0] == 1 and self.shape[1] == other.shape[1]:
other = dia_matrix((other.toarray().ravel(), [0]),
shape=(other.shape[1], other.shape[1]))
return self._mul_sparse_matrix(other)
# self is a row.
elif self.shape[0] == 1 and self.shape[1] == other.shape[1]:
copy = dia_matrix((self.toarray().ravel(), [0]),
shape=(self.shape[1], self.shape[1]))
return other._mul_sparse_matrix(copy)
# Column vector times matrix. other is a column.
elif other.shape[1] == 1 and self.shape[0] == other.shape[0]:
other = dia_matrix((other.toarray().ravel(), [0]),
shape=(other.shape[0], other.shape[0]))
return other._mul_sparse_matrix(self)
# self is a column.
elif self.shape[1] == 1 and self.shape[0] == other.shape[0]:
copy = dia_matrix((self.toarray().ravel(), [0]),
shape=(self.shape[0], self.shape[0]))
return copy._mul_sparse_matrix(other)
else:
raise ValueError("inconsistent shapes")
# Dense matrix.
if isdense(other):
if self.shape == other.shape:
ret = self.tocoo()
ret.data = np.multiply(ret.data, other[ret.row, ret.col]
).view(np.ndarray).ravel()
return ret
# Single element.
elif other.size == 1:
return self._mul_scalar(other.flat[0])
# Anything else.
return np.multiply(self.todense(), other)
def is_reversible(T, mu=None, tol=1e-15):
r"""Check reversibility of the given transition matrix.
Parameters
----------
T : (M, M) ndarray or scipy.sparse matrix
Transition matrix
mu : (M,) ndarray (optional)
Test reversibility with respect to this vector
tol : float (optional)
Floating point tolerance to check with
Returns
-------
is_reversible : bool
True, if T is reversible, False otherwise
Notes
-----
A transition matrix :math:`T=(t_{ij})` is reversible with respect
to a probability vector :math:`\mu=(\mu_i)` if the follwing holds,
.. math:: \mu_i \, t_{ij}= \mu_j \, t_{ji}.
In this case :math:`\mu` is the stationary vector for :math:`T`,
so that :math:`\mu^T T = \mu^T`.
If the stationary vector is unknown it is computed from :math:`T`
before reversibility is checked.
A reversible transition matrix has purely real eigenvalues. The
left eigenvectors :math:`(l_i)` can be computed from right
eigenvectors :math:`(r_i)` via :math:`l_i=\mu_i r_i`.
Examples
--------
>>> from pyemma.msm.analysis import is_reversible
>>> P=np.array([[0.8, 0.1, 0.1], [0.5, 0.0, 0.5], [0.0, 0.1, 0.9]])
>>> is_reversible(P)
False
>>> T=np.array([[0.9, 0.1, 0.0], [0.5, 0.0, 0.5], [0.0, 0.1, 0.9]])
is_reversible(T)
True
"""
if issparse(T):
return sparse.assessment.is_reversible(T, mu, tol)
elif isdense(T):
return dense.assessment.is_reversible(T, mu, tol)
else:
raise _type_not_supported
def largest_connected_set(C, directed=True):
r"""Largest connected component for a directed graph with edge-weights
given by the count matrix.
Parameters
----------
C : scipy.sparse matrix
Count matrix specifying edge weights.
directed : bool, optional
Whether to compute connected components for a directed or
undirected graph. Default is True.
Returns
-------
lcc : array of integers
The largest connected component of the directed graph.
See also
--------
connected_sets
Notes
-----
Viewing the count matrix as the adjacency matrix of a (directed)
graph the largest connected set is the largest connected set of
nodes of the corresponding graph. The largest connected set of a graph
can be efficiently computed using Tarjan's algorithm.
References
----------
.. [1] Tarjan, R E. 1972. Depth-first search and linear graph
algorithms. SIAM Journal on Computing 1 (2): 146-160.
Examples
--------
>>> import numpy as np
>>> from msmtools.estimation import largest_connected_set
>>> C = np.array([[10, 1, 0], [2, 0, 3], [0, 0, 4]])
>>> lcc_directed = largest_connected_set(C)
>>> lcc_directed
array([0, 1])
>>> lcc_undirected = largest_connected_set(C, directed=False)
>>> lcc_undirected
array([0, 1, 2])
"""
if isdense(C):
return sparse.connectivity.largest_connected_set(csr_matrix(C), directed=directed)
else:
return sparse.connectivity.largest_connected_set(C, directed=directed)
def prior_rev(C, alpha=-1.0):
r"""Prior counts for sampling of reversible transition
matrices.
Prior is defined as
b_ij= alpha if i<=j
b_ij=0 else
Parameters
----------
C : (M, M) ndarray or scipy.sparse matrix
Count matrix
alpha : float (optional)
Value of prior counts
Returns
-------
B : (M, M) ndarray
Matrix of prior counts
Notes
-----
The reversible prior is a matrix with -1 on the upper triangle.
Adding this prior respects the fact that
for a reversible transition matrix the degrees of freedom
correspond essentially to the upper triangular part of the matrix.
The prior is defined as
.. math:: b_{ij} = \left \{ \begin{array}{rl}
\alpha & i \leq j \\
0 & \text{elsewhere}
\end{array} \right .
Examples
--------
>>> from pyemma.msm.estimation import prior_rev
>>> C=np.array([10, 1, 0], [2, 0, 3], [0, 1, 4]])
>>> B=prior_const(C)
>>> B
array([[-1., -1., -1.],
[ 0., -1., -1.],
[ 0., 0., -1.]])
"""
if isdense(C):
return sparse.prior.prior_rev(C, alpha=alpha)
else:
warnings.warn("Prior will be a dense matrix for sparse input")
return sparse.prior.prior_rev(C, alpha=alpha)
def connected_sets(C, directed=True):
r"""Compute connected sets of microstates.
Connected components for a directed graph with edge-weights
given by the count matrix.
Parameters
----------
C : scipy.sparse matrix
Count matrix specifying edge weights.
directed : bool, optional
Whether to compute connected components for a directed or
undirected graph. Default is True.
Returns
-------
cc : list of arrays of integers
Each entry is an array containing all vertices (states) in the
corresponding connected component. The list is sorted
according to the size of the individual components. The
largest connected set is the first entry in the list, lcc=cc[0].
Notes
-----
Viewing the count matrix as the adjacency matrix of a (directed) graph
the connected components are given by the connected components of that
graph. Connected components of a graph can be efficiently computed
using Tarjan's algorithm.
References
----------
.. [1] Tarjan, R E. 1972. Depth-first search and linear graph
algorithms. SIAM Journal on Computing 1 (2): 146-160.
Examples
--------
>>> from pyemma.msm.estimation import connected_sets
>>> C=np.array([10, 1, 0], [2, 0, 3], [0, 0, 4]])
>>> cc_directed=connected_sets(C)
>>> cc_directed
[array([0, 1]), array([2])]
>>> cc_undirected=connected_sets(C, directed=False)
>>> cc_undirected
[array([0, 1, 2])]
"""
if isdense(C):
return sparse.connectivity.connected_sets(csr_matrix(C), directed=directed)
else:
return sparse.connectivity.connected_sets(C, directed=directed)
def prior_rev(C, alpha=-1.0):
r"""Prior counts for sampling of reversible transition
matrices.
Prior is defined as
b_ij= alpha if i<=j
b_ij=0 else
Parameters
----------
C : (M, M) ndarray or scipy.sparse matrix
Count matrix
alpha : float (optional)
Value of prior counts
Returns
-------
B : (M, M) ndarray
Matrix of prior counts
Notes
-----
The reversible prior is a matrix with -1 on the upper triangle.
Adding this prior respects the fact that
for a reversible transition matrix the degrees of freedom
correspond essentially to the upper triangular part of the matrix.
The prior is defined as
.. math:: b_{ij} = \left \{ \begin{array}{rl}
\alpha & i \leq j \\
0 & \text{elsewhere}
\end{array} \right .
Examples
--------
>>> from pyemma.msm.estimation import prior_rev
>>> C=np.array([10, 1, 0], [2, 0, 3], [0, 1, 4]])
>>> B=prior_const(C)
>>> B
array([[-1., -1., -1.],
[ 0., -1., -1.],
[ 0., 0., -1.]])
"""
if isdense(C):
return sparse.prior.prior_rev(C, alpha=alpha)
else:
warnings.warn("Prior will be a dense matrix for sparse input")
return sparse.prior.prior_rev(C, alpha=alpha)
def largest_connected_set(C, directed=True):
r"""Largest connected component for a directed graph with edge-weights
given by the count matrix.
Parameters
----------
C : scipy.sparse matrix
Count matrix specifying edge weights.
directed : bool, optional
Whether to compute connected components for a directed or
undirected graph. Default is True.
Returns
-------
lcc : array of integers
The largest connected component of the directed graph.
See also
--------
connected_sets
Notes
-----
Viewing the count matrix as the adjacency matrix of a (directed)
graph the largest connected set is the largest connected set of
nodes of the corresponding graph. The largest connected set of a graph
can be efficiently computed using Tarjan's algorithm :footcite:`tarjan1972depth`.
References
----------
.. footbibliography::
Examples
--------
>>> import numpy as np
>>> from deeptime.markov.tools.estimation import largest_connected_set
>>> C = np.array([[10, 1, 0], [2, 0, 3], [0, 0, 4]])
>>> lcc_directed = largest_connected_set(C)
>>> lcc_directed
array([0, 1])
>>> lcc_undirected = largest_connected_set(C, directed=False)
>>> lcc_undirected
array([0, 1, 2])
"""
if isdense(C):
return sparse.connectivity.largest_connected_set(csr_matrix(C),
directed=directed)
else:
return sparse.connectivity.largest_connected_set(C, directed=directed)
def connected_sets(C, directed=True):
r"""Compute connected sets of microstates.
Connected components for a directed graph with edge-weights
given by the count matrix.
Parameters
----------
C : scipy.sparse matrix
Count matrix specifying edge weights.
directed : bool, optional
Whether to compute connected components for a directed or
undirected graph. Default is True.
Returns
-------
cc : list of arrays of integers
Each entry is an array containing all vertices (states) in the
corresponding connected component. The list is sorted
according to the size of the individual components. The
largest connected set is the first entry in the list, lcc=cc[0].
Notes
-----
Viewing the count matrix as the adjacency matrix of a (directed) graph
the connected components are given by the connected components of that
graph. Connected components of a graph can be efficiently computed
using Tarjan's algorithm.
References
----------
.. [1] Tarjan, R E. 1972. Depth-first search and linear graph
algorithms. SIAM Journal on Computing 1 (2): 146-160.
Examples
--------
>>> from pyemma.msm.estimation import connected_sets
>>> C=np.array([10, 1, 0], [2, 0, 3], [0, 0, 4]])
>>> cc_directed=connected_sets(C)
>>> cc_directed
[array([0, 1]), array([2])]
>>> cc_undirected=connected_sets(C, directed=False)
>>> cc_undirected
[array([0, 1, 2])]
"""
if isdense(C):
return sparse.connectivity.connected_sets(csr_matrix(C), directed=directed)
else:
return sparse.connectivity.connected_sets(C, directed=directed)
def largest_connected_set(C, directed=True):
r"""Largest connected component for a directed graph with edge-weights
given by the count matrix.
Parameters
----------
C : scipy.sparse matrix
Count matrix specifying edge weights.
directed : bool, optional
Whether to compute connected components for a directed or
undirected graph. Default is True.
Returns
-------
lcc : array of integers
The largest connected component of the directed graph.
See also
--------
connected_sets
Notes
-----
Viewing the count matrix as the adjacency matrix of a (directed)
graph the largest connected set is the largest connected set of
nodes of the corresponding graph. The largest connected set of a graph
can be efficiently computed using Tarjan's algorithm.
References
----------
.. [1] Tarjan, R E. 1972. Depth-first search and linear graph
algorithms. SIAM Journal on Computing 1 (2): 146-160.
Examples
--------
>>> from pyemma.msm.estimation import largest_connected_set
>>> C=np.array([10, 1, 0], [2, 0, 3], [0, 0, 4]])
>>> lcc_directed=largest_connected_set(C)
>>> lcc_directed
array([0, 1])
>>> lcc_undirected=largest_connected_set(C, directed=False)
>>> lcc_undirected
array([0, 1, 2])
"""
if isdense(C):
return sparse.connectivity.largest_connected_set(csr_matrix(C), directed=directed)
else:
return sparse.connectivity.largest_connected_set(C, directed=directed)
def is_connected(C, directed=True):
"""Check connectivity of the given matrix.
Parameters
----------
C : scipy.sparse matrix
Count matrix specifying edge weights.
directed : bool, optional
Whether to compute connected components for a directed or
undirected graph. Default is True.
Returns
-------
is_connected: bool
True if C is connected, False otherwise.
See also
--------
largest_connected_submatrix
Notes
-----
A count matrix is connected if the graph having the count matrix
as adjacency matrix has a single connected component. Connectivity
of a graph can be efficiently checked using Tarjan's algorithm :cite:`tools-est-is-conn-tarjan1972depth`.
References
----------
.. bibliography:: /references.bib
:style: unsrt
:filter: docname in docnames
:keyprefix: tools-est-is-conn-
Examples
--------
>>> import numpy as np
>>> from deeptime.markov.tools.estimation import is_connected
>>> C = np.array([[10, 1, 0], [2, 0, 3], [0, 0, 4]])
>>> is_connected(C)
False
>>> is_connected(C, directed=False)
True
"""
if isdense(C):
return sparse.connectivity.is_connected(csr_matrix(C),
directed=directed)
else:
return sparse.connectivity.is_connected(C, directed=directed)
def __rsub__(self,other): # other - self
# note: this can't be replaced by other + (-self) for unsigned types
if isscalarlike(other):
if other == 0:
return -self.copy()
else: # Now we would add this scalar to every element.
raise NotImplementedError('adding a nonzero scalar to a '
'sparse matrix is not supported')
elif isdense(other):
# Convert this matrix to a dense matrix and subtract them
return other - self.todense()
else:
return NotImplemented
def mfpt(T, target):
r"""Mean first passage time to target state.
Parameters
----------
T : ndarray, shape=(n,n)
Transition matrix.
target : int
Target state for mfpt calculation.
Returns
-------
m_t : ndarray, shape=(n,)
Vector of mean first passage times to target state t.
Notes
-----
The mean first passage time :math:`\mathbf{E}_x[T_y]` is the expected
htting time of state :math:`y` starting in state :math:`x`.
For a fixed target state :math:`y` it is given by
.. math :: \mathbb{E}_x[T_y] = \left \{ \begin{array}{cc}
0 & x=y \\
1+\sum_{z} T_{x,z} \mathbb{E}_z[T_y] & x \neq y
\end{array} \right.
References
----------
.. [1] Hoel, P G and S C Port and C J Stone. 1972. Introduction to
Stochastic Processes.
Examples
--------
>>> from pyemma.msm.analysis import mfpt
>>> T=np.array([[0.9, 0.1, 0.0], [0.5, 0.0, 0.5], [0.0, 0.1, 0.9]])
>>> m_t=mfpt(T)
>>> m_t
array([ 0., 12., 22.])
"""
if issparse(T):
return sparse.mean_first_passage_time.mfpt(T, target)
elif isdense(T):
return dense.mean_first_passage_time.mfpt(T, target)
else:
raise _type_not_supported
def pathways(F, A, B, fraction=1.0, maxiter=1000):
r"""Decompose flux network into dominant reaction paths.
Parameters
----------
F : (M, M) scipy.sparse matrix
The flux network (matrix of netflux values)
A : array_like
The set of starting states
B : array_like
The set of end states
fraction : float, optional
Fraction of total flux to assemble in pathway decomposition
maxiter : int, optional
Maximum number of pathways for decomposition
Returns
-------
paths : list
List of dominant reaction pathways
capacities: list
List of capacities corresponding to each reactions pathway in paths
Notes
-----
The default value for fraction is 1.0, i.e. all dominant reaction
pathways for the flux network are computed. For large netorks the
number of possible reaction paths can increase rapidly so that it
becomes prohibitevely expensive to compute all possible reaction
paths. To prevent this from happening maxiter sets the maximum
number of reaction pathways that will be computed.
For large flux networks it might be necessary to decrease fraction
or to increase maxiter. It is advisable to begin with a small
value for fraction and monitor the number of pathways returned
when increasing the value of fraction.
References
----------
.. [1] P. Metzner, C. Schuette and E. Vanden-Eijnden.
Transition Path Theory for Markov Jump Processes.
Multiscale Model Simul 7: 1192-1219 (2009)
"""
if issparse(F):
return sparse.pathways.pathways(F, A, B, fraction=fraction, maxiter=maxiter)
elif isdense(F):
return sparse.pathways.pathways(csr_matrix(F), A, B, fraction=fraction, maxiter=maxiter)
else:
raise _type_not_supported
def is_connected(C, directed=True):
"""Check connectivity of the given matrix.
Parameters
----------
C : scipy.sparse matrix
Count matrix specifying edge weights.
directed : bool, optional
Whether to compute connected components for a directed or
undirected graph. Default is True.
Returns
-------
is_connected: bool
True if C is connected, False otherwise.
See also
--------
largest_connected_submatrix
Notes
-----
A count matrix is connected if the graph having the count matrix
as adjacency matrix has a single connected component. Connectivity
of a graph can be efficiently checked using Tarjan's algorithm.
References
----------
.. [1] Tarjan, R E. 1972. Depth-first search and linear graph
algorithms. SIAM Journal on Computing 1 (2): 146-160.
Examples
--------
>>> import numpy as np
>>> from msmtools.estimation import is_connected
>>> C = np.array([[10, 1, 0], [2, 0, 3], [0, 0, 4]])
>>> is_connected(C)
False
>>> is_connected(C, directed=False)
True
"""
if isdense(C):
return sparse.connectivity.is_connected(csr_matrix(C),
directed=directed)
else:
return sparse.connectivity.is_connected(C, directed=directed)
def expected_counts_stationary(T, N, mu=None):
r"""Expected transition counts for Markov chain in equilibrium.
Parameters
----------
T : (M, M) ndarray or sparse matrix
Transition matrix.
N : int
Number of steps for chain.
mu : (M,) ndarray (optional)
Stationary distribution for T. If mu is not specified it will be
computed from T.
Returns
-------
EC : (M, M) ndarray or sparse matrix
Expected value for transition counts after N steps.
Notes
-----
Since :math:`\mu` is stationary for :math:`T` we have
.. math::
\mathbb{E}[C^{(N)}]=N D_{\mu}T.
:math:`D_{\mu}` is a diagonal matrix. Elements on the diagonal are
given by the stationary vector :math:`\mu`
Examples
--------
>>> from pyemma.msm.analysis import expected_counts
>>> T=np.array([[0.9, 0.1, 0.0], [0.5, 0.0, 0.5], [0.0, 0.1, 0.9]])
>>> N=100
>>> EC=expected_counts_stationary(T, N)
>>> EC
array([[ 40.90909091, 4.54545455, 0. ],
[ 4.54545455, 0. , 4.54545455],
[ 0. , 4.54545455, 40.90909091]])
"""
if issparse(T):
return sparse.expectations.expected_counts_stationary(T, N, mu=mu)
elif isdense(T):
return dense.expectations.expected_counts_stationary(T, N, mu=mu)
else:
_type_not_supported
def stationary_distribution(T):
r"""Compute stationary distribution of stochastic matrix T.
Parameters
----------
T : (M, M) ndarray or scipy.sparse matrix
Transition matrix
Returns
-------
mu : (M,) ndarray
Vector of stationary probabilities.
Notes
-----
The stationary distribution :math:`\mu` is the left eigenvector
corresponding to the non-degenerate eigenvalue :math:`\lambda=1`,
.. math:: \mu^T T =\mu^T.
Examples
--------
>>> from pyemma.msm.analysis import stationary_distribution
>>> T=np.array([[0.9, 0.1, 0.0], [0.4, 0.2, 0.4], [0.0, 0.1, 0.9]])
>>> mu=stationary_distribution(T)
>>> mu
array([0.44444444, 0.11111111, 0.44444444])
"""
# is this a transition matrix?
if not is_transition_matrix(T):
raise ValueError("Input matrix is not a transition matrix."
"Cannot compute stationary distribution")
# is the stationary distribution unique?
if not is_connected(T):
raise ValueError("Input matrix is not connected. "
"Therefore it has no unique stationary "
"distribution. Separate disconnected components "
"and handle them separately")
# we're good to go...
if issparse(T):
return sparse.decomposition.stationary_distribution_from_backward_iteration(T)
elif isdense(T):
return dense.decomposition.stationary_distribution_from_backward_iteration(T)
else:
raise _type_not_supported
def is_connected(C, directed=True):
"""Check connectivity of the given matrix.
Parameters
----------
C : scipy.sparse matrix
Count matrix specifying edge weights.
directed : bool, optional
Whether to compute connected components for a directed or
undirected graph. Default is True.
Returns
-------
is_connected: bool
True if C is connected, False otherwise.
See also
--------
largest_connected_submatrix
Notes
-----
A count matrix is connected if the graph having the count matrix
as adjacency matrix has a single connected component. Connectivity
of a graph can be efficiently checked using Tarjan's algorithm.
References
----------
.. [1] Tarjan, R E. 1972. Depth-first search and linear graph
algorithms. SIAM Journal on Computing 1 (2): 146-160.
Examples
--------
>>> from pyemma.msm.estimation import is_connected
>>> C=np.array([10, 1, 0], [2, 0, 3], [0, 0, 4]])
>>> is_connected(C)
False
>>> is_connected(C)
True
"""
if isdense(C):
return sparse.connectivity.is_connected(csr_matrix(C), directed=directed)
else:
return sparse.connectivity.is_connected(C, directed=directed)
def timescales(T, tau=1, k=None, ncv=None):
r"""Compute implied time scales of given transition matrix.
Parameters
----------
T : (M, M) ndarray or scipy.sparse matrix
Transition matrix
tau : int (optional)
The time-lag (in elementary time steps of the microstate
trajectory) at which the given transition matrix was
constructed.
k : int (optional)
Compute the first `k` implied time scales.
ncv : int (optional, for sparse T only)
The number of Lanczos vectors generated, `ncv` must be greater than k;
it is recommended that ncv > 2*k
Returns
-------
ts : (M,) ndarray
The implied time scales of the transition matrix. If `k` is
not None then the shape of `ts` is (k,).
Notes
-----
The implied time scale :math:`t_i` is defined as
.. math:: t_i=-\frac{\tau}{\log \lvert \lambda_i \rvert}
Examples
--------
>>> from pyemma.msm.analysis import timescales
>>> T=np.array([[0.9, 0.1, 0.0], [0.5, 0.0, 0.5], [0.0, 0.1, 0.9]])
>>> ts=timescales(T)
>>> ts
array([ inf, 9.49122158, 0.43429448])
"""
if issparse(T):
return sparse.decomposition.timescales(T, tau=tau, k=k, ncv=ncv)
elif isdense(T):
return dense.decomposition.timescales(T, tau=tau, k=k)
else:
raise _type_not_supported
def expected_counts(T, p0, N):
r"""Compute expected transition counts for Markov chain with n steps.
Parameters
----------
T : (M, M) ndarray or sparse matrix
Transition matrix
p0 : (M,) ndarray
Initial (probability) vector
N : int
Number of steps to take
Returns
--------
EC : (M, M) ndarray or sparse matrix
Expected value for transition counts after N steps
Notes
-----
Expected counts can be computed via the following expression
.. math::
\mathbb{E}[C^{(N)}]=\sum_{k=0}^{N-1} \text{diag}(p^{T} T^{k}) T
Examples
--------
>>> from pyemma.msm.analysis import expected_counts
>>> T=np.array([[0.9, 0.1, 0.0], [0.5, 0.0, 0.5], [0.0, 0.1, 0.9]])
>>> p0=np.array([1.0, 0.0, 0.0])
>>> N=100
>>> EC=expected_counts(T, p0, N)
>>> EC
array([[ 45.44616147, 5.0495735 , 0. ],
[ 4.50413223, 0. , 4.50413223],
[ 0. , 4.04960006, 36.44640052]])
"""
if issparse(T):
return sparse.expectations.expected_counts(p0, T, N)
elif isdense(T):
return dense.expectations.expected_counts(p0, T, N)
else:
_type_not_supported
def __sub__(self,other):
# First check if argument is a scalar
if isscalarlike(other):
if other == 0:
return self.copy()
else: # Now we would add this scalar to every element.
raise NotImplementedError('adding a nonzero scalar to a '
'sparse matrix is not supported')
elif isspmatrix(other):
if (other.shape != self.shape):
raise ValueError("inconsistent shapes")
return self._binopt(other,'_minus_')
elif isdense(other):
# Convert this matrix to a dense matrix and subtract them
return self.todense() - other
else:
return NotImplemented
def prior_neighbor(C, alpha=0.001):
r"""Neighbor prior for the given count matrix.
Parameters
----------
C : (M, M) ndarray or scipy.sparse matrix
Count matrix
alpha : float (optional)
Value of prior counts
Returns
-------
B : (M, M) ndarray or scipy.sparse matrix
Prior count matrix
Notes
------
The neighbor prior :math:`b_{ij}` is defined as
.. math:: b_{ij}=\left \{ \begin{array}{rl}
\alpha & c_{ij}+c_{ji}>0 \\
0 & \text{else}
\end{array} \right .
Examples
--------
>>> import numpy as np
>>> from msmtools.estimation import prior_neighbor
>>> C = np.array([[10, 1, 0], [2, 0, 3], [0, 1, 4]])
>>> B = prior_neighbor(C)
>>> B
array([[ 0.001, 0.001, 0. ],
[ 0.001, 0. , 0.001],
[ 0. , 0.001, 0.001]])
"""
if isdense(C):
B = sparse.prior.prior_neighbor(csr_matrix(C), alpha=alpha)
return B.toarray()
else:
return sparse.prior.prior_neighbor(C, alpha=alpha)
def is_transition_matrix(T, tol=1e-15):
r"""Check if the given matrix is a transition matrix.
Parameters
----------
T : (M, M) ndarray or scipy.sparse matrix
Matrix to check
tol : float (optional)
Floating point tolerance to check with
Returns
-------
is_transition_matrix : bool
True, if T is a valid transition matrix, False otherwise
Notes
-----
A valid transition matrix :math:`P=(p_{ij})` has non-negative
elements, :math:`p_{ij} \geq 0`, and elements of each row sum up
to one, :math:`\sum_j p_{ij} = 1`. Matrices wit this property are
also called stochastic matrices.
Examples
--------
>>> from pyemma.msm.analysis import is_transition_matrix
>>> A=np.array([[0.4, 0.5, 0.3], [0.2, 0.4, 0.4], [-1, 1, 1]])
>>> is_transition_matrix(A)
False
>>> T=np.array([[0.9, 0.1, 0.0], [0.5, 0.0, 0.5], [0.0, 0.1, 0.9]])
>>> is_transition_matrix(T)
True
"""
if issparse(T):
return sparse.assessment.is_transition_matrix(T, tol)
elif isdense(T):
return dense.assessment.is_transition_matrix(T, tol)
else:
raise _type_not_supported
def is_rate_matrix(K, tol=1e-15):
r"""Check if the given matrix is a rate matrix.
Parameters
----------
K : (M, M) ndarray or scipy.sparse matrix
Matrix to check
tol : float (optional)
Floating point tolerance to check with
Returns
-------
is_rate_matrix : bool
True, if K is a valid rate matrix, False otherwise
Notes
-----
A valid rate matrix :math:`K=(k_{ij})` has non-positive off
diagonal elements, :math:`k_{ij} \leq 0`, for :math:`i \neq j`,
and elements of each row sum up to zero, :math:`\sum_{j}
k_{ij}=0`.
Examples
--------
>>> from pyemma.msm.analysis import is_rate_matrix
>>> A=np.array([[0.5, -0.5, -0.2], [-0.3, 0.6, -0.3], [-0.2, 0.2, 0.0]])
>>> is_rate_matrix(A)
False
>>> K=np.array([[0.3, -0.2, -0.1], [-0.5, 0.5, 0.0], [-0.1, -0.1, 0.2]])
>>> is_rate_matrix(K)
True
"""
if issparse(K):
return sparse.assessment.is_rate_matrix(K, tol)
elif isdense(K):
return dense.assessment.is_rate_matrix(K, tol)
else:
raise _type_not_supported
def prior_neighbor(C, alpha = 0.001):
r"""Neighbor prior for the given count matrix.
Parameters
----------
C : (M, M) ndarray or scipy.sparse matrix
Count matrix
alpha : float (optional)
Value of prior counts
Returns
-------
B : (M, M) ndarray or scipy.sparse matrix
Prior count matrix
Notes
------
The neighbor prior :math:`b_{ij}` is defined as
.. math:: b_{ij}=\left \{ \begin{array}{rl}
\alpha & c_{ij}+c_{ji}>0 \\
0 & \text{else}
\end{array} \right .
Examples
--------
>>> from pyemma.msm.estimation import prior_neighbor
>>> C=np.array([10, 1, 0], [2, 0, 3], [0, 1, 4]])
>>> B=prior_neighbor(C)
>>> B
array([[ 0.001, 0.001, 0. ],
[ 0.001, 0. , 0.001],
[ 0. , 0.001, 0.001]])
"""
if isdense(C):
B=sparse.prior.prior_neighbor(csr_matrix(C), alpha=alpha)
return B.toarray()
else:
return sparse.prior.prior_neighbor(C, alpha=alpha)
def __eq__(self, other):
# Scalar other.
if isscalarlike(other):
if np.isnan(other):
return self.__class__(self.shape, dtype=np.bool_)
if other == 0:
warn(
"Comparing a sparse matrix with 0 using == is inefficient"
", try using != instead.",
SparseEfficiencyWarning,
stacklevel=3)
all_true = self.__class__(np.ones(self.shape, dtype=np.bool_))
inv = self._scalar_binopt(other, operator.ne)
return all_true - inv
else:
return self._scalar_binopt(other, operator.eq)
# Dense other.
elif isdense(other):
return self.todense() == other
# Pydata sparse other.
elif is_pydata_spmatrix(other):
return NotImplemented
# Sparse other.
elif isspmatrix(other):
warn(
"Comparing sparse matrices using == is inefficient, try using"
" != instead.",
SparseEfficiencyWarning,
stacklevel=3)
# TODO sparse broadcasting
if self.shape != other.shape:
return False
elif self.format != other.format:
other = other.asformat(self.format)
res = self._binopt(other, '_ne_')
all_true = self.__class__(np.ones(self.shape, dtype=np.bool_))
return all_true - res
else:
return False
def prior_const(C, alpha=0.001):
r"""Constant prior for given count matrix.
Parameters
----------
C : (M, M) ndarray or scipy.sparse matrix
Count matrix
alpha : float (optional)
Value of prior counts
Returns
-------
B : (M, M) ndarray
Prior count matrix
Notes
-----
The prior is defined as
.. math:: \begin{array}{rl} b_{ij}= \alpha & \forall i, j \end{array}
Examples
--------
>>> import numpy as np
>>> from msmtools.estimation import prior_const
>>> C = np.array([[10, 1, 0], [2, 0, 3], [0, 1, 4]])
>>> B = prior_const(C)
>>> B
array([[ 0.001, 0.001, 0.001],
[ 0.001, 0.001, 0.001],
[ 0.001, 0.001, 0.001]])
"""
if isdense(C):
return sparse.prior.prior_const(C, alpha=alpha)
else:
warnings.warn("Prior will be a dense matrix for sparse input")
return sparse.prior.prior_const(C, alpha=alpha)
def to_netflux(flux):
r"""Compute the netflux from the gross flux.
Parameters
----------
flux : (M, M) ndarray
Matrix of flux values between pairs of states.
Returns
-------
netflux : (M, M) ndarray
Matrix of netflux values between pairs of states.
Notes
-----
The netflux or effective current is defined as
.. math::
f_{ij}^{+}=\max \{ f_{ij}-f_{ji}, 0 \},
see [1]_.
:math:`f_{ij}` is the flux for the transition from :math:`A` to
:math:`B`.
References
----------
.. [1] P. Metzner, C. Schuette and E. Vanden-Eijnden.
Transition Path Theory for Markov Jump Processes.
Multiscale Model Simul 7: 1192-1219 (2009)
"""
if issparse(flux):
return sparse.tpt.to_netflux(flux)
elif isdense(flux):
return dense.tpt.to_netflux(flux)
else:
raise _type_not_supported
def transition_matrix(C,
reversible=False,
mu=None,
method='auto',
maxiter: int = 1000000,
maxerr: float = 1e-8,
rev_pisym: bool = False,
return_statdist: bool = False,
warn_not_converged: bool = True):
r"""Estimate the transition matrix from the given countmatrix. :footcite:`prinz2011markov`
:footcite:`bowman2009progress` :footcite:`trendelkamp2015estimation`
Parameters
----------
C : numpy ndarray or scipy.sparse matrix
Count matrix
reversible : bool (optional)
If True restrict the ensemble of transition matrices
to those having a detailed balance symmetry otherwise
the likelihood optimization is carried out over the whole
space of stochastic matrices.
mu : array_like
The stationary distribution of the MLE transition matrix.
method : str
Select which implementation to use for the estimation.
One of 'auto', 'dense' and 'sparse', optional, default='auto'.
'dense' always selects the dense implementation, 'sparse' always selects
the sparse one.
'auto' selects the most efficient implementation according to
the sparsity structure of the matrix: if the occupation of the C
matrix is less then one third, select sparse. Else select dense.
The type of the T matrix returned always matches the type of the
C matrix, irrespective of the method that was used to compute it.
maxiter : int, optional, default=1000000
Optional parameter with reversible = True.
maximum number of iterations before the method exits
maxerr : float, optional, default=1e-8
Optional parameter with reversible = True.
convergence tolerance for transition matrix estimation.
This specifies the maximum change of the Euclidean norm of relative
stationary probabilities (:math:`x_i = \sum_k x_{ik}`). The relative stationary probability changes
:math:`e_i = (x_i^{(1)} - x_i^{(2)})/(x_i^{(1)} + x_i^{(2)})` are used in order to track changes in small
probabilities. The Euclidean norm of the change vector, :math:`|e_i|_2`, is compared to maxerr.
rev_pisym : bool, optional, default=False
Fast computation of reversible transition matrix by normalizing
:math:`x_{ij} = \pi_i p_{ij} + \pi_j p_{ji}`. :math:`p_{ij}` is the direct
(nonreversible) estimate and :math:`\pi_i` is its stationary distribution.
This estimator is asympotically unbiased but not maximum likelihood.
return_statdist : bool, optional, default=False
Optional parameter with reversible = True.
If set to true, the stationary distribution is also returned
return_conv : bool, optional, default=False
Optional parameter with reversible = True.
If set to true, the likelihood history and the pi_change history is returned.
warn_not_converged : bool, optional, default=True
Prints a warning if not converged.
Returns
-------
result : Union[array_like, Tuple[array_like, np.ndarray]]
The MLE transition matrix and optionally the stationary distribution if `return_statist=True`.
The transition matrix has the same data type (dense or sparse) as the input matrix C.
Notes
-----
The transition matrix is a maximum likelihood estimate (MLE) of
the probability distribution of transition matrices with
parameters given by the count matrix.
References
----------
.. footbibliography::
Examples
--------
>>> import numpy as np
>>> from deeptime.markov.tools.estimation import transition_matrix
>>> C = np.array([[10, 1, 1], [2, 0, 3], [0, 1, 4]])
Non-reversible estimate
>>> T_nrev = transition_matrix(C)
>>> print(np.array_str(T_nrev, precision=3))
[[0.833 0.083 0.083]
[0.4 0. 0.6 ]
[0. 0.2 0.8 ]]
Reversible estimate
>>> T_rev = transition_matrix(C, reversible=True)
>>> print(np.array_str(T_rev, precision=3))
[[0.833 0.104 0.063]
[0.351 0. 0.649]
[0.049 0.151 0.8 ]]
Reversible estimate with given stationary vector
>>> mu = np.array([0.7, 0.01, 0.29])
>>> T_mu = transition_matrix(C, reversible=True, mu=mu)
>>> print(np.array_str(T_mu, precision=3))
[[0.948 0.006 0.046]
[0.429 0. 0.571]
[0.111 0.02 0.869]]
"""
if issparse(C):
sparse_input_type = True
elif isdense(C):
sparse_input_type = False
else:
raise NotImplementedError('C has an unknown type.')
if method == 'dense':
sparse_computation = False
elif method == 'sparse':
sparse_computation = True
elif method == 'auto':
# heuristically determine whether is't more efficient to do a dense of sparse computation
if sparse_input_type:
dof = C.getnnz()
else:
dof = np.count_nonzero(C)
dimension = C.shape[0]
if dimension * dimension < 3 * dof:
sparse_computation = False
else:
sparse_computation = True
else:
raise ValueError(('method="%s" is no valid choice. It should be one of'
'"dense", "sparse" or "auto".') % method)
# convert input type
if sparse_computation and not sparse_input_type:
C = coo_matrix(C)
if not sparse_computation and sparse_input_type:
C = C.toarray()
if reversible:
if mu is None:
if sparse_computation:
if rev_pisym:
result = sparse.transition_matrix.transition_matrix_reversible_pisym(
C, return_statdist=return_statdist)
else:
result = sparse.mle.mle_trev(
C,
maxerr=maxerr,
maxiter=maxiter,
warn_not_converged=warn_not_converged,
return_statdist=return_statdist)
else:
if rev_pisym:
result = dense.transition_matrix.transition_matrix_reversible_pisym(
C, return_statdist=return_statdist)
else:
result = dense.mle.mle_trev(
C,
maxerr=maxerr,
maxiter=maxiter,
warn_not_converged=warn_not_converged,
return_statdist=return_statdist)
else:
if sparse_computation:
# Sparse, reversible, fixed pi (currently using dense with sparse conversion)
result = sparse.mle.mle_trev_given_pi(
C,
mu,
maxerr=maxerr,
maxiter=maxiter,
warn_not_converged=warn_not_converged)
else:
result = dense.mle.mle_trev_given_pi(C,
mu,
maxerr=maxerr,
maxiter=maxiter)
else: # nonreversible estimation
if mu is None:
if sparse_computation:
# Sparse, nonreversible
result = sparse.transition_matrix.transition_matrix_non_reversible(
C)
else:
# Dense, nonreversible
result = dense.transition_matrix.transition_matrix_non_reversible(
C)
# Both methods currently do not have an iterate of pi, so we compute it here for consistency.
if return_statdist:
from ..analysis import stationary_distribution
mu = stationary_distribution(result)
else:
raise NotImplementedError(
'nonreversible mle with fixed stationary distribution not implemented.'
)
if return_statdist and isinstance(result, tuple):
T, mu = result
else:
T = result
# convert return type
if sparse_computation and not sparse_input_type:
T = T.toarray()
elif not sparse_computation and sparse_input_type:
T = csr_matrix(T)
if return_statdist:
return T, mu
return T
def to_dense(a):
if not (isdense(a) or issparse(a)):
a = array(a)
if a.ndim == 1:
a = a.reshape(1, a.shape[0])
return a.toarray() if issparse(a) else a
def _fit_single(self, X, random_state, y=None):
"""Perform one run of co-clustering.
Parameters
----------
X : numpy array or scipy sparse matrix, shape=(n_samples, n_features)
Matrix to be analyzed
"""
X_trace = X ## X_trace garde la trace des valeurs null dans X avant l'imputation.
K = self.n_row_clusters
L = self.n_col_clusters
if self.init is None:
W = self.random_init(L, X.shape[1], random_state)
else:
W = np.matrix(self.init, dtype=float)
X = sp.csr_matrix(X)
N = float(
X.sum()
) # Dans le cas ou la matrice contient que des 1 et des 0, N = le nombre de 1 donc le nombre de données non null.
X = X.multiply(1. / N) # Normalisation
Z = sp.lil_matrix(self.random_init(
K, X.shape[0], self.random_state)) # K: Nombre de lignes
W = sp.csr_matrix(W)
# Imputation pour l'initialisation
# vu que c une phase d'initialisation pourquoi faire un random si on peut utiliser le KNN imputeur qui pourra accelerer la convergences.
X = random_imput(X.toarray())
X = sp.csr_matrix(X)
# Initial delta
p_il = X * W # columns
# p_il = p_il # matrix m,l ; column l' contains the p_il'
p_kj = X.T * Z # matrix j,k
p_kd = p_kj.sum(axis=0) # array containing the p_k.
p_dl = p_il.sum(axis=0) # array containing the p_.l
# p_k. p_.l ; transpose because p_kd is "horizontal"
p_kd_times_p_dl = p_kd.T * p_dl
min_p_kd_times_p_dl = np.nanmin(
p_kd_times_p_dl[np.nonzero(p_kd_times_p_dl)])
p_kd_times_p_dl[p_kd_times_p_dl == 0.] = min_p_kd_times_p_dl * 0.01
p_kd_times_p_dl_inv = 1. / p_kd_times_p_dl
p_kl = (Z.T * X) * W
delta_kl = p_kl.multiply(p_kd_times_p_dl_inv)
change = True
news = []
n_iters = self.max_iter
pkl_mi_previous = float(-np.inf)
# Loop
while change and n_iters > 0:
change = False
## X' = X
## Imputation(X)
# Update Z
p_il = X * W # matrix m,l ; column l' contains the p_il'
if not isdense(delta_kl):
delta_kl = delta_kl.todense()
delta_kl[delta_kl == 0.] = 0.0001 # to prevent log(0)
log_delta_kl = np.log(delta_kl.T)
log_delta_kl = sp.lil_matrix(log_delta_kl)
# p_il * (d_kl)T ; we examine each cluster
Z1 = p_il * log_delta_kl
Z1 = Z1.toarray()
Z = np.zeros_like(Z1)
# Z[(line index 1...), (max col index for 1...)]
Z[np.arange(len(Z1)), Z1.argmax(1)] = 1
Z = sp.lil_matrix(Z)
# Update delta
# matrice d, k ; column k' contains the p_jk'
p_kj = X.T * Z
# p_il unchanged
p_dl = p_il.sum(axis=0) # array l containing the p_.l
p_kd = p_kj.sum(axis=0) # array k containing the p_k.
# p_k. p_.l ; transpose because p_kd is "horizontal"
p_kd_times_p_dl = p_kd.T * p_dl
min_p_kd_times_p_dl = np.nanmin(
p_kd_times_p_dl[np.nonzero(p_kd_times_p_dl)])
p_kd_times_p_dl[p_kd_times_p_dl == 0.] = min_p_kd_times_p_dl * 0.01
p_kd_times_p_dl_inv = 1. / p_kd_times_p_dl
p_kl = (Z.T * X) * W
delta_kl = p_kl.multiply(p_kd_times_p_dl_inv)
#Imputation partie 2
X = imput_block(X.toarray(),
Z.toarray().argmax(axis=1),
W.toarray().argmax(axis=1), X_trace)
X = sp.csr_matrix(X)
#Update W
p_kj = X.T * Z # matrice m,l ; column l' contains the p_il'
if not isdense(delta_kl):
delta_kl = delta_kl.todense()
delta_kl[delta_kl == 0.] = 0.0001 # to prevent log(0)
log_delta_kl = np.log(delta_kl)
log_delta_kl = sp.lil_matrix(log_delta_kl)
W1 = p_kj * log_delta_kl # p_kj * d_kl ; we examine each cluster
W1 = W1.toarray()
W = np.zeros_like(W1)
W[np.arange(len(W1)), W1.argmax(1)] = 1
W = sp.lil_matrix(W)
# Update delta
p_il = X * W # matrix d,k ; column k' contains the p_jk'
# p_kj unchanged
p_dl = p_il.sum(axis=0) # array l containing the p_.l
p_kd = p_kj.sum(axis=0) # array k containing the p_k.
# p_k. p_.l ; transpose because p_kd is "horizontal"
p_kd_times_p_dl = p_kd.T * p_dl
min_p_kd_times_p_dl = np.nanmin(
p_kd_times_p_dl[np.nonzero(p_kd_times_p_dl)])
p_kd_times_p_dl[p_kd_times_p_dl == 0.] = min_p_kd_times_p_dl * 0.01
p_kd_times_p_dl_inv = 1. / p_kd_times_p_dl
p_kl = (Z.T * X) * W
delta_kl = p_kl.multiply(p_kd_times_p_dl_inv)
#Imputation partie 4
X = imput_block(X.toarray(),
Z.toarray().argmax(axis=1),
W.toarray().argmax(axis=1), X_trace)
X = sp.csr_matrix(X)
# to prevent log(0) when computing criterion
if not isdense(delta_kl):
delta_kl = delta_kl.todense()
delta_kl[delta_kl == 0.] = 0.0001
# Criterion
pkl_mi = sp.lil_matrix(p_kl).multiply(
sp.lil_matrix(np.log(delta_kl)))
pkl_mi = pkl_mi.sum()
if np.abs(pkl_mi - pkl_mi_previous) > self.tol:
pkl_mi_previous = pkl_mi
change = True
news.append(pkl_mi)
n_iters -= 1
self.criterions = news
self.criterion = pkl_mi
self.row_labels_ = Z.toarray().argmax(axis=1).tolist()
self.column_labels_ = W.toarray().argmax(axis=1).tolist()
self.delta_kl_ = delta_kl
self.X = X
self.Z = Z
self.W = W
def largest_connected_submatrix(C, directed=True, lcc=None):
r"""Compute the count matrix on the largest connected set.
Parameters
----------
C : scipy.sparse matrix
Count matrix specifying edge weights.
directed : bool, optional
Whether to compute connected components for a directed or
undirected graph. Default is True
lcc : (M,) ndarray, optional
The largest connected set
Returns
-------
C_cc : scipy.sparse matrix
Count matrix of largest completely
connected set of vertices (states)
See also
--------
largest_connected_set
Notes
-----
Viewing the count matrix as the adjacency matrix of a (directed)
graph the larest connected submatrix is the adjacency matrix of
the largest connected set of the corresponding graph. The largest
connected submatrix can be efficiently computed using Tarjan's algorithm :cite:`tools-est-lcsm-tarjan1972depth`.
References
----------
.. bibliography:: /references.bib
:style: unsrt
:filter: docname in docnames
:keyprefix: tools-est-lcsm-
Examples
--------
>>> import numpy as np
>>> from deeptime.markov.tools.estimation import largest_connected_submatrix
>>> C = np.array([[10, 1, 0], [2, 0, 3], [0, 0, 4]])
>>> C_cc_directed = largest_connected_submatrix(C)
>>> C_cc_directed # doctest: +ELLIPSIS
array([[10, 1],
[ 2, 0]]...)
>>> C_cc_undirected = largest_connected_submatrix(C, directed=False)
>>> C_cc_undirected # doctest: +ELLIPSIS
array([[10, 1, 0],
[ 2, 0, 3],
[ 0, 0, 4]]...)
"""
if isdense(C):
return sparse.connectivity.largest_connected_submatrix(
csr_matrix(C), directed=directed, lcc=lcc).toarray()
else:
return sparse.connectivity.largest_connected_submatrix(
C, directed=directed, lcc=lcc)
def save_matrix(filename, A, mode='default'):
r"""Save matrix as binary file.
Parameters
----------
filename : str
Relative or absolute pathname of the output file.
A : (M, N) ndarray or sparse matrix
mode : {'default', 'dense', 'sparse'}
========== ===================================================
mode
========== ===================================================
'default' Use the type of A to determine the format\
name.xxx (dense), name.coo.xxx (sparse)
'dense' Enforce conversion to a dense representation\
and store the corresponding ndarray
'sparse' Convert to sparse matrix in COO-format\
and store the coordinate list as ndarray
========== ===================================================
See also
--------
load_matrix
Notes
-----
(M, N) dense matrices are stored as ndarrays in numpy .npy binary
format. Sparse matrices are converted to coordinate list (COO)
format. The coordinate list [...,(row, col, value),...] is then
stored as a (K, 3) ndarray in numpy .npy binary format. K is the
number of nonzero entries in the sparse matrix.
Using the naming scheme name.npy for dense matrices
and name.coo.npy for sparse matrices will allow
load_matrix to automatically infer the appropriate matrix
type from the given filename.
Examples
--------
>>> from tempfile import NamedTemporaryFile
>>> from pyemma.msm.io import load_matrix, save_matrix
**dense**
Use temporary file with ending '.npy'
>>> tmpfile=NamedTemporaryFile(suffix='.npy')
Dense (3, 2) matrix
>>> A=np.array([[3, 1], [2, 1], [1, 1]])
>>> write_matrix(tmpfile.name, A)
Load from disk
>>> X=load_matrix(tmpfile.name)
>>> X
array([[ 3., 1.],
[ 2., 1.],
[ 1., 1.]])
**sparse**
>>> from scipy.sparse import csr_matrix
Use temporary file with ending '.coo.dat'
>>> tmpfile=NamedTemporaryFile(suffix='.coo.npy')
Sparse (3, 3) matrix
>>> A=csr_matrix(np.eye(3))
>>> write_matrix(tmpfile.name, A)
Load from disk
>>> X=load_matrix(tmpfile.name)
>>> X
array([[ 1., 0., 0.],
[ 0., 1., 0.],
[ 0., 0., 1.]])
"""
if mode=='dense':
A=matrix.todense(A)
matrix.save_matrix_dense(filename, A)
elif mode=='sparse':
A=matrix.tosparse(A)
matrix.save_matrix_sparse(filename, A)
else:
if isdense(A):
matrix.save_matrix_dense(filename, A)
elif issparse(A):
matrix.save_matrix_sparse(filename, A)
else:
raise TypeError('A is not a numpy.ndarray or a scipy.sparse matrix.')
def _fit_single(self, X, random_state, y=None):
"""Perform one run of co-clustering.
Parameters
----------
X : numpy array or scipy sparse matrix, shape=(n_samples, n_features)
Matrix to be analyzed
"""
K = self.n_row_clusters
L = self.n_col_clusters
if self.init is None:
W = random_init(L, X.shape[1], random_state)
else:
W = np.matrix(self.init, dtype=float)
X = sp.csr_matrix(X)
N = float(X.sum())
X = X.multiply(1. / N)
Z = sp.lil_matrix(random_init(K, X.shape[0], self.random_state))
W = sp.csr_matrix(W)
# Initial delta
p_il = X * W
# p_il = p_il # matrix m,l ; column l' contains the p_il'
p_kj = X.T * Z # matrix j,k
p_kd = p_kj.sum(axis=0) # array containing the p_k.
p_dl = p_il.sum(axis=0) # array containing the p_.l
# p_k. p_.l ; transpose because p_kd is "horizontal"
p_kd_times_p_dl = p_kd.T * p_dl
min_p_kd_times_p_dl = np.nanmin(
p_kd_times_p_dl[np.nonzero(p_kd_times_p_dl)])
p_kd_times_p_dl[p_kd_times_p_dl == 0.] = min_p_kd_times_p_dl * 0.01
p_kd_times_p_dl_inv = 1. / p_kd_times_p_dl
p_kl = (Z.T * X) * W
delta_kl = p_kl.multiply(p_kd_times_p_dl_inv)
change = True
news = []
n_iters = self.max_iter
pkl_mi_previous = float(-np.inf)
# Loop
while change and n_iters > 0:
change = False
# Update Z
p_il = X * W # matrix m,l ; column l' contains the p_il'
if not isdense(delta_kl):
delta_kl = delta_kl.todense()
delta_kl[delta_kl == 0.] = 0.0001 # to prevent log(0)
log_delta_kl = np.log(delta_kl.T)
log_delta_kl = sp.lil_matrix(log_delta_kl)
# p_il * (d_kl)T ; we examine each cluster
Z1 = p_il * log_delta_kl
Z1 = Z1.toarray()
Z = np.zeros_like(Z1)
# Z[(line index 1...), (max col index for 1...)]
Z[np.arange(len(Z1)), Z1.argmax(1)] = 1
Z = sp.lil_matrix(Z)
# Update delta
# matrice d, k ; column k' contains the p_jk'
p_kj = X.T * Z
# p_il unchanged
p_dl = p_il.sum(axis=0) # array l containing the p_.l
p_kd = p_kj.sum(axis=0) # array k containing the p_k.
# p_k. p_.l ; transpose because p_kd is "horizontal"
p_kd_times_p_dl = p_kd.T * p_dl
min_p_kd_times_p_dl = np.nanmin(
p_kd_times_p_dl[np.nonzero(p_kd_times_p_dl)])
p_kd_times_p_dl[p_kd_times_p_dl == 0.] = min_p_kd_times_p_dl * 0.01
p_kd_times_p_dl_inv = 1. / p_kd_times_p_dl
p_kl = (Z.T * X) * W
delta_kl = p_kl.multiply(p_kd_times_p_dl_inv)
# Update W
p_kj = X.T * Z # matrice m,l ; column l' contains the p_il'
if not isdense(delta_kl):
delta_kl = delta_kl.todense()
delta_kl[delta_kl == 0.] = 0.0001 # to prevent log(0)
log_delta_kl = np.log(delta_kl)
log_delta_kl = sp.lil_matrix(log_delta_kl)
W1 = p_kj * log_delta_kl # p_kj * d_kl ; we examine each cluster
W1 = W1.toarray()
W = np.zeros_like(W1)
W[np.arange(len(W1)), W1.argmax(1)] = 1
W = sp.lil_matrix(W)
# Update delta
p_il = X * W # matrix d,k ; column k' contains the p_jk'
# p_kj unchanged
p_dl = p_il.sum(axis=0) # array l containing the p_.l
p_kd = p_kj.sum(axis=0) # array k containing the p_k.
# p_k. p_.l ; transpose because p_kd is "horizontal"
p_kd_times_p_dl = p_kd.T * p_dl
min_p_kd_times_p_dl = np.nanmin(
p_kd_times_p_dl[np.nonzero(p_kd_times_p_dl)])
p_kd_times_p_dl[p_kd_times_p_dl == 0.] = min_p_kd_times_p_dl * 0.01
p_kd_times_p_dl_inv = 1. / p_kd_times_p_dl
p_kl = (Z.T * X) * W
delta_kl = p_kl.multiply(p_kd_times_p_dl_inv)
# to prevent log(0) when computing criterion
if not isdense(delta_kl):
delta_kl = delta_kl.todense()
delta_kl[delta_kl == 0.] = 0.0001
# Criterion
pkl_mi = sp.lil_matrix(p_kl).multiply(
sp.lil_matrix(np.log(delta_kl)))
pkl_mi = pkl_mi.sum()
if np.abs(pkl_mi - pkl_mi_previous) > self.tol:
pkl_mi_previous = pkl_mi
change = True
news.append(pkl_mi)
n_iters -= 1
self.criterions = news
self.criterion = pkl_mi
self.row_labels_ = Z.toarray().argmax(axis=1).tolist()
self.column_labels_ = W.toarray().argmax(axis=1).tolist()
self.delta_kl_ = delta_kl
self.Z = Z
self.W = W
def transition_matrix(C, reversible=False, mu=None, **kwargs):
r"""Estimate the transition matrix from the given countmatrix.
Parameters
----------
C : numpy ndarray or scipy.sparse matrix
Count matrix
reversible : bool (optional)
If True restrict the ensemble of transition matrices
to those having a detailed balance symmetry otherwise
the likelihood optimization is carried out over the whole
space of stochastic matrices.
mu : array_like
The stationary distribution of the MLE transition matrix.
**kwargs: Optional algorithm-specific parameters. See below for special cases
eps = 1E-6 : float
Optional parameter with reversible = True and mu!=None.
Regularization parameter for the interior point method. This value is added
to the diagonal elements of C that are zero.
Xinit : (M, M) ndarray
Optional parameter with reversible = True.
initial value for the matrix of absolute transition probabilities. Unless set otherwise,
will use X = diag(pi) t, where T is a nonreversible transition matrix estimated from C,
i.e. T_ij = c_ij / sum_k c_ik, and pi is its stationary distribution.
maxiter = 1000000 : int
Optional parameter with reversible = True.
maximum number of iterations before the method exits
maxerr = 1e-8 : float
Optional parameter with reversible = True.
convergence tolerance for transition matrix estimation.
This specifies the maximum change of the Euclidean norm of relative
stationary probabilities (:math:`x_i = \sum_k x_{ik}`). The relative stationary probability changes
:math:`e_i = (x_i^{(1)} - x_i^{(2)})/(x_i^{(1)} + x_i^{(2)})` are used in order to track changes in small
probabilities. The Euclidean norm of the change vector, :math:`|e_i|_2`, is compared to maxerr.
return_statdist = False : Boolean
Optional parameter with reversible = True.
If set to true, the stationary distribution is also returned
return_conv = False : Boolean
Optional parameter with reversible = True.
If set to true, the likelihood history and the pi_change history is returned.
Returns
-------
P : (M, M) ndarray or scipy.sparse matrix
The MLE transition matrix. P has the same data type (dense or sparse)
as the input matrix C.
The reversible estimator returns by default only P, but may also return
(P,pi) or (P,lhist,pi_changes) or (P,pi,lhist,pi_changes) depending on the return settings
P : ndarray (n,n)
transition matrix. This is the only return for return_statdist = False, return_conv = False
(pi) : ndarray (n)
stationary distribution. Only returned if return_statdist = True
(lhist) : ndarray (k)
likelihood history. Has the length of the number of iterations needed.
Only returned if return_conv = True
(pi_changes) : ndarray (k)
history of likelihood history. Has the length of the number of iterations needed.
Only returned if return_conv = True
Notes
-----
The transition matrix is a maximum likelihood estimate (MLE) of
the probability distribution of transition matrices with
parameters given by the count matrix.
References
----------
.. [1] Prinz, J H, H Wu, M Sarich, B Keller, M Senne, M Held, J D
Chodera, C Schuette and F Noe. 2011. Markov models of
molecular kinetics: Generation and validation. J Chem Phys
134: 174105
.. [2] Bowman, G R, K A Beauchamp, G Boxer and V S Pande. 2009.
Progress and challenges in the automated construction of Markov state models for full protein systems.
J. Chem. Phys. 131: 124101
Examples
--------
>>> from pyemma.msm.estimation import transition_matrix
>>> C = np.array([10, 1, 1], [2, 0, 3], [0, 1, 4]])
Non-reversible estimate
>>> T_nrev = transition_matrix(C)
>>> T_nrev
array([[ 0.83333333, 0.08333333, 0.08333333],
[ 0.33333333, 0.16666667, 0.5 ],
[ 0. , 0.2 , 0.8 ]])
Reversible estimate
>>> T_rev = transition_matrix(C)
>>> T_rev
array([[ 0.83333333, 0.10385552, 0.06281115],
[ 0.29228896, 0.16666667, 0.54104437],
[ 0.04925323, 0.15074676, 0.80000001]])
Reversible estimate with given stationary vector
>>> mu = np.array([0.7, 0.01, 0.29])
>>> T_mu = transition_matrix(C, reversible=True, mu=mu)
>>> T_mu
array([[ 0.94841372, 0.00534691, 0.04623938],
[ 0.37428347, 0.12715063, 0.4985659 ],
[ 0.11161229, 0.01719193, 0.87119578]])
"""
if issparse(C):
sparse_mode = True
elif isdense(C):
sparse_mode = False
else:
raise NotImplementedError('C has an unknown type.')
if reversible:
if mu is None:
if sparse_mode:
return sparse.mle_trev.mle_trev(C, **kwargs)
else:
return dense.transition_matrix.estimate_transition_matrix_reversible(
C, **kwargs)
else:
if sparse_mode:
# Sparse, reversible, fixed pi (currently using dense with sparse conversion)
return sparse.mle_trev_given_pi.mle_trev_given_pi(
C, mu, **kwargs)
else:
return dense.mle_trev_given_pi.mle_trev_given_pi(
C, mu, **kwargs)
else: # nonreversible estimation
if mu is None:
if sparse_mode:
# Sparse, nonreversible
return sparse.transition_matrix.transition_matrix_non_reversible(
C)
else:
# Dense, nonreversible
return dense.transition_matrix.transition_matrix_non_reversible(
C)
else:
raise NotImplementedError(
'nonreversible mle with fixed stationary distribution not implemented.'
)
def write_matrix(filename, A, mode='default', fmt='%.18e', header='', comments='#'):
r"""Write matrix to ascii file.
Parameters
----------
filename : str
Relative or absolute pathname of the output file.
A : (M, N) ndarray or sparse matrix
mode : {'default', 'dense', 'sparse'}
How to determine the storage format
========== ===============================================
mode
========== ===============================================
'default' Use the type of A to determine the format
'dense' Enforce conversion to a dense representation\
and store the corresponding ndarray
'sparse' Convert to sparse matrix in COO-format and\
and store the coordinate list as ndarray
========== ===============================================
fmt : str or sequence of strs, optional
A single format (%10.5f), a sequence of formats, or a multi-format
string, e.g. 'Iteration %d - %10.5f', in which case delimiter is
ignored.
header : str, optional
String that will be written at the beginning of the file.
comments : str, optional
String that will be prepended to the header strings,
to mark them as comments. Default: '# '.
See also
--------
read_matrix
Notes
-----
(M, N) dense matrices are stored as ascii file with M rows and N
columns. Sparse matrices are converted to coordinate list (COO)
format. The coordinate list [...,(row, col, value),...] is then
stored as a dense (K, 3) ndarray. K is the number of nonzero
entries in the sparse matrix.
Using the naming scheme name.xxx for dense matrices and
name.coo.xxx for sparse matrices will allow read_matrix to
automatically infer the appropriate matrix type from the given
filename.
Examples
--------
>>> from tempfile import NamedTemporaryFile
>>> from pyemma.msm.io import read_matrix, write_matrix
**dense**
Use temporary file with ending '.dat'
>>> tmpfile=NamedTemporaryFile(suffix='.dat')
Dense (3, 2) matrix
>>> A=np.array([[3, 1], [2, 1], [1, 1]])
>>> write_matrix(tmpfile.name, A)
Load from disk
>>> X=load_matrix(tmpfile.name)
>>> X
array([[ 3., 1.],
[ 2., 1.],
[ 1., 1.]])
**sparse**
>>> from scipy.sparse import csr_matrix
Use temporary file with ending '.coo.dat'
>>> tmpfile=NamedTemporaryFile(suffix='.coo.dat')
Sparse (3, 3) matrix
>>> A=csr_matrix(np.eye(3))
>>> write_matrix(tmpfile.name, A)
Load from disk
>>> X=load_matrix(tmpfile.name)
>>> X
array([[ 1., 0., 0.],
[ 0., 1., 0.],
[ 0., 0., 1.]])
"""
if mode=='dense':
A=matrix.todense(A)
matrix.write_matrix_dense(filename, A, fmt=fmt, header=header, comments=comments)
elif mode=='sparse':
A=matrix.tosparse(A)
matrix.write_matrix_sparse(filename, A, fmt=fmt, header=header, comments=comments)
else:
if isdense(A):
matrix.write_matrix_dense(filename, A, fmt=fmt, header=header, comments=comments)
elif issparse(A):
matrix.write_matrix_sparse(filename, A, fmt=fmt, header=header, comments=comments)
else:
raise TypeError('A is not a numpy.ndarray or a scipy.sparse matrix.')
def test_isdense(self):
assert_equal(sputils.isdense(np.array([1])), True)
assert_equal(sputils.isdense(np.matrix([1])), True)
def _fit_single(self, X,impute_func,na_rows, na_cols, random_state, y=None):
"""Perform one run of co-clustering.
Parameters
----------
X : numpy array or scipy sparse matrix, shape=(n_samples, n_features)
Matrix to be analyzed
"""
K = self.n_row_clusters
L = self.n_col_clusters
if self.init is None:
W = random_init(L, X.shape[1], random_state)
else:
W = np.matrix(self.init, dtype=float)
w_ = np.argmax(W, axis=1)[:, np.newaxis]
X = sp.csr_matrix(X)
N = float(X.sum())
X = X.multiply(1. / N)
Z = sp.lil_matrix(random_init(K, X.shape[0], self.random_state))
W = sp.csr_matrix(W)
# Initial delta
p_il = X * W
# p_il = p_il # matrix i,l ; column l' contains the p_il'
p_kj = X.T * Z # matrix j,k
p_kd = p_kj.sum(axis=0) # array containing the p_k.
p_dl = p_il.sum(axis=0) # array containing the p_.l
# p_k. p_.l ; transpose because p_kd is "horizontal"
p_kd_times_p_dl = p_kd.T * p_dl
min_p_kd_times_p_dl = np.nanmin(
p_kd_times_p_dl[
np.nonzero(p_kd_times_p_dl)])
p_kd_times_p_dl[p_kd_times_p_dl == 0.] = min_p_kd_times_p_dl * 0.01
p_kd_times_p_dl_inv = 1. / p_kd_times_p_dl
p_kl = (Z.T * X) * W
delta_kl = p_kl.multiply(p_kd_times_p_dl_inv)
change = True
news = []
n_iters = self.max_iter
pkl_mi_previous = float(-np.inf)
# Loop
while change and n_iters > 0:
change = False
# Update Z
p_il = X * W # CSR matrix i,l
if not isdense(delta_kl):
delta_kl = delta_kl.todense()
delta_kl[delta_kl == 0.] = 0.0001 # to prevent log(0)
log_delta_kl = np.log(delta_kl.T)
log_delta_kl = sp.csr_matrix(log_delta_kl)
# p_il matmul (log d_kl)T ; we examine each cluster
Z1 = p_il @ log_delta_kl
# Z1 is CSR which is bad for support item assignment. So convert Z1
Z1 = Z1.toarray()
Z = np.zeros_like(Z1)
# Z[(line index 1...), (max col index for 1...)]
Z[np.arange(len(Z1)), Z1.argmax(1)] = 1
# impute missing value
z_ = np.argmax(Z, axis=1)[:, np.newaxis]
X = sp.csr_matrix(impute_func(X.toarray(), Z, W.toarray(), z_, w_, na_rows, na_cols))
Z = sp.csr_matrix(Z)
# Update delta
# matrice d, k ; column k' contains the p_jk'
p_kj = X.T * Z
# p_il unchanged so no need for p_dl = p_il.sum(axis=0)
p_kd = p_kj.sum(axis=0) # array k containing the p_k.
# p_k. p_.l ; transpose because p_kd is "horizontal"
p_kd_times_p_dl = p_kd.T * p_dl
min_p_kd_times_p_dl = np.nanmin(
p_kd_times_p_dl[
np.nonzero(p_kd_times_p_dl)])
p_kd_times_p_dl[p_kd_times_p_dl == 0.] = min_p_kd_times_p_dl * 0.01
p_kd_times_p_dl_inv = 1. / p_kd_times_p_dl
p_kl = (Z.T * X) * W
delta_kl = p_kl.multiply(p_kd_times_p_dl_inv)
# Update W
p_kj = X.T * Z # CSR matrice j,k
if not isdense(delta_kl): # delta_kl should be a sparse coo here
delta_kl = delta_kl.todense()
delta_kl[delta_kl == 0.] = 0.0001 # to prevent log(0)
log_delta_kl = np.log(delta_kl)
log_delta_kl = sp.lil_matrix(log_delta_kl)
W1 = p_kj * log_delta_kl # p_kj * d_kl ; we examine each cluster
# W1 is CSR which is bad for support item assignment. So convert W1
W1 = W1.toarray()
W = np.zeros_like(W1)
W[np.arange(len(W1)), W1.argmax(1)] = 1
# impute missing value
w_ = np.argmax(W, axis=1)[:, np.newaxis]
X = sp.csr_matrix(impute_func(X.toarray(), Z.toarray(), W, z_, w_, na_rows, na_cols))
W = sp.csr_matrix(W)
# Update delta
p_il = X * W # matrix d,k ; column k' contains the p_jk'
# p_kj unchanged so no need for p_kd = p_kj.sum(axis=0)
p_dl = p_il.sum(axis=0) # array l containing the p_.l
# p_k. p_.l ; transpose because p_kd is "horizontal"
p_kd_times_p_dl = p_kd.T * p_dl
min_p_kd_times_p_dl = np.nanmin(
p_kd_times_p_dl[
np.nonzero(p_kd_times_p_dl)])
p_kd_times_p_dl[p_kd_times_p_dl == 0.] = min_p_kd_times_p_dl * 0.01
p_kd_times_p_dl_inv = 1. / p_kd_times_p_dl
p_kl = (Z.T * X) * W
delta_kl = p_kl.multiply(p_kd_times_p_dl_inv)
# to prevent log(0) when computing criterion
# but note that delta_kl = csr eletwise-mult array is a coo
# which does not support item assignment
if not isdense(delta_kl):
delta_kl = delta_kl.todense()
delta_kl[delta_kl == 0.] = 0.0001
# Criterion
# p_kl is csr_matrix; delta_kl is np.matrix
# just in case, note that their matmul results in a COO matrix !
pkl_mi = p_kl.multiply(np.log(delta_kl) )
pkl_mi = pkl_mi.sum()
if np.abs(pkl_mi - pkl_mi_previous) > self.tol:
pkl_mi_previous = pkl_mi
change = True
news.append(pkl_mi)
n_iters -= 1
self.criterions = news
self.criterion = pkl_mi
self.row_labels_ = Z.toarray().argmax(axis=1).tolist()
self.column_labels_ = W.toarray().argmax(axis=1).tolist()
self.delta_kl_ = delta_kl
self.Z = Z
self.W = W
self.X_ = X
def largest_connected_submatrix(C, directed=True, lcc=None):
r"""Compute the count matrix on the largest connected set.
Parameters
----------
C : scipy.sparse matrix
Count matrix specifying edge weights.
directed : bool, optional
Whether to compute connected components for a directed or
undirected graph. Default is True
lcc : (M,) ndarray, optional
The largest connected set
Returns
-------
C_cc : scipy.sparse matrix
Count matrix of largest completely
connected set of vertices (states)
See also
--------
largest_connected_set
Notes
-----
Viewing the count matrix as the adjacency matrix of a (directed)
graph the larest connected submatrix is the adjacency matrix of
the largest connected set of the corresponding graph. The largest
connected submatrix can be efficiently computed using Tarjan's algorithm.
References
----------
.. [1] Tarjan, R E. 1972. Depth-first search and linear graph
algorithms. SIAM Journal on Computing 1 (2): 146-160.
Examples
--------
>>> import numpy as np
>>> from msmtools.estimation import largest_connected_submatrix
>>> C = np.array([[10, 1, 0], [2, 0, 3], [0, 0, 4]])
>>> C_cc_directed = largest_connected_submatrix(C)
>>> C_cc_directed # doctest: +ELLIPSIS
array([[10, 1],
[ 2, 0]]...)
>>> C_cc_undirected = largest_connected_submatrix(C, directed=False)
>>> C_cc_undirected # doctest: +ELLIPSIS
array([[10, 1, 0],
[ 2, 0, 3],
[ 0, 0, 4]]...)
"""
if isdense(C):
return sparse.connectivity.largest_connected_submatrix(
csr_matrix(C), directed=directed, lcc=lcc).toarray()
else:
return sparse.connectivity.largest_connected_submatrix(
C, directed=directed, lcc=lcc)
def transition_matrix(C, reversible=False, mu=None, method='auto', **kwargs):
r"""Estimate the transition matrix from the given countmatrix.
Parameters
----------
C : numpy ndarray or scipy.sparse matrix
Count matrix
reversible : bool (optional)
If True restrict the ensemble of transition matrices
to those having a detailed balance symmetry otherwise
the likelihood optimization is carried out over the whole
space of stochastic matrices.
mu : array_like
The stationary distribution of the MLE transition matrix.
method : str
Select which implementation to use for the estimation.
One of 'auto', 'dense' and 'sparse', optional, default='auto'.
'dense' always selects the dense implementation, 'sparse' always selects
the sparse one.
'auto' selects the most efficient implementation according to
the sparsity structure of the matrix: if the occupation of the C
matrix is less then one third, select sparse. Else select dense.
The type of the T matrix returned always matches the type of the
C matrix, irrespective of the method that was used to compute it.
**kwargs: Optional algorithm-specific parameters. See below for special cases
Xinit : (M, M) ndarray
Optional parameter with reversible = True.
initial value for the matrix of absolute transition probabilities. Unless set otherwise,
will use X = diag(pi) t, where T is a nonreversible transition matrix estimated from C,
i.e. T_ij = c_ij / sum_k c_ik, and pi is its stationary distribution.
maxiter : 1000000 : int
Optional parameter with reversible = True.
maximum number of iterations before the method exits
maxerr : 1e-8 : float
Optional parameter with reversible = True.
convergence tolerance for transition matrix estimation.
This specifies the maximum change of the Euclidean norm of relative
stationary probabilities (:math:`x_i = \sum_k x_{ik}`). The relative stationary probability changes
:math:`e_i = (x_i^{(1)} - x_i^{(2)})/(x_i^{(1)} + x_i^{(2)})` are used in order to track changes in small
probabilities. The Euclidean norm of the change vector, :math:`|e_i|_2`, is compared to maxerr.
rev_pisym : bool, default=False
Fast computation of reversible transition matrix by normalizing
:math:`x_{ij} = \pi_i p_{ij} + \pi_j p_{ji}`. :math:`p_{ij}` is the direct
(nonreversible) estimate and :math:`pi_i` is its stationary distribution.
This estimator is asympotically unbiased but not maximum likelihood.
return_statdist : bool, default=False
Optional parameter with reversible = True.
If set to true, the stationary distribution is also returned
return_conv : bool, default=False
Optional parameter with reversible = True.
If set to true, the likelihood history and the pi_change history is returned.
warn_not_converged : bool, default=True
Prints a warning if not converged.
Returns
-------
P : (M, M) ndarray or scipy.sparse matrix
The MLE transition matrix. P has the same data type (dense or sparse)
as the input matrix C.
The reversible estimator returns by default only P, but may also return
(P,pi) or (P,lhist,pi_changes) or (P,pi,lhist,pi_changes) depending on the return settings
P : ndarray (n,n)
transition matrix. This is the only return for return_statdist = False, return_conv = False
(pi) : ndarray (n)
stationary distribution. Only returned if return_statdist = True
(lhist) : ndarray (k)
likelihood history. Has the length of the number of iterations needed.
Only returned if return_conv = True
(pi_changes) : ndarray (k)
history of likelihood history. Has the length of the number of iterations needed.
Only returned if return_conv = True
Notes
-----
The transition matrix is a maximum likelihood estimate (MLE) of
the probability distribution of transition matrices with
parameters given by the count matrix.
References
----------
.. [1] Prinz, J H, H Wu, M Sarich, B Keller, M Senne, M Held, J D
Chodera, C Schuette and F Noe. 2011. Markov models of
molecular kinetics: Generation and validation. J Chem Phys
134: 174105
.. [2] Bowman, G R, K A Beauchamp, G Boxer and V S Pande. 2009.
Progress and challenges in the automated construction of Markov state models for full protein systems.
J. Chem. Phys. 131: 124101
.. [3] Trendelkamp-Schroer, B, H Wu, F Paul and F. Noe. 2015
Estimation and uncertainty of reversible Markov models.
J. Chem. Phys. 143: 174101
Examples
--------
>>> import numpy as np
>>> from msmtools.estimation import transition_matrix
>>> C = np.array([[10, 1, 1], [2, 0, 3], [0, 1, 4]])
Non-reversible estimate
>>> T_nrev = transition_matrix(C)
>>> T_nrev
array([[ 0.83333333, 0.08333333, 0.08333333],
[ 0.4 , 0. , 0.6 ],
[ 0. , 0.2 , 0.8 ]])
Reversible estimate
>>> T_rev = transition_matrix(C, reversible=True)
>>> T_rev
array([[ 0.83333333, 0.10385551, 0.06281115],
[ 0.35074677, 0. , 0.64925323],
[ 0.04925323, 0.15074677, 0.8 ]])
Reversible estimate with given stationary vector
>>> mu = np.array([0.7, 0.01, 0.29])
>>> T_mu = transition_matrix(C, reversible=True, mu=mu)
>>> T_mu
array([[ 0.94771371, 0.00612645, 0.04615984],
[ 0.42885157, 0. , 0.57114843],
[ 0.11142031, 0.01969477, 0.86888491]])
"""
if issparse(C):
sparse_input_type = True
elif isdense(C):
sparse_input_type = False
else:
raise NotImplementedError('C has an unknown type.')
if method == 'dense':
sparse_computation = False
elif method == 'sparse':
sparse_computation = True
elif method == 'auto':
# heuristically determine whether is't more efficient to do a dense of sparse computation
if sparse_input_type:
dof = C.getnnz()
else:
dof = np.count_nonzero(C)
dimension = C.shape[0]
if dimension * dimension < 3 * dof:
sparse_computation = False
else:
sparse_computation = True
else:
raise ValueError(('method="%s" is no valid choice. It should be one of'
'"dense", "sparse" or "auto".') % method)
# convert input type
if sparse_computation and not sparse_input_type:
C = coo_matrix(C)
if not sparse_computation and sparse_input_type:
C = C.toarray()
if reversible:
rev_pisym = kwargs.pop('rev_pisym', False)
if mu is None:
if sparse_computation:
if rev_pisym:
T = sparse.transition_matrix.transition_matrix_reversible_pisym(
C)
else:
T = sparse.mle_trev.mle_trev(C, **kwargs)
else:
if rev_pisym:
T = dense.transition_matrix.transition_matrix_reversible_pisym(
C)
else:
T = dense.mle_trev.mle_trev(C, **kwargs)
else:
if sparse_computation:
# Sparse, reversible, fixed pi (currently using dense with sparse conversion)
T = sparse.mle_trev_given_pi.mle_trev_given_pi(C, mu, **kwargs)
else:
T = dense.mle_trev_given_pi.mle_trev_given_pi(C, mu, **kwargs)
else: # nonreversible estimation
if mu is None:
if sparse_computation:
# Sparse, nonreversible
T = sparse.transition_matrix.transition_matrix_non_reversible(
C)
else:
# Dense, nonreversible
T = dense.transition_matrix.transition_matrix_non_reversible(C)
else:
raise NotImplementedError(
'nonreversible mle with fixed stationary distribution not implemented.'
)
# convert return type
return_statdist = 'return_statdist' in kwargs
if sparse_computation and not sparse_input_type:
if return_statdist:
if mu is not None:
raise NotImplementedError()
return T[0].toarray(), T[1]
else:
return T.toarray()
elif not sparse_computation and sparse_input_type:
if return_statdist:
if mu is not None:
raise NotImplementedError()
return csr_matrix(T[0]), T[1]
else:
return csr_matrix(T)
else:
return T
def flux_matrix(T, pi, qminus, qplus, netflux=True):
r"""Compute the TPT flux network for the reaction A-->B.
Parameters
----------
T : (M, M) ndarray
transition matrix
pi : (M,) ndarray
Stationary distribution corresponding to T
qminus : (M,) ndarray
Backward comittor
qplus : (M,) ndarray
Forward committor
netflux : boolean
True: net flux matrix will be computed
False: gross flux matrix will be computed
Returns
-------
flux : (M, M) ndarray
Matrix of flux values between pairs of states.
Notes
-----
Computation of the flux network relies on transition path theory
(TPT) [1]. Here we use discrete transition path theory [2] in
the transition matrix formulation [3].
See also
--------
committor.forward_committor, committor.backward_committor
Notes
-----
Computation of the flux network relies on transition path theory
(TPT). The central object used in transition path theory is the
forward and backward comittor function.
The TPT (gross) flux is defined as
.. math:: f_{ij}=\left \{ \begin{array}{rl}
\pi_i q_i^{(-)} p_{ij} q_j^{(+)} & i \neq j \\
0 & i=j\
\end{array} \right .
The TPT net flux is then defined as
.. math:: f_{ij}=\max\{f_{ij} - f_{ji}, 0\} \:\:\:\forall i,j.
References
----------
.. [1] W. E and E. Vanden-Eijnden.
Towards a theory of transition paths.
J. Stat. Phys. 123: 503-523 (2006)
.. [2] P. Metzner, C. Schuette and E. Vanden-Eijnden.
Transition Path Theory for Markov Jump Processes.
Multiscale Model Simul 7: 1192-1219 (2009)
.. [3] F. Noe, Ch. Schuette, E. Vanden-Eijnden, L. Reich and
T. Weikl: Constructing the Full Ensemble of Folding Pathways
from Short Off-Equilibrium Simulations.
Proc. Natl. Acad. Sci. USA, 106, 19011-19016 (2009)
"""
if issparse(T):
return sparse.tpt.flux_matrix(T, pi, qminus, qplus, netflux=netflux)
elif isdense(T):
return dense.tpt.flux_matrix(T, pi, qminus, qplus, netflux=netflux)
else:
raise _type_not_supported
