def fingerprint_correlation(self, a, b=None, k=None, ncv=None):
# basic checks for a and b
a = _types.ensure_ndarray(a, ndim=1, kind='numeric')
b = _types.ensure_ndarray_or_None(b,
ndim=1,
kind='numeric',
size=len(a))
# are we on microstates space?
if len(a) == self.nstates_obs:
a = _np.dot(self.observation_probabilities, a)
if b is not None:
b = _np.dot(self.observation_probabilities, b)
# now we are on macrostate space, or something is wrong
if len(a) == self.nstates:
return _MSM.fingerprint_correlation(self, a, b=b)
else:
raise ValueError(
'observable vectors have size %s which is incompatible with both hidden (%s)'
' and observed states (%s)' %
(len(a), self.nstates, self.nstates_obs))
def __init__(self, lag=1, reversible=True, statdist_constraint=None,
count_mode='sliding', sparse=False,
connectivity='largest', dt_traj='1 step', maxiter=1000000,
maxerr=1e-8):
r"""Maximum likelihood estimator for MSMs given discrete trajectory statistics
Parameters
----------
lag : int
lag time at which transitions are counted and the transition matrix is
estimated.
reversible : bool, optional, default = True
If true compute reversible MSM, else non-reversible MSM
statdist : (M,) ndarray, optional
Stationary vector on the full set of states. Estimation will be
made such the the resulting transition matrix has this distribution
as an equilibrium distribution. Set probabilities to zero if these
states should be excluded from the analysis.
count_mode : str, optional, default='sliding'
mode to obtain count matrices from discrete trajectories. Should be
one of:
* 'sliding' : A trajectory of length T will have :math:`T-tau` counts
at time indexes
.. math::
(0 \rightarrow \tau), (1 \rightarrow \tau+1), ..., (T-\tau-1 \rightarrow T-1)
* 'effective' : Uses an estimate of the transition counts that are
statistically uncorrelated. Recommended when used with a
Bayesian MSM.
* 'sample' : A trajectory of length T will have :math:`T/tau` counts
at time indexes
.. math::
(0 \rightarrow \tau), (\tau \rightarrow 2 \tau), ..., (((T/tau)-1) \tau \rightarrow T)
sparse : bool, optional, default = False
If true compute count matrix, transition matrix and all derived
quantities using sparse matrix algebra. In this case python sparse
matrices will be returned by the corresponding functions instead of
numpy arrays. This behavior is suggested for very large numbers of
states (e.g. > 4000) because it is likely to be much more efficient.
connectivity : str, optional, default = 'largest'
Connectivity mode. Three methods are intended (currently only 'largest'
is implemented)
* 'largest' : The active set is the largest reversibly connected set.
All estimation will be done on this subset and all quantities
(transition matrix, stationary distribution, etc) are only defined
on this subset and are correspondingly smaller than the full set
of states
* 'all' : The active set is the full set of states. Estimation will be
conducted on each reversibly connected set separately. That means
the transition matrix will decompose into disconnected submatrices,
the stationary vector is only defined within subsets, etc.
Currently not implemented.
* 'none' : The active set is the full set of states. Estimation will
be conducted on the full set of
states without ensuring connectivity. This only permits
nonreversible estimation. Currently not implemented.
dt_traj : str, optional, default='1 step'
Description of the physical time of the input trajectories. May be used
by analysis algorithms such as plotting tools to pretty-print the axes.
By default '1 step', i.e. there is no physical time unit. Specify by a
number, whitespace and unit. Permitted units are (* is an arbitrary
string):
| 'fs', 'femtosecond*'
| 'ps', 'picosecond*'
| 'ns', 'nanosecond*'
| 'us', 'microsecond*'
| 'ms', 'millisecond*'
| 's', 'second*'
maxiter: int, optioanl, default = 1000000
Optional parameter with reversible = True. maximum number of iterations
before the transition matrix estimation method exits
maxerr : float, optional, default = 1e-8
Optional parameter with reversible = True.
convergence tolerance for transition matrix estimation.
This specifies the maximum change of the Euclidean norm of relative
stationary probabilities (:math:`x_i = \sum_k x_{ik}`). The relative
stationary probability changes
:math:`e_i = (x_i^{(1)} - x_i^{(2)})/(x_i^{(1)} + x_i^{(2)})` are used
in order to track changes in small probabilities. The Euclidean norm
of the change vector, :math:`|e_i|_2`, is compared to maxerr.
"""
self.lag = lag
# set basic parameters
self.reversible = reversible
self.statdist_constraint = _types.ensure_ndarray_or_None(statdist_constraint, ndim=None, kind='numeric')
if self.statdist_constraint is not None: # renormalize
self.statdist_constraint /= self.statdist_constraint.sum()
# sparse matrix computation wanted?
self.sparse = sparse
# store counting mode (lowercase)
self.count_mode = str(count_mode).lower()
if self.count_mode not in ('sliding', 'effective', 'sample'):
raise ValueError('count mode ' + count_mode + ' is unknown.')
# store connectivity mode (lowercase)
self.connectivity = connectivity.lower()
if self.connectivity == 'largest':
pass # this is the current default. no need to do anything
elif self.connectivity == 'all':
raise NotImplementedError('MSM estimation with connectivity=\'all\' is currently not implemented.')
elif self.connectivity == 'none':
raise NotImplementedError('MSM estimation with connectivity=\'none\' is currently not implemented.')
else:
raise ValueError('connectivity mode ' + str(connectivity) + ' is unknown.')
# time step
self.dt_traj = dt_traj
self.timestep_traj = _TimeUnit(dt_traj)
# convergence parameters
self.maxiter = maxiter
self.maxerr = maxerr
def cktest(self,
n_observables=None,
observables='phi',
statistics='psi',
mlags=10,
n_jobs=1,
show_progress=True,
iterable=None):
r"""Do the Chapman-Kolmogorov test by computing predictions for higher lag times and by performing estimations at higher lag times.
Notes
-----
This method computes two sets of time-lagged covariance matrices
* estimates at higher lag times :
.. math::
\left\langle \mathbf{K}(n\tau)g_{i},f_{j}\right\rangle_{\rho_{0}}
where :math:`\rho_{0}` is the empirical distribution implicitly defined
by all data points from time steps 0 to T-tau in all trajectories,
:math:`\mathbf{K}(n\tau)` is a rank-reduced Koopman matrix estimated
at the lag-time n*tau and g and f are some functions of the data.
Rank-reduction of the Koopman matrix is controlled by the `dim`
parameter of :func:`vamp <pyemma.coordinates.vamp>`.
* predictions at higher lag times :
.. math::
\left\langle \mathbf{K}^{n}(\tau)g_{i},f_{j}\right\rangle_{\rho_{0}}
where :math:`\mathbf{K}^{n}` is the n'th power of the rank-reduced
Koopman matrix contained in self.
The Champan-Kolmogorov test is to compare the predictions to the
estimates.
Parameters
----------
n_observables : int, optional, default=None
Limit the number of default observables (and of default statistics)
to this number.
Only used if `observables` are None or `statistics` are None.
observables : np.ndarray((input_dimension, n_observables)) or 'phi'
Coefficients that express one or multiple observables :math:`g`
in the basis of the input features.
This parameter can be 'phi'. In that case, the dominant
right singular functions of the Koopman operator estimated
at the smallest lag time are used as default observables.
statistics : np.ndarray((input_dimension, n_statistics)) or 'psi'
Coefficients that express one or multiple statistics :math:`f`
in the basis of the input features.
This parameter can be 'psi'. In that case, the dominant
left singular functions of the Koopman operator estimated
at the smallest lag time are used as default statistics.
mlags : int or int-array, default=10
multiples of lag times for testing the Model, e.g. range(10).
A single int will trigger a range, i.e. mlags=10 maps to
mlags=range(10).
Note that you need to be able to do a model prediction for each
of these lag time multiples, e.g. the value 0 only make sense
if model.expectation(lag_multiple=0) will work.
n_jobs : int, default=1
how many jobs to use during calculation
show_progress : bool, default=True
Show progressbars for calculation?
iterable : any data format that `pyemma.coordinates.vamp()` accepts as input, optional
It `iterable` is None, the same data source with which VAMP
was initialized will be used for all estimation.
Otherwise, all estimates (not predictions) from data will be computed
from the data contained in `iterable`.
Returns
-------
vckv : :class:`VAMPChapmanKolmogorovValidator <pyemma.coordinates.transform.VAMPChapmanKolmogorovValidator>`
Contains the estimated and the predicted covarince matrices.
The object can be plotted with :func:`plot_cktest <pyemma.plots.plot_cktest>` with the option `y01=False`.
"""
if n_observables is not None:
if n_observables > self.dimension():
warnings.warn(
'Selected singular functions as observables but dimension '
'is lower than requested number of observables.')
n_observables = self.dimension()
else:
n_observables = self.dimension()
if isinstance(observables, str) and observables == 'phi':
observables = self.singular_vectors_right[:, 0:n_observables]
observables_mean_free = True
else:
ensure_ndarray(observables, ndim=2)
observables_mean_free = False
if isinstance(statistics, str) and statistics == 'psi':
statistics = self.singular_vectors_left[:, 0:n_observables]
statistics_mean_free = True
else:
ensure_ndarray_or_None(statistics, ndim=2)
statistics_mean_free = False
ck = VAMPChapmanKolmogorovValidator(self.model,
self,
observables,
statistics,
observables_mean_free,
statistics_mean_free,
mlags=mlags,
n_jobs=n_jobs,
show_progress=show_progress)
if iterable is None:
iterable = self.data_producer
ck.estimate(iterable)
return ck
def __init__(self, lag=1, reversible=True, statdist_constraint=None,
count_mode='sliding', sparse=False,
connectivity='largest', dt_traj='1 step', maxiter=1000000,
maxerr=1e-8, score_method='VAMP2', score_k=10,
mincount_connectivity='1/n', core_set=None, milestoning_method='last_core'):
r"""Maximum likelihood estimator for MSMs given discrete trajectory statistics
Parameters
----------
lag : int
lag time at which transitions are counted and the transition matrix is
estimated.
reversible : bool, optional, default = True
If true compute reversible MSM, else non-reversible MSM
statdist : (M,) ndarray, optional
Stationary vector on the full set of states. Estimation will be
made such the the resulting transition matrix has this distribution
as an equilibrium distribution. Set probabilities to zero if these
states should be excluded from the analysis.
count_mode : str, optional, default='sliding'
mode to obtain count matrices from discrete trajectories. Should be
one of:
* 'sliding' : A trajectory of length T will have :math:`T-tau` counts
at time indexes
.. math::
(0 \rightarrow \tau), (1 \rightarrow \tau+1), ..., (T-\tau-1 \rightarrow T-1)
* 'effective' : Uses an estimate of the transition counts that are
statistically uncorrelated. Recommended when used with a
Bayesian MSM.
* 'sample' : A trajectory of length T will have :math:`T/tau` counts
at time indexes
.. math::
(0 \rightarrow \tau), (\tau \rightarrow 2 \tau), ..., (((T/tau)-1) \tau \rightarrow T)
sparse : bool, optional, default = False
If true compute count matrix, transition matrix and all derived
quantities using sparse matrix algebra. In this case python sparse
matrices will be returned by the corresponding functions instead of
numpy arrays. This behavior is suggested for very large numbers of
states (e.g. > 4000) because it is likely to be much more efficient.
connectivity : str, optional, default = 'largest'
Connectivity mode. Three methods are intended (currently only 'largest'
is implemented)
* 'largest' : The active set is the largest reversibly connected set.
All estimation will be done on this subset and all quantities
(transition matrix, stationary distribution, etc) are only defined
on this subset and are correspondingly smaller than the full set
of states
* 'all' : The active set is the full set of states. Estimation will be
conducted on each reversibly connected set separately. That means
the transition matrix will decompose into disconnected submatrices,
the stationary vector is only defined within subsets, etc.
Currently not implemented.
* 'none' : The active set is the full set of states. Estimation will
be conducted on the full set of
states without ensuring connectivity. This only permits
nonreversible estimation. Currently not implemented.
dt_traj : str, optional, default='1 step'
Description of the physical time of the input trajectories. May be used
by analysis algorithms such as plotting tools to pretty-print the axes.
By default '1 step', i.e. there is no physical time unit. Specify by a
number, whitespace and unit. Permitted units are (* is an arbitrary
string):
| 'fs', 'femtosecond*'
| 'ps', 'picosecond*'
| 'ns', 'nanosecond*'
| 'us', 'microsecond*'
| 'ms', 'millisecond*'
| 's', 'second*'
maxiter: int, optioanl, default = 1000000
Optional parameter with reversible = True. maximum number of iterations
before the transition matrix estimation method exits
maxerr : float, optional, default = 1e-8
Optional parameter with reversible = True.
convergence tolerance for transition matrix estimation.
This specifies the maximum change of the Euclidean norm of relative
stationary probabilities (:math:`x_i = \sum_k x_{ik}`). The relative
stationary probability changes
:math:`e_i = (x_i^{(1)} - x_i^{(2)})/(x_i^{(1)} + x_i^{(2)})` are used
in order to track changes in small probabilities. The Euclidean norm
of the change vector, :math:`|e_i|_2`, is compared to maxerr.
score_method : str, optional, default='VAMP2'
Score to be used with score function. Available are:
| 'VAMP1' [1]_
| 'VAMP2' [1]_
| 'VAMPE' [1]_
score_k : int or None
The maximum number of eigenvalues or singular values used in the
score. If set to None, all available eigenvalues will be used.
mincount_connectivity : float or '1/n'
minimum number of counts to consider a connection between two states.
Counts lower than that will count zero in the connectivity check and
may thus separate the resulting transition matrix. The default
evaluates to 1/nstates.
core_set : None (default) or array like, dtype=int
Definition of core set for milestoning MSMs.
If set to None, replaces state -1 (if found in discrete trajectories) and
performs milestone counting. No effect for Voronoi-discretized trajectories (default).
If a list or np.ndarray is supplied, discrete trajectories will be assigned
accordingly.
milestoning_method : str
Method to use for counting transitions in trajectories with unassigned frames.
Currently available:
| 'last_core', assigns unassigned frames to last visited core
References
----------
.. [1] H. Wu and F. Noe: Variational approach for learning Markov processes from time series data
(in preparation)
"""
super(MaximumLikelihoodMSM, self).__init__(lag=lag, reversible=reversible, count_mode=count_mode,
sparse=sparse, connectivity=connectivity, dt_traj=dt_traj,
score_method=score_method, score_k=score_k,
mincount_connectivity=mincount_connectivity,
core_set=core_set, milestoning_method=milestoning_method)
self.statdist_constraint = _types.ensure_ndarray_or_None(statdist_constraint, ndim=None, kind='numeric')
if self.statdist_constraint is not None: # renormalize
self.statdist_constraint /= self.statdist_constraint.sum()
# convergence parameters
self.maxiter = maxiter
self.maxerr = maxerr
