def fingerprint_correlation(self, a, b=None, k=None, ncv=None): # basic checks for a and b a = _types.ensure_ndarray(a, ndim=1, kind='numeric') b = _types.ensure_ndarray_or_None(b, ndim=1, kind='numeric', size=len(a)) # are we on microstates space? if len(a) == self.nstates_obs: a = _np.dot(self.observation_probabilities, a) if b is not None: b = _np.dot(self.observation_probabilities, b) # now we are on macrostate space, or something is wrong if len(a) == self.nstates: return _MSM.fingerprint_correlation(self, a, b=b) else: raise ValueError( 'observable vectors have size %s which is incompatible with both hidden (%s)' ' and observed states (%s)' % (len(a), self.nstates, self.nstates_obs))
def __init__(self, lag=1, reversible=True, statdist_constraint=None, count_mode='sliding', sparse=False, connectivity='largest', dt_traj='1 step', maxiter=1000000, maxerr=1e-8): r"""Maximum likelihood estimator for MSMs given discrete trajectory statistics Parameters ---------- lag : int lag time at which transitions are counted and the transition matrix is estimated. reversible : bool, optional, default = True If true compute reversible MSM, else non-reversible MSM statdist : (M,) ndarray, optional Stationary vector on the full set of states. Estimation will be made such the the resulting transition matrix has this distribution as an equilibrium distribution. Set probabilities to zero if these states should be excluded from the analysis. count_mode : str, optional, default='sliding' mode to obtain count matrices from discrete trajectories. Should be one of: * 'sliding' : A trajectory of length T will have :math:`T-tau` counts at time indexes .. math:: (0 \rightarrow \tau), (1 \rightarrow \tau+1), ..., (T-\tau-1 \rightarrow T-1) * 'effective' : Uses an estimate of the transition counts that are statistically uncorrelated. Recommended when used with a Bayesian MSM. * 'sample' : A trajectory of length T will have :math:`T/tau` counts at time indexes .. math:: (0 \rightarrow \tau), (\tau \rightarrow 2 \tau), ..., (((T/tau)-1) \tau \rightarrow T) sparse : bool, optional, default = False If true compute count matrix, transition matrix and all derived quantities using sparse matrix algebra. In this case python sparse matrices will be returned by the corresponding functions instead of numpy arrays. This behavior is suggested for very large numbers of states (e.g. > 4000) because it is likely to be much more efficient. connectivity : str, optional, default = 'largest' Connectivity mode. Three methods are intended (currently only 'largest' is implemented) * 'largest' : The active set is the largest reversibly connected set. All estimation will be done on this subset and all quantities (transition matrix, stationary distribution, etc) are only defined on this subset and are correspondingly smaller than the full set of states * 'all' : The active set is the full set of states. Estimation will be conducted on each reversibly connected set separately. That means the transition matrix will decompose into disconnected submatrices, the stationary vector is only defined within subsets, etc. Currently not implemented. * 'none' : The active set is the full set of states. Estimation will be conducted on the full set of states without ensuring connectivity. This only permits nonreversible estimation. Currently not implemented. dt_traj : str, optional, default='1 step' Description of the physical time of the input trajectories. May be used by analysis algorithms such as plotting tools to pretty-print the axes. By default '1 step', i.e. there is no physical time unit. Specify by a number, whitespace and unit. Permitted units are (* is an arbitrary string): | 'fs', 'femtosecond*' | 'ps', 'picosecond*' | 'ns', 'nanosecond*' | 'us', 'microsecond*' | 'ms', 'millisecond*' | 's', 'second*' maxiter: int, optioanl, default = 1000000 Optional parameter with reversible = True. maximum number of iterations before the transition matrix estimation method exits maxerr : float, optional, default = 1e-8 Optional parameter with reversible = True. convergence tolerance for transition matrix estimation. This specifies the maximum change of the Euclidean norm of relative stationary probabilities (:math:`x_i = \sum_k x_{ik}`). The relative stationary probability changes :math:`e_i = (x_i^{(1)} - x_i^{(2)})/(x_i^{(1)} + x_i^{(2)})` are used in order to track changes in small probabilities. The Euclidean norm of the change vector, :math:`|e_i|_2`, is compared to maxerr. """ self.lag = lag # set basic parameters self.reversible = reversible self.statdist_constraint = _types.ensure_ndarray_or_None(statdist_constraint, ndim=None, kind='numeric') if self.statdist_constraint is not None: # renormalize self.statdist_constraint /= self.statdist_constraint.sum() # sparse matrix computation wanted? self.sparse = sparse # store counting mode (lowercase) self.count_mode = str(count_mode).lower() if self.count_mode not in ('sliding', 'effective', 'sample'): raise ValueError('count mode ' + count_mode + ' is unknown.') # store connectivity mode (lowercase) self.connectivity = connectivity.lower() if self.connectivity == 'largest': pass # this is the current default. no need to do anything elif self.connectivity == 'all': raise NotImplementedError('MSM estimation with connectivity=\'all\' is currently not implemented.') elif self.connectivity == 'none': raise NotImplementedError('MSM estimation with connectivity=\'none\' is currently not implemented.') else: raise ValueError('connectivity mode ' + str(connectivity) + ' is unknown.') # time step self.dt_traj = dt_traj self.timestep_traj = _TimeUnit(dt_traj) # convergence parameters self.maxiter = maxiter self.maxerr = maxerr
def cktest(self, n_observables=None, observables='phi', statistics='psi', mlags=10, n_jobs=1, show_progress=True, iterable=None): r"""Do the Chapman-Kolmogorov test by computing predictions for higher lag times and by performing estimations at higher lag times. Notes ----- This method computes two sets of time-lagged covariance matrices * estimates at higher lag times : .. math:: \left\langle \mathbf{K}(n\tau)g_{i},f_{j}\right\rangle_{\rho_{0}} where :math:`\rho_{0}` is the empirical distribution implicitly defined by all data points from time steps 0 to T-tau in all trajectories, :math:`\mathbf{K}(n\tau)` is a rank-reduced Koopman matrix estimated at the lag-time n*tau and g and f are some functions of the data. Rank-reduction of the Koopman matrix is controlled by the `dim` parameter of :func:`vamp <pyemma.coordinates.vamp>`. * predictions at higher lag times : .. math:: \left\langle \mathbf{K}^{n}(\tau)g_{i},f_{j}\right\rangle_{\rho_{0}} where :math:`\mathbf{K}^{n}` is the n'th power of the rank-reduced Koopman matrix contained in self. The Champan-Kolmogorov test is to compare the predictions to the estimates. Parameters ---------- n_observables : int, optional, default=None Limit the number of default observables (and of default statistics) to this number. Only used if `observables` are None or `statistics` are None. observables : np.ndarray((input_dimension, n_observables)) or 'phi' Coefficients that express one or multiple observables :math:`g` in the basis of the input features. This parameter can be 'phi'. In that case, the dominant right singular functions of the Koopman operator estimated at the smallest lag time are used as default observables. statistics : np.ndarray((input_dimension, n_statistics)) or 'psi' Coefficients that express one or multiple statistics :math:`f` in the basis of the input features. This parameter can be 'psi'. In that case, the dominant left singular functions of the Koopman operator estimated at the smallest lag time are used as default statistics. mlags : int or int-array, default=10 multiples of lag times for testing the Model, e.g. range(10). A single int will trigger a range, i.e. mlags=10 maps to mlags=range(10). Note that you need to be able to do a model prediction for each of these lag time multiples, e.g. the value 0 only make sense if model.expectation(lag_multiple=0) will work. n_jobs : int, default=1 how many jobs to use during calculation show_progress : bool, default=True Show progressbars for calculation? iterable : any data format that `pyemma.coordinates.vamp()` accepts as input, optional It `iterable` is None, the same data source with which VAMP was initialized will be used for all estimation. Otherwise, all estimates (not predictions) from data will be computed from the data contained in `iterable`. Returns ------- vckv : :class:`VAMPChapmanKolmogorovValidator <pyemma.coordinates.transform.VAMPChapmanKolmogorovValidator>` Contains the estimated and the predicted covarince matrices. The object can be plotted with :func:`plot_cktest <pyemma.plots.plot_cktest>` with the option `y01=False`. """ if n_observables is not None: if n_observables > self.dimension(): warnings.warn( 'Selected singular functions as observables but dimension ' 'is lower than requested number of observables.') n_observables = self.dimension() else: n_observables = self.dimension() if isinstance(observables, str) and observables == 'phi': observables = self.singular_vectors_right[:, 0:n_observables] observables_mean_free = True else: ensure_ndarray(observables, ndim=2) observables_mean_free = False if isinstance(statistics, str) and statistics == 'psi': statistics = self.singular_vectors_left[:, 0:n_observables] statistics_mean_free = True else: ensure_ndarray_or_None(statistics, ndim=2) statistics_mean_free = False ck = VAMPChapmanKolmogorovValidator(self.model, self, observables, statistics, observables_mean_free, statistics_mean_free, mlags=mlags, n_jobs=n_jobs, show_progress=show_progress) if iterable is None: iterable = self.data_producer ck.estimate(iterable) return ck
def __init__(self, lag=1, reversible=True, statdist_constraint=None, count_mode='sliding', sparse=False, connectivity='largest', dt_traj='1 step', maxiter=1000000, maxerr=1e-8, score_method='VAMP2', score_k=10, mincount_connectivity='1/n', core_set=None, milestoning_method='last_core'): r"""Maximum likelihood estimator for MSMs given discrete trajectory statistics Parameters ---------- lag : int lag time at which transitions are counted and the transition matrix is estimated. reversible : bool, optional, default = True If true compute reversible MSM, else non-reversible MSM statdist : (M,) ndarray, optional Stationary vector on the full set of states. Estimation will be made such the the resulting transition matrix has this distribution as an equilibrium distribution. Set probabilities to zero if these states should be excluded from the analysis. count_mode : str, optional, default='sliding' mode to obtain count matrices from discrete trajectories. Should be one of: * 'sliding' : A trajectory of length T will have :math:`T-tau` counts at time indexes .. math:: (0 \rightarrow \tau), (1 \rightarrow \tau+1), ..., (T-\tau-1 \rightarrow T-1) * 'effective' : Uses an estimate of the transition counts that are statistically uncorrelated. Recommended when used with a Bayesian MSM. * 'sample' : A trajectory of length T will have :math:`T/tau` counts at time indexes .. math:: (0 \rightarrow \tau), (\tau \rightarrow 2 \tau), ..., (((T/tau)-1) \tau \rightarrow T) sparse : bool, optional, default = False If true compute count matrix, transition matrix and all derived quantities using sparse matrix algebra. In this case python sparse matrices will be returned by the corresponding functions instead of numpy arrays. This behavior is suggested for very large numbers of states (e.g. > 4000) because it is likely to be much more efficient. connectivity : str, optional, default = 'largest' Connectivity mode. Three methods are intended (currently only 'largest' is implemented) * 'largest' : The active set is the largest reversibly connected set. All estimation will be done on this subset and all quantities (transition matrix, stationary distribution, etc) are only defined on this subset and are correspondingly smaller than the full set of states * 'all' : The active set is the full set of states. Estimation will be conducted on each reversibly connected set separately. That means the transition matrix will decompose into disconnected submatrices, the stationary vector is only defined within subsets, etc. Currently not implemented. * 'none' : The active set is the full set of states. Estimation will be conducted on the full set of states without ensuring connectivity. This only permits nonreversible estimation. Currently not implemented. dt_traj : str, optional, default='1 step' Description of the physical time of the input trajectories. May be used by analysis algorithms such as plotting tools to pretty-print the axes. By default '1 step', i.e. there is no physical time unit. Specify by a number, whitespace and unit. Permitted units are (* is an arbitrary string): | 'fs', 'femtosecond*' | 'ps', 'picosecond*' | 'ns', 'nanosecond*' | 'us', 'microsecond*' | 'ms', 'millisecond*' | 's', 'second*' maxiter: int, optioanl, default = 1000000 Optional parameter with reversible = True. maximum number of iterations before the transition matrix estimation method exits maxerr : float, optional, default = 1e-8 Optional parameter with reversible = True. convergence tolerance for transition matrix estimation. This specifies the maximum change of the Euclidean norm of relative stationary probabilities (:math:`x_i = \sum_k x_{ik}`). The relative stationary probability changes :math:`e_i = (x_i^{(1)} - x_i^{(2)})/(x_i^{(1)} + x_i^{(2)})` are used in order to track changes in small probabilities. The Euclidean norm of the change vector, :math:`|e_i|_2`, is compared to maxerr. score_method : str, optional, default='VAMP2' Score to be used with score function. Available are: | 'VAMP1' [1]_ | 'VAMP2' [1]_ | 'VAMPE' [1]_ score_k : int or None The maximum number of eigenvalues or singular values used in the score. If set to None, all available eigenvalues will be used. mincount_connectivity : float or '1/n' minimum number of counts to consider a connection between two states. Counts lower than that will count zero in the connectivity check and may thus separate the resulting transition matrix. The default evaluates to 1/nstates. core_set : None (default) or array like, dtype=int Definition of core set for milestoning MSMs. If set to None, replaces state -1 (if found in discrete trajectories) and performs milestone counting. No effect for Voronoi-discretized trajectories (default). If a list or np.ndarray is supplied, discrete trajectories will be assigned accordingly. milestoning_method : str Method to use for counting transitions in trajectories with unassigned frames. Currently available: | 'last_core', assigns unassigned frames to last visited core References ---------- .. [1] H. Wu and F. Noe: Variational approach for learning Markov processes from time series data (in preparation) """ super(MaximumLikelihoodMSM, self).__init__(lag=lag, reversible=reversible, count_mode=count_mode, sparse=sparse, connectivity=connectivity, dt_traj=dt_traj, score_method=score_method, score_k=score_k, mincount_connectivity=mincount_connectivity, core_set=core_set, milestoning_method=milestoning_method) self.statdist_constraint = _types.ensure_ndarray_or_None(statdist_constraint, ndim=None, kind='numeric') if self.statdist_constraint is not None: # renormalize self.statdist_constraint /= self.statdist_constraint.sum() # convergence parameters self.maxiter = maxiter self.maxerr = maxerr