def _prune_states(self, lattice_frame, maxrank, beamlogprob):
""" Returns indices of the active states in `lattice_frame`
after rank and beam pruning.
"""
# Beam pruning
threshlogprob = logsum(lattice_frame) + beamlogprob
# Rank pruning
if maxrank:
# How big should our rank pruning histogram be?
nbins = 3 * len(lattice_frame)
lattice_min = lattice_frame[lattice_frame > ZEROLOGPROB].min() - 1
hst, cdf = np.histogram(lattice_frame,
bins=nbins,
range=(lattice_min, lattice_frame.max()))
# Want to look at the high ranks.
hst = hst[::-1].cumsum()
cdf = cdf[::-1]
rankthresh = cdf[hst >= min(maxrank, self._n_states)].max()
# Only change the threshold if it is stricter than the beam
# threshold.
threshlogprob = max(threshlogprob, rankthresh)
# Which states are active?
state_idx, = np.nonzero(lattice_frame >= threshlogprob)
return state_idx
def _do_forward_pass(self,
framelogprob,
maxrank=None,
beamlogprob=-np.Inf):
nobs = len(framelogprob)
fwdlattice = np.zeros((nobs, self._n_states))
fwdlattice[0] = self._log_startprob + framelogprob[0]
for n in range(1, nobs):
idx = self._prune_states(fwdlattice[n - 1], maxrank, beamlogprob)
fwdlattice[n] = (logsum(
self._log_transmat[idx].T + fwdlattice[n - 1, idx], axis=1) +
framelogprob[n])
fwdlattice[fwdlattice <= ZEROLOGPROB] = -np.Inf
return logsum(fwdlattice[-1]), fwdlattice
def _accumulate_sufficient_statistics(self, stats, seq, framelogprob,
posteriors, fwdlattice, bwdlattice,
params):
stats['nobs'] += 1
if 's' in params:
stats['start'] += posteriors[0]
if 't' in params:
for t in range(len(framelogprob)):
zeta = (fwdlattice[t - 1][:, np.newaxis] + self._log_transmat +
framelogprob[t] + bwdlattice[t])
stats['trans'] += np.exp(zeta - logsum(zeta))
def eval(self, obs, maxrank=None, beamlogprob=-np.Inf):
"""Compute the log probability under the model and compute posteriors
Implements rank and beam pruning in the forward-backward
algorithm to speed up inference in large models.
Parameters
----------
obs : array_like, shape (n, n_features)
Sequence of n_features-dimensional data points. Each row
corresponds to a single point in the sequence.
maxrank : int
Maximum rank to evaluate for rank pruning. If not None,
only consider the top `maxrank` states in the inner
sum of the forward algorithm recursion. Defaults to None
(no rank pruning). See The HTK Book for more details.
beamlogprob : float
Width of the beam-pruning beam in log-probability units.
Defaults to -numpy.Inf (no beam pruning). See The HTK
Book for more details.
Returns
-------
logprob : array_like, shape (n,)
Log probabilities of the sequence `observation`
posteriors: array_like, shape (n, n_states)
Posterior probabilities of each state for each
observation
See Also
--------
score : Compute the log probability under the model
decode : Find most likely state sequence corresponding to a `observation`
"""
obs = np.asanyarray(obs)
framelogprob = self._compute_log_likelihood(obs)
logprob, fwdlattice = self._do_forward_pass(framelogprob, maxrank,
beamlogprob)
bwdlattice = self._do_backward_pass(framelogprob, fwdlattice, maxrank,
beamlogprob)
gamma = fwdlattice + bwdlattice
# gamma is guaranteed to be correctly normalized by logprob at
# all frames, unless we do approximate inference using pruning.
# So, we will normalize each frame explicitly in case we
# pruned too aggressively.
posteriors = np.exp(gamma.T - logsum(gamma, axis=1)).T
return logprob, posteriors
def _do_backward_pass(self,
framelogprob,
fwdlattice,
maxrank=None,
beamlogprob=-np.Inf):
nobs = len(framelogprob)
bwdlattice = np.zeros((nobs, self._n_states))
for n in range(nobs - 1, 0, -1):
# Do HTK style pruning (p. 137 of HTK Book version 3.4).
# Don't bother computing backward probability if
# fwdlattice * bwdlattice is more than a certain distance
# from the total log likelihood.
idx = self._prune_states(bwdlattice[n] + fwdlattice[n], None, -50)
#beamlogprob)
#-numpy.Inf)
bwdlattice[n -
1] = logsum(self._log_transmat[:, idx] +
bwdlattice[n, idx] + framelogprob[n, idx],
axis=1)
bwdlattice[bwdlattice <= ZEROLOGPROB] = -np.Inf
return bwdlattice
def fit(self,
obs,
n_iter=10,
thresh=1e-2,
params=string.ascii_letters,
init_params=string.ascii_letters,
maxrank=None,
beamlogprob=-np.Inf,
**kwargs):
"""Estimate model parameters.
An initialization step is performed before entering the EM
algorithm. If you want to avoid this step, set the keyword
argument init_params to the empty string ''. Likewise, if you
would like just to do an initialization, call this method with
n_iter=0.
Parameters
----------
obs : list
List of array-like observation sequences (shape (n_i, n_features)).
n_iter : int, optional
Number of iterations to perform.
thresh : float, optional
Convergence threshold.
params : string, optional
Controls which parameters are updated in the training
process. Can contain any combination of 's' for startprob,
't' for transmat, 'm' for means, and 'c' for covars, etc.
Defaults to all parameters.
init_params : string, optional
Controls which parameters are initialized prior to
training. Can contain any combination of 's' for
startprob, 't' for transmat, 'm' for means, and 'c' for
covars, etc. Defaults to all parameters.
maxrank : int, optional
Maximum rank to evaluate for rank pruning. If not None,
only consider the top `maxrank` states in the inner
sum of the forward algorithm recursion. Defaults to None
(no rank pruning). See "The HTK Book" for more details.
beamlogprob : float, optional
Width of the beam-pruning beam in log-probability units.
Defaults to -numpy.Inf (no beam pruning). See "The HTK
Book" for more details.
Notes
-----
In general, `logprob` should be non-decreasing unless
aggressive pruning is used. Decreasing `logprob` is generally
a sign of overfitting (e.g. a covariance parameter getting too
small). You can fix this by getting more training data, or
decreasing `covars_prior`.
"""
self._init(obs, init_params)
logprob = []
for i in range(n_iter):
# Expectation step
stats = self._initialize_sufficient_statistics()
curr_logprob = 0
for seq in obs:
framelogprob = self._compute_log_likelihood(seq)
lpr, fwdlattice = self._do_forward_pass(
framelogprob, maxrank, beamlogprob)
bwdlattice = self._do_backward_pass(framelogprob, fwdlattice,
maxrank, beamlogprob)
gamma = fwdlattice + bwdlattice
posteriors = np.exp(gamma.T - logsum(gamma, axis=1)).T
curr_logprob += lpr
self._accumulate_sufficient_statistics(stats, seq,
framelogprob,
posteriors, fwdlattice,
bwdlattice, params)
logprob.append(curr_logprob)
# Check for convergence.
if i > 0 and abs(logprob[-1] - logprob[-2]) < thresh:
break
# Maximization step
self._do_mstep(stats, params, **kwargs)
return self