def trainCGandEM(distInit, accumulate, ps = d.getDefaultParamSpec(), createAccEM = d.getDefaultCreateAcc(), estimateTotAux = d.getDefaultEstimateTotAux(), iterations = 5, length = -50, afterEst = None, verbosity = 0):
"""Re-estimates a distribution using conjugate gradients and EM.
See the note in the docstring for this module for information on how the
log likelihood is scaled. This scaling is presumed to have only a small
impact on the dist returned by this function (via its impact on trainCG).
"""
assert iterations >= 1
dist = distInit
for it in range(1, iterations + 1):
if verbosity >= 1:
print 'trainCGandEM: starting it =', it, 'of CG and EM'
dist = (timed(trainCG) if verbosity >= 2 else trainCG)(dist, accumulate, ps = ps, length = length, verbosity = verbosity)
dist, _, _, _ = expectationMaximization(dist, accumulate, createAcc = createAccEM, estimateTotAux = estimateTotAux, verbosity = verbosity)
if afterEst is not None:
afterEst(dist = dist, it = it)
if verbosity >= 1:
print 'trainCGandEM: finished it =', it, 'of CG and EM'
print 'trainCGandEM:'
return dist
def expectationMaximization(distPrev,
accumulate,
createAcc=d.getDefaultCreateAcc(),
estimateTotAux=d.getDefaultEstimateTotAux(),
afterAcc=None,
monotoneAux=True,
verbosity=0):
"""Performs one step of expectation maximization.
See the note in the docstring for this module for information on how the
log likelihood is scaled. This scaling has no effect on the dist returned
by this function.
"""
acc = createAcc(distPrev)
accumulate(acc)
if afterAcc is not None:
afterAcc(acc)
logLikePrev = acc.logLike()
count = acc.count()
count = max(count, 1.0)
dist, (aux, auxRat) = estimateTotAux(acc)
if monotoneAux and aux < logLikePrev:
raise RuntimeError(
're-estimated auxiliary value (%s) less than previous log likelihood (%s) during expectation-maximization (count = %s)'
% (aux / count, logLikePrev / count, count))
if verbosity >= 2:
print 'trainEM: logLikePrev = %s -> aux = %s (%s) (%s count)' % (
logLikePrev / count, aux / count, d.Rat.toString(auxRat), count)
return dist, logLikePrev, (aux, auxRat), count
def getDeltaIter(labelsRoot,
accForLabel,
distRoot,
splitIter,
createAcc=d.getDefaultCreateAcc()):
def getLogProb(dist, labels):
acc = createAcc(dist)
for label in labels:
d.addAcc(acc, accForLabel(label))
return acc.logLike()
logProbDict = dict()
logProbDict[()] = getLogProb(distRoot, labelsRoot)
labelsDict = dict()
labelsDict[()] = labelsRoot
for answerSeq, fullQuestion, distForAnswer in splitIter:
labelsForAnswer = partitionLabels(labelsDict[answerSeq], fullQuestion)
_, question = fullQuestion
delta = -logProbDict[answerSeq]
for answer, labels, dist in zip(question.codomain(), labelsForAnswer,
distForAnswer):
logProb = getLogProb(dist, labels)
delta += logProb
logProbDict[answerSeq + (answer, )] = logProb
labelsDict[answerSeq + (answer, )] = labels
yield delta
def getDeltaIter(labelsRoot, accForLabel, distRoot, splitIter,
createAcc = d.getDefaultCreateAcc()):
def getLogProb(dist, labels):
acc = createAcc(dist)
for label in labels:
d.addAcc(acc, accForLabel(label))
return acc.logLike()
logProbDict = dict()
logProbDict[()] = getLogProb(distRoot, labelsRoot)
labelsDict = dict()
labelsDict[()] = labelsRoot
for answerSeq, fullQuestion, distForAnswer in splitIter:
labelsForAnswer = partitionLabels(labelsDict[answerSeq], fullQuestion)
_, question = fullQuestion
delta = -logProbDict[answerSeq]
for answer, labels, dist in zip(question.codomain(), labelsForAnswer,
distForAnswer):
logProb = getLogProb(dist, labels)
delta += logProb
logProbDict[answerSeq + (answer,)] = logProb
labelsDict[answerSeq + (answer,)] = labels
yield delta
def expectationMaximization(distPrev, accumulate, createAcc = d.getDefaultCreateAcc(), estimateTotAux = d.getDefaultEstimateTotAux(), afterAcc = None, monotoneAux = True, verbosity = 0):
"""Performs one step of expectation maximization.
See the note in the docstring for this module for information on how the
log likelihood is scaled. This scaling has no effect on the dist returned
by this function.
"""
acc = createAcc(distPrev)
accumulate(acc)
if afterAcc is not None:
afterAcc(acc)
logLikePrev = acc.logLike()
count = acc.count()
count = max(count, 1.0)
dist, (aux, auxRat) = estimateTotAux(acc)
if monotoneAux and aux < logLikePrev:
raise RuntimeError('re-estimated auxiliary value (%s) less than previous log likelihood (%s) during expectation-maximization (count = %s)' % (aux / count, logLikePrev / count, count))
if verbosity >= 2:
print 'trainEM: logLikePrev = %s -> aux = %s (%s) (%s count)' % (logLikePrev / count, aux / count, d.Rat.toString(auxRat), count)
return dist, logLikePrev, (aux, auxRat), count
def trainEM(distInit, accumulate, createAcc = d.getDefaultCreateAcc(), estimateTotAux = d.getDefaultEstimateTotAux(), logLikePrevInit = float('-inf'), deltaThresh = 1e-8, minIterations = 1, maxIterations = None, beforeAcc = None, afterAcc = None, afterEst = None, monotone = False, monotoneAux = True, verbosity = 0):
"""Re-estimates a distribution using expectation maximization.
See the note in the docstring for this module for information on how the
log likelihood is scaled. This scaling only affects the dist returned by
this function to the extent that it effectively scales the deltaThresh
threshold used to assess convergence, and so may sometimes affect the number
of iterations of expectation maximization performed.
"""
assert minIterations >= 1
assert maxIterations is None or maxIterations >= minIterations
dist = distInit
logLikePrev = logLikePrevInit
it = 0
converged = False
while it < minIterations or (not converged) and (maxIterations is None or it < maxIterations):
if beforeAcc is not None:
beforeAcc(dist)
logLikePrevPrev = logLikePrev
if verbosity >= 2:
print 'trainEM: it %s:' % (it + 1)
dist, logLikePrev, (aux, auxRat), count = expectationMaximization(dist, accumulate, createAcc = createAcc, estimateTotAux = estimateTotAux, afterAcc = afterAcc, monotoneAux = monotoneAux, verbosity = verbosity)
deltaLogLikePrev = logLikePrev - logLikePrevPrev
if monotone and deltaLogLikePrev < 0.0:
raise RuntimeError('log likelihood decreased during expectation-maximization')
if verbosity >= 2:
print 'trainEM: deltaLogLikePrev = %s' % (deltaLogLikePrev / count)
if afterEst is not None:
afterEst(dist = dist, it = it)
converged = (abs(deltaLogLikePrev) <= deltaThresh * count)
it += 1
if verbosity >= 1:
if converged:
print 'trainEM: converged at thresh', deltaThresh, 'in', it, 'iterations'
else:
print 'trainEM: did NOT converge at thresh', deltaThresh, 'in', it, 'iterations'
return dist
def trainCGandEM(distInit,
accumulate,
ps=d.getDefaultParamSpec(),
createAccEM=d.getDefaultCreateAcc(),
estimateTotAux=d.getDefaultEstimateTotAux(),
iterations=5,
length=-50,
afterEst=None,
verbosity=0):
"""Re-estimates a distribution using conjugate gradients and EM.
See the note in the docstring for this module for information on how the
log likelihood is scaled. This scaling is presumed to have only a small
impact on the dist returned by this function (via its impact on trainCG).
"""
assert iterations >= 1
dist = distInit
for it in range(1, iterations + 1):
if verbosity >= 1:
print 'trainCGandEM: starting it =', it, 'of CG and EM'
dist = (timed(trainCG) if verbosity >= 2 else trainCG)(
dist, accumulate, ps=ps, length=length, verbosity=verbosity)
dist, _, _, _ = expectationMaximization(dist,
accumulate,
createAcc=createAccEM,
estimateTotAux=estimateTotAux,
verbosity=verbosity)
if afterEst is not None:
afterEst(dist=dist, it=it)
if verbosity >= 1:
print 'trainCGandEM: finished it =', it, 'of CG and EM'
print 'trainCGandEM:'
return dist
def trainEM(distInit,
accumulate,
createAcc=d.getDefaultCreateAcc(),
estimateTotAux=d.getDefaultEstimateTotAux(),
logLikePrevInit=float('-inf'),
deltaThresh=1e-8,
minIterations=1,
maxIterations=None,
beforeAcc=None,
afterAcc=None,
afterEst=None,
monotone=False,
monotoneAux=True,
verbosity=0):
"""Re-estimates a distribution using expectation maximization.
See the note in the docstring for this module for information on how the
log likelihood is scaled. This scaling only affects the dist returned by
this function to the extent that it effectively scales the deltaThresh
threshold used to assess convergence, and so may sometimes affect the number
of iterations of expectation maximization performed.
"""
assert minIterations >= 1
assert maxIterations is None or maxIterations >= minIterations
dist = distInit
logLikePrev = logLikePrevInit
it = 0
converged = False
while it < minIterations or (not converged) and (maxIterations is None
or it < maxIterations):
if beforeAcc is not None:
beforeAcc(dist)
logLikePrevPrev = logLikePrev
if verbosity >= 2:
print 'trainEM: it %s:' % (it + 1)
dist, logLikePrev, (aux, auxRat), count = expectationMaximization(
dist,
accumulate,
createAcc=createAcc,
estimateTotAux=estimateTotAux,
afterAcc=afterAcc,
monotoneAux=monotoneAux,
verbosity=verbosity)
deltaLogLikePrev = logLikePrev - logLikePrevPrev
if monotone and deltaLogLikePrev < 0.0:
raise RuntimeError(
'log likelihood decreased during expectation-maximization')
if verbosity >= 2:
print 'trainEM: deltaLogLikePrev = %s' % (deltaLogLikePrev /
count)
if afterEst is not None:
afterEst(dist=dist, it=it)
converged = (abs(deltaLogLikePrev) <= deltaThresh * count)
it += 1
if verbosity >= 1:
if converged:
print 'trainEM: converged at thresh', deltaThresh, 'in', it, 'iterations'
else:
print 'trainEM: did NOT converge at thresh', deltaThresh, 'in', it, 'iterations'
return dist
