def forQuestion(self, valueToAcc, question):
accForAnswer = [self.createAcc() for _ in question.codomain()]
for labelValue, acc in valueToAcc.iteritems():
answer = question(labelValue)
d.addAcc(accForAnswer[answer], acc)
return accForAnswer
def forQuestion(self, valueToAcc, question):
accForAnswer = [ self.createAcc() for _ in question.codomain() ]
for labelValue, acc in valueToAcc.iteritems():
answer = question(labelValue)
d.addAcc(accForAnswer[answer], acc)
return accForAnswer
def forQuestion(self, labels, fullQuestion):
labelValuer, question = fullQuestion
accForLabel = self.accForLabel
accForAnswer = [ self.createAcc() for _ in question.codomain() ]
for label in labels:
acc = accForLabel(label)
answer = question(labelValuer(label))
d.addAcc(accForAnswer[answer], acc)
return accForAnswer
def forQuestion(self, labels, fullQuestion):
labelValuer, question = fullQuestion
accForLabel = self.accForLabel
accForAnswer = [self.createAcc() for _ in question.codomain()]
for label in labels:
acc = accForLabel(label)
answer = question(labelValuer(label))
d.addAcc(accForAnswer[answer], acc)
return accForAnswer
def estimate(distPrev, createAcc, estimateDist, afterAcc, verbosity, *accs):
accTot = createAcc(distPrev)
for acc in accs:
d.addAcc(accTot, acc)
if afterAcc is not None:
afterAcc(accTot)
logLikePrev = accTot.logLike()
count = accTot.count()
count = max(count, 1.0)
dist = estimateDist(accTot)
if verbosity >= 2:
print ('trainEM: logLikePrev = %s (%s count)' %
(logLikePrev / count, count))
return dist
def estimate(distPrev, createAcc, estimateDist, afterAcc, verbosity, *accs):
accTot = createAcc(distPrev)
for acc in accs:
d.addAcc(accTot, acc)
if afterAcc is not None:
afterAcc(accTot)
logLikePrev = accTot.logLike()
count = accTot.count()
count = max(count, 1.0)
dist = estimateDist(accTot)
if verbosity >= 2:
print('trainEM: logLikePrev = %s (%s count)' %
(logLikePrev / count, count))
return dist
def getQgToValueToAcc(self, labels, questionGroups):
accForLabel = self.accForLabel
numQuestionGroups = len(questionGroups)
labelValuers = [ labelValuer
for labelValuer, questions in questionGroups ]
qgToValueToAcc = [ defaultdict(self.createAcc)
for _ in range(numQuestionGroups) ]
for label in labels:
acc = accForLabel(label)
for qgIndex in range(numQuestionGroups):
labelValuer = labelValuers[qgIndex]
valueToAcc = qgToValueToAcc[qgIndex]
d.addAcc(valueToAcc[labelValuer(label)], acc)
return qgToValueToAcc
def getLeafToQgToValueToAcc(self, numLeaves, labelToLeafIndex,
questionGroups):
leafToQgToValueToAcc = [[
defaultdict(self.createAcc) for _ in questionGroups
] for _ in range(numLeaves)]
for labelledAccs in self.labelledAccChunks:
for label, acc in labelledAccs:
try:
leafIndex = labelToLeafIndex[label]
except KeyError:
pass
else:
for qgIndex, (labelValuer, _) in enumerate(questionGroups):
valueToAcc = leafToQgToValueToAcc[leafIndex][qgIndex]
d.addAcc(valueToAcc[labelValuer(label)], acc)
return leafToQgToValueToAcc
def getQgToValueToAcc(self, labels, questionGroups):
accForLabel = self.accForLabel
numQuestionGroups = len(questionGroups)
labelValuers = [
labelValuer for labelValuer, questions in questionGroups
]
qgToValueToAcc = [
defaultdict(self.createAcc) for _ in range(numQuestionGroups)
]
for label in labels:
acc = accForLabel(label)
for qgIndex in range(numQuestionGroups):
labelValuer = labelValuers[qgIndex]
valueToAcc = qgToValueToAcc[qgIndex]
d.addAcc(valueToAcc[labelValuer(label)], acc)
return qgToValueToAcc
def getLeafToQgToValueToAcc(self, numLeaves, labelToLeafIndex,
questionGroups):
leafToQgToValueToAcc = [
[
defaultdict(self.createAcc)
for _ in questionGroups
]
for _ in range(numLeaves)
]
for labelledAccs in self.labelledAccChunks:
for label, acc in labelledAccs:
try:
leafIndex = labelToLeafIndex[label]
except KeyError:
pass
else:
for qgIndex, (labelValuer, _) in enumerate(questionGroups):
valueToAcc = leafToQgToValueToAcc[leafIndex][qgIndex]
d.addAcc(valueToAcc[labelValuer(label)], acc)
return leafToQgToValueToAcc
def estimateWithTotAux(distPrev, createAcc, estimateTotAux, afterAcc,
monotoneAux, verbosity, *accs):
accTot = createAcc(distPrev)
for acc in accs:
d.addAcc(accTot, acc)
if afterAcc is not None:
afterAcc(accTot)
logLikePrev = accTot.logLike()
count = accTot.count()
count = max(count, 1.0)
dist, (aux, auxRat) = estimateTotAux(accTot)
if monotoneAux and aux < logLikePrev:
raise RuntimeError('re-estimated auxiliary value (%s) less than'
' previous log likelihood (%s) during'
' expectation-maximization (count = %s)' %
(aux / count, logLikePrev / count, count))
if verbosity >= 2:
auxRatString = d.Rat.toString(auxRat)
print('trainEM: logLikePrev = %s -> aux = %s (%s) (%s count)' %
(logLikePrev / count, aux / count, auxRatString, count))
return dist
def estimateWithTotAux(distPrev, createAcc, estimateTotAux, afterAcc,
monotoneAux, verbosity, *accs):
accTot = createAcc(distPrev)
for acc in accs:
d.addAcc(accTot, acc)
if afterAcc is not None:
afterAcc(accTot)
logLikePrev = accTot.logLike()
count = accTot.count()
count = max(count, 1.0)
dist, (aux, auxRat) = estimateTotAux(accTot)
if monotoneAux and aux < logLikePrev:
raise RuntimeError('re-estimated auxiliary value (%s) less than'
' previous log likelihood (%s) during'
' expectation-maximization (count = %s)' %
(aux / count, logLikePrev / count, count))
if verbosity >= 2:
auxRatString = d.Rat.toString(auxRat)
print ('trainEM: logLikePrev = %s -> aux = %s (%s) (%s count)' %
(logLikePrev / count, aux / count, auxRatString, count))
return dist
def all(self, labels):
accForLabel = self.accForLabel
accTot = self.createAcc()
for label in labels:
d.addAcc(accTot, accForLabel(label))
return accTot
def all(self):
accTot = self.createAcc()
for labelledAccs in self.labelledAccChunks:
for _, acc in labelledAccs:
d.addAcc(accTot, acc)
return accTot
def getLogProb(dist, labels):
acc = createAcc(dist)
for label in labels:
d.addAcc(acc, accForLabel(label))
return acc.logLike()
def getLogProb(dist, labels):
acc = createAcc(dist)
for label in labels:
d.addAcc(acc, accForLabel(label))
return acc.logLike()
def all(self, labels):
accForLabel = self.accForLabel
accTot = self.createAcc()
for label in labels:
d.addAcc(accTot, accForLabel(label))
return accTot
def all(self):
accTot = self.createAcc()
for labelledAccs in self.labelledAccChunks:
for _, acc in labelledAccs:
d.addAcc(accTot, acc)
return accTot
