def probabilities(m):
for history, probabilities in self.data[m]:
gram = tuple(map(self.vocabulary.symbol, reversed(history)))
for predicted, probability in probabilities:
if predicted is not self.vocabulary.noneIndex:
yield gram + (
self.vocabulary.symbol(predicted), ), probability
def frobnicateWithTrueWordRange(self, rawWords): for i in range(len(rawWords)): history = [ f for w in rawWords[max(0, i - self.order) : i] for f in self.frobnicateWord(w) ] for f in self.frobnicateWord(rawWords[i]): yield tuple(reversed(history)), f history.append(f)
def frobnicateWithTrueWordRange(self, rawWords): for i in xrange(len(rawWords)): history = [ f for w in rawWords[max(0, i - self.order) : i] for f in self.frobnicateWord(w) ] for f in self.frobnicateWord(rawWords[i]): yield tuple(reversed(history)), f history.append(f)
def visitIf(self, node):
# eliminate if-const
#
removables = []
trueconst = None
visit = self.visit
for test, body in node.tests:
mark_targeted(test)
visit(test)
visit(body)
if test.isconst:
if not test.value:
removables.append((test, body))
elif not trueconst:
trueconst = test, body
elif bool_final_not(test):
#
# convert "if a and not b --> if not (not a or b)"
# we have a difficulty removing the UNARY_NOT in the first case...
#
make_copy(
test,
ast.Not((isinstance(test, ast.And) and ast.Or
or ast.And)([neg_AST(i) for i in test.nodes])))
if node.else_:
visit(node.else_)
for i in removables:
node.tests.remove(i)
if not node.tests:
if node.else_:
make_copy(node, node.else_)
else:
make_copy(node, ast.Pass(node.lineno))
elif trueconst:
if trueconst == node.tests[0]:
make_copy(node, trueconst[1])
else:
node.else_ = trueconst[1]
node.tests = node.tests[:node.tests.index(trueconst)]
if not isinstance(node, ast.If):
return
node.fallthrough = False
if node.else_:
for test, stmt in node.tests:
if not stmts_eq_base(stmt, node.else_):
break
else:
## print '*', repr (node)
basenode = stmts_get_base(node.else_.nodes[0])
N = stmts_get_diff(node.else_.nodes[0])
for test, stmt in reversed(node.tests):
N = ast.Conditional(test, stmts_get_diff(stmt.nodes[0]), N)
optimizeConditional(N)
basenode = basenode(N)
make_copy(node, basenode)
## print '*>>', repr (node)
opt_progress(':?:')
else:
# fallthrough means that the end of the if block must not JUMP_FORWARD
# past the POP_TOP.
if isinstance(node.tests[-1][1].nodes[-1], ast.Discard):
node.fallthrough = True
opt_progress('fth')
node.tests[-1][1].nodes[-1] = node.tests[-1][1].nodes[-1].expr
def backOffs(m):
for history, probabilities in self.data[m]:
if self.vocabulary.noneIndex in probabilities:
gram = tuple(map(self.vocabulary.symbol,
reversed(history)))
yield gram, probabilities[self.vocabulary.noneIndex]
def backOffs(m): for history, probabilities in self.data[m]: if self.vocabulary.noneIndex in probabilities: gram = tuple(map(self.vocabulary.symbol, reversed(history))) yield gram, probabilities[self.vocabulary.noneIndex]
def probabilities(m): for history, probabilities in self.data[m]: gram = tuple(map(self.vocabulary.symbol, reversed(history))) for predicted, probability in probabilities: if predicted is not self.vocabulary.noneIndex: yield gram + (self.vocabulary.symbol(predicted),), probability
