def probabilities(m): for history, probabilities in self.data[m]: gram = tuple(map(self.vocabulary.symbol, reversed(history))) for predicted, probability in probabilities: if predicted is not self.vocabulary.noneIndex: yield gram + ( self.vocabulary.symbol(predicted), ), probability
def frobnicateWithTrueWordRange(self, rawWords): for i in range(len(rawWords)): history = [ f for w in rawWords[max(0, i - self.order) : i] for f in self.frobnicateWord(w) ] for f in self.frobnicateWord(rawWords[i]): yield tuple(reversed(history)), f history.append(f)
def frobnicateWithTrueWordRange(self, rawWords): for i in xrange(len(rawWords)): history = [ f for w in rawWords[max(0, i - self.order) : i] for f in self.frobnicateWord(w) ] for f in self.frobnicateWord(rawWords[i]): yield tuple(reversed(history)), f history.append(f)
def visitIf(self, node): # eliminate if-const # removables = [] trueconst = None visit = self.visit for test, body in node.tests: mark_targeted(test) visit(test) visit(body) if test.isconst: if not test.value: removables.append((test, body)) elif not trueconst: trueconst = test, body elif bool_final_not(test): # # convert "if a and not b --> if not (not a or b)" # we have a difficulty removing the UNARY_NOT in the first case... # make_copy( test, ast.Not((isinstance(test, ast.And) and ast.Or or ast.And)([neg_AST(i) for i in test.nodes]))) if node.else_: visit(node.else_) for i in removables: node.tests.remove(i) if not node.tests: if node.else_: make_copy(node, node.else_) else: make_copy(node, ast.Pass(node.lineno)) elif trueconst: if trueconst == node.tests[0]: make_copy(node, trueconst[1]) else: node.else_ = trueconst[1] node.tests = node.tests[:node.tests.index(trueconst)] if not isinstance(node, ast.If): return node.fallthrough = False if node.else_: for test, stmt in node.tests: if not stmts_eq_base(stmt, node.else_): break else: ## print '*', repr (node) basenode = stmts_get_base(node.else_.nodes[0]) N = stmts_get_diff(node.else_.nodes[0]) for test, stmt in reversed(node.tests): N = ast.Conditional(test, stmts_get_diff(stmt.nodes[0]), N) optimizeConditional(N) basenode = basenode(N) make_copy(node, basenode) ## print '*>>', repr (node) opt_progress(':?:') else: # fallthrough means that the end of the if block must not JUMP_FORWARD # past the POP_TOP. if isinstance(node.tests[-1][1].nodes[-1], ast.Discard): node.fallthrough = True opt_progress('fth') node.tests[-1][1].nodes[-1] = node.tests[-1][1].nodes[-1].expr
def backOffs(m): for history, probabilities in self.data[m]: if self.vocabulary.noneIndex in probabilities: gram = tuple(map(self.vocabulary.symbol, reversed(history))) yield gram, probabilities[self.vocabulary.noneIndex]
def probabilities(m): for history, probabilities in self.data[m]: gram = tuple(map(self.vocabulary.symbol, reversed(history))) for predicted, probability in probabilities: if predicted is not self.vocabulary.noneIndex: yield gram + (self.vocabulary.symbol(predicted),), probability