def compute_proposal_probability(self, grammar, t1, t2, resampleProbability=lambdaOne, recurse=True): # NOTE: This is not strictly necessary since we don't actually have to sum over trees # if we use an auxiliary variable argument. But this fits nicely with the other proposers # and is not much slower. chosen_node1 , chosen_node2 = least_common_difference(t1,t2) lps = [] if chosen_node1 is None: # any node in the tree could have been regenerated for node in t1: lp_of_choosing_node = t1.sampling_log_probability(node,resampleProbability=resampleProbability) with BVRuleContextManager(grammar, node.parent, recurse_up=True): lp_of_generating_tree = grammar.log_probability(node) lps += [lp_of_choosing_node + lp_of_generating_tree] else: # we have a specific path up the tree while chosen_node1: lp_of_choosing_node = t1.sampling_log_probability(chosen_node1,resampleProbability=resampleProbability) with BVRuleContextManager(grammar, chosen_node2.parent, recurse_up=True): lp_of_generating_tree = grammar.log_probability(chosen_node2) lps += [lp_of_choosing_node + lp_of_generating_tree] if recurse: chosen_node1 = chosen_node1.parent chosen_node2 = chosen_node2.parent else: chosen_node1 = None return logsumexp(lps)
def single_probability(self, t): # in this tree, in its context (recursing up), what is the probability of this single expansion? with BVRuleContextManager(self, t, recurse_up=True): z = log(sum([ r.p for r in self.get_rules(t.returntype) ])) r = self.get_matching_rule(t) return log(r.p)-z
def propose_tree(self, t, resampleProbability=lambdaOne): """ Propose to a tree, returning the new tree and the prob. of sampling it. """ newt = copy(t) try: # sample a subnode n, lp = newt.sample_subnode( resampleProbability=resampleProbability) except NodeSamplingException: # If we've been given resampleProbability that can't sample raise ProposalFailedException assert getattr(n, "resampleProbability", 1.0) > 0.0, "*** Error in propose_tree %s ; %s" % ( resampleProbability(t), t) # In the context of the parent, resample n according to the grammar # We recurse_up in order to add all the parent's rules with BVRuleContextManager(self.grammar, n.parent, recurse_up=True): n.setto(self.grammar.generate(n.returntype)) # compute the forward/backward probability f = lp + self.grammar.log_probability(newt) b = (log(1.0*resampleProbability(n)) - log(newt.sample_node_normalizer(resampleProbability=resampleProbability)))\ + self.grammar.log_probability(t) return [newt, f - b]
def generate(self, x=None): """Generate from the grammar Arguments: x (string): What we start from -- can be None and then we use Grammar.start. """ # print "# Calling Grammar.generate", type(x), x # Decide what to start from based on the default if start is not specified if x is None: x = self.start assert self.start in self.nonterminals(), \ "The default start symbol %s is not a defined nonterminal" % self.start # Dispatch different kinds of generation if isinstance(x,list): return map(lambda xi: self.generate(x=xi), x) # If we get a list, just map along it to generate. elif self.is_nonterminal(x): # sample a grammar rule rules = self.get_rules(x) assert len(rules) > 0, "*** No rules in x=%s"%x # sample the rule r = weighted_sample(rules, probs=lambda x: x.p, log=False) # Make a stub for this functionNode fn = r.make_FunctionNodeStub(self, None) # Define a new context that is the grammar with the rule added # Then, when we exit, it's still right. with BVRuleContextManager(self, fn, recurse_up=False): # not sure why we can't use with/as: # Can't recurse on None or else we genreate from self.start if fn.args is not None: # and generate below *in* this context (e.g. with the new rules added) try: fn.args = self.generate(fn.args) except RuntimeError as e: print "*** Runtime error in %s" % fn raise e # and set the parents for a in fn.argFunctionNodes(): a.parent = fn return fn elif isinstance(x, FunctionNode): # this will let us finish generation of a partial tree x.args = [ self.generate(a) for a in x.args] for a in x.argFunctionNodes(): a.parent = x return x else: # must be a terminal assert isinstance(x, str), ("*** Terminal must be a string! x="+x) return x
def generate_with_counts(self, x=None, nc=None): if x is None: x = self.start assert self.start in self.nonterminals(), \ "The default start symbol %s is not a defined nonterminal" % self.start if nc is None: nc = np.zeros(np.shape(self.alphas)) x = self.generate_with_counts(x, nc) return x, nc try: if isinstance(x, list): return map(lambda xi: self.generate_with_counts(x=xi, nc=nc), x) elif self.is_nonterminal(x): rules = self.get_rules(x) assert len(rules) > 0, "*** No rules in x=%s" % x r = weighted_sample(rules, probs=lambda x: x.p, log=False) rule_ix = self.sorted_rules.index(x) sym_ix = rules.index(r) nc[rule_ix, sym_ix] += 1.0 fn = r.make_FunctionNodeStub(self, None) with BVRuleContextManager(self, fn, recurse_up=False): if fn.args is not None: try: fn.args = self.generate_with_counts(fn.args, nc) except RuntimeError as e: print "*** Runtime error in %s" % fn raise e for a in fn.argFunctionNodes(): a.parent = fn return fn elif isinstance(x, FunctionNode): x.args = [self.generate_with_counts(a, nc) for a in x.args] for a in x.argFunctionNodes(): a.parent = x return x else: assert isinstance( x, str), ("*** Terminal must be a string! x=" + x) return x except AssertionError as e: print "***Assertion error in %s" % x raise e
def propose_tree(self, grammar, t, resampleProbability=lambdaOne): """Propose, returning the new tree""" new_t = copy(t) try: # to sample a subnode n, lp = new_t.sample_subnode(resampleProbability=resampleProbability) except NodeSamplingException: # when no nodes can be sampled raise ProposalFailedException # In the context of the parent, resample n according to the # grammar. recurse_up in order to add all the parent's rules with BVRuleContextManager(grammar, n.parent, recurse_up=True): n.setto(grammar.generate(n.returntype)) return new_t
def make_children(self): assert self.children is None ## Otherwise we should never call this root = self.value.value if root.count_nodes() >= self.value.maxnodes: raise StatePruneException # Now make the copy newfn = copy(root) ## find the first unfilled Node: the argi'th argument of fn in our new copy try: fn, argi = None, None # the index of the fn. This will be used to find it in the copy for j, x in enumerate(newfn): if x.args is not None: for i, a in enumerate(x.args): if self.grammar.is_nonterminal(a): fn, argi = x, i raise LoopsBreakException except LoopsBreakException: pass assert fn is not None, "Cannot call make_children on a terminal. This must be avoided in State.next()" # Now make the children below children = [] with BVRuleContextManager(self.grammar, fn, recurse_up=True): rules = self.grammar.get_rules(fn.args[argi]) lZ = log(sum([r.p for r in rules])) for r in rules: fn.args[argi] = r.make_FunctionNodeStub(self.grammar, fn) # copy the type in self.value newh = self.value.__copy__(value=None) newh.set_value( copy(newfn), f=lambdaAssertFalse ) # Need to copy so different r give different fn; don't use set_value or it compiles # and make it into a State s = type(self)(newh, data=self.data, grammar=self.grammar, hole_penalty=self.hole_penalty, parent=self) children.append(s) return children
def iterate_subnodes(self, t, d=0, predicate=lambdaTrue, do_bv=True, yield_depth=False): """ Iterate through all subnodes of node *t*, while updating the added rules (bound variables) so that at each subnode, the grammar is accurate to what it was. if *do_bv*=False, we don't do bound variables (useful for things like counting nodes, instead of having to update the grammar) *yield_depth*: if True, we return (node, depth) instead of node *predicate*: filter only the ones that match this NOTE: if you DON'T iterate all the way through, you end up acculmulating bv rules so NEVER stop this iteration in the middle! TODO: Make this more elegant -- use BVCM """ if isFunctionNode(t): # print "iterate subnode: ", t, t.added_rule if predicate(t): yield (t, d) if yield_depth else t #Define a new context that is the grammar with the rule added. Then, when we exit, it's still right with BVRuleContextManager(self, t.added_rule): if t.args is not None: for g in self.iterate_subnodes( t.args, d=d + 1, do_bv=do_bv, yield_depth=yield_depth, predicate=predicate ): # pass up anything from below yield g elif isinstance(t, list): for a in t: for g in self.iterate_subnodes(a, d=d, do_bv=do_bv, yield_depth=yield_depth, predicate=predicate): yield g
def generate(self, x=None): """ Generate from the PCFG -- default is to start from x - either a nonterminal or a FunctionNode. Returns a FunctionNode. """ #print "# Calling grammar.generate", d, type(x), x # Decide what to start from based on the default if start is not specified if x is None: x = self.start assert self.start in self.rules, "The default start symbol %s is not a defined nonterminal" % self.start # Dispatch different kinds of generation if isinstance(x, list): # If we get a list, just map along it to generate. We don't count lists as depth--only FunctionNodes return map(lambda xi: self.generate(x=xi), x) elif self.is_nonterminal(x): # sample a grammar rule r, gp = weighted_sample(self.rules[x], probs=lambda x: x.p, return_probability=True, log=False) #print "SAMPLED:", gp, r, type(r) # Make a stub for this functionNode fn = r.make_FunctionNodeStub( self, gp) ## NOT SURE WHY BU TCOPY IS NECESSARY HERE # Define a new context that is the grammar with the rule added. Then, when we exit, it's still right with BVRuleContextManager( self, fn.added_rule): # not sure why I can't use with/as: if fn.args is not None: # Can't recurse on None or else we genreate from self.start fn.args = self.generate( fn.args ) # and generate below *in* this context (e.g. with the new rules added) return fn else: # must be a terminal assert isinstance(x, str), ("*** Terminal must be a string! x=" + x) return x
def unpack_ascii_rec(self, s, x, idx2rule): """ Unpack a string into a tree. Follows the format of Grammar.generate, but indexes the choices with s """ assert x is not None # should have been given by unpack_ascii # Dispatch different kinds of generation if isinstance(x, list): return map(lambda xi: self.unpack_ascii_rec(s, xi, idx2rule), x) elif self.is_nonterminal(x): rules = self.get_rules(x) # index # instead of sampling a rule, get it from the string i = pack_string.index(s[0]) del s[0] # remove (works since s is a list) r = idx2rule[i] # Make a stub for this functionNode fn = r.make_FunctionNodeStub(self, None) with BVRuleContextManager(self, fn, recurse_up=False): # add rule (to idx2rule) if isinstance(r, BVAddGrammarRule): idx = max(idx2rule.keys()) + 1 idx2rule[idx] = fn.added_rule # recurse if fn.args is not None: fn.args = self.unpack_ascii_rec(s, fn.args, idx2rule) # remove rule (as we now do in packing, too) if isinstance(r, BVAddGrammarRule): idx = max(idx2rule.keys()) del idx2rule[idx] for a in fn.argFunctionNodes(): a.parent = fn return fn else: # must be a terminal return x
def compute_proposal_probability(self, grammar, t1, t2, resampleProbability=lambdaOne): node_1, node_2 = least_common_difference(t1, t2) if (node_1 and node_2 and any([ nodes_are_roughly_equal(arg, node_1) for arg in None2Empty(node_2.args) ])): lp_choosing_node_1 = t1.sampling_log_probability( node_1, resampleProbability=lambda t: can_insert_FunctionNode( t, grammar) * resampleProbability(t)) lp_choosing_rule = -nicelog( len( filter(can_insert_GrammarRule, grammar.rules[node_1.returntype]))) lp_choosing_replacement = -nicelog( len( filter( lambda i: node_2.args[i].returntype == node_1. returntype, xrange(len(node_2.args))))) lp_generation = [] for arg in node_2.args: if not (arg.name == node_1.name and arg.returntype == node_1.returntype and arg.args == node_1.args ): # if the nodes are significantly different with BVRuleContextManager(grammar, node_2, recurse_up=True): lp_generation += [grammar.log_probability(arg)] lp_copy_making_node_2 = lp_choosing_rule + lp_choosing_replacement + sum( lp_generation) return lp_choosing_node_1 + lp_copy_making_node_2 else: return -Infinity # the trees cannot be identical if we performed an insertion
def iterate_subnodes(self, grammar, t=None, d=0, predicate=lambdaTrue, yield_depth=False, recurse_up=False): """Iterate through all subnodes of node *t*, while updating the added rules (bound variables) so that at each subnode, the grammar is accurate to what it was. Arguments --------- grammar : LOTlib.Grammar This is the grammar we're iterating through t : FunctionNode The tree we will iterate over yield_depth : bool If True, we return (node, depth) instead of node. predicate : function Filter only the nodes that match this function (i.e. eval (function(fn) == True) on each fn). recurse_up : bool Do we recurse all the way up and add all above nodes too? """ if not t: t = self if predicate(t): yield (t, d) if yield_depth else t # Define a new context that is the grammar with the rule added. # Then, when we exit, it's still right. with BVRuleContextManager(grammar, t, recurse_up=recurse_up): for a in t.argFunctionNodes(): # Pass up anything from below for g in self.iterate_subnodes( grammar, a, d=d + 1, yield_depth=yield_depth, predicate=predicate, recurse_up=False): # we never have to recurse up yield g
def propose_tree(self, grammar, tree, resampleProbability=lambdaOne): new_t = copy(tree) try: # to choose a node to insert on ni, lp = new_t.sample_subnode(lambda t: can_insert_FunctionNode( t, grammar) * resampleProbability(t)) except NodeSamplingException: raise ProposalFailedException # is there a rule that expands from ni.returntype to some ni.returntype? replicating_rules = filter(can_insert_GrammarRule, grammar.rules[ni.returntype]) if len(replicating_rules) == 0: raise ProposalFailedException # sample a rule r = sample1(replicating_rules) # the functionNode we are building fn = r.make_FunctionNodeStub(grammar, ni.parent) # figure out which arg will be the existing ni replicatingindices = filter(lambda i: fn.args[i] == ni.returntype, xrange(len(fn.args))) if len(replicatingindices) <= 0: # should never happen raise ProposalFailedException # choose the one to replace replace_i = sample1(replicatingindices) ## Now expand the other args, with the right rules in the grammar with BVRuleContextManager(grammar, fn, recurse_up=True): for i, a in enumerate(fn.args): fn.args[i] = copy(ni) if ( i == replace_i) else grammar.generate(a) # perform the insertion ni.setto(fn) return new_t
def log_probability(self, t): """ Returns the log probability of t, recomputing everything (as we do now) This is overall about half as fast, but it means we don't have to store generation_probability """ assert isinstance(t, FunctionNode) z = log(sum([ r.p for r in self.get_rules(t.returntype) ])) # Find the one that matches. While it may seem like we should store this, that is hard to make work # with multiple grammar objects across loading/saving, because the objects will change. This way, # we always look it up. lp = -Infinity r = self.get_matching_rule(t) assert r is not None, "Failed to find matching rule at %s %s" % (t, r) lp = log(r.p) - z with BVRuleContextManager(self, t): for a in t.argFunctionNodes(): lp += self.log_probability(a) return lp
def lp_propose(self, x, y, resampleProbability=lambdaOne, xZ=None): """ Returns a log probability of starting at x and ending up at y from a regeneration move. Any node is a candidate if the trees are identical except for what's below those nodes (although what's below *can* be identical!) NOTE: This does NOT take into account insert/delete NOTE: Not so simple because we must count multiple paths NOTE: This is currently not correct because it will mess up with bound variables, which now have unique names. Also it seems to add too many rules to the grammar, probably via recurse_up """ RP = -Infinity if isinstance(x, FunctionNode) and isinstance( y, FunctionNode) and x.returntype == y.returntype: # compute the normalizer if xZ is None: xZ = x.sample_node_normalizer( resampleProbability=resampleProbability) # Well we could select x's root to go to Y, but we must recompute y under the current grammar with BVRuleContextManager(self.grammar, x, recurse_up=True): RP = logplusexp( RP, log(1.0 * resampleProbability(x)) - log(xZ) + self.grammar.log_probability(y)) if x.name == y.name and x.args is not None and y.args is not None and len( x.args) == len(y.args): # how many kids are not equal, and where was the last? mismatch_count, mismatch_index = 0, 0 for i, xa, ya in zip(xrange(len(x.args)), x.args, y.args): if xa != ya: # checks whole subtree! mismatch_count += 1 mismatch_index = i if mismatch_count > 1: break # can't win if mismatch_count > 1: # We have to have only selected x,y to regenerate pass elif mismatch_count == 1: # we could propose to x, or x.args[mismatch_index], but nothing else (nothing else will fix the mismatch) with BVRuleContextManager( self.grammar, x, recurse_up=False): # recurse, but keep track of bv RP = logplusexp( RP, self.lp_propose( x.args[mismatch_index], y.args[mismatch_index], resampleProbability=resampleProbability, xZ=xZ)) else: # identical trees -- we could propose to any, so that's just the tree probability below convolved with the resample p for xi in x.iterate_subnodes(self.grammar, recurse_up=True): if xi is not x: # but we already counted ourself (NOTE: Must be "is", not ==) # Here we use grammar.log_probability since the grammar may have changed with bv RP = logplusexp( RP, log(resampleProbability(xi) * 1.0) - log(xZ) + self.grammar.log_probability(xi)) return RP
def enumerate_at_depth(self, d, nt=None, leaves=True): """Generate trees at depth d, no deeper or shallower. Parameters d (int): the depth of trees you want to generate nt (str): the type of the nonterminal you want to return (None reverts to self.start) leaves (bool): do we put terminals in the leaves or leave nonterminal types? This is useful in PartitionMCMC. This returns trees of depth d-1! Return: yields the ... """ if nt is None: nt = self.start # handle garbage that may be passed in here if not self.is_nonterminal(nt): yield nt raise StopIteration if d == 0: if leaves: # Note: can NOT use filter here, or else it doesn't include added rules for r in self.rules[nt]: if self.is_terminal_rule(r): yield r.make_FunctionNodeStub(self, None) else: # If not leaves, we just put the nonterminal type in the leaves yield nt else: # Note: can NOT use filter here, or else it doesn't include added rules. No sorting either! for r in self.rules[nt]: # No good since it won't be deep enough if self.is_terminal_rule(r): continue fn = r.make_FunctionNodeStub(self, None) # The possible depths for the i'th child # Here we just ensure that nonterminals vary up to d, and otherwise child_i_depths = lambda i: xrange(d) if self.is_nonterminal(fn.args[i]) else [0] # The depths of each kid for cd in lazyproduct(map(child_i_depths, xrange(len(fn.args))), child_i_depths): # One must be equal to d-1 # TODO: can be made more efficient via permutations. Also can skip terminals in args. if max(cd) < d-1: continue assert max(cd) == d-1 myiter = lazyproduct( [self.enumerate_at_depth(di, nt=a, leaves=leaves) for di, a in zip(cd, fn.args)], lambda i: self.enumerate_at_depth(cd[i], nt=fn.args[i], leaves=leaves)) try: while True: # Make a copy so we don't modify anything yieldfn = copy(fn) # BVRuleContextManager here makes us remove the rule BEFORE yielding, # or else this will be incorrect. Wasteful but necessary. with BVRuleContextManager(self, fn, recurse_up=False): yieldfn.args = myiter.next() for a in yieldfn.argFunctionNodes(): # Update parents a.parent = yieldfn yield copy(yieldfn) except StopIteration: # Catch this here so we continue in this loop over rules pass
def propose_tree(self, t): newt = copy(t) if random() < 0.5: # So we insert # Choose a node at random to insert on # TODO: We could precompute the nonterminals we can do this move on, if we wanted try: ni, lp = newt.sample_subnode(isNotBVAddFunctionNode) except NodeSamplingException: raise ProposalFailedException # Since it's an insert, see if there is a (replicating) rule that expands # from ni.returntype to some ni.returntype replicating_rules = filter(is_replicating_GrammarRule, self.grammar.rules[ni.returntype]) if len(replicating_rules) == 0: return [newt, fb] # sample a rule and compute its probability (not under the predicate) r = sample1(replicating_rules) # the functionNode we are building fn = r.make_FunctionNodeStub(self, ni.parent) # figure out which arg will be the existing ni replicatingindices = filter(lambda i: fn.args[i] == ni.returntype, xrange(len(fn.args))) assert replicatingindices > 0 # since that's what a replicating rule is replace_i = sample1( replicatingindices) # choose the one to replace fn.args[replace_i] = copy(ni) # the one we replace ## Now expand the other args, with the right rules in the grammar with BVRuleContextManager(self.grammar, fn, recurse_up=True): # and generate the args below for i, a in enumerate(fn.args): if i != replace_i: fn.args[i] = self.grammar.generate( a) #else generate like normalized # we need a count of how many kids are the same afterwards after_same_children = sum([x == ni for x in fn.args]) ni.setto(fn) with BVRuleContextManager(self.grammar, fn, recurse_up=True): # what is the prob mass of the new stuff? new_lp_below = sum([ self.grammar.log_probability(fn.args[i]) if (i != replace_i and isFunctionNode(fn.args[i])) else 0. for i in xrange(len(fn.args)) ]) # What is the new normalizer? newZ = newt.sample_node_normalizer(isNotBVAddFunctionNode) assert newZ > 0 # To sample forward: choose the node ni, choose the replicating rule, choose which "to" to expand (we could have put it on any of the replicating rules that are identical), and genreate the rest of the tree f = lp + (-log(len(replicating_rules))) + ( log(after_same_children) - log(len(replicatingindices))) + new_lp_below # To go backwards, choose the inserted rule, and any of the identical children, out of all replicators b = (log(1.0 * isNotBVAddFunctionNode(fn)) - log(newZ)) + ( log(after_same_children) - log(len(fn.args))) else: # A delete move! # Sample a node at random try: ni, lp = newt.sample_subnode( isNotBVAddFunctionNode) # this could raise exception # Really, it had to be not None if ni.args is None: raise NodeSamplingException except NodeSamplingException: raise ProposalFailedException # Figure out which of my children have the same type as me replicating_kid_indices = filter( lambda i: isFunctionNode(ni.args[i]) and ni.args[i].returntype == ni.returntype, range(len(ni.args))) nrk = len(replicating_kid_indices) # how many replicating kids if nrk == 0: raise ProposalFailedException replicating_rules = filter(is_replicating_GrammarRule, self.grammar.rules[ni.returntype]) assert len(replicating_rules ) > 0 # better be some or where did ni come from? samplei = sample1( replicating_kid_indices ) # who to promote; NOTE: not done via any weighting # We need to be in the right grammar state to evaluate log_probability with BVRuleContextManager(self.grammar, ni.args[samplei], recurse_up=True): # Now we must count the multiple ways we could go forward or back # Here, we could have sampled any of them equivalent to ni.args[i] before_same_children = sum([ x == ni.args[samplei] for x in ni.args ]) # how many are the same after? # the lp of everything we'd have to create going backwards old_lp_below = sum([ self.grammar.log_probability(ni.args[i]) if (i != samplei and isFunctionNode(ni.args[i])) else 0. for i in xrange(len(ni.args)) ]) # and replace it ni.args[ samplei].parent = ni.parent # update this first ;; TODO: IS THIS NECSESARY? ni.setto(ni.args[samplei]) # And compute f/b probs newZ = newt.sample_node_normalizer( resampleProbability=isNotBVAddFunctionNode) # To go forward, choose the node, and then from all equivalent children f = lp + (log(before_same_children) - log(nrk)) # To go back, choose the node, choose the replicating rule, choose where to put it, and generate the rest of the tree b = (log(1.0 * isNotBVAddFunctionNode(ni)) - log(newZ)) + -log(len(replicating_rules)) + ( log(before_same_children) - log(nrk)) + old_lp_below return [newt, f - b]
def increment_tree_(self, x=None, depth=0, max_depth=Infinity, depthdict=None): """ A lazy version of tree enumeration. Here, we generate all trees, starting from a rule or a nonterminal symbol and going up to max_depth This is constant memory and should produce each tree *once* (However: if a grammar has multiple derivations of the same str(tree), then you will see repeats!). TODO: CHANGE THIS TO ENUMERATE SHALLOW->DEEP *x*: A node in the tree *depth*: Depth of the tree *depthdict* : memoizes depth_to_terminal so that we can order rules in order to make enumeration small->large """ # wrap no specification for x if depth >= max_depth: raise StopIteration if isFunctionNode(x): # NOTE: WE don't need to handle BV here since they are handled below when we use the rule original_x = copy(x) # go all odometer on the kids below:: iters = [ self.increment_tree_( x=y, depth=depth, max_depth=max_depth, depthdict=depthdict) if self.is_nonterminal(y) else None for y in x.args ] if len(iters) == 0: yield copy(x) else: #print "HERE", iters for i in xrange(len(iters)): if iters[i] is not None: x.args[i] = iters[i].next() # the index of the last terminal symbol (may not be len(iters)-1), last_terminal_idx = max([ i if iters[i] is not None else -1 for i in xrange(len(iters)) ]) ## Now loop through the args, counting them up while True: yield copy( x ) # yield the initial tree, and then each successive tree # and then process each carry: for carry_pos in xrange( len(iters) ): # index into which tree we are incrementing if iters[ carry_pos] is not None: # we are not a terminal symbol (mixed in) ## NOTE: This *MUST* go here in order to prevent adding a rule and then not removing it when you carry (thus introducing a bv of a1 into a2) with BVRuleContextManager(self, x.added_rule): try: x.args[carry_pos] = iters[carry_pos].next() break # if we increment successfully, no carry, so break out of i loop except StopIteration: # if so, then "carry" if carry_pos == last_terminal_idx: raise StopIteration elif iters[carry_pos] is not None: # reset the incrementer since we just carried iters[ carry_pos] = self.increment_tree_( x=original_x.args[carry_pos], depth=depth, max_depth=max_depth, depthdict=depthdict) x.args[carry_pos] = iters[ carry_pos].next() # reset this # and just continue your loop over i (which processes the carry) elif self.is_nonterminal(x): # just a single nonterminal ## TODO: somewhat inefficient since we do this each time: ## Here we change the order of rules to be terminals *first* terminals = [] nonterminals = [] for k in self.rules[x]: if not self.is_terminal_rule( k ): #AAH this used to be called "x" and that ruined the scope of the outer "x" nonterminals.append(k) else: terminals.append(k) # sort by probability, so high probability trees *tend* to come first terminals = sorted( terminals, key=lambda r: self.depth_to_terminal(r, current_d=depthdict)) nonterminals = sorted( nonterminals, key=lambda r: self.depth_to_terminal(r, current_d=depthdict)) Z = logsumexp([log(r.p) for r in self.rules[x]]) # normalizer #print terminals #print nonterminals #print "---------------------------------------" # yield each of the rules that lead to terminals -- always do this since depth>=0 (above) for r in terminals: fn = r.make_FunctionNodeStub(self, (log(r.p) - Z)) # Do not need to set added_rule since they can't exist here yield fn if depth < max_depth: # if we can go deeper for r in nonterminals: #expand each nonterminals fn = r.make_FunctionNodeStub(self, (log(r.p) - Z)) for q in self.increment_tree_(x=fn, depth=depth + 1, max_depth=max_depth, depthdict=depthdict): yield q else: yield x