def combiner(item, childobjs): graph = leaf(item) for nt, cgraph in childobjs.items(): p,r,c = graph.find_nt_edge(*nt) fragment = Hgraph.from_triples([(p,r,c)],graph.node_to_concepts) try: graph = graph.replace_fragment(fragment, cgraph) except AssertionError, e: raise DerivationException, "Incompatible hyperedge type for nonterminal %s." % str(nt[0])
def combiner(item, childobjs): graph = leaf(item) for nt, cgraph in childobjs.items(): p, r, c = graph.find_nt_edge(*nt) fragment = Hgraph.from_triples([(p, r, c)], graph.node_to_concepts) try: graph = graph.replace_fragment(fragment, cgraph) except AssertionError, e: raise DerivationException, "Incompatible hyperedge type for nonterminal %s." % str( nt[0])
def get_binarized_partitions(graph, edgesets): if len(edgesets) == 1: yield [graph.triples()] return gen = get_partitions(graph, edgesets[0], [edge for edgeset in edgesets[1:] for edge in edgeset]) for left_edges, right_edges in gen: possibilities = get_binarized_partitions(Hgraph.from_triples(right_edges, {}, warn=False), edgesets[1:]) poss_list = list(possibilities) for partitions in poss_list: yield [left_edges] + partitions
def compute_chart( tree, graph, prefix="" ): # Recursively compute the chart. Graph is the sub-graph we're considering, tree is the tree of spans. count[0] += 1 triples = set(graph.triples()) edge_vector = tuple([1 if x in triples else 0 for x in graph_edge_list]) leaves = tree.leaves() # if not isinstance(tree,fancy_tree.FancyTree): # triples = set(graph.triples()) # edge_vector= tuple(1 for x in graph_edge_list if x in triples else 0) # return Partition(tree.node, leaves[0], leaves[-1], edge_vector) # else: if len(tree) == 1 and not isinstance(tree[0], fancy_tree.FancyTree): return Partition(tree.node, leaves[0], leaves[-1], edge_vector) # First get the set of aligned edges for this constituent and it's children aligned_edges_for_span = set([edge for token in tree.leaves() for edge in rev_alignments[token]]) partition_object = Partition(tree.node, leaves[0], leaves[-1], edge_vector) if partition_object not in chart: try: possibilities = [] child_edgesets = [] # Compute edge set for each child for t in tree: edgeset = [] for l in t.leaves(): edgeset.extend(rev_alignments[l]) child_edgesets.append(edgeset) # For each possible partitioning for cparts in get_binarized_partitions(graph, child_edgesets): child_forests = [] for i in range(len(tree)): childgraph = Hgraph.from_triples(cparts[i], {}, warn=False) sub_forest = compute_chart(tree[i], childgraph, prefix=prefix + " ") if len(chart) > MAX_CHART_SIZE: raise ChartTooBigException, "Chart size exceeded 5000 entries. dropping this sentence." child_forests.append(sub_forest) possibilities.append(child_forests) chart[partition_object] = possibilities except IncompatibleAlignmentException: chart.inconsistent_alignment = (tree.node, leaves[0], leaves[-1]) return partition_object return partition_object
def get_binarized_partitions(graph, edgesets): if len(edgesets) == 1: yield [graph.triples()] return gen = get_partitions( graph, edgesets[0], [edge for edgeset in edgesets[1:] for edge in edgeset]) for left_edges, right_edges in gen: possibilities = get_binarized_partitions( Hgraph.from_triples(right_edges, {}, warn=False), edgesets[1:]) poss_list = list(possibilities) for partitions in poss_list: yield [left_edges] + partitions
def merge_string_nonterminals(self, string, amr, next_id): """ Binarizes a string-graph pair consisting entirely of nonterminals, ensuring correct visit order for parsing. """ rules = [] stack = [] tokens = list(reversed([s for s in string if s])) # standard shift-reduce binarization algorithm # TODO add citation after paper is published while tokens: next_tok = tokens.pop() next_tok_triple_l = [t for t in amr.triples() if str(t[1]) == next_tok] assert len(next_tok_triple_l) == 1 next_tok_triple = next_tok_triple_l[0] if not stack: stack.append(next_tok) continue stack_top = stack.pop() stack_top_triple = [t for t in amr.triples() if str(t[1]) == stack_top][0] if (stack_top_triple[0] not in next_tok_triple[2]) and \ (next_tok_triple[0] not in stack_top_triple[2]): # can't merge, so shift stack.append(stack_top) stack.append(next_tok) continue # can merge, so reduce rule_amr = Hgraph.from_triples([stack_top_triple, next_tok_triple]) assert len(rule_amr.roots) == 1 rule_string = [stack_top, next_tok] fictitious_tree = self.make_fictitious_tree(string, rule_string) new_rule, tree, amr, next_id = self.make_rule(fictitious_tree, amr, Tree('X', rule_string), rule_amr, next_id) string = tree.leaves() tokens.append('#%s' % new_rule.symbol) rules.append(new_rule) if len(stack) > 1: raise BinarizationException return string, amr, rules, next_id
def replace_instance_edges(graph, tree): t = [] alignments = {} tree_leaves = tree.leaves() for e in graph.triples(): if e in graph.edge_alignments: p, r, ch = e token = graph.edge_alignments[e][0] # TODO: not sure what to do with multiple tokens new_edge = (p, "%s'" % tree_leaves[token], ch) t.append(new_edge) alignments[new_edge] = [token] else: t.append(e) res = Hgraph.from_triples(t, {}, warn=False) res.edge_alignments = alignments res.node_alignments = graph.node_alignments res.roots = graph.roots res.external_nodes = graph.external_nodes return res
def collapse_amr_terminals(self, tree, amr, next_id): """ Creates new rules by merging terminal subgraphs with their closest nonterminal edge. """ # triples returns in breadth-first order, so first triples in the list are # closest to the root of the AMR nonterminals = list(reversed([t for t in amr.triples() if isinstance(t[1], NonterminalLabel)])) rules = [] first = True while nonterminals: nt = nonterminals.pop() # in general, we will attach to a given nonterminal edge all of the # terminal edges reachable from its tail nodes attached_terminals = self.terminal_search(nt, amr.triples()) if first: # we still have to handle terminal edges that are higher than any # nonterminal edge # because the first nonterminal edge is closest to the root of the AMR, # it must be reachable from the root without passing through any other # nonterminal, so we can attach all the high terminals (those reachable # from the root) to the first nonterminal attached_terminals |= self.terminal_search(amr.root_edges()[0], amr.triples()) attached_terminals |= {amr.root_edges()[0]} first = False # don't bother making a rule when there's nothing to collapse if not attached_terminals: continue rule_amr = Hgraph.from_triples({nt} | attached_terminals) rule_tree = str(nt[1]) assert len(rule_amr.roots) == 1 new_rule, tree, amr, next_id = self.make_rule(tree, amr, rule_tree, rule_amr, next_id) rules.append(new_rule) return tree, amr, rules, next_id
def replace_instance_edges(graph, tree): t = [] alignments = {} tree_leaves = tree.leaves() for e in graph.triples(): if e in graph.edge_alignments: p, r, ch = e token = graph.edge_alignments[e][ 0] # TODO: not sure what to do with multiple tokens new_edge = (p, "%s'" % tree_leaves[token], ch) t.append(new_edge) alignments[new_edge] = [token] else: t.append(e) res = Hgraph.from_triples(t, {}, warn=False) res.edge_alignments = alignments res.node_alignments = graph.node_alignments res.roots = graph.roots res.external_nodes = graph.external_nodes return res
def merge_tree_symbols(self, tree, amr, next_id): """ Binarizes a tree-graph pair according to the binariziation dictated by the tree. WILL FAIL OFTEN IF TREE IS NOT BINARIZED. """ rules = [] while True: if not isinstance(tree, Tree): assert len(amr.triples()) == 1 return tree, amr, rules, next_id # a collapsible subtree consists of # 1. many terminals # 2. one nonterminal and many terminals # 3. two nonterminals collapsible_subtrees = [] for st in tree.subtrees(): terminals = [t for t in st.leaves() if t[0] == '#'] if len(terminals) == 1: collapsible_subtrees.append(st) elif len(terminals) == 2 and len(st.leaves()) == 2: collapsible_subtrees.append(st) # if there are no subtrees to collapse, this rule isn't binarizable if len(collapsible_subtrees) == 0: raise BinarizationException rule_tree = max(collapsible_subtrees, key=lambda x: x.height()) terminals = [t for t in rule_tree.leaves() if t[0] == '#'] rule_edge_l = [t for t in amr.triples() if str(t[1]) in terminals] rule_amr = Hgraph.from_triples(rule_edge_l) # if the induced graph is disconnected, this rule isn't binarizable if len(rule_amr.roots) != 1: raise BinarizationException new_rule, tree, amr, next_id = self.make_rule(tree, amr, rule_tree, rule_amr, next_id) rules.append(new_rule) return tree, amr, rules, next_id
def collapse_string_terminals(self, string, amr, next_id): """ Creates new rules by merging terminal tokens with their closest nonterminal. All terminals attach to the left (except for terminals left of the first nonterminal, which attach right). """ nonterminals = list(reversed([t for t in string if t[0] == '#'])) rules = [] # attach first terminals to the right slice_from = 0 while nonterminals: nt = nonterminals.pop() if nonterminals: slice_to = string.index(nonterminals[-1]) else: slice_to = len(string) if slice_to - slice_from == 1: # there are no terminals to attach here, so skip ahead slice_from = slice_to continue rule_string = string[slice_from:slice_to] nt_edge_l = [e for e in amr.triples(nodelabels = self.nodelabels) if str(e[1]) == nt] assert len(nt_edge_l) == 1 rule_amr = Hgraph.from_triples(nt_edge_l) # hallucinate a tree with acceptable structure for make_rule fictitious_tree = self.make_fictitious_tree(string, rule_string) new_rule, tree, amr, next_id = self.make_rule(fictitious_tree, amr, Tree('X', rule_string), rule_amr, next_id) string = tree.leaves() rules.append(new_rule) slice_from = slice_from + 1 return string, amr, rules, next_id
def convert_chart(partition, external_nodes, nt, first=False): nt = NonterminalLabel(nt.label) # Get rid of the index if partition in seen: node = seen[partition] result.use_counts[node] += 1 return node leaves = chart.tree.leaves() edges_in_partition = [graph_edge_list[i] for i in range(len(partition.edges)) if partition.edges[i] == 1] if not partition in chart: # leaf graph = Hgraph.from_triples(edges_in_partition, {}, warn=False) graph.roots = graph.find_roots() graph.roots.sort(lambda x, y: node_order[x] - node_order[y]) graph.external_nodes = external_nodes str_rhs = [leaves[i] for i in range(partition.str_start, partition.str_end + 1)] rule = Rule(0, nt.label, graph, tuple(str_rhs), 1) rule_id = self.add_rule(rule) fragment = fragment_counter[0] result[fragment] = [(rule_id, [])] result.use_counts[fragment] += 1 seen[partition] = fragment fragment_counter[0] += 1 return fragment poss = [] count = 0 for possibility in chart[partition]: count += 1 partition_graph = Hgraph.from_triples(edges_in_partition, {}, warn=False) # This is the parent graph partition_graph.roots = partition_graph.find_roots() partition_graph.roots.sort(lambda x, y: node_order[x] - node_order[y]) partition_graph.external_nodes = external_nodes children = [] # print partition_graph.to_amr_string() spans_to_nt = {} old_pgraph = partition_graph index = 1 for subpartition in possibility: # These are the different sub-constituents edges_in_subpartition = [ graph_edge_list[i] for i in range(len(subpartition.edges)) if subpartition.edges[i] == 1 ] if edges_in_subpartition: # Some constituents do not have any edges aligned to them sub_graph = Hgraph.from_triples(edges_in_subpartition, {}, warn=False) sub_graph.roots = sub_graph.find_roots() sub_graph.roots.sort(lambda x, y: node_order[x] - node_order[y]) external_node_list = partition_graph.find_external_nodes2(sub_graph) external_node_list.sort(lambda x, y: node_order[x] - node_order[y]) sub_external_nodes = dict([(k, v) for v, k in enumerate(external_node_list)]) sub_graph.external_nodes = sub_external_nodes sub_nt = NonterminalLabel("%s%i" % (subpartition.phrase, len(sub_external_nodes)), index) children.append(convert_chart(subpartition, sub_external_nodes, sub_nt)) # Recursive call old_pgraph = partition_graph partition_graph = partition_graph.collapse_fragment2( sub_graph, sub_nt, external=external_node_list, warn=False ) spans_to_nt[subpartition.str_start] = (sub_nt, subpartition.str_end) else: sub_nt = NonterminalLabel(subpartition.phrase, index) # assert partition_graph.is_connected() index += 1 partition_graph.roots = partition_graph.find_roots() partition_graph.roots.sort(lambda x, y: node_order[x] - node_order[y]) # Assemble String rule str_rhs = [] i = partition.str_start while i <= partition.str_end: if i in spans_to_nt: new_nt, i = spans_to_nt[i] str_rhs.append(new_nt) else: str_rhs.append(leaves[i]) i = i + 1 rule = Rule(0, nt.label, partition_graph, tuple(str_rhs), 1) rule_id = self.add_rule(rule) poss.append((rule_id, children)) fragment = fragment_counter[0] result[fragment] = poss result.use_counts[fragment] += 1 seen[partition] = fragment fragment_counter[0] += 1 return fragment
def get_graph(self, graph): trips = graph.triples() return Hgraph.from_triples([trips[i] for i in range(len(trips)) if self.edges[i] == 1], {}, warn=False)
def compute_chart( tree, graph, prefix="" ): # Recursively compute the chart. Graph is the sub-graph we're considering, tree is the tree of spans. count[0] += 1 triples = set(graph.triples()) edge_vector = tuple( [1 if x in triples else 0 for x in graph_edge_list]) leaves = tree.leaves() #if not isinstance(tree,fancy_tree.FancyTree): # triples = set(graph.triples()) # edge_vector= tuple(1 for x in graph_edge_list if x in triples else 0) # return Partition(tree.node, leaves[0], leaves[-1], edge_vector) #else: if len(tree) == 1 and not isinstance(tree[0], fancy_tree.FancyTree): return Partition(tree.node, leaves[0], leaves[-1], edge_vector) # First get the set of aligned edges for this constituent and it's children aligned_edges_for_span = set([ edge for token in tree.leaves() for edge in rev_alignments[token] ]) partition_object = Partition(tree.node, leaves[0], leaves[-1], edge_vector) if partition_object not in chart: try: possibilities = [] child_edgesets = [] # Compute edge set for each child for t in tree: edgeset = [] for l in t.leaves(): edgeset.extend(rev_alignments[l]) child_edgesets.append(edgeset) # For each possible partitioning for cparts in get_binarized_partitions( graph, child_edgesets): child_forests = [] for i in range(len(tree)): childgraph = Hgraph.from_triples(cparts[i], {}, warn=False) sub_forest = compute_chart(tree[i], childgraph, prefix=prefix + " ") if len(chart) > MAX_CHART_SIZE: raise ChartTooBigException, "Chart size exceeded 5000 entries. dropping this sentence." child_forests.append(sub_forest) possibilities.append(child_forests) chart[partition_object] = possibilities except IncompatibleAlignmentException: chart.inconsistent_alignment = (tree.node, leaves[0], leaves[-1]) return partition_object return partition_object
def convert_chart(partition, external_nodes, nt, first=False): nt = NonterminalLabel(nt.label) # Get rid of the index if partition in seen: node = seen[partition] result.use_counts[node] += 1 return node leaves = chart.tree.leaves() edges_in_partition = [ graph_edge_list[i] for i in range(len(partition.edges)) if partition.edges[i] == 1 ] if not partition in chart: # leaf graph = Hgraph.from_triples(edges_in_partition, {}, warn=False) graph.roots = graph.find_roots() graph.roots.sort(lambda x, y: node_order[x] - node_order[y]) graph.external_nodes = external_nodes str_rhs = [ leaves[i] for i in range(partition.str_start, partition.str_end + 1) ] rule = Rule(0, nt.label, graph, tuple(str_rhs), 1) rule_id = self.add_rule(rule) fragment = fragment_counter[0] result[fragment] = [(rule_id, [])] result.use_counts[fragment] += 1 seen[partition] = fragment fragment_counter[0] += 1 return fragment poss = [] count = 0 for possibility in chart[partition]: count += 1 partition_graph = Hgraph.from_triples( edges_in_partition, {}, warn=False) # This is the parent graph partition_graph.roots = partition_graph.find_roots() partition_graph.roots.sort( lambda x, y: node_order[x] - node_order[y]) partition_graph.external_nodes = external_nodes children = [] #print partition_graph.to_amr_string() spans_to_nt = {} old_pgraph = partition_graph index = 1 for subpartition in possibility: #These are the different sub-constituents edges_in_subpartition = [ graph_edge_list[i] for i in range(len(subpartition.edges)) if subpartition.edges[i] == 1 ] if edges_in_subpartition: # Some constituents do not have any edges aligned to them sub_graph = Hgraph.from_triples(edges_in_subpartition, {}, warn=False) sub_graph.roots = sub_graph.find_roots() sub_graph.roots.sort( lambda x, y: node_order[x] - node_order[y]) external_node_list = partition_graph.find_external_nodes2( sub_graph) external_node_list.sort( lambda x, y: node_order[x] - node_order[y]) sub_external_nodes = dict([ (k, v) for v, k in enumerate(external_node_list) ]) sub_graph.external_nodes = sub_external_nodes sub_nt = NonterminalLabel( "%s%i" % (subpartition.phrase, len(sub_external_nodes)), index) children.append( convert_chart(subpartition, sub_external_nodes, sub_nt)) # Recursive call old_pgraph = partition_graph partition_graph = partition_graph.collapse_fragment2( sub_graph, sub_nt, external=external_node_list, warn=False) spans_to_nt[subpartition.str_start] = ( sub_nt, subpartition.str_end) else: sub_nt = NonterminalLabel(subpartition.phrase, index) #assert partition_graph.is_connected() index += 1 partition_graph.roots = partition_graph.find_roots() partition_graph.roots.sort( lambda x, y: node_order[x] - node_order[y]) # Assemble String rule str_rhs = [] i = partition.str_start while i <= partition.str_end: if i in spans_to_nt: new_nt, i = spans_to_nt[i] str_rhs.append(new_nt) else: str_rhs.append(leaves[i]) i = i + 1 rule = Rule(0, nt.label, partition_graph, tuple(str_rhs), 1) rule_id = self.add_rule(rule) poss.append((rule_id, children)) fragment = fragment_counter[0] result[fragment] = poss result.use_counts[fragment] += 1 seen[partition] = fragment fragment_counter[0] += 1 return fragment
def get_graph(self, graph): trips = graph.triples() return Hgraph.from_triples( [trips[i] for i in range(len(trips)) if self.edges[i] == 1], {}, warn=False)