def combiner(item, childobjs):
graph = leaf(item)
for nt, cgraph in childobjs.items():
p,r,c = graph.find_nt_edge(*nt)
fragment = Hgraph.from_triples([(p,r,c)],graph.node_to_concepts)
try:
graph = graph.replace_fragment(fragment, cgraph)
except AssertionError, e:
raise DerivationException, "Incompatible hyperedge type for nonterminal %s." % str(nt[0])
def combiner(item, childobjs):
graph = leaf(item)
for nt, cgraph in childobjs.items():
p, r, c = graph.find_nt_edge(*nt)
fragment = Hgraph.from_triples([(p, r, c)], graph.node_to_concepts)
try:
graph = graph.replace_fragment(fragment, cgraph)
except AssertionError, e:
raise DerivationException, "Incompatible hyperedge type for nonterminal %s." % str(
nt[0])
def get_binarized_partitions(graph, edgesets):
if len(edgesets) == 1:
yield [graph.triples()]
return
gen = get_partitions(graph, edgesets[0], [edge for edgeset in edgesets[1:] for edge in edgeset])
for left_edges, right_edges in gen:
possibilities = get_binarized_partitions(Hgraph.from_triples(right_edges, {}, warn=False), edgesets[1:])
poss_list = list(possibilities)
for partitions in poss_list:
yield [left_edges] + partitions
def compute_chart(
tree, graph, prefix=""
): # Recursively compute the chart. Graph is the sub-graph we're considering, tree is the tree of spans.
count[0] += 1
triples = set(graph.triples())
edge_vector = tuple([1 if x in triples else 0 for x in graph_edge_list])
leaves = tree.leaves()
# if not isinstance(tree,fancy_tree.FancyTree):
# triples = set(graph.triples())
# edge_vector= tuple(1 for x in graph_edge_list if x in triples else 0)
# return Partition(tree.node, leaves[0], leaves[-1], edge_vector)
# else:
if len(tree) == 1 and not isinstance(tree[0], fancy_tree.FancyTree):
return Partition(tree.node, leaves[0], leaves[-1], edge_vector)
# First get the set of aligned edges for this constituent and it's children
aligned_edges_for_span = set([edge for token in tree.leaves() for edge in rev_alignments[token]])
partition_object = Partition(tree.node, leaves[0], leaves[-1], edge_vector)
if partition_object not in chart:
try:
possibilities = []
child_edgesets = []
# Compute edge set for each child
for t in tree:
edgeset = []
for l in t.leaves():
edgeset.extend(rev_alignments[l])
child_edgesets.append(edgeset)
# For each possible partitioning
for cparts in get_binarized_partitions(graph, child_edgesets):
child_forests = []
for i in range(len(tree)):
childgraph = Hgraph.from_triples(cparts[i], {}, warn=False)
sub_forest = compute_chart(tree[i], childgraph, prefix=prefix + " ")
if len(chart) > MAX_CHART_SIZE:
raise ChartTooBigException, "Chart size exceeded 5000 entries. dropping this sentence."
child_forests.append(sub_forest)
possibilities.append(child_forests)
chart[partition_object] = possibilities
except IncompatibleAlignmentException:
chart.inconsistent_alignment = (tree.node, leaves[0], leaves[-1])
return partition_object
return partition_object
def get_binarized_partitions(graph, edgesets):
if len(edgesets) == 1:
yield [graph.triples()]
return
gen = get_partitions(
graph, edgesets[0],
[edge for edgeset in edgesets[1:] for edge in edgeset])
for left_edges, right_edges in gen:
possibilities = get_binarized_partitions(
Hgraph.from_triples(right_edges, {}, warn=False), edgesets[1:])
poss_list = list(possibilities)
for partitions in poss_list:
yield [left_edges] + partitions
def merge_string_nonterminals(self, string, amr, next_id):
"""
Binarizes a string-graph pair consisting entirely of nonterminals, ensuring
correct visit order for parsing.
"""
rules = []
stack = []
tokens = list(reversed([s for s in string if s]))
# standard shift-reduce binarization algorithm
# TODO add citation after paper is published
while tokens:
next_tok = tokens.pop()
next_tok_triple_l = [t for t in amr.triples() if str(t[1]) == next_tok]
assert len(next_tok_triple_l) == 1
next_tok_triple = next_tok_triple_l[0]
if not stack:
stack.append(next_tok)
continue
stack_top = stack.pop()
stack_top_triple = [t for t in amr.triples() if str(t[1]) == stack_top][0]
if (stack_top_triple[0] not in next_tok_triple[2]) and \
(next_tok_triple[0] not in stack_top_triple[2]):
# can't merge, so shift
stack.append(stack_top)
stack.append(next_tok)
continue
# can merge, so reduce
rule_amr = Hgraph.from_triples([stack_top_triple, next_tok_triple])
assert len(rule_amr.roots) == 1
rule_string = [stack_top, next_tok]
fictitious_tree = self.make_fictitious_tree(string, rule_string)
new_rule, tree, amr, next_id = self.make_rule(fictitious_tree, amr,
Tree('X', rule_string), rule_amr, next_id)
string = tree.leaves()
tokens.append('#%s' % new_rule.symbol)
rules.append(new_rule)
if len(stack) > 1:
raise BinarizationException
return string, amr, rules, next_id
def replace_instance_edges(graph, tree):
t = []
alignments = {}
tree_leaves = tree.leaves()
for e in graph.triples():
if e in graph.edge_alignments:
p, r, ch = e
token = graph.edge_alignments[e][0] # TODO: not sure what to do with multiple tokens
new_edge = (p, "%s'" % tree_leaves[token], ch)
t.append(new_edge)
alignments[new_edge] = [token]
else:
t.append(e)
res = Hgraph.from_triples(t, {}, warn=False)
res.edge_alignments = alignments
res.node_alignments = graph.node_alignments
res.roots = graph.roots
res.external_nodes = graph.external_nodes
return res
def collapse_amr_terminals(self, tree, amr, next_id):
"""
Creates new rules by merging terminal subgraphs with their closest
nonterminal edge.
"""
# triples returns in breadth-first order, so first triples in the list are
# closest to the root of the AMR
nonterminals = list(reversed([t for t in amr.triples() if isinstance(t[1],
NonterminalLabel)]))
rules = []
first = True
while nonterminals:
nt = nonterminals.pop()
# in general, we will attach to a given nonterminal edge all of the
# terminal edges reachable from its tail nodes
attached_terminals = self.terminal_search(nt, amr.triples())
if first:
# we still have to handle terminal edges that are higher than any
# nonterminal edge
# because the first nonterminal edge is closest to the root of the AMR,
# it must be reachable from the root without passing through any other
# nonterminal, so we can attach all the high terminals (those reachable
# from the root) to the first nonterminal
attached_terminals |= self.terminal_search(amr.root_edges()[0],
amr.triples())
attached_terminals |= {amr.root_edges()[0]}
first = False
# don't bother making a rule when there's nothing to collapse
if not attached_terminals:
continue
rule_amr = Hgraph.from_triples({nt} | attached_terminals)
rule_tree = str(nt[1])
assert len(rule_amr.roots) == 1
new_rule, tree, amr, next_id = self.make_rule(tree, amr, rule_tree,
rule_amr, next_id)
rules.append(new_rule)
return tree, amr, rules, next_id
def replace_instance_edges(graph, tree):
t = []
alignments = {}
tree_leaves = tree.leaves()
for e in graph.triples():
if e in graph.edge_alignments:
p, r, ch = e
token = graph.edge_alignments[e][
0] # TODO: not sure what to do with multiple tokens
new_edge = (p, "%s'" % tree_leaves[token], ch)
t.append(new_edge)
alignments[new_edge] = [token]
else:
t.append(e)
res = Hgraph.from_triples(t, {}, warn=False)
res.edge_alignments = alignments
res.node_alignments = graph.node_alignments
res.roots = graph.roots
res.external_nodes = graph.external_nodes
return res
def merge_tree_symbols(self, tree, amr, next_id):
"""
Binarizes a tree-graph pair according to the binariziation dictated by the
tree. WILL FAIL OFTEN IF TREE IS NOT BINARIZED.
"""
rules = []
while True:
if not isinstance(tree, Tree):
assert len(amr.triples()) == 1
return tree, amr, rules, next_id
# a collapsible subtree consists of
# 1. many terminals
# 2. one nonterminal and many terminals
# 3. two nonterminals
collapsible_subtrees = []
for st in tree.subtrees():
terminals = [t for t in st.leaves() if t[0] == '#']
if len(terminals) == 1:
collapsible_subtrees.append(st)
elif len(terminals) == 2 and len(st.leaves()) == 2:
collapsible_subtrees.append(st)
# if there are no subtrees to collapse, this rule isn't binarizable
if len(collapsible_subtrees) == 0:
raise BinarizationException
rule_tree = max(collapsible_subtrees, key=lambda x: x.height())
terminals = [t for t in rule_tree.leaves() if t[0] == '#']
rule_edge_l = [t for t in amr.triples() if str(t[1]) in terminals]
rule_amr = Hgraph.from_triples(rule_edge_l)
# if the induced graph is disconnected, this rule isn't binarizable
if len(rule_amr.roots) != 1:
raise BinarizationException
new_rule, tree, amr, next_id = self.make_rule(tree, amr, rule_tree,
rule_amr, next_id)
rules.append(new_rule)
return tree, amr, rules, next_id
def collapse_string_terminals(self, string, amr, next_id):
"""
Creates new rules by merging terminal tokens with their closest nonterminal.
All terminals attach to the left (except for terminals left of the first
nonterminal, which attach right).
"""
nonterminals = list(reversed([t for t in string if t[0] == '#']))
rules = []
# attach first terminals to the right
slice_from = 0
while nonterminals:
nt = nonterminals.pop()
if nonterminals:
slice_to = string.index(nonterminals[-1])
else:
slice_to = len(string)
if slice_to - slice_from == 1:
# there are no terminals to attach here, so skip ahead
slice_from = slice_to
continue
rule_string = string[slice_from:slice_to]
nt_edge_l = [e for e in amr.triples(nodelabels = self.nodelabels) if str(e[1]) == nt]
assert len(nt_edge_l) == 1
rule_amr = Hgraph.from_triples(nt_edge_l)
# hallucinate a tree with acceptable structure for make_rule
fictitious_tree = self.make_fictitious_tree(string, rule_string)
new_rule, tree, amr, next_id = self.make_rule(fictitious_tree, amr,
Tree('X', rule_string), rule_amr, next_id)
string = tree.leaves()
rules.append(new_rule)
slice_from = slice_from + 1
return string, amr, rules, next_id
def convert_chart(partition, external_nodes, nt, first=False):
nt = NonterminalLabel(nt.label) # Get rid of the index
if partition in seen:
node = seen[partition]
result.use_counts[node] += 1
return node
leaves = chart.tree.leaves()
edges_in_partition = [graph_edge_list[i] for i in range(len(partition.edges)) if partition.edges[i] == 1]
if not partition in chart: # leaf
graph = Hgraph.from_triples(edges_in_partition, {}, warn=False)
graph.roots = graph.find_roots()
graph.roots.sort(lambda x, y: node_order[x] - node_order[y])
graph.external_nodes = external_nodes
str_rhs = [leaves[i] for i in range(partition.str_start, partition.str_end + 1)]
rule = Rule(0, nt.label, graph, tuple(str_rhs), 1)
rule_id = self.add_rule(rule)
fragment = fragment_counter[0]
result[fragment] = [(rule_id, [])]
result.use_counts[fragment] += 1
seen[partition] = fragment
fragment_counter[0] += 1
return fragment
poss = []
count = 0
for possibility in chart[partition]:
count += 1
partition_graph = Hgraph.from_triples(edges_in_partition, {}, warn=False) # This is the parent graph
partition_graph.roots = partition_graph.find_roots()
partition_graph.roots.sort(lambda x, y: node_order[x] - node_order[y])
partition_graph.external_nodes = external_nodes
children = []
# print partition_graph.to_amr_string()
spans_to_nt = {}
old_pgraph = partition_graph
index = 1
for subpartition in possibility: # These are the different sub-constituents
edges_in_subpartition = [
graph_edge_list[i] for i in range(len(subpartition.edges)) if subpartition.edges[i] == 1
]
if edges_in_subpartition: # Some constituents do not have any edges aligned to them
sub_graph = Hgraph.from_triples(edges_in_subpartition, {}, warn=False)
sub_graph.roots = sub_graph.find_roots()
sub_graph.roots.sort(lambda x, y: node_order[x] - node_order[y])
external_node_list = partition_graph.find_external_nodes2(sub_graph)
external_node_list.sort(lambda x, y: node_order[x] - node_order[y])
sub_external_nodes = dict([(k, v) for v, k in enumerate(external_node_list)])
sub_graph.external_nodes = sub_external_nodes
sub_nt = NonterminalLabel("%s%i" % (subpartition.phrase, len(sub_external_nodes)), index)
children.append(convert_chart(subpartition, sub_external_nodes, sub_nt)) # Recursive call
old_pgraph = partition_graph
partition_graph = partition_graph.collapse_fragment2(
sub_graph, sub_nt, external=external_node_list, warn=False
)
spans_to_nt[subpartition.str_start] = (sub_nt, subpartition.str_end)
else:
sub_nt = NonterminalLabel(subpartition.phrase, index)
# assert partition_graph.is_connected()
index += 1
partition_graph.roots = partition_graph.find_roots()
partition_graph.roots.sort(lambda x, y: node_order[x] - node_order[y])
# Assemble String rule
str_rhs = []
i = partition.str_start
while i <= partition.str_end:
if i in spans_to_nt:
new_nt, i = spans_to_nt[i]
str_rhs.append(new_nt)
else:
str_rhs.append(leaves[i])
i = i + 1
rule = Rule(0, nt.label, partition_graph, tuple(str_rhs), 1)
rule_id = self.add_rule(rule)
poss.append((rule_id, children))
fragment = fragment_counter[0]
result[fragment] = poss
result.use_counts[fragment] += 1
seen[partition] = fragment
fragment_counter[0] += 1
return fragment
def get_graph(self, graph):
trips = graph.triples()
return Hgraph.from_triples([trips[i] for i in range(len(trips)) if self.edges[i] == 1], {}, warn=False)
def compute_chart(
tree,
graph,
prefix=""
): # Recursively compute the chart. Graph is the sub-graph we're considering, tree is the tree of spans.
count[0] += 1
triples = set(graph.triples())
edge_vector = tuple(
[1 if x in triples else 0 for x in graph_edge_list])
leaves = tree.leaves()
#if not isinstance(tree,fancy_tree.FancyTree):
# triples = set(graph.triples())
# edge_vector= tuple(1 for x in graph_edge_list if x in triples else 0)
# return Partition(tree.node, leaves[0], leaves[-1], edge_vector)
#else:
if len(tree) == 1 and not isinstance(tree[0],
fancy_tree.FancyTree):
return Partition(tree.node, leaves[0], leaves[-1], edge_vector)
# First get the set of aligned edges for this constituent and it's children
aligned_edges_for_span = set([
edge for token in tree.leaves()
for edge in rev_alignments[token]
])
partition_object = Partition(tree.node, leaves[0], leaves[-1],
edge_vector)
if partition_object not in chart:
try:
possibilities = []
child_edgesets = []
# Compute edge set for each child
for t in tree:
edgeset = []
for l in t.leaves():
edgeset.extend(rev_alignments[l])
child_edgesets.append(edgeset)
# For each possible partitioning
for cparts in get_binarized_partitions(
graph, child_edgesets):
child_forests = []
for i in range(len(tree)):
childgraph = Hgraph.from_triples(cparts[i], {},
warn=False)
sub_forest = compute_chart(tree[i],
childgraph,
prefix=prefix + " ")
if len(chart) > MAX_CHART_SIZE:
raise ChartTooBigException, "Chart size exceeded 5000 entries. dropping this sentence."
child_forests.append(sub_forest)
possibilities.append(child_forests)
chart[partition_object] = possibilities
except IncompatibleAlignmentException:
chart.inconsistent_alignment = (tree.node, leaves[0],
leaves[-1])
return partition_object
return partition_object
def convert_chart(partition, external_nodes, nt, first=False):
nt = NonterminalLabel(nt.label) # Get rid of the index
if partition in seen:
node = seen[partition]
result.use_counts[node] += 1
return node
leaves = chart.tree.leaves()
edges_in_partition = [
graph_edge_list[i] for i in range(len(partition.edges))
if partition.edges[i] == 1
]
if not partition in chart: # leaf
graph = Hgraph.from_triples(edges_in_partition, {}, warn=False)
graph.roots = graph.find_roots()
graph.roots.sort(lambda x, y: node_order[x] - node_order[y])
graph.external_nodes = external_nodes
str_rhs = [
leaves[i]
for i in range(partition.str_start, partition.str_end + 1)
]
rule = Rule(0, nt.label, graph, tuple(str_rhs), 1)
rule_id = self.add_rule(rule)
fragment = fragment_counter[0]
result[fragment] = [(rule_id, [])]
result.use_counts[fragment] += 1
seen[partition] = fragment
fragment_counter[0] += 1
return fragment
poss = []
count = 0
for possibility in chart[partition]:
count += 1
partition_graph = Hgraph.from_triples(
edges_in_partition, {},
warn=False) # This is the parent graph
partition_graph.roots = partition_graph.find_roots()
partition_graph.roots.sort(
lambda x, y: node_order[x] - node_order[y])
partition_graph.external_nodes = external_nodes
children = []
#print partition_graph.to_amr_string()
spans_to_nt = {}
old_pgraph = partition_graph
index = 1
for subpartition in possibility: #These are the different sub-constituents
edges_in_subpartition = [
graph_edge_list[i]
for i in range(len(subpartition.edges))
if subpartition.edges[i] == 1
]
if edges_in_subpartition: # Some constituents do not have any edges aligned to them
sub_graph = Hgraph.from_triples(edges_in_subpartition,
{},
warn=False)
sub_graph.roots = sub_graph.find_roots()
sub_graph.roots.sort(
lambda x, y: node_order[x] - node_order[y])
external_node_list = partition_graph.find_external_nodes2(
sub_graph)
external_node_list.sort(
lambda x, y: node_order[x] - node_order[y])
sub_external_nodes = dict([
(k, v) for v, k in enumerate(external_node_list)
])
sub_graph.external_nodes = sub_external_nodes
sub_nt = NonterminalLabel(
"%s%i" %
(subpartition.phrase, len(sub_external_nodes)),
index)
children.append(
convert_chart(subpartition, sub_external_nodes,
sub_nt)) # Recursive call
old_pgraph = partition_graph
partition_graph = partition_graph.collapse_fragment2(
sub_graph,
sub_nt,
external=external_node_list,
warn=False)
spans_to_nt[subpartition.str_start] = (
sub_nt, subpartition.str_end)
else:
sub_nt = NonterminalLabel(subpartition.phrase, index)
#assert partition_graph.is_connected()
index += 1
partition_graph.roots = partition_graph.find_roots()
partition_graph.roots.sort(
lambda x, y: node_order[x] - node_order[y])
# Assemble String rule
str_rhs = []
i = partition.str_start
while i <= partition.str_end:
if i in spans_to_nt:
new_nt, i = spans_to_nt[i]
str_rhs.append(new_nt)
else:
str_rhs.append(leaves[i])
i = i + 1
rule = Rule(0, nt.label, partition_graph, tuple(str_rhs), 1)
rule_id = self.add_rule(rule)
poss.append((rule_id, children))
fragment = fragment_counter[0]
result[fragment] = poss
result.use_counts[fragment] += 1
seen[partition] = fragment
fragment_counter[0] += 1
return fragment
def get_graph(self, graph):
trips = graph.triples()
return Hgraph.from_triples(
[trips[i] for i in range(len(trips)) if self.edges[i] == 1], {},
warn=False)
