def test_rst_to_dt(self):
lw_trees = [
"(R:rel (S x) (N y))", """
(R:rel
(S x)
(N:rel (N h) (S t)))
""", """
(R:r
(S x)
(N:r (N:r (S t1) (N h))
(S t2)))
"""
]
for lstr in lw_trees:
rst1 = parse_lightweight_tree(lstr)
dep = RstDepTree.from_simple_rst_tree(rst1)
rst2 = deptree_to_simple_rst_tree(dep)
self.assertEqual(rst1, rst2, "round-trip on " + lstr)
for name, tree in self._test_trees().items():
rst1 = SimpleRSTTree.from_rst_tree(tree)
dep = RstDepTree.from_simple_rst_tree(rst1)
rst2 = deptree_to_simple_rst_tree(dep)
self.assertEqual(
treenode(rst1).span,
treenode(rst2).span, "span equality on " + name)
self.assertEqual(
treenode(rst1).edu_span,
treenode(rst2).edu_span, "edu span equality on " + name)
def test_rst_to_dt(self):
lw_trees = ["(R:rel (S x) (N y))",
"""
(R:rel
(S x)
(N:rel (N h) (S t)))
""",
"""
(R:r
(S x)
(N:r (N:r (S t1) (N h))
(S t2)))
"""
]
for lstr in lw_trees:
rst1 = parse_lightweight_tree(lstr)
dep = RstDepTree.from_simple_rst_tree(rst1)
rst2 = deptree_to_simple_rst_tree(dep)
self.assertEqual(rst1, rst2, "round-trip on " + lstr)
for name, tree in self._test_trees().items():
rst1 = SimpleRSTTree.from_rst_tree(tree)
dep = RstDepTree.from_simple_rst_tree(rst1)
rst2 = deptree_to_simple_rst_tree(dep)
self.assertEqual(treenode(rst1).span,
treenode(rst2).span,
"span equality on " + name)
self.assertEqual(treenode(rst1).edu_span,
treenode(rst2).edu_span,
"edu span equality on " + name)
def convert(corpus, multinuclear, odir):
"""
Convert every RST tree in the corpus to a dependency tree
(and back, but simplified using a set of relation types
that will be systematically treated as multinuclear)
"""
bin_dir = os.path.join(odir, "rst-binarised")
dt_dir = os.path.join(odir, "rst-to-dt")
rst2_dir = os.path.join(odir, "dt-to-rst")
for subdir in [bin_dir, dt_dir, rst2_dir]:
if not os.path.exists(subdir):
os.makedirs(subdir)
for k in corpus:
suffix = os.path.splitext(k.doc)[0]
stree = SimpleRSTTree.from_rst_tree(corpus[k])
with open(os.path.join(bin_dir, suffix), 'w') as fout:
fout.write(str(stree))
dtree = RstDepTree.from_simple_rst_tree(stree)
with open(os.path.join(dt_dir, suffix), 'w') as fout:
fout.write(str(dtree))
stree2 = deptree_to_simple_rst_tree(dtree)
with open(os.path.join(rst2_dir, suffix), 'w') as fout:
fout.write(str(stree2))
def convert(corpus, multinuclear, odir):
"""
Convert every RST tree in the corpus to a dependency tree
(and back, but simplified using a set of relation types
that will be systematically treated as multinuclear)
"""
bin_dir = os.path.join(odir, "rst-binarised")
dt_dir = os.path.join(odir, "rst-to-dt")
rst2_dir = os.path.join(odir, "dt-to-rst")
for subdir in [bin_dir, dt_dir, rst2_dir]:
if not os.path.exists(subdir):
os.makedirs(subdir)
for k in corpus:
suffix = os.path.splitext(k.doc)[0]
stree = SimpleRSTTree.from_rst_tree(corpus[k])
with open(os.path.join(bin_dir, suffix), 'w') as fout:
fout.write(str(stree))
dtree = RstDepTree.from_simple_rst_tree(stree)
with open(os.path.join(dt_dir, suffix), 'w') as fout:
fout.write(str(dtree))
stree2 = deptree_to_simple_rst_tree(dtree)
with open(os.path.join(rst2_dir, suffix), 'w') as fout:
fout.write(str(stree2))
def test_dt_to_rst_order(self):
lw_trees = [
"(R:r (N:r (N h) (S r1)) (S r2))",
"(R:r (S:r (S l2) (N l1)) (N h))",
"(R:r (N:r (S l1) (N h)) (S r1))",
"""
(R:r
(N:r
(N:r (S l2)
(N:r (S l1)
(N h)))
(S r1))
(S r2))
""", # ((l2 <- l1 <- h) -> r1 -> r2)
"""
(R:r
(N:r
(S l2)
(N:r (N:r (S l1)
(N h))
(S r1)))
(S r2))
""", # (l2 <- ((l1 <- h) -> r1)) -> r2
]
for lstr in lw_trees:
rst1 = parse_lightweight_tree(lstr)
dep = RstDepTree.from_simple_rst_tree(rst1)
dep_a = dep
rst2a = deptree_to_simple_rst_tree(dep_a)
self.assertEqual(rst1, rst2a, "round-trip on " + lstr)
dep_b = copy.deepcopy(dep)
dep_b.deps(0).reverse()
rst2b = deptree_to_simple_rst_tree(dep_b)
# TODO assertion on rst2b?
dep_c = copy.deepcopy(dep)
random.shuffle(dep_c.deps(0))
rst2c = deptree_to_simple_rst_tree(dep_c)
def test_rst_to_dt_nuclearity_loss(self):
"""
Test that we still get sane tree structure with
nuclearity loss
"""
tricky = """
(R:r (S t) (N h))
"""
nuked = """
(R:r (N t) (N h))
"""
# tricky = """
# (R:r
# (S x)
# (N:r (N:r (S t1) (N h))
# (S t2)))
# """
#
# nuked = """
# (R:r
# (N x)
# (N:r (N:r (N t1) (N h))
# (N t2)))
# """
rst0 = parse_lightweight_tree(nuked)
rst1 = parse_lightweight_tree(tricky)
# a little sanity check first
dep0 = RstDepTree.from_simple_rst_tree(rst0)
rev0 = deptree_to_simple_rst_tree(dep0) # was:, ['r'])
self.assertEqual(rst0, rev0, "same structure " + nuked) # sanity
# now the real test
dep1 = RstDepTree.from_simple_rst_tree(rst1)
rev1 = deptree_to_simple_rst_tree(dep1) # was:, ['r'])
def test_rst_to_dt_nuclearity_loss(self):
"""
Test that we still get sane tree structure with
nuclearity loss
"""
tricky = """
(R:r (S t) (N h))
"""
nuked = """
(R:r (N t) (N h))
"""
# tricky = """
# (R:r
# (S x)
# (N:r (N:r (S t1) (N h))
# (S t2)))
# """
#
# nuked = """
# (R:r
# (N x)
# (N:r (N:r (N t1) (N h))
# (N t2)))
# """
rst0 = parse_lightweight_tree(nuked)
rst1 = parse_lightweight_tree(tricky)
# a little sanity check first
dep0 = RstDepTree.from_simple_rst_tree(rst0)
rev0 = deptree_to_simple_rst_tree(dep0) # was:, ['r'])
self.assertEqual(rst0, rev0, "same structure " + nuked) # sanity
# now the real test
dep1 = RstDepTree.from_simple_rst_tree(rst1)
rev1 = deptree_to_simple_rst_tree(dep1) # was:, ['r'])
def get_oracle_ctrees(dep_edges,
att_edus,
nuc_strategy="unamb_else_most_frequent",
rank_strategy="closest-intra-rl-inter-rl",
prioritize_same_unit=True,
strict=False):
"""Build the oracle constituency tree(s) for a dependency tree.
Parameters
----------
dep_edges: dict(string, [(string, string, string)])
Edges for each document, indexed by doc name
Cf. type of return value from
irit-rst-dt/ctree.py:load_attelo_output_file()
att_edus: cf return type of attelo.io.load_edus
EDUs as they are known to attelo
strict: boolean, True by default
If True, any link from ROOT to an EDU that is neither 'ROOT' nor
UNRELATED raises an exception, otherwise a warning is issued.
Returns
-------
ctrees: list of RstTree
There can be several e.g. for leaky sentences.
"""
# rebuild educe EDUs from their attelo description
# and group them by doc_name
educe_edus = defaultdict(list)
edu2sent_idx = defaultdict(dict)
gid2num = dict()
for att_edu in att_edus:
# doc name
doc_name = att_edu.grouping
# EDU info
# skip ROOT (automatically added by RstDepTree.__init__)
if att_edu.id == 'ROOT':
continue
edu_num = int(att_edu.id.rsplit('_', 1)[1])
edu_span = EduceSpan(att_edu.start, att_edu.end)
edu_text = att_edu.text
educe_edus[doc_name].append(EduceEDU(edu_num, edu_span, edu_text))
# map global id of EDU to num of EDU inside doc
gid2num[att_edu.id] = edu_num
# map EDU to sentence
try:
sent_idx = int(att_edu.subgrouping.split('_sent')[1])
except IndexError:
# this EDU could not be attached to any sentence (ex: missing
# text in the PTB), so a default subgrouping identifier was used ;
# we aim for consistency with educe and map these to "None"
sent_idx = None
edu2sent_idx[doc_name][edu_num] = sent_idx
# check that our info covers only one document
assert len(educe_edus) == 1
# then restrict to this document
doc_name = educe_edus.keys()[0]
educe_edus = educe_edus[doc_name]
edu2sent_idx = edu2sent_idx[doc_name]
# sort EDUs by num
educe_edus = list(sorted(educe_edus, key=lambda e: e.num))
# rebuild educe-style edu2sent ; prepend 0 for the fake root
edu2sent = [0] + [edu2sent_idx[e.num] for e in educe_edus]
# classifiers for nuclearity and ranking
# FIXME declare, fit and predict upstream...
# nuclearity
nuc_classifier = DummyNuclearityClassifier(strategy=nuc_strategy)
nuc_classifier.fit([], []) # empty X and y for dummy fit
# ranking classifier
rank_classifier = InsideOutAttachmentRanker(
strategy=rank_strategy, prioritize_same_unit=prioritize_same_unit)
# rebuild RstDepTrees
dtree = RstDepTree(educe_edus)
for src_id, tgt_id, lbl in dep_edges:
if src_id == 'ROOT':
if lbl not in ['ROOT', UNKNOWN]:
err_msg = 'weird root label: {} {} {}'.format(
src_id, tgt_id, lbl)
if strict:
raise ValueError(err_msg)
else:
print('W: {}, using ROOT instead'.format(err_msg))
dtree.set_root(gid2num[tgt_id])
else:
dtree.add_dependency(gid2num[src_id], gid2num[tgt_id], lbl)
# add nuclearity: heuristic baseline
dtree.nucs = nuc_classifier.predict([dtree])[0]
# add rank: some strategies require a mapping from EDU to sentence
# EXPERIMENTAL attach array of sentence index for each EDU in tree
dtree.sent_idx = edu2sent
# end EXPERIMENTAL
dtree.ranks = rank_classifier.predict([dtree])[0]
# end NEW
# create pred ctree
try:
bin_srtrees = deptree_to_simple_rst_tree(dtree, allow_forest=True)
if False: # EXPERIMENTAL
# currently False to run on output that already has
# labels embedding nuclearity
bin_srtrees = [
SimpleRSTTree.incorporate_nuclearity_into_label(bin_srtree)
for bin_srtree in bin_srtrees
]
bin_rtrees = [
SimpleRSTTree.to_binary_rst_tree(bin_srtree)
for bin_srtree in bin_srtrees
]
except RstDtException as rst_e:
print(rst_e)
if False:
print('\n'.join('{}: {}'.format(edu.text_span(), edu)
for edu in educe_edus[doc_name]))
raise
ctrees = bin_rtrees
return ctrees
def get_oracle_ctrees(dep_edges, att_edus,
nuc_strategy="unamb_else_most_frequent",
rank_strategy="closest-intra-rl-inter-rl",
prioritize_same_unit=True,
strict=False):
"""Build the oracle constituency tree(s) for a dependency tree.
Parameters
----------
dep_edges: dict(string, [(string, string, string)])
Edges for each document, indexed by doc name
Cf. type of return value from
irit-rst-dt/ctree.py:load_attelo_output_file()
att_edus: cf return type of attelo.io.load_edus
EDUs as they are known to attelo
strict: boolean, True by default
If True, any link from ROOT to an EDU that is neither 'ROOT' nor
UNRELATED raises an exception, otherwise a warning is issued.
Returns
-------
ctrees: list of RstTree
There can be several e.g. for leaky sentences.
"""
# rebuild educe EDUs from their attelo description
# and group them by doc_name
educe_edus = defaultdict(list)
edu2sent_idx = defaultdict(dict)
gid2num = dict()
for att_edu in att_edus:
# doc name
doc_name = att_edu.grouping
# EDU info
# skip ROOT (automatically added by RstDepTree.__init__)
if att_edu.id == 'ROOT':
continue
edu_num = int(att_edu.id.rsplit('_', 1)[1])
edu_span = EduceSpan(att_edu.start, att_edu.end)
edu_text = att_edu.text
educe_edus[doc_name].append(EduceEDU(edu_num, edu_span, edu_text))
# map global id of EDU to num of EDU inside doc
gid2num[att_edu.id] = edu_num
# map EDU to sentence
try:
sent_idx = int(att_edu.subgrouping.split('_sent')[1])
except IndexError:
# this EDU could not be attached to any sentence (ex: missing
# text in the PTB), so a default subgrouping identifier was used ;
# we aim for consistency with educe and map these to "None"
sent_idx = None
edu2sent_idx[doc_name][edu_num] = sent_idx
# check that our info covers only one document
assert len(educe_edus) == 1
# then restrict to this document
doc_name = educe_edus.keys()[0]
educe_edus = educe_edus[doc_name]
edu2sent_idx = edu2sent_idx[doc_name]
# sort EDUs by num
educe_edus = list(sorted(educe_edus, key=lambda e: e.num))
# rebuild educe-style edu2sent ; prepend 0 for the fake root
edu2sent = [0] + [edu2sent_idx[e.num] for e in educe_edus]
# classifiers for nuclearity and ranking
# FIXME declare, fit and predict upstream...
# nuclearity
nuc_classifier = DummyNuclearityClassifier(strategy=nuc_strategy)
nuc_classifier.fit([], []) # empty X and y for dummy fit
# ranking classifier
rank_classifier = InsideOutAttachmentRanker(
strategy=rank_strategy,
prioritize_same_unit=prioritize_same_unit)
# rebuild RstDepTrees
dtree = RstDepTree(educe_edus)
for src_id, tgt_id, lbl in dep_edges:
if src_id == 'ROOT':
if lbl not in ['ROOT', UNKNOWN]:
err_msg = 'weird root label: {} {} {}'.format(
src_id, tgt_id, lbl)
if strict:
raise ValueError(err_msg)
else:
print('W: {}, using ROOT instead'.format(err_msg))
dtree.set_root(gid2num[tgt_id])
else:
dtree.add_dependency(gid2num[src_id], gid2num[tgt_id], lbl)
# add nuclearity: heuristic baseline
dtree.nucs = nuc_classifier.predict([dtree])[0]
# add rank: some strategies require a mapping from EDU to sentence
# EXPERIMENTAL attach array of sentence index for each EDU in tree
dtree.sent_idx = edu2sent
# end EXPERIMENTAL
dtree.ranks = rank_classifier.predict([dtree])[0]
# end NEW
# create pred ctree
try:
bin_srtrees = deptree_to_simple_rst_tree(dtree, allow_forest=True)
if False: # EXPERIMENTAL
# currently False to run on output that already has
# labels embedding nuclearity
bin_srtrees = [SimpleRSTTree.incorporate_nuclearity_into_label(
bin_srtree) for bin_srtree in bin_srtrees]
bin_rtrees = [SimpleRSTTree.to_binary_rst_tree(bin_srtree)
for bin_srtree in bin_srtrees]
except RstDtException as rst_e:
print(rst_e)
if False:
print('\n'.join('{}: {}'.format(edu.text_span(), edu)
for edu in educe_edus[doc_name]))
raise
ctrees = bin_rtrees
return ctrees
