def hybrid_tree_1():
tree = HybridTree()
tree.add_node('v1', CoNLLToken('Piet', '_', 'NP', 'NP', '_', 'SBJ'), True)
tree.add_node('v21', CoNLLToken('Marie', '_', 'N', 'N', '_', 'OBJ'), True)
tree.add_node('v', CoNLLToken('helpen', '_', 'V', 'V', '_', 'ROOT'), True)
tree.add_node('v2', CoNLLToken('lezen', '_', 'V', 'V', '_', 'VBI'), True)
tree.add_child('v', 'v2')
tree.add_child('v', 'v1')
tree.add_child('v2', 'v21')
tree.add_to_root('v')
tree.reorder()
return tree
def multi_dep_tree():
tree = HybridTree('multi')
tree.add_node('1', CoNLLToken('A', '_', 'pA', 'pA', '_', 'dA'), True)
tree.add_node('211', CoNLLToken('B', '_', 'pB', 'pB', '_', 'dB'), True)
tree.add_node('11', CoNLLToken('C', '_', 'pC', 'pC', '_', 'dC'), True)
tree.add_node('2', CoNLLToken('D', '_', 'pD', 'pD', '_', 'dD'), True)
tree.add_node('21', CoNLLToken('E', '_', 'pE', 'pE', '_', 'dE'), True)
tree.add_to_root('2')
tree.add_to_root('1')
for c in ['21', '211']:
tree.add_child('2', c)
tree.add_child('1', '11')
tree.reorder()
return tree
def derivation_to_hybrid_tree(der,
poss,
ordered_labels,
construct_token,
disconnected=None):
"""
:param der:
:type der: LCFRSDerivation
:param poss: list of POS-tags
:type poss: list[str]
:param ordered_labels: list of words
:type ordered_labels: list[str]
:param disconnected: list of positions in ordered_labels that are disconnected
:type disconnected: list[object]
:rtype: GeneralHybridTree
Turn a derivation tree into a hybrid tree. Assuming poss and ordered_labels to have equal length.
"""
if not disconnected:
disconnected = []
tree = HybridTree()
j = 1
for i in range(len(ordered_labels)):
token = construct_token(ordered_labels[i], poss[i], True)
if i in disconnected:
tree.add_node("d" + str(i), token, True, False)
else:
tree.add_node("c" + str(j), token, True, True)
j += 1
for id in der.ids():
token = construct_token(der.getRule(id).lhs().nont(), '_', False)
tree.add_node(id, token)
for child in der.child_ids(id):
tree.add_child(id, child)
for position in der.terminal_positions(id):
tree.add_child(id, "c" + str(position))
tree.add_to_root(der.root_id())
tree.reorder()
return tree
def test_fst_compilation_right(self):
if not test_pynini:
return
tree = hybrid_tree_1()
tree2 = hybrid_tree_2()
terminal_labeling = the_terminal_labeling_factory().get_strategy('pos')
(_, grammar) = induce_grammar(
[tree, tree2],
the_labeling_factory().create_simple_labeling_strategy(
'empty', 'pos'), terminal_labeling.token_label,
[right_branching], 'START')
a, rules = compile_wfst_from_right_branching_grammar(grammar)
print(repr(a))
symboltable = a.input_symbols()
string = 'NP N V V V'.split(' ')
token_sequence = [
construct_conll_token(form, lemma) for form, lemma in zip(
'Piet Marie helpen leren lezen'.split(' '), string)
]
fsa = fsa_from_list_of_symbols(string, symboltable)
self.assertEqual(
'0\t1\tNP\tNP\n1\t2\tN\tN\n2\t3\tV\tV\n3\t4\tV\tV\n4\t5\tV\tV\n5\n',
fsa.text().decode('utf-8'))
b = compose(fsa, a)
print(b.input_symbols())
for i in b.input_symbols():
print(i)
print("Input Composition")
print(b.text(symboltable, symboltable).decode('utf-8'))
i = 0
for path in paths(b):
print(i, "th path:", path, end=' ')
r = list(map(rules.index_object, path))
d = PolishDerivation(r[1::])
dcp = DCP_evaluator(d).getEvaluation()
h = HybridTree()
dcp_to_hybridtree(h, dcp, token_sequence, False,
construct_conll_token)
h.reorder()
if h == tree2:
print("correct")
else:
print("incorrect")
i += 1
stats = defaultdict(lambda: 0)
local_rule_stats(b, stats, 15)
print(stats)
print("Shortest path probability")
best = shortestpath(b)
best.topsort()
self.assertAlmostEqual(1.80844898756e-05,
pow(e, -float(shortestdistance(best)[-1])))
print(best.text())
polish_rules = retrieve_rules(best)
self.assertSequenceEqual(polish_rules, [8, 7, 1, 6, 2, 5, 3, 10, 3, 3])
polish_rules = list(map(rules.index_object, polish_rules))
print(polish_rules)
der = PolishDerivation(polish_rules[1::])
print(der)
print(
derivation_to_hybrid_tree(der, string,
"Piet Marie helpen lezen leren".split(),
construct_conll_token))
dcp = DCP_evaluator(der).getEvaluation()
h_tree_2 = HybridTree()
dcp_to_hybridtree(h_tree_2, dcp, token_sequence, False,
construct_conll_token)
print(h_tree_2)
class GeneralHybridTreeTestCase(unittest.TestCase):
tree = None
def setUp(self):
self.tree = HybridTree()
self.tree.add_node("v1", construct_conll_token("Piet", "NP"), True)
self.tree.add_node("v21", construct_conll_token("Marie", "N"), True)
self.tree.add_node("v", construct_conll_token("helpen", "VP"), True)
self.tree.add_node("v2", construct_conll_token("lezen", "V"), True)
self.tree.add_child("v", "v2")
self.tree.add_child("v", "v1")
self.tree.add_child("v2", "v21")
self.tree.add_node("v3", construct_conll_token(".", "Punc"), True,
False)
self.tree.add_to_root("v")
def test_children(self):
self.assertListEqual(self.tree.children('v'), ['v2', 'v1'])
self.tree.reorder()
self.assertListEqual(self.tree.children('v'), ['v1', 'v2'])
def test_fringe(self):
self.tree.reorder()
self.assertListEqual(self.tree.fringe('v'), [2, 0, 3, 1])
self.assertListEqual(self.tree.fringe('v2'), [3, 1])
def test_n_spans(self):
self.tree.reorder()
self.assertEqual(self.tree.n_spans('v'), 1)
self.assertEqual(self.tree.n_spans('v2'), 2)
def test_n_gaps(self):
self.tree.reorder()
self.assertEqual(self.tree.n_gaps(), 1)
def test_node_ids(self):
self.tree.reorder()
self.assertListEqual(sorted(self.tree.nodes()),
sorted(['v', 'v1', 'v2', 'v21', 'v3']))
def test_complete(self):
self.tree.reorder()
self.assertEqual(self.tree.complete(), True)
def test_unlabelled_structure(self):
self.tree.reorder()
self.assertTupleEqual(self.tree.unlabelled_structure(),
({0, 1, 2, 3}, [({0}, []),
({1, 3}, [({1}, [])])]))
def test_max_n_spans(self):
self.tree.reorder()
self.assertEqual(self.tree.max_n_spans(), 2)
def test_labelled_yield(self):
self.tree.reorder()
self.assertListEqual(
[token.form() for token in self.tree.token_yield()],
"Piet Marie helpen lezen".split(' '))
def test_full_labelled_yield(self):
self.tree.reorder()
self.assertListEqual(
[token.form() for token in self.tree.full_token_yield()],
"Piet Marie helpen lezen .".split(' '))
def test_full_yield(self):
self.tree.reorder()
self.assertListEqual(self.tree.full_yield(),
'v1 v21 v v2 v3'.split(' '))
# def test_labelled_spans(self):
# self.tree.reorder()
# self.assertListEqual(self.tree.labelled_spans(), [])
def test_pos_yield(self):
self.tree.reorder()
self.assertListEqual(
[token.pos() for token in self.tree.token_yield()],
"NP N VP V".split(' '))
def test_recursive_partitioning(self):
self.tree.reorder()
self.assertEqual(self.tree.recursive_partitioning(),
({0, 1, 2, 3}, [({0}, []),
({1, 3}, [({1}, []), ({3}, [])]),
({2}, [])]))