def test_own_induction(self):
# trees = [self.get_single_tree()]
# trees = [hybrid_tree_1()]
trees = self.get_whole_corpus()
# print(trees[0])
partition_builders = []
for choice_function in [
choose_min,
choose_max,
choose_middle,
# choose_random
]:
for split_function in [
monadic_split,
spans_split,
# flat_wide_split,
sorted_even_2_split,
# sorted_even_3_split,
# random_split
]:
partition_builders.append(
PartitionBuilder(choice_function=choice_function,
split_function=split_function))
terminal_labelings = [
the_terminal_labeling_factory().get_strategy('form'),
the_terminal_labeling_factory().get_strategy('pos')
]
for partition_builder in partition_builders:
for terminal_labeling in terminal_labelings:
print('make induction with:\n'
f'partition builder: {partition_builder}\n'
f'terminal_labelling: {terminal_labeling}')
nonterminal_counts = {}
grammar = induce(trees=trees,
partition_builder=partition_builder,
terminal_labeling=terminal_labeling,
nonterminal_counts=nonterminal_counts)
rules = grammar.rules()
n_rules = len(rules)
for i, rule in enumerate(rules):
# if not i % 10_000:
# print(f'update rule {i} of {n_rules}')
weight = rule.weight()
nonterminal = rule.lhs().nont()
assert nonterminal in nonterminal_counts
weight /= nonterminal_counts[nonterminal]
rule.set_weight(weight)
# for rule in sorted(rules, key=lambda r: r.weight(), reverse=True)[100:]:
# print(rule)
# print(f'grammar:\n{grammar}')
classes = rule_classes_created_terminals(grammar)
class_sizes = [len(classes[key]) for key in classes]
class_sizes.sort(reverse=True)
print(f'min: {min(class_sizes)})\n'
f'max: {max(class_sizes)}\n'
f'avg: {sum(class_sizes) / len(class_sizes)}\n')
def test_multiroot(self):
tree = multi_dep_tree()
term_pos = the_terminal_labeling_factory().get_strategy(
'pos').token_label
fanout_1 = the_recursive_partitioning_factory().get_partitioning(
'fanout-1')
for top_level_labeling_strategy in ['strict', 'child']:
labeling_strategy = the_labeling_factory(
).create_simple_labeling_strategy(top_level_labeling_strategy,
'pos+deprel')
for recursive_partitioning in [[direct_extraction], fanout_1,
[left_branching]]:
(_, grammar) = induce_grammar([tree], labeling_strategy,
term_pos, recursive_partitioning,
'START')
print(grammar)
parser = LCFRS_parser(grammar, 'pA pB pC pD pE'.split(' '))
print(parser.best_derivation_tree())
cleaned_tokens = copy.deepcopy(tree.full_token_yield())
for token in cleaned_tokens:
token.set_edge_label('_')
hybrid_tree = HybridTree()
hybrid_tree = parser.dcp_hybrid_tree_best_derivation(
hybrid_tree, cleaned_tokens, True, construct_conll_token)
print(hybrid_tree)
self.assertEqual(tree, hybrid_tree)
def test_conll_grammar_induction():
db_connection = experiment_database.initialize_database(SAMPLE_DB)
root_default_deprel = 'ROOT'
disconnected_default_deprel = 'PUNC'
terminal_labeling_strategy = the_terminal_labeling_factory().get_strategy(
'pos')
for ignore_punctuation in [True, False]:
for top_level, node_to_string in itertools.product(['strict', 'child'],
['pos', 'deprel']):
nont_labelling = label.the_labeling_factory(
).create_simple_labeling_strategy(top_level, node_to_string)
for rec_par_s in [
'direct_extraction', 'left_branching', 'right_branching',
'fanout-1', 'fanout_2'
]:
rec_par = grammar.induction.recursive_partitioning.the_recursive_partitioning_factory(
).get_partitioning(rec_par_s)
grammar, experiment = induce_grammar_from_file(
CONLL_TRAIN, db_connection, nont_labelling,
terminal_labeling_strategy.token_label, rec_par,
sys.maxsize, False, 'START', ignore_punctuation)
print()
parse_sentences_from_file(
grammar, experiment, db_connection, CONLL_TEST,
terminal_labeling_strategy.prepare_parser_input, 20,
sys.maxsize, False, ignore_punctuation,
root_default_deprel, disconnected_default_deprel)
experiment_database.finalize_database(db_connection)
def test_basic_split_merge(self):
tree = hybrid_tree_1()
tree2 = hybrid_tree_2()
terminal_labeling = the_terminal_labeling_factory().get_strategy('pos')
(_, grammar) = induce_grammar(
[tree, tree2],
the_labeling_factory().create_simple_labeling_strategy(
'empty', 'pos'), terminal_labeling.token_label, [cfg], 'START')
for rule in grammar.rules():
print(rule, file=stderr)
print("call S/M Training", file=stderr)
new_grammars = split_merge_training(grammar,
terminal_labeling, [tree, tree2],
3,
5,
merge_threshold=0.5,
debug=False)
for new_grammar in new_grammars:
for i, rule in enumerate(new_grammar.rules()):
print(i, rule, file=stderr)
print(file=stderr)
print("finished S/M Training", file=stderr)
def test_conll_grammar_induction():
ignore_punctuation = True
trees = parse_conll_corpus(TEST_FILE, False)
trees = disconnect_punctuation(trees)
terminal_labeling = the_terminal_labeling_factory().get_strategy('pos')
nonterminal_labeling = the_labeling_factory(
).create_simple_labeling_strategy('child', 'pos')
(_, grammar) = d_i.induce_grammar(trees, nonterminal_labeling,
terminal_labeling.token_label,
[direct_extraction], 'START')
trees2 = parse_conll_corpus(TEST_FILE_MODIFIED, False)
trees2 = disconnect_punctuation(trees2)
for tree in trees2:
parser = LCFRS_parser(
grammar,
terminal_labeling.prepare_parser_input(tree.token_yield()))
cleaned_tokens = copy.deepcopy(tree.full_token_yield())
for token in cleaned_tokens:
token.set_edge_label('_')
h_tree = HybridTree()
h_tree = parser.dcp_hybrid_tree_best_derivation(
h_tree, cleaned_tokens, ignore_punctuation,
construct_conll_token)
# print h_tree
print('input -> hybrid-tree -> output')
print(tree_to_conll_str(tree))
print('parsed tokens')
print(list(map(str, h_tree.full_token_yield())))
print('test_parser output')
print(tree_to_conll_str(h_tree))
def test_basic_em_training(self):
tree = hybrid_tree_1()
tree2 = hybrid_tree_2()
terminal_labeling = the_terminal_labeling_factory().get_strategy('pos')
(_, grammar) = induce_grammar(
[tree, tree2],
the_labeling_factory().create_simple_labeling_strategy(
'empty', 'pos'), terminal_labeling.token_label, [cfg], 'START')
for rule in grammar.rules():
print(rule, file=stderr)
print("compute reducts", file=stderr)
trace = compute_reducts(grammar, [tree, tree2], terminal_labeling)
print("call em Training", file=stderr)
emTrainer = PyEMTrainer(trace)
emTrainer.em_training(grammar, n_epochs=10)
print("finished em Training", file=stderr)
for rule in grammar.rules():
print(rule, file=stderr)
def test_corpus_split_merge_training(self):
train = 'res/dependency_conll/german/tiger/train/german_tiger_train.conll'
limit_train = 100
test = train
# test = '../../res/dependency_conll/german/tiger/test/german_tiger_test.conll'
trees = parse_conll_corpus(train, False, limit_train)
primary_labelling = the_labeling_factory().create_simple_labeling_strategy("childtop", "deprel")
term_labelling = the_terminal_labeling_factory().get_strategy('pos')
start = 'START'
recursive_partitioning = [cfg]
(n_trees, grammar_prim) = induce_grammar(trees, primary_labelling, term_labelling.token_label,
recursive_partitioning, start)
# for rule in grammar.rules():
# print >>stderr, rule
trees = parse_conll_corpus(train, False, limit_train)
print("call S/M Training", file=stderr)
new_grammars = split_merge_training(grammar_prim, term_labelling, trees, 4, 10, tie_breaking=True, init="equal",
sigma=0.05, seed=50, merge_threshold=0.1)
print("finished S/M Training", file=stderr)
for new_grammar in new_grammars:
for i, rule in enumerate(new_grammar.rules()):
print(i, rule, file=stderr)
print(file=stderr)
def test_corpus_em_training(self):
train = 'res/dependency_conll/german/tiger/train/german_tiger_train.conll'
limit_train = 200
test = train
# test = '../../res/dependency_conll/german/tiger/test/german_tiger_test.conll'
trees = parse_conll_corpus(train, False, limit_train)
primary_labelling = the_labeling_factory(
).create_simple_labeling_strategy("childtop", "deprel")
term_labelling = the_terminal_labeling_factory().get_strategy('pos')
start = 'START'
recursive_partitioning = [cfg]
(n_trees, grammar_prim) = induce_grammar(trees, primary_labelling,
term_labelling.token_label,
recursive_partitioning, start)
# for rule in grammar.rules():
# print >>stderr, rule
trees = parse_conll_corpus(train, False, limit_train)
print("compute reducts", file=stderr)
trace = compute_reducts(grammar_prim, trees, term_labelling)
print("call em Training", file=stderr)
emTrainer = PyEMTrainer(trace)
emTrainer.em_training(grammar_prim,
20,
tie_breaking=True,
init="equal",
sigma=0.05,
seed=50)
print("finished em Training", file=stderr)
def test_trace_serialization(self):
tree = hybrid_tree_1()
tree2 = hybrid_tree_2()
terminal_labeling = the_terminal_labeling_factory().get_strategy('pos')
(_, grammar) = induce_grammar(
[tree, tree2],
the_labeling_factory().create_simple_labeling_strategy(
'empty', 'pos'), terminal_labeling.token_label, [cfg], 'START')
for rule in grammar.rules():
print(rule, file=stderr)
trace = compute_reducts(grammar, [tree, tree2], terminal_labeling)
trace.serialize(b"/tmp/reducts.p")
grammar_load = grammar
trace2 = PySDCPTraceManager(grammar_load, terminal_labeling)
trace2.load_traces_from_file(b"/tmp/reducts.p")
trace2.serialize(b"/tmp/reducts2.p")
with open(b"/tmp/reducts.p", "r") as f1, open(b"/tmp/reducts2.p",
"r") as f2:
for e1, e2 in zip(f1, f2):
self.assertEqual(e1, e2)
def test_minimum_risk_parsing(self):
limit_train = 20
limit_test = 10
train = 'res/dependency_conll/german/tiger/train/german_tiger_train.conll'
test = train
parser_type = GFParser_k_best
# test = '../../res/dependency_conll/german/tiger/test/german_tiger_test.conll'
trees = parse_conll_corpus(train, False, limit_train)
primary_labelling = the_labeling_factory(
).create_simple_labeling_strategy("childtop", "deprel")
term_labelling = the_terminal_labeling_factory().get_strategy('pos')
start = 'START'
recursive_partitioning = [cfg]
(n_trees, grammar_prim) = induce_grammar(trees, primary_labelling,
term_labelling.token_label,
recursive_partitioning, start)
parser_type.preprocess_grammar(grammar_prim)
tree_yield = term_labelling.prepare_parser_input
trees = parse_conll_corpus(test, False, limit_test)
for i, tree in enumerate(trees):
print("Parsing sentence ", i, file=stderr)
# print >>stderr, tree
parser = parser_type(grammar_prim,
tree_yield(tree.token_yield()),
k=50)
self.assertTrue(parser.recognized())
derivations = [der for der in parser.k_best_derivation_trees()]
print("# derivations: ", len(derivations), file=stderr)
h_trees = []
current_weight = 0
weights = []
derivation_list = []
for weight, der in derivations:
self.assertTrue(not der in derivation_list)
derivation_list.append(der)
dcp = DCP_evaluator(der).getEvaluation()
h_tree = HybridTree()
cleaned_tokens = copy.deepcopy(tree.full_token_yield())
dcp_to_hybridtree(h_tree, dcp, cleaned_tokens, False,
construct_conll_token)
h_trees.append(h_tree)
weights.append(weight)
if True:
min_risk_tree = compute_minimum_risk_tree(h_trees, weights)
if not min_risk_tree.__eq__(h_trees[0]):
print(h_trees[0])
print(min_risk_tree)
def test_grammar_export(self):
tree = hybrid_tree_1()
tree2 = hybrid_tree_2()
terminal_labeling = the_terminal_labeling_factory().get_strategy('pos')
_, grammar = induce_grammar(
[tree, tree2],
the_labeling_factory().create_simple_labeling_strategy(
'empty', 'pos'),
# the_labeling_factory().create_simple_labeling_strategy('child', 'pos+deprel'),
terminal_labeling.token_label,
[direct_extraction],
'START')
print(max([grammar.fanout(nont) for nont in grammar.nonts()]))
print(grammar)
prefix = '/tmp/'
name = 'tmpGrammar'
name_ = export(grammar, prefix, name)
self.assertEqual(0, compile_gf_grammar(prefix, name_))
GFParser.preprocess_grammar(grammar)
string = ["NP", "N", "V", "V", "V"]
parser = GFParser(grammar, string)
self.assertTrue(parser.recognized())
der = parser.best_derivation_tree()
self.assertTrue(
der.check_integrity_recursive(der.root_id(), grammar.start()))
print(der)
print(
derivation_to_hybrid_tree(der, string,
"Piet Marie helpen lezen leren".split(),
construct_conll_token))
dcp = DCP_evaluator(der).getEvaluation()
h_tree_2 = HybridTree()
token_sequence = [
construct_conll_token(form, lemma)
for form, lemma in zip('Piet Marie helpen lezen leren'.split(' '),
'NP N V V V'.split(' '))
]
dcp_to_hybridtree(h_tree_2, dcp, token_sequence, False,
construct_conll_token)
print(h_tree_2)
def test_non_lexicalized_indcution(self):
trees = self.get_whole_corpus(n=1)
nonterminal_counts = {}
grammar = induce(
trees=trees,
partition_builder=PartitionBuilder(
choice_function=old_choice,
split_function=sorted_even_2_split),
terminal_labeling=the_terminal_labeling_factory().get_strategy(
'form'),
nonterminal_counts=nonterminal_counts)
print(grammar)
def _test_dependency_induction(self):
tree = self.get_single_tree()
grammar = dependency_induce_grammar(
trees=[tree],
nont_labelling=the_labeling_factory(
).create_simple_labeling_strategy('empty', 'pos'),
term_labelling=the_terminal_labeling_factory().get_strategy(
'form').token_label,
recursive_partitioning=[
lambda tree: PartitionBuilder(choice_function=min,
split_function=monadic_split)
(tree)
])
print(grammar)
def run_experiment(db_file, training_corpus, test_corpus, do_parse,
ignore_punctuation, length_limit, labeling,
terminal_labeling, partitioning, root_default_deprel,
disconnected_default_deprel, max_training, max_test):
labeling_choices = labeling.split('-')
if len(labeling_choices) == 2:
nont_labelling = label.the_labeling_factory(
).create_simple_labeling_strategy(labeling_choices[0],
labeling_choices[1])
elif len(labeling_choices) > 2:
nont_labelling = label.the_labeling_factory(
).create_complex_labeling_strategy(labeling_choices)
# labeling == 'strict-pos-leaf:dep':
# labeling == 'child-pos-leaf:dep':
else:
print("Error: Invalid labeling strategy: " + labeling)
exit(1)
rec_par = the_recursive_partitioning_factory().get_partitioning(
partitioning)
if rec_par is None:
print("Error: Invalid recursive partitioning strategy: " +
partitioning)
exit(1)
term_labeling_strategy = the_terminal_labeling_factory().get_strategy(
terminal_labeling)
if term_labeling_strategy is None:
print("Error: Invalid recursive partitioning strategy: " +
partitioning)
exit(1)
parser_type = the_parser_factory().getParser(partitioning)
if parser_type is None:
print("Error: Invalid parser type: " + partitioning)
exit(1)
connection = experiment_database.initialize_database(db_file)
grammar, experiment = induce_grammar_from_file(
training_corpus, connection, nont_labelling, term_labeling_strategy,
rec_par, max_training, False, 'START', ignore_punctuation)
if do_parse:
parse_sentences_from_file(grammar, parser_type, experiment, connection,
test_corpus,
term_labeling_strategy.prepare_parser_input,
length_limit, max_test, False,
ignore_punctuation, root_default_deprel,
disconnected_default_deprel)
experiment_database.finalize_database(connection)
def test_dcp_evaluation_with_induced_dependency_grammar(self):
tree = hybrid_tree_1()
print(tree)
tree2 = hybrid_tree_2()
print(tree2)
# print tree.recursive_partitioning()
labeling = the_labeling_factory().create_simple_labeling_strategy(
'child', 'pos')
term_pos = the_terminal_labeling_factory().get_strategy(
'pos').token_label
(_, grammar) = induce_grammar([tree, tree2], labeling, term_pos,
[direct_extraction], 'START')
# print grammar
self.assertEqual(grammar.well_formed(), None)
self.assertEqual(grammar.ordered()[0], True)
# print max([grammar.fanout(nont) for nont in grammar.nonts()])
print(grammar)
parser = Parser(grammar, 'NP N V V'.split(' '))
self.assertEqual(parser.recognized(), True)
for item in parser.successful_root_items():
der = Derivation()
derivation_tree(der, item, None)
print(der)
hybrid_tree = derivation_to_hybrid_tree(
der, 'NP N V V'.split(' '),
'Piet Marie helpen lezen'.split(' '),
construct_constituent_token)
print(hybrid_tree)
dcp = DCP_evaluator(der).getEvaluation()
h_tree_2 = HybridTree()
token_sequence = [
construct_conll_token(form, lemma)
for form, lemma in zip('Piet Marie helpen lezen'.split(' '),
'NP N V V'.split(' '))
]
dcp_to_hybridtree(h_tree_2, dcp, token_sequence, False,
construct_conll_token)
def test_cfg_parser(self):
tree = hybrid_tree_1()
tree2 = hybrid_tree_2()
terminal_labeling = the_terminal_labeling_factory().get_strategy('pos')
(_, grammar) = induce_grammar(
[tree, tree2],
the_labeling_factory().create_simple_labeling_strategy(
'empty', 'pos'), terminal_labeling.token_label, [cfg], 'START')
for parser_class in [LCFRS_parser, CFGParser]:
parser_class.preprocess_grammar(grammar)
string = ["NP", "N", "V", "V", "V"]
parser = parser_class(grammar, string)
self.assertTrue(parser.recognized())
der = parser.best_derivation_tree()
self.assertTrue(
der.check_integrity_recursive(der.root_id(), grammar.start()))
print(der)
print(
derivation_to_hybrid_tree(
der, string, "Piet Marie helpen lezen leren".split(),
construct_conll_token))
dcp = DCP_evaluator(der).getEvaluation()
h_tree_2 = HybridTree()
token_sequence = [
construct_conll_token(form, lemma) for form, lemma in zip(
'Piet Marie helpen lezen leren'.split(' '),
'NP N V V V'.split(' '))
]
dcp_to_hybridtree(h_tree_2, dcp, token_sequence, False,
construct_conll_token)
print(h_tree_2)
def test_basic_sdcp_parsing_dependency(self):
tree1 = hybrid_tree_1()
tree2 = hybrid_tree_2()
terminal_labeling = the_terminal_labeling_factory().get_strategy('pos')
(_, grammar) = induce_grammar(
[tree1, tree2],
the_labeling_factory().create_simple_labeling_strategy(
'empty', 'pos'), terminal_labeling.token_label, [cfg], 'START')
print("grammar induced. Printing rules...", file=stderr)
for rule in grammar.rules():
print(rule, file=stderr)
parser_type = LCFRS_sDCP_Parser
print("preprocessing grammar", file=stderr)
parser_type.preprocess_grammar(grammar, terminal_labeling)
print("invoking parser", file=stderr)
parser = parser_type(grammar, tree1)
print("listing derivations", file=stderr)
for der in parser.all_derivation_trees():
print(der)
output_tree = HybridTree()
tokens = tree1.token_yield()
dcp_to_hybridtree(output_tree,
DCP_evaluator(der).getEvaluation(), tokens,
False, construct_conll_token)
print(tree1)
print(output_tree)
print("completed test", file=stderr)
def test_fst_compilation_left(self):
if not test_pynini:
return
tree = hybrid_tree_1()
tree2 = hybrid_tree_2()
terminal_labeling = the_terminal_labeling_factory().get_strategy('pos')
(_, grammar) = induce_grammar(
[tree, tree2],
the_labeling_factory().create_simple_labeling_strategy(
'empty', 'pos'), terminal_labeling.token_label,
[left_branching], 'START')
fst, rules = compile_wfst_from_left_branching_grammar(grammar)
print(repr(fst))
symboltable = fst.input_symbols()
string = ["NP", "N", "V", "V", "V"]
fsa = fsa_from_list_of_symbols(string, symboltable)
self.assertEqual(
fsa.text().decode('utf-8'),
'0\t1\tNP\tNP\n1\t2\tN\tN\n2\t3\tV\tV\n3\t4\tV\tV\n4\t5\tV\tV\n5\n'
)
b = compose(fsa, fst)
print(b.text(symboltable, symboltable))
print("Shortest path probability", end=' ')
best = shortestpath(b)
best.topsort()
# self.assertAlmostEquals(pow(e, -float(shortestdistance(best)[-1])), 1.80844898756e-05)
print(best.text())
polish_rules = retrieve_rules(best)
self.assertSequenceEqual(polish_rules, [1, 2, 3, 4, 5, 4, 9, 4, 7, 8])
polish_rules = list(map(rules.index_object, polish_rules))
for rule in polish_rules:
print(rule)
print()
der = ReversePolishDerivation(polish_rules[0:-1])
self.assertTrue(der.check_integrity_recursive(der.root_id()))
print(der)
LeftBranchingFSTParser.preprocess_grammar(grammar)
parser = LeftBranchingFSTParser(grammar, string)
der_ = parser.best_derivation_tree()
print(der_)
self.assertTrue(der_.check_integrity_recursive(der_.root_id()))
print(
derivation_to_hybrid_tree(der, string,
"Piet Marie helpen lezen leren".split(),
construct_conll_token))
print(
derivation_to_hybrid_tree(der_, string,
"Piet Marie helpen lezen leren".split(),
construct_conll_token))
dcp = DCP_evaluator(der).getEvaluation()
h_tree_2 = HybridTree()
token_sequence = [
construct_conll_token(form, lemma)
for form, lemma in zip('Piet Marie helpen lezen leren'.split(' '),
'NP N V V V'.split(' '))
]
dcp_to_hybridtree(h_tree_2, dcp, token_sequence, False,
construct_conll_token)
print(h_tree_2)
def test_single_root_induction(self):
tree = hybrid_tree_1()
# print tree.children("v")
# print tree
#
# for id_set in ['v v1 v2 v21'.split(' '), 'v1 v2'.split(' '),
# 'v v21'.split(' '), ['v'], ['v1'], ['v2'], ['v21']]:
# print id_set, 'top:', top(tree, id_set), 'bottom:', bottom(tree, id_set)
# print id_set, 'top_max:', max(tree, top(tree, id_set)), 'bottom_max:', max(tree, bottom(tree, id_set))
#
# print "some rule"
# for mem, arg in [(-1, 0), (0,0), (1,0)]:
# print create_DCP_rule(mem, arg, top_max(tree, ['v','v1','v2','v21']), bottom_max(tree, ['v','v1','v2','v21']),
# [(top_max(tree, l), bottom_max(tree, l)) for l in [['v1', 'v2'], ['v', 'v21']]])
#
#
# print "some other rule"
# for mem, arg in [(-1,1),(1,0)]:
# print create_DCP_rule(mem, arg, top_max(tree, ['v1','v2']), bottom_max(tree, ['v1','v2']),
# [(top_max(tree, l), bottom_max(tree, l)) for l in [['v1'], ['v2']]])
#
# print 'strict:' , strict_labeling(tree, top_max(tree, ['v','v21']), bottom_max(tree, ['v','v21']))
# print 'child:' , child_labeling(tree, top_max(tree, ['v','v21']), bottom_max(tree, ['v','v21']))
# print '---'
# print 'strict: ', strict_labeling(tree, top_max(tree, ['v1','v21']), bottom_max(tree, ['v1','v21']))
# print 'child: ', child_labeling(tree, top_max(tree, ['v1','v21']), bottom_max(tree, ['v1','v21']))
# print '---'
# print 'strict:' , strict_labeling(tree, top_max(tree, ['v','v1', 'v21']), bottom_max(tree, ['v','v1', 'v21']))
# print 'child:' , child_labeling(tree, top_max(tree, ['v','v1', 'v21']), bottom_max(tree, ['v','v1', 'v21']))
tree2 = hybrid_tree_2()
# print tree2.children("v")
# print tree2
#
# print 'siblings v211', tree2.siblings('v211')
# print top(tree2, ['v','v1', 'v211'])
# print top_max(tree2, ['v','v1', 'v211'])
#
# print '---'
# print 'strict:' , strict_labeling(tree2, top_max(tree2, ['v','v1', 'v211']), bottom_max(tree2, ['v','v11', 'v211']))
# print 'child:' , child_labeling(tree2, top_max(tree2, ['v','v1', 'v211']), bottom_max(tree2, ['v','v11', 'v211']))
# rec_par = ('v v1 v2 v21'.split(' '),
# [('v1 v2'.split(' '), [(['v1'],[]), (['v2'],[])])
# ,('v v21'.split(' '), [(['v'],[]), (['v21'],[])])
# ])
#
# grammar = LCFRS(nonterminal_str(tree, top_max(tree, rec_par[0]), bottom_max(tree, rec_par[0]), 'strict'))
#
# add_rules_to_grammar_rec(tree, rec_par, grammar, 'child')
#
# grammar.make_proper()
# print grammar
print(tree.recursive_partitioning())
terminal_labeling = the_terminal_labeling_factory().get_strategy('pos')
(_, grammar) = induce_grammar(
[tree, tree2],
the_labeling_factory().create_simple_labeling_strategy(
'empty', 'pos'),
# the_labeling_factory().create_simple_labeling_strategy('child', 'pos+deprel'),
terminal_labeling.token_label,
[direct_extraction],
'START')
print(max([grammar.fanout(nont) for nont in grammar.nonts()]))
print(grammar)
parser = LCFRS_parser(grammar, 'NP N V V'.split(' '))
print(parser.best_derivation_tree())
tokens = [
construct_conll_token(form, pos) for form, pos in zip(
'Piet Marie helpen lezen'.split(' '), 'NP N V V'.split(' '))
]
hybrid_tree = HybridTree()
hybrid_tree = parser.dcp_hybrid_tree_best_derivation(
hybrid_tree, tokens, True, construct_conll_token)
print(list(map(str, hybrid_tree.full_token_yield())))
print(hybrid_tree)
string = "foo"
dcp_string = DCP_string(string)
dcp_string.set_edge_label("bar")
print(dcp_string, dcp_string.edge_label())
linearize(
grammar,
the_labeling_factory().create_simple_labeling_strategy(
'child', 'pos+deprel'),
the_terminal_labeling_factory().get_strategy('pos'), sys.stdout)
def generic_parsing_test(self, parser_type, limit_train, limit_test,
compare_order):
def filter_by_id(n, trees):
j = 0
for tree in trees:
if j in n:
yield tree
j += 1
#params
train = 'res/dependency_conll/german/tiger/train/german_tiger_train.conll'
test = train
# test = 'res/dependency_conll/german/tiger/test/german_tiger_test.conll'
trees = parse_conll_corpus(train, False, limit_train)
primary_labelling = the_labeling_factory(
).create_simple_labeling_strategy("childtop", "deprel")
term_labelling = the_terminal_labeling_factory().get_strategy('pos')
start = 'START'
recursive_partitioning = [cfg]
(n_trees, grammar_prim) = induce_grammar(trees, primary_labelling,
term_labelling.token_label,
recursive_partitioning, start)
parser_type.preprocess_grammar(grammar_prim, term_labelling)
trees = parse_conll_corpus(test, False, limit_test)
count_derivs = {}
no_complete_match = 0
for i, tree in enumerate(trees):
print("Parsing tree for ", i, file=stderr)
print(tree, file=stderr)
parser = parser_type(grammar_prim, tree)
self.assertTrue(parser.recognized())
count_derivs[i] = 0
print("Found derivations for ", i, file=stderr)
j = 0
derivations = []
for der in parser.all_derivation_trees():
self.assertTrue(
der.check_integrity_recursive(der.root_id(), start))
print(count_derivs[i], file=stderr)
print(der, file=stderr)
output_tree = HybridTree()
tokens = tree.token_yield()
the_yield = der.compute_yield()
# print >>stderr, the_yield
tokens2 = list(
map(lambda pos: construct_conll_token('_', pos),
the_yield))
dcp_to_hybridtree(output_tree,
DCP_evaluator(der).getEvaluation(),
tokens2,
False,
construct_conll_token,
reorder=False)
print(tree, file=stderr)
print(output_tree, file=stderr)
self.compare_hybrid_trees(tree, output_tree, compare_order)
count_derivs[i] += 1
derivations.append(der)
self.assertTrue(
sDCPParserTest.pairwise_different(
derivations, sDCPParserTest.compare_derivations))
self.assertEqual(len(derivations), count_derivs[i])
if count_derivs[i] == 0:
no_complete_match += 1
for key in count_derivs:
print(key, count_derivs[key])
print("# trees with no complete match:", no_complete_match)
from hybridtree.monadic_tokens import construct_conll_token
import dependency.induction as d_i
import dependency.labeling as d_l
import time
import parser.parser_factory
import copy
from parser.sDCP_parser.sdcp_parser_wrapper import LCFRS_sDCP_Parser
from playground_rparse.process_rparse_grammar import fall_back_left_branching_token
import subprocess
import re
import argparse
from grammar.induction.terminal_labeling import the_terminal_labeling_factory
result = 'recursive-partitoning-results.conll'
start = 'START'
term_labelling = the_terminal_labeling_factory().get_strategy('pos')
tree_parser = LCFRS_sDCP_Parser # tree parser to count derivations per hybrid tree
tree_yield = term_labelling.prepare_parser_input
train_limit = 2000
test_limit = 5100
# add command line arguments
argParser = argparse.ArgumentParser(
description=
'Train a hybrid grammar using different strategies for recursive partitioning transformation.'
)
argParser.add_argument('-s',
nargs='*',
def trainAndEval(strategy,
labelling1,
labelling2,
fanout,
parser_type,
train,
test,
cDT,
parseStrings,
ignore_punctuation=False):
file = open('results.txt', 'a')
term_labelling = the_terminal_labeling_factory().get_strategy('pos')
recursive_partitioning = d_i.the_recursive_partitioning_factory(
).get_partitioning('fanout-' + str(fanout) + strategy)
primary_labelling = d_l.the_labeling_factory(
).create_simple_labeling_strategy(labelling1, labelling2)
trees = parse_conll_corpus(train, False, train_limit)
if ignore_punctuation:
trees = disconnect_punctuation(trees)
(n_trees, grammar) = d_i.induce_grammar(trees, primary_labelling,
term_labelling.token_label,
recursive_partitioning, start)
# write current transformation strategy and hyperparameters to results.txt
if strategy == '':
file.write('rtl ' + labelling1 + ' ' + labelling2 +
' maximal fanout:' + fanout)
else:
splitList = strategy.split('-')
if splitList[1] == 'left':
file.write('ltr ' + labelling1 + ' ' + labelling2 +
' maximal fanout:' + fanout)
elif splitList[1] == 'random':
file.write('random seed:' + splitList[2] + ' ' + labelling1 + ' ' +
labelling2 + ' maximal fanout:' + fanout)
elif splitList[1] == 'no':
if splitList[4] == 'random':
file.write('nnont fallback:random seed:' + splitList[5] + ' ' +
labelling1 + ' ' + labelling2 + ' maximal fanout:' +
fanout)
elif splitList[4] == 'ltr':
file.write('nnont fallback:ltr' + ' ' + labelling1 + ' ' +
labelling2 + ' maximal fanout:' + fanout)
elif splitList[4] == 'rtl':
file.write('nnont fallback:rtl' + ' ' + labelling1 + ' ' +
labelling2 + ' maximal fanout:' + fanout)
else:
file.write('nnont fallback:argmax' + ' ' + labelling1 + ' ' +
labelling2 + ' maximal fanout:' + fanout)
else: #argmax
file.write('argmax ' + labelling1 + ' ' + labelling2 +
' maximal fanout:' + fanout)
file.write('\n')
res = ''
res += '#nonts:' + str(len(grammar.nonts()))
res += ' #rules:' + str(len(grammar.rules()))
file.write(res)
res = ''
# The following code is to count the number of derivations for a hypergraph (tree parser required)
if cDT == True:
tree_parser.preprocess_grammar(grammar, term_labelling)
trees = parse_conll_corpus(train, False, train_limit)
if ignore_punctuation:
trees = disconnect_punctuation(trees)
derCount = 0
derMax = 0
for tree in trees:
parser = tree_parser(grammar, tree) # if tree parser is used
der = parser.count_derivation_trees()
if der > derMax:
derMax = der
derCount += der
res += "\n#derivation trees: average: " + str(
1.0 * derCount / n_trees)
res += " maximal: " + str(derMax)
file.write(res)
res = ''
total_time = 0.0
# The following code works for string parsers for evaluating
if parseStrings == True:
parser_type.preprocess_grammar(grammar)
trees = parse_conll_corpus(test, False, test_limit)
if ignore_punctuation:
trees = disconnect_punctuation(trees)
i = 0
with open(result, 'w') as result_file:
failures = 0
for tree in trees:
time_stamp = time.clock()
i += i
#if (i % 100 == 0):
#print '.',
#sys.stdout.flush()
parser = parser_type(grammar, tree_yield(tree.token_yield()))
time_stamp = time.clock() - time_stamp
total_time += time_stamp
cleaned_tokens = copy.deepcopy(tree.full_token_yield())
for token in cleaned_tokens:
token.set_edge_label('_')
h_tree = HybridTree(tree.sent_label())
h_tree = parser.dcp_hybrid_tree_best_derivation(
h_tree, cleaned_tokens, ignore_punctuation,
construct_conll_token)
if h_tree:
result_file.write(tree_to_conll_str(h_tree))
result_file.write('\n\n')
else:
failures += 1
forms = [token.form() for token in tree.full_token_yield()]
poss = [token.pos() for token in tree.full_token_yield()]
result_file.write(
tree_to_conll_str(
fall_back_left_branching_token(cleaned_tokens)))
result_file.write('\n\n')
res += "\nattachment scores:\nno punctuation: "
out = subprocess.check_output(
["perl", "../util/eval.pl", "-g", test, "-s", result, "-q"])
match = re.search(r'[^=]*= (\d+\.\d+)[^=]*= (\d+.\d+).*', out)
res += ' labelled:' + match.group(1) #labeled attachment score
res += ' unlabelled:' + match.group(2) #unlabeled attachment score
res += "\npunctation: "
out = subprocess.check_output(
["perl", "../util/eval.pl", "-g", test, "-s", result, "-q", "-p"])
match = re.search(r'[^=]*= (\d+\.\d+)[^=]*= (\d+.\d+).*', out)
res += ' labelled:' + match.group(1)
res += ' unlabelled:' + match.group(2)
res += "\nparse time: " + str(total_time)
file.write(res)
file.write('\n\n\n')
file.close()
def test_direct_induction(self):
tree = self.get_single_tree()
terminal_labeling = the_terminal_labeling_factory().get_strategy('pos')
lcfrs = direct_extract_lcfrs(tree=tree,
term_labeling=terminal_labeling)
print(lcfrs)
def main():
# induce grammar from a corpus
trees = parse_conll_corpus(train, False, limit_train)
nonterminal_labelling = the_labeling_factory(
).create_simple_labeling_strategy("childtop", "deprel")
term_labelling = the_terminal_labeling_factory().get_strategy('pos')
start = 'START'
recursive_partitioning = [cfg]
_, grammar = induce_grammar(trees, nonterminal_labelling,
term_labelling.token_label,
recursive_partitioning, start)
# compute some derivations
derivations = obtain_derivations(grammar, term_labelling)
# create derivation manager and add derivations
manager = PyDerivationManager(grammar)
manager.convert_derivations_to_hypergraphs(derivations)
manager.serialize(b"/tmp/derivations.txt")
# build and configure split/merge trainer and supplementary objects
rule_to_nonterminals = []
for i in range(0, len(grammar.rule_index())):
rule = grammar.rule_index(i)
nonts = [
manager.get_nonterminal_map().object_index(rule.lhs().nont())
] + [
manager.get_nonterminal_map().object_index(nont)
for nont in rule.rhs()
]
rule_to_nonterminals.append(nonts)
grammarInfo = PyGrammarInfo(grammar, manager.get_nonterminal_map())
storageManager = PyStorageManager()
builder = PySplitMergeTrainerBuilder(manager, grammarInfo)
builder.set_em_epochs(20)
builder.set_percent_merger(60.0)
splitMergeTrainer = builder.build()
latentAnnotation = [
build_PyLatentAnnotation_initial(grammar, grammarInfo, storageManager)
]
for i in range(max_cycles + 1):
latentAnnotation.append(
splitMergeTrainer.split_merge_cycle(latentAnnotation[-1]))
# pickle.dump(map(lambda la: la.serialize(), latentAnnotation), open(sm_info_path, 'wb'))
smGrammar = build_sm_grammar(latentAnnotation[i],
grammar,
grammarInfo,
rule_pruning=0.0001,
rule_smoothing=0.01)
print("Cycle: ", i, "Rules: ", len(smGrammar.rules()))
if parsing:
parser = GFParser(smGrammar)
trees = parse_conll_corpus(test, False, limit_test)
for tree in trees:
parser.set_input(
term_labelling.prepare_parser_input(tree.token_yield()))
parser.parse()
if parser.recognized():
print(
derivation_to_hybrid_tree(
parser.best_derivation_tree(),
[token.pos() for token in tree.token_yield()],
[token.form() for token in tree.token_yield()],
construct_constituent_token))
def test_k_best_parsing(self):
limit_train = 20
limit_test = 10
train = 'res/dependency_conll/german/tiger/train/german_tiger_train.conll'
test = train
parser_type = GFParser_k_best
# test = '../../res/dependency_conll/german/tiger/test/german_tiger_test.conll'
trees = parse_conll_corpus(train, False, limit_train)
primary_labelling = the_labeling_factory(
).create_simple_labeling_strategy("childtop", "deprel")
term_labelling = the_terminal_labeling_factory().get_strategy('pos')
start = 'START'
recursive_partitioning = [cfg]
(n_trees, grammar_prim) = induce_grammar(trees, primary_labelling,
term_labelling.token_label,
recursive_partitioning, start)
parser_type.preprocess_grammar(grammar_prim)
tree_yield = term_labelling.prepare_parser_input
trees = parse_conll_corpus(test, False, limit_test)
for i, tree in enumerate(trees):
print("Parsing sentence ", i, file=stderr)
# print >>stderr, tree
parser = parser_type(grammar_prim,
tree_yield(tree.token_yield()),
k=50)
self.assertTrue(parser.recognized())
derivations = [der for der in parser.k_best_derivation_trees()]
print("# derivations: ", len(derivations), file=stderr)
h_trees = []
current_weight = 0
weights = []
derivation_list = []
for weight, der in derivations:
# print >>stderr, exp(-weight)
# print >>stderr, der
self.assertTrue(not der in derivation_list)
derivation_list.append(der)
# TODO this should hold, but it looks like a GF bug!
# self.assertGreaterEqual(weight, current_weight)
current_weight = weight
dcp = DCP_evaluator(der).getEvaluation()
h_tree = HybridTree()
cleaned_tokens = copy.deepcopy(tree.full_token_yield())
dcp_to_hybridtree(h_tree, dcp, cleaned_tokens, False,
construct_conll_token)
h_trees.append(h_tree)
weights.append(weight)
# print >>stderr, h_tree
# print a matrix indicating which derivations result
# in the same hybrid tree
if True:
for i, h_tree1 in enumerate(h_trees):
for h_tree2 in h_trees:
if h_tree1 == h_tree2:
print("x", end=' ', file=stderr)
else:
print("", end=' ', file=stderr)
print(weights[i], file=stderr)
print(file=stderr)
def test_best_trees(self):
limit_train = 5000
limit_test = 100
train = 'res/dependency_conll/german/tiger/train/german_tiger_train.conll'
test = train
parser_type = GFParser_k_best
# test = '../../res/dependency_conll/german/tiger/test/german_tiger_test.conll'
trees = parse_conll_corpus(train, False, limit_train)
primary_labelling = the_labeling_factory(
).create_simple_labeling_strategy("child", "pos+deprel")
term_labelling = the_terminal_labeling_factory().get_strategy('pos')
start = 'START'
recursive_partitioning = [cfg]
(n_trees, grammar_prim) = induce_grammar(trees, primary_labelling,
term_labelling.token_label,
recursive_partitioning, start)
parser_type.preprocess_grammar(grammar_prim)
tree_yield = term_labelling.prepare_parser_input
trees = parse_conll_corpus(test, False, limit_test)
for i, tree in enumerate(trees):
print("Parsing sentence ", i, file=stderr)
parser = parser_type(grammar_prim,
tree_yield(tree.token_yield()),
k=200)
self.assertTrue(parser.recognized())
viterbi_weight = parser.viterbi_weight()
viterbi_deriv = parser.viterbi_derivation()
der_to_tree = lambda der: dcp_to_hybridtree(
HybridTree(),
DCP_evaluator(der).getEvaluation(),
copy.deepcopy(tree.full_token_yield()), False,
construct_conll_token)
viterbi_tree = der_to_tree(viterbi_deriv)
ordered_parse_trees = parser.best_trees(der_to_tree)
best_tree, best_weight, best_witnesses = ordered_parse_trees[0]
for i, (parsed_tree, _, _) in enumerate(ordered_parse_trees):
if parsed_tree.__eq__(tree):
print("Gold tree is ",
i + 1,
" in best tree list",
file=stderr)
break
if (not viterbi_tree.__eq__(best_tree)
and viterbi_weight != best_weight):
print("viterbi and k-best tree differ", file=stderr)
print("viterbi: ", viterbi_weight, file=stderr)
print("k-best: ", best_weight, best_witnesses, file=stderr)
if False:
print(viterbi_tree, file=stderr)
print(tree_to_conll_str(viterbi_tree), file=stderr)
print(best_tree, file=stderr)
print(tree_to_conll_str(best_tree), file=stderr)
print("gold tree", file=stderr)
print(tree, file=stderr)
print(tree_to_conll_str(tree), file=stderr)
def test_fst_compilation_right(self):
if not test_pynini:
return
tree = hybrid_tree_1()
tree2 = hybrid_tree_2()
terminal_labeling = the_terminal_labeling_factory().get_strategy('pos')
(_, grammar) = induce_grammar(
[tree, tree2],
the_labeling_factory().create_simple_labeling_strategy(
'empty', 'pos'), terminal_labeling.token_label,
[right_branching], 'START')
a, rules = compile_wfst_from_right_branching_grammar(grammar)
print(repr(a))
symboltable = a.input_symbols()
string = 'NP N V V V'.split(' ')
token_sequence = [
construct_conll_token(form, lemma) for form, lemma in zip(
'Piet Marie helpen leren lezen'.split(' '), string)
]
fsa = fsa_from_list_of_symbols(string, symboltable)
self.assertEqual(
'0\t1\tNP\tNP\n1\t2\tN\tN\n2\t3\tV\tV\n3\t4\tV\tV\n4\t5\tV\tV\n5\n',
fsa.text().decode('utf-8'))
b = compose(fsa, a)
print(b.input_symbols())
for i in b.input_symbols():
print(i)
print("Input Composition")
print(b.text(symboltable, symboltable).decode('utf-8'))
i = 0
for path in paths(b):
print(i, "th path:", path, end=' ')
r = list(map(rules.index_object, path))
d = PolishDerivation(r[1::])
dcp = DCP_evaluator(d).getEvaluation()
h = HybridTree()
dcp_to_hybridtree(h, dcp, token_sequence, False,
construct_conll_token)
h.reorder()
if h == tree2:
print("correct")
else:
print("incorrect")
i += 1
stats = defaultdict(lambda: 0)
local_rule_stats(b, stats, 15)
print(stats)
print("Shortest path probability")
best = shortestpath(b)
best.topsort()
self.assertAlmostEqual(1.80844898756e-05,
pow(e, -float(shortestdistance(best)[-1])))
print(best.text())
polish_rules = retrieve_rules(best)
self.assertSequenceEqual(polish_rules, [8, 7, 1, 6, 2, 5, 3, 10, 3, 3])
polish_rules = list(map(rules.index_object, polish_rules))
print(polish_rules)
der = PolishDerivation(polish_rules[1::])
print(der)
print(
derivation_to_hybrid_tree(der, string,
"Piet Marie helpen lezen leren".split(),
construct_conll_token))
dcp = DCP_evaluator(der).getEvaluation()
h_tree_2 = HybridTree()
dcp_to_hybridtree(h_tree_2, dcp, token_sequence, False,
construct_conll_token)
print(h_tree_2)
