def test_induction_from_corpus_tree(self): dsg = sentence_names_to_deep_syntax_graphs(["s26954"], "res/tiger/tiger_s26954.xml", hold=False)[0] def label_edge(edge): if isinstance(edge.label, ConstituentTerminal): return edge.label.pos() else: return edge.label labeling = lambda nodes, dsg: simple_labeling(nodes, dsg, label_edge) term_labeling_token = PosTerminals() def term_labeling(token): if isinstance(token, ConstituentTerminal): return term_labeling_token.token_label(token) else: return token rec_part_strategy = the_recursive_partitioning_factory().get_partitioning('cfg')[0] rec_part = rec_part_strategy(dsg) dcmp = compute_decomposition(dsg, rec_part) grammar = induce_grammar_from(dsg, rec_part, dcmp, labeling=labeling, terminal_labeling=term_labeling) print(grammar) parser = LCFRS_parser(grammar) parser.set_input(term_labeling_token.prepare_parser_input(dsg.sentence)) parser.parse() self.assertTrue(parser.recognized()) derivation = parser.best_derivation_tree() self.assertNotEqual(derivation, None)
def test_lcfrs_sdcp_parsing(self): def tree1(): tree = ConstituentTree("1") for i, t in enumerate(["a", "b", "c", "d"]): tree.add_leaf(str(i), "P" + t, t) tree.set_label('r0', 'C') tree.set_label('r1', 'A') tree.set_label('r2', 'B') tree.add_to_root('r0') tree.add_child('r0', 'r1') tree.add_child('r0', 'r2') tree.add_child('r1', '0') tree.add_child('r1', '2') tree.add_child('r2', '1') tree.add_child('r2', '3') print(tree, tree.word_yield()) return tree def tree2(): tree = ConstituentTree("1") for i, t in enumerate(["a", "b", "d", "c"]): tree.add_leaf(str(i), "P" + t, t) tree.set_label('r0', 'C') tree.set_label('r1', 'A') tree.set_label('r2', 'B') tree.add_to_root('r0') tree.add_child('r0', 'r1') tree.add_child('r0', 'r2') tree.add_child('r1', '0') tree.add_child('r1', '3') tree.add_child('r2', '1') tree.add_child('r2', '2') print(tree, tree.word_yield()) return tree t1 = tree1() t2 = tree2() grammar = direct_extract_lcfrs(t1) grammar.add_gram(direct_extract_lcfrs(t2)) print(grammar) # LCFRS_sDCP_Parser.preprocess_grammar(grammar, PosTerminals(), debug=True) parser = LCFRS_sDCP_Parser(grammar, terminal_labelling=PosTerminals(), debug=True) for t in [t1, t2]: # parser = LCFRS_sDCP_Parser(grammar, t) parser.set_input(t) parser.parse() self.assertTrue(parser.recognized()) derivs = list(parser.all_derivation_trees()) for der in derivs: print(der) self.assertEqual(1, len(derivs)) parser.clear()
def fringe_extract_lcfrs(tree, fringes, naming='strict', term_labeling=PosTerminals(), isolate_pos=False, feature_logging=None): """ :type tree: ConstituentTree :param fringes: recursive partitioning :param naming: 'strict' or 'child' :type naming: str :type term_labeling: ConstituentTerminalLabeling :rtype: LCFRS Get LCFRS for tree. """ gram = LCFRS(start=START) first = None if len(tree.id_yield()) == 1 and isolate_pos: idx = tree.id_yield()[0] if tree.root[0] != idx: c_nont, c_spans, c_id_seq, c_nont_feat \ = fringe_extract_lcfrs_recur(tree, fringes, gram, naming, term_labeling, isolate_pos, feature_logging, yield_one_check=False) fringe = fringes[0] spans = join_spans(fringe) args = [] term_to_pos = {} # maps input position to position in LCFRS rule for span in spans: args += [span_to_arg(span, [c_spans], tree, term_to_pos, term_labeling)] id_seq = make_id_seq(tree, tree.root[0], fringe) dcp_rules = [] for (i, seq) in enumerate(id_seq): dcp_rhs = make_fringe_terms(tree, seq, [c_id_seq], term_to_pos, term_labeling) dcp_lhs = DCP_var(-1, i) dcp_rule = DCP_rule(dcp_lhs, dcp_rhs) dcp_rules += [dcp_rule] nont = id_nont(id_seq, tree, naming) + '/' + str(len(spans)) nont_feat = feats(id_seq, tree) lhs = LCFRS_lhs(nont) for arg in args: lhs.add_arg(arg) rule = gram.add_rule(lhs, [c_nont], dcp=dcp_rules) if feature_logging is not None: feature_logging[(nont, nont_feat)] += 1 feature_logging[(rule.get_idx(), nont_feat, tuple([c_nont_feat]))] += 1 first = nont if first is None: (first, _, _, _) = fringe_extract_lcfrs_recur(tree, fringes, gram, naming, term_labeling, isolate_pos, feature_logging) lhs = LCFRS_lhs(START) lhs.add_arg([LCFRS_var(0, 0)]) dcp_rule = DCP_rule(DCP_var(-1, 0), [DCP_var(0, 0)]) gram.add_rule(lhs, [first], dcp=[dcp_rule]) return gram
def test_fanout_marking(self): label = 's1813' path = "res/tiger/tiger_8000.xml" dsgs = sentence_names_to_deep_syntax_graphs([label], path, hold=False, reorder_children=True) term_labeling_token = PosTerminals() def term_labeling(token): if isinstance(token, ConstituentTerminal): return term_labeling_token.token_label(token) else: return token def rec_part_strategy(direction, subgrouping, fanout): if direction == "right-to-left": return lambda dsg: fanout_limited_partitioning(dsg.recursive_partitioning(subgrouping), fanout) else: return lambda dsg: fanout_limited_partitioning_left_to_right(dsg.recursive_partitioning(subgrouping), fanout) def label_edge(edge): if isinstance(edge.label, ConstituentTerminal): return edge.label.pos() else: return edge.label def stupid_edge(edge): return "X" def label_child(edge, j): return edge.get_function(j) def simple_nonterminal_labeling(nodes, dsg): return simple_labeling(nodes, dsg, label_edge) def bot_stupid_nonterminal_labeling(nodes, dsg): return top_bot_labeling(nodes, dsg, label_edge, stupid_edge) def missing_child_nonterminal_labeling(nodes, dsg): return missing_child_labeling(nodes, dsg, label_edge, label_child) rec_part = rec_part_strategy("left-to-right", True, 2)(dsgs[0]) print(rec_part) dcmp = compute_decomposition(dsgs[0], rec_part) print(dcmp) grammar = induce_grammar_from(dsgs[0], rec_part, dcmp, labeling=missing_child_nonterminal_labeling, terminal_labeling=term_labeling) print(grammar) for rule in grammar.rules(): print(rule)
def __init__(self): self.normalize = False self.disconnect_punctuation = True self.terminal_labeling = PosTerminals() # self.nont_labeling = NonterminalsWithFunctions() self.nont_labeling = BasicNonterminalLabeling() self.binarize = True self.isolate_pos = True self.hmarkov = 0 self.use_discodop_binarization = False self.discodop_binarization_params = [ "--headrules=util/negra.headrules", "--binarize", "-h 1", "-v 1" ]
def direct_extract_lcfrs_from_prebinarized_corpus(tree, term_labeling=PosTerminals(), nont_labeling=BasicNonterminalLabeling(), isolate_pos=True): gram = LCFRS(start=START) root = tree.root[0] if tree.is_leaf(root): lhs = LCFRS_lhs(START) label = term_labeling.token_label(tree.node_token(root)) lhs.add_arg([label]) dcp_rule = DCP_rule(DCP_var(-1, 0), [DCP_term(DCP_index(0, edge_label=tree.node_token(root).edge(), pos=tree.node_token(root).pos()), [])]) gram.add_rule(lhs, [], dcp=[dcp_rule]) else: first = direct_extract_lcfrs_prebinarized_recur(tree, root, gram, term_labeling, nont_labeling, isolate_pos) lhs = LCFRS_lhs(START) lhs.add_arg([LCFRS_var(0, 0)]) dcp_rule = DCP_rule(DCP_var(-1, 0), [DCP_var(0, 0)]) gram.add_rule(lhs, [first], dcp=[dcp_rule]) return gram
filter=test_input_filter, type=corpus_type_test) return train, dev, test, test_input def my_feature_filter(elem): base_feats = ["number", "person", "tense", "mood", "case", "degree", "category", "pos", "gender"] feat_set = {feat: value for feat, value in elem[0]} if "pos" in feat_set and feat_set["pos"] in {"APPR", "APPRART"}: return extract_feat(elem[0], features=base_feats + ["lemma"]) return extract_feat(elem[0], features=base_feats) FINE_TERMINAL_LABELING = FeatureTerminals(token_to_features, feature_filter=my_feature_filter) FINE_TERMINAL_LABELING = CompositionalTerminalLabeling(FormTerminals(), PosTerminals()) FALLBACK_TERMINAL_LABELING = PosTerminals() DEFAULT_RARE_WORD_THRESHOLD = 10 def terminal_labeling(corpus, threshold=DEFAULT_RARE_WORD_THRESHOLD): return FrequencyBiasedTerminalLabeling(FINE_TERMINAL_LABELING, FALLBACK_TERMINAL_LABELING, corpus, threshold) MULTI_OBJECTIVES = "multi-objectives" MULTI_OBJECTIVES_INDEPENDENT = "multi-objectives-independent" BASE_GRAMMAR = "base-grammar" # use base grammar for parsing (no annotations LA) MAX_RULE_PRODUCT_ONLY = "max-rule-product-only" TEST_SECOND_HALF = False
def direct_extract_lcfrs(tree, term_labeling=PosTerminals(), nont_labeling=BasicNonterminalLabeling(), binarize=False, isolate_pos=False, hmarkov=0): """ :type tree: ConstituentTree :type term_labeling: ConstituentTerminalLabeling :type binarize: bool :type isolate_pos: bool :type hmarkov: int :rtype: LCFRS Extract LCFRS directly from hybrid tree. """ assert not binarize or isolate_pos # Binarization without POS isolation requires a more sophisticated sDCP handling # with more than one variable per nonterminal. This is not implemented. # see, e.g., TIGER sentence 328: # # ROOT # ┌──────────────────────┬─────────────────────┼────────────────────────────────────────────────────────────────┐ # │ │ S │ # │ │ ┌────┬───┬───────┴───────────┬──────┬──────────────────────────────────┐ │ # │ │ │ NP │ │ │ │ │ # │ ┌──────┬────┬─── │ ── │ ───┴── │ ───────────────── │ ──── │ ────────────┐ │ │ # │ │ │ │ │ │ NP │ │ PP PP │ # │ │ │ │ │ │ ┌───┴───────┐ │ │ ┌──────┼──────┐ ┌─────┴────┐ │ # $[ ADV ADV CARD $[ VAFIN ART NN ADV ADV APPR CARD NN APPRART NN $. # │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ # `` Deutlich über 5000 '' hat die SPD-Stadtregieru jetzt jeweils binnen zwölf Monaten im Visier . # ng # # [a] S/1(LCFRS_var(mem=0, arg=0) LCFRS_var(mem=1, arg=0) ADV ADV LCFRS_var(mem=0, arg=1) LCFRS_var(mem=1, arg=1)) # -> NP/2 BAR/S/2 <0>=S:{--}(<0,0> <1,0> [0:{MO}]() [1:{MO}]()) # [b] BAR/S/2(VAFIN LCFRS_var(mem=0, arg=0); LCFRS_var(mem=1, arg=0)) # -> NP/1 PP/1 <0>=[0:{HD}]() <0,0> <1,0> # here the <1,0> variable in rule [a] needs to be split into <1,0> and <1,1>, because the NP occurs before "ADV ADV" # in the canonically ordered tree, but PP occurs afterwards # # TODO: Claim: This only affects BAR/.. nonterminals. Not more than fanout(BAR/...) nonterminals are needed. # TODO: Thus, each BAR/.. nonterminal gets a uniform number of sDCP arguments, some may be empty. # TODO: The term.args() string can be analyzed to construct appropriate sDCP rules for the binarization artifacts. gram = LCFRS(start=START) root = tree.root[0] if tree.is_leaf(root): lhs = LCFRS_lhs(START) label = term_labeling.token_label(tree.node_token(root)) lhs.add_arg([label]) dcp_rule = DCP_rule(DCP_var(-1, 0), [DCP_term(DCP_index(0, edge_label=tree.node_token(root).edge(), pos=tree.node_token(root).pos()), [])]) gram.add_rule(lhs, [], dcp=[dcp_rule]) else: first = direct_extract_lcfrs_from(tree, root, gram, term_labeling, nont_labeling, binarize, isolate_pos, hmarkov=hmarkov) lhs = LCFRS_lhs(START) lhs.add_arg([LCFRS_var(0, 0)]) dcp_rule = DCP_rule(DCP_var(-1, 0), [DCP_var(0, 0)]) gram.add_rule(lhs, [first], dcp=[dcp_rule]) return gram
import plac from constituent.induction import direct_extract_lcfrs, BasicNonterminalLabeling, \ direct_extract_lcfrs_from_prebinarized_corpus from grammar.induction.terminal_labeling import PosTerminals, FrequencyBiasedTerminalLabeling, \ FormTerminals, CompositionalTerminalLabeling, UNK4, Suffix from experiment.resources import TRAINING, VALIDATION, TESTING, TESTING_INPUT, RESULT, CorpusFile from experiment.split_merge_experiment import SplitMergeExperiment from experiment.hg_constituent_experiment import ConstituentExperiment, ScoringExperiment, ScorerAndWriter, \ setup_corpus_resources, MULTI_OBJECTIVES, MULTI_OBJECTIVES_INDEPENDENT, BASE_GRAMMAR, MAX_RULE_PRODUCT_ONLY TEST_SECOND_HALF = False # parse second half of test set # FINE_TERMINAL_LABELING = FeatureTerminals(token_to_features, feature_filter=my_feature_filter) # FINE_TERMINAL_LABELING = FormTerminals() FINE_TERMINAL_LABELING = CompositionalTerminalLabeling(FormTerminals(), PosTerminals()) FALLBACK_TERMINAL_LABELING = PosTerminals() DEFAULT_RARE_WORD_THRESHOLD = 10 def terminal_labeling(corpus, threshold=DEFAULT_RARE_WORD_THRESHOLD): # return FrequencyBiasedTerminalLabeling(FINE_TERMINAL_LABELING, FALLBACK_TERMINAL_LABELING, corpus, threshold) # return UNK4(trees=corpus, threshold=threshold, use_pos=True) return Suffix(trees=corpus, threshold=threshold, suffix_length=2) class InductionSettings: def __init__(self): self.normalize = False self.disconnect_punctuation = True self.terminal_labeling = PosTerminals()
import unittest from corpora.negra_parse import sentence_names_to_hybridtrees from constituent.induction import direct_extract_lcfrs_from_prebinarized_corpus from grammar.induction.terminal_labeling import FormTerminals, PosTerminals, CompositionalTerminalLabeling, \ FrequencyBiasedTerminalLabeling from parser.naive.parsing import LCFRS_parser from hybridtree.constituent_tree import HybridTree from hybridtree.monadic_tokens import construct_constituent_token from copy import deepcopy from parser.sDCP_parser.sdcp_parser_wrapper import PysDCPParser, LCFRS_sDCP_Parser from parser.sDCPevaluation.evaluator import DCP_evaluator, dcp_to_hybridtree from parser.discodop_parser.parser import DiscodopKbestParser fine_terminal_labeling = CompositionalTerminalLabeling(FormTerminals(), PosTerminals()) fallback_terminal_labeling = PosTerminals() terminal_threshold = 10 def terminal_labeling(corpus, threshold=terminal_threshold): return FrequencyBiasedTerminalLabeling(fine_terminal_labeling, fallback_terminal_labeling, corpus, threshold) class TestDiscodopLCFRSInduction(unittest.TestCase): def test_something(self): normal_corpus = 'res/tiger/tiger_8000.export' binarized_corpus = 'res/tiger/tiger_8000_bin.export' limit = 55000
def construct_terminal_labeling(labeling, corpus, threshold=DEFAULT_RARE_WORD_THRESHOLD): if labeling == "form+pos": return FrequencyBiasedTerminalLabeling(FINE_TERMINAL_LABELING, FALLBACK_TERMINAL_LABELING, corpus, threshold) elif labeling == "suffixes": return FrequentSuffixTerminalLabeling(corpus, threshold) elif labeling == 'suffixes+pos': return CompositionalTerminalLabeling(FrequentSuffixTerminalLabeling(corpus, threshold), PosTerminals()) elif labeling == 'stanford4' or 'stanford4de': unk_model = 'unknownword4%s' % ("de" if labeling == 'stanford4de' else "") return StanfordUNKing(corpus, unknown_threshold=threshold, openclass_threshold=50, unk_model=unk_model) else: raise Exception("Unknown terminal labeling \"%s\"" % labeling)
def test_primary_tree_violation_workaround(self): label = 's150' label2 = 's6516' path = "res/tiger/tiger_8000.xml" train_dsgs = sentence_names_to_deep_syntax_graphs([label, label2], path, hold=False, reorder_children=True) binarize = True # Grammar induction term_labeling_token = PosTerminals() def label_edge(edge): if isinstance(edge.label, ConstituentTerminal): return edge.label.pos() else: return edge.label def term_labeling(token): if isinstance(token, ConstituentTerminal): return term_labeling_token.token_label(token) else: return token if binarize: def modify_token(token): if isinstance(token, ConstituentCategory): token_new = deepcopy(token) token_new.set_category(token.category() + '-BAR') return token_new elif isinstance(token, str): return token + '-BAR' else: assert False train_dsgs = [dsg.binarize(bin_modifier=modify_token) for dsg in train_dsgs] def is_bin(token): if isinstance(token, ConstituentCategory): if token.category().endswith('-BAR'): return True elif isinstance(token, str): if token.endswith('-BAR'): return True return False def debinarize(dsg): return dsg.debinarize(is_bin=is_bin) else: debinarize = id def rec_part_strategy(direction, subgrouping, fanout): if direction == "right-to-left": return lambda dsg: fanout_limited_partitioning(dsg.recursive_partitioning(subgrouping), fanout) else: return lambda dsg: fanout_limited_partitioning_left_to_right( dsg.recursive_partitioning(subgrouping, weak=True), fanout) the_rec_part_strategy = rec_part_strategy("left-to-right", True, 1) def simple_nonterminal_labeling(nodes, dsg): return simple_labeling(nodes, dsg, label_edge) # render_and_view_dog(train_dsgs[0].dog, 'train_dsg_tmp') grammar = induction_on_a_corpus(train_dsgs, the_rec_part_strategy, simple_nonterminal_labeling, term_labeling)
def test_json_corpus_grammar_export(self): start = 1 stop = 50 # path = "res/tiger/tiger_release_aug07.corrected.16012013.utf8.xml" path = "res/tiger/tiger_8000.xml" exclude = [] dsgs = sentence_names_to_deep_syntax_graphs( ['s' + str(i) for i in range(start, stop + 1) if i not in exclude] , path , hold=False) rec_part_strategy = the_recursive_partitioning_factory().get_partitioning('cfg')[0] def label_edge(edge): if isinstance(edge.label, ConstituentTerminal): return edge.label.pos() else: return edge.label nonterminal_labeling = lambda nodes, dsg: simple_labeling(nodes, dsg, label_edge) term_labeling_token = PosTerminals() def term_labeling(token): if isinstance(token, ConstituentTerminal): return term_labeling_token.token_label(token) else: return token grammar = induction_on_a_corpus(dsgs, rec_part_strategy, nonterminal_labeling, term_labeling) grammar.make_proper() terminals = Enumerator() data = export_dog_grammar_to_json(grammar, terminals) grammar_path = '/tmp/json_grammar.json' with open(grammar_path, 'w') as file: json.dump(data, file) corpus_path = '/tmp/json_corpus.json' with open(corpus_path, 'w') as file: json.dump(export_corpus_to_json(dsgs, terminals, terminal_labeling=term_labeling), file) with open('/tmp/enumerator.enum', 'w') as file: terminals.print_index(file) reduct_dir = '/tmp/reduct_grammars' if os.path.isdir(reduct_dir): shutil.rmtree(reduct_dir) os.makedirs(reduct_dir) p = subprocess.Popen([' '.join( ["java", "-jar", os.path.join("util", SCHICK_PARSER_JAR), 'dog-reduct', '-g', grammar_path, '-t', corpus_path, "-o", reduct_dir])], shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) print("stdout", p.stdout.name) while True: nextline = p.stdout.readline() if nextline == b'' and p.poll() is not None: break print(nextline.decode('unicode_escape'), end='') # sys.stdout.write(nextline) # sys.stdout.flush() p.wait() p.stdout.close() self.assertEqual(0, p.returncode) rtgs = [] for i in range(1, len(dsgs) + 1): rtgs.append(read_rtg('/tmp/reduct_grammars/' + str(i) + '.gra')) derivation_manager = PyDerivationManager(grammar) derivation_manager.convert_rtgs_to_hypergraphs(rtgs) derivation_manager.serialize(bytes('/tmp/reduct_manager.trace', encoding='utf8')) f = lambda token: token.pos() if isinstance(token, ConstituentTerminal) else token for i, (rtg, dsg) in enumerate(zip(rtgs, dsgs)): derivations = [LCFRSDerivationWrapper(der) for der in derivation_manager.enumerate_derivations(i, grammar)] self.assertGreaterEqual(len(derivations), 1) if len(derivations) > 1: print("Sentence", i) for der in derivations: print(der) for der in derivations: dog, sync = dog_evaluation(der) dsg2 = DeepSyntaxGraph(der.compute_yield(), dog, sync) dsg.dog.project_labels(f) dsg.sentence = list(map(f, dsg.sentence)) self.assertEqual(dsg.sentence, dsg2.sentence) morphs = dsg.dog.compute_isomorphism(dsg2.dog) self.assertFalse(morphs is None) self.assertListEqual([[morphs[0].get(node, node) for node in syncs] for syncs in dsg.synchronization], dsg2.synchronization) pass
def test_induction_on_a_corpus(self): interactive = False start = 1 stop = 50 path = "res/tiger/tiger_release_aug07.corrected.16012013.utf8.xml" # path = "res/tiger/tiger_8000.xml" exclude = [] dsgs = sentence_names_to_deep_syntax_graphs( ['s' + str(i) for i in range(start, stop + 1) if i not in exclude] , path , hold=False) rec_part_strategy = the_recursive_partitioning_factory().get_partitioning('cfg')[0] def label_edge(edge): if isinstance(edge.label, ConstituentTerminal): return edge.label.pos() else: return edge.label nonterminal_labeling = lambda nodes, dsg: simple_labeling(nodes, dsg, label_edge) term_labeling_token = PosTerminals() def term_labeling(token): if isinstance(token, ConstituentTerminal): return term_labeling_token.token_label(token) else: return token grammar = induction_on_a_corpus(dsgs, rec_part_strategy, nonterminal_labeling, term_labeling, normalize=True) grammar.make_proper() parser = CFGParser(grammar) scorer = PredicateArgumentScoring() for dsg in dsgs: parser.set_input(term_labeling_token.prepare_parser_input(dsg.sentence)) parser.parse() self.assertTrue(parser.recognized()) derivation = parser.best_derivation_tree() dog, sync = dog_evaluation(derivation) dsg2 = DeepSyntaxGraph(dsg.sentence, dog, sync) f = lambda token: token.pos() if isinstance(token, ConstituentTerminal) else token dsg.dog.project_labels(f) parser.clear() scorer.add_accuracy_frames( dsg.labeled_frames(guard=lambda x: len(x[1]) > 0), dsg2.labeled_frames(guard=lambda x: len(x[1]) > 0) ) # print('dsg: ', dsg.dog, '\n', [dsg.get_graph_position(i) for i in range(len(dsg.sentence))], # '\n\n parsed: ', dsg2.dog, '\n', [dsg2.get_graph_position(i+1) for i in range(len(dsg2.sentence))]) # print() if interactive: if dsg.label == 's50': pass if dsg.dog != dog: z1 = render_and_view_dog(dsg.dog, "corpus_" + dsg.label) z2 = render_and_view_dog(dog, "parsed_" + dsg.label) z1.communicate() z2.communicate() print("Labeled frames:") print("P", scorer.labeled_frame_scorer.precision(), "R", scorer.labeled_frame_scorer.recall(), "F1", scorer.labeled_frame_scorer.fmeasure()) print("Labeled dependencies:") print("P", scorer.labeled_dependency_scorer.precision(), "R", scorer.labeled_dependency_scorer.recall(), "F1", scorer.labeled_dependency_scorer.fmeasure())
def main2(): induction_settings = InductionSettings() # terminal labeling induction_settings.terminal_labeling_token = PosTerminals() def term_labeling(token): if isinstance(token, ConstituentTerminal): return induction_settings.terminal_labeling_token.token_label( token) else: return token induction_settings.terminal_labeling = term_labeling # recursive partitioning def rec_part_strategy(direction, subgrouping, fanout, binarize): if direction == "right-to-left": return lambda dsg: fanout_limited_partitioning( dsg.recursive_partitioning(subgrouping, weak=binarize), fanout) else: return lambda dsg: fanout_limited_partitioning_left_to_right( dsg.recursive_partitioning(subgrouping, weak=binarize), fanout) induction_settings.binarize = True induction_settings.direction = "left-to-right" induction_settings.subgrouping = False induction_settings.fanout = 1 induction_settings.rec_part_strategy = rec_part_strategy( induction_settings.direction, induction_settings.subgrouping, induction_settings.fanout, induction_settings.binarize) # Nonterminal Labeling induction_settings.start = "START" def label_edge(edge): if isinstance(edge.label, ConstituentTerminal): return edge.label.pos() else: return edge.label def stupid_edge(edge): return "X" def label_child(edge, j): return edge.get_function(j) def simple_nonterminal_labeling(nodes, dsg): return simple_labeling(nodes, dsg, label_edge) def bot_stupid_nonterminal_labeling(nodes, dsg): return top_bot_labeling(nodes, dsg, label_edge, stupid_edge) def missing_child_nonterminal_labeling(nodes, dsg): return missing_child_labeling(nodes, dsg, label_edge, label_child) induction_settings.nonterminal_labeling = simple_nonterminal_labeling induction_settings.normalize = True experiment = DOGExperiment(induction_settings) # Corpora start = 1 stop = 2000 test_start = 7001 test_stop = 7200 # path = "res/tiger/tiger_release_aug07.corrected.16012013.utf8.xml" corpus_path = "res/tiger/tiger_8000.xml" exclude = [] experiment.resources[TRAINING] = CorpusFile(corpus_path, start, stop) experiment.resources[TESTING] = CorpusFile(corpus_path, test_start, test_stop) experiment.oracle_parsing = True experiment.purge_rule_freq = None # 1.0 experiment.k_best = 100 experiment.run_experiment()
def run_experiment(rec_part_strategy, nonterminal_labeling, exp, reorder_children, binarize=True): start = 1 stop = 7000 test_start = 7001 test_stop = 7200 # path = "res/tiger/tiger_release_aug07.corrected.16012013.utf8.xml" corpus_path = "res/tiger/tiger_8000.xml" exclude = [] train_dsgs = sentence_names_to_deep_syntax_graphs( ['s' + str(i) for i in range(start, stop + 1) if i not in exclude], corpus_path, hold=False, reorder_children=reorder_children) test_dsgs = sentence_names_to_deep_syntax_graphs( [ 's' + str(i) for i in range(test_start, test_stop + 1) if i not in exclude ], corpus_path, hold=False, reorder_children=reorder_children) # Grammar induction term_labeling_token = PosTerminals() def term_labeling(token): if isinstance(token, ConstituentTerminal): return term_labeling_token.token_label(token) else: return token if binarize: def modify_token(token): if isinstance(token, ConstituentCategory): token_new = deepcopy(token) token_new.set_category(token.category() + '-BAR') return token_new elif isinstance(token, str): return token + '-BAR' else: assert False train_dsgs = [ dsg.binarize(bin_modifier=modify_token) for dsg in train_dsgs ] def is_bin(token): if isinstance(token, ConstituentCategory): if token.category().endswith('-BAR'): return True elif isinstance(token, str): if token.endswith('-BAR'): return True return False def debinarize(dsg): return dsg.debinarize(is_bin=is_bin) else: debinarize = id grammar = induction_on_a_corpus(train_dsgs, rec_part_strategy, nonterminal_labeling, term_labeling) grammar.make_proper() print("Nonterminals", len(grammar.nonts()), "Rules", len(grammar.rules())) parser = GFParser_k_best(grammar, k=500) return do_parsing(parser, test_dsgs, term_labeling_token, oracle=True, debinarize=debinarize) # Compute reducts, i.e., intersect grammar with each training dsg basedir = path.join('/tmp/dog_experiments', 'exp' + str(exp)) reduct_dir = path.join(basedir, 'reduct_grammars') terminal_map = Enumerator() if not os.path.isdir(basedir): os.makedirs(basedir) data = export_dog_grammar_to_json(grammar, terminal_map) grammar_path = path.join(basedir, 'grammar.json') with open(grammar_path, 'w') as file: json.dump(data, file) corpus_path = path.join(basedir, 'corpus.json') with open(corpus_path, 'w') as file: json.dump( export_corpus_to_json(train_dsgs, terminal_map, terminal_labeling=term_labeling), file) with open(path.join(basedir, 'enumerator.enum'), 'w') as file: terminal_map.print_index(file) if os.path.isdir(reduct_dir): shutil.rmtree(reduct_dir) os.makedirs(reduct_dir) p = subprocess.Popen([ ' '.join([ "java", "-jar", os.path.join("util", SCHICK_PARSER_JAR), 'dog-reduct', '-g', grammar_path, '-t', corpus_path, "-o", reduct_dir ]) ], shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) while True: nextline = p.stdout.readline() if nextline == '' and p.poll() is not None: break sys.stdout.write(nextline) sys.stdout.flush() p.wait() p.stdout.close() rtgs = [] for i in range(1, len(train_dsgs) + 1): rtgs.append(read_rtg(path.join(reduct_dir, str(i) + '.gra'))) derivation_manager = PyDerivationManager(grammar) derivation_manager.convert_rtgs_to_hypergraphs(rtgs) derivation_manager.serialize(path.join(basedir, 'reduct_manager.trace')) # Training ## prepare EM training em_epochs = 20 seed = 0 smoothing_factor = 0.01 split_randomization = 0.01 sm_cycles = 2 merge_percentage = 50.0 grammarInfo = PyGrammarInfo(grammar, derivation_manager.get_nonterminal_map()) storageManager = PyStorageManager() em_builder = PySplitMergeTrainerBuilder(derivation_manager, grammarInfo) em_builder.set_em_epochs(em_epochs) em_builder.set_simple_expector(threads=THREADS) emTrainer = em_builder.build() # randomize initial weights and do em training la_no_splits = build_PyLatentAnnotation_initial(grammar, grammarInfo, storageManager) la_no_splits.add_random_noise(seed=seed) emTrainer.em_train(la_no_splits) la_no_splits.project_weights(grammar, grammarInfo) do_parsing(CFGParser(grammar), test_dsgs, term_labeling_token) return ## prepare SM training builder = PySplitMergeTrainerBuilder(derivation_manager, grammarInfo) builder.set_em_epochs(em_epochs) builder.set_split_randomization(1.0, seed + 1) builder.set_simple_expector(threads=THREADS) builder.set_smoothing_factor(smoothingFactor=smoothing_factor) builder.set_split_randomization(percent=split_randomization) # builder.set_scc_merger(-0.2) builder.set_percent_merger(merge_percentage) splitMergeTrainer = builder.build() # splitMergeTrainer.setMaxDrops(validationDropIterations, mode="smoothing") splitMergeTrainer.setEMepochs(em_epochs, mode="smoothing") # set initial latent annotation latentAnnotation = [la_no_splits] # carry out split/merge training and do parsing parsing_method = "filter-ctf" # parsing_method = "single-best-annotation" k_best = 50 for i in range(1, sm_cycles + 1): splitMergeTrainer.reset_random_seed(seed + i + 1) latentAnnotation.append( splitMergeTrainer.split_merge_cycle(latentAnnotation[-1])) print("Cycle: ", i) if parsing_method == "single-best-annotation": smGrammar = latentAnnotation[i].build_sm_grammar( grammar, grammarInfo, rule_pruning=0.0001, rule_smoothing=0.1) print("Rules in smoothed grammar: ", len(smGrammar.rules())) parser = GFParser(smGrammar) elif parsing_method == "filter-ctf": latentAnnotation[-1].project_weights(grammar, grammarInfo) parser = Coarse_to_fine_parser( grammar, latentAnnotation[-1], grammarInfo, derivation_manager.get_nonterminal_map(), base_parser_type=GFParser_k_best, k=k_best) else: raise (Exception()) do_parsing(parser, test_dsgs, term_labeling_token) del parser