def compute_oracle_derivation(parser, dsg, mapping=id): validationMethod = "F1" best_der = None best_f1 = -1 best_prec = -1 best_rec = -1 relevant = dsg.labeled_frames(guard=lambda x: x[1] > 0) for _, derivation in parser.k_best_derivation_trees(): dog, sync = dog_evaluation(derivation) dsg2 = DeepSyntaxGraph(dsg.sentence, mapping(dog), sync) retrieved = dsg2.labeled_frames(guard=lambda x: x[1] > 0) inters = retrieved & relevant # in case of parse failure there are two options here: # - parse failure -> no spans at all, thus precision = 1 # - parse failure -> a dummy tree with all spans wrong, thus precision = 0 precision = 1.0 * len(inters) / len(retrieved) \ if len(retrieved) > 0 else 0 recall = 1.0 * len(inters) / len(relevant) \ if len(relevant) > 0 else 0 fmeasure = 2.0 * precision * recall / (precision + recall) \ if precision + recall > 0 else 0 if (validationMethod == "F1" and fmeasure > best_f1)\ or (validationMethod == "Precision" and precision > best_prec)\ or (validationMethod == "Recall"and recall > best_rec): best_der, best_f1, best_prec, best_rec = derivation, fmeasure, precision, recall return best_der
def do_parsing(parser, test_dsgs, term_labeling_token, oracle=False, debinarize=id): interactive = True # False scorer = PredicateArgumentScoring() not_output_connected = 0 start = time.time() for dsg in test_dsgs: parser.set_input(term_labeling_token.prepare_parser_input( dsg.sentence)) parser.parse() f = lambda token: token.pos() if isinstance(token, ConstituentTerminal ) else token dsg.dog.project_labels(f) if parser.recognized(): if oracle: derivation = compute_oracle_derivation(parser, dsg, debinarize) else: derivation = parser.best_derivation_tree() dog, sync = dog_evaluation(derivation) if not dog.output_connected(): not_output_connected += 1 if interactive: z2 = render_and_view_dog(dog, "parsed_" + dsg.label) # z2.communicate() dsg2 = DeepSyntaxGraph(dsg.sentence, debinarize(dog), sync) scorer.add_accuracy_frames( dsg.labeled_frames(guard=lambda x: len(x[1]) > 0), dsg2.labeled_frames(guard=lambda x: len(x[1]) > 0)) # print('dsg: ', dsg.dog, '\n', [dsg.get_graph_position(i) for i in range(len(dsg.sentence))], '\n\n parsed: ', dsg2.dog, '\n', [dsg2.get_graph_position(i+1) for i in range(len(dsg2.sentence))]) # print() if False and interactive: if dsg.label == 's50': pass if dsg.dog != dog: z1 = render_and_view_dog(dsg.dog, "corpus_" + dsg.label) z2 = render_and_view_dog(dog, "parsed_" + dsg.label) z1.communicate() z2.communicate() else: scorer.add_failure( dsg.labeled_frames(guard=lambda x: len(x[1]) > 0)) parser.clear() print("Completed parsing in", time.time() - start, "seconds.") print("Parse failures:", scorer.labeled_frame_scorer.n_failures()) print("Not output connected", not_output_connected) print("Labeled frames:") print("P", scorer.labeled_frame_scorer.precision(), "R", scorer.labeled_frame_scorer.recall(), "F1", scorer.labeled_frame_scorer.fmeasure(), "EM", scorer.labeled_frame_scorer.exact_match()) print("Unlabeled frames:") print("P", scorer.unlabeled_frame_scorer.precision(), "R", scorer.unlabeled_frame_scorer.recall(), "F1", scorer.unlabeled_frame_scorer.fmeasure(), "EM", scorer.unlabeled_frame_scorer.exact_match()) print("Labeled dependencies:") print("P", scorer.labeled_dependency_scorer.precision(), "R", scorer.labeled_dependency_scorer.recall(), "F1", scorer.labeled_dependency_scorer.fmeasure(), "EM", scorer.labeled_dependency_scorer.exact_match()) print("Unlabeled dependencies:") print("P", scorer.unlabeled_dependency_scorer.precision(), "R", scorer.unlabeled_dependency_scorer.recall(), "F1", scorer.unlabeled_dependency_scorer.fmeasure(), "EM", scorer.unlabeled_dependency_scorer.exact_match()) return scorer