def test(nlp, src, gen, bert=False, print_annotations=False, print_latex=False, verbose=False): if print_annotations: print("source:", src_line[:50]) print("summary:", gen_line[:50]) src = nlp(src) gen = nlp(gen) if verbose: print("clusters:", src._.coref_clusters, gen._.coref_clusters) ce = CompoundEquivalency() spe = SpeakerPronounEquivalency() spe.register(src) spe.register(gen) kg = KnowledgeGraph(nlp, use_bert=bert, equivalencies=[ce, spe], verbose=verbose) if print_annotations: annotator = Annotator(src, gen, latex=print_latex) kg.add_document(src) contained = 0 contained_bert = 0 missing = 0 missing_verb = 0 missing_actors = 0 missing_acteds = 0 contradiction = 0 contradiction_bert = 0 invalid_simplification = 0 total = 0 for token in gen: if token.pos_ == "VERB": total += 1 relation = kg.get_relation(token) r = kg.query_relation(relation) if r[0] == KnowledgeGraph.entailment: if print_annotations: print(util.format("contained", "blue", latex=print_latex), "|", relation, "|", r[1]) contained += 1 if r[0] == KnowledgeGraph.entailment_bert: if print_annotations: print( util.format("contained (BERT)", "blue", latex=print_latex), "|", relation, "|", r[1]) contained_bert += 1 if r[0] == KnowledgeGraph.contradiction_bert: if print_annotations: print( util.format("contradiction (BERT)", "red", latex=print_latex), "|", relation, "|", r[1]) contradiction_bert += 1 elif r[0] == KnowledgeGraph.missing_dependencies: missing += 1 if print_annotations: print( util.format("generic missing dependency", "yellow", latex=print_latex), "|", relation, "|", r[1]) elif r[0] == KnowledgeGraph.missing_actors: missing_actors += 1 if print_annotations: print( util.format("missing actors", "magenta", latex=print_latex), "|", relation, "|", r[1]) elif r[0] == KnowledgeGraph.missing_acteds: missing_acteds += 1 if print_annotations: print( util.format("missing acteds", "magenta", latex=print_latex), "|", relation, "|", r[1]) elif r[0] == KnowledgeGraph.missing_verb: missing_verb += 1 if print_annotations: print( util.format("missing verb", "magenta", latex=print_latex), "|", relation, "|", r[1]) elif r[0] == KnowledgeGraph.invalid_simplification: invalid_simplification += 1 if print_annotations: print( util.format("invalid simplification", "magenta", latex=print_latex), "|", relation, "|", r[1]) elif r[0] == KnowledgeGraph.contradiction: contradiction += 1 if print_annotations: print( util.format("contradiction", "red", latex=print_latex), "|", relation, "|", r[1]) if print_annotations: annotator.annotate(relation, r) if print_annotations: annotated_document, annotated_summary = annotator.annotated() print("Document:", " ".join(annotated_document)) print("Summary:", " ".join(annotated_summary)) if total == 0: return 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 return 100.0 * contained / total, \ 100.0 * contained_bert / total, \ 100.0 * missing / total, \ 100.0 * missing_verb / total, \ 100.0 * missing_actors / total, \ 100.0 * missing_acteds / total, \ 100.0 * contradiction / total, \ 100.0 * contradiction_bert / total, \ 100.0 * invalid_simplification / total