def test_number_of_tagged_entities(): if run_tests: dataset_parts = read_corpus("LocText", corpus_percentage=1.0) dataset_whole = read_corpus("LocText", corpus_percentage=1.0) assert 0 == num_predicted_entities( dataset_parts) == num_predicted_entities(dataset_whole) TAGGER_SEND_PARTS.annotate(dataset_parts) TAGGER_SEND_WHOLE.annotate(dataset_whole) for docid, document in dataset_parts.documents.items(): if docid in [ "23543752", # Special analysis as it contains unicode chracters (μ) "23150645" # also unicode and annotation of reductase where reductase as string alone is not tagged ]: for e in document.predicted_entities(): print(e) print() num_preds_with_parts = num_predicted_entities(dataset_parts) num_preds_with_whole = num_predicted_entities(dataset_whole) print("Numbers, real:", len(list(dataset_parts.entities())), "vs. pred: ", num_preds_with_parts, num_preds_with_whole) # Equality holds if all tagger entity types are explicitly given, otherwise the tagging with parts yield less predicted entities assert num_preds_with_parts == num_preds_with_whole for pred_part, pred_whole in zip(dataset_parts.predicted_entities(), dataset_whole.predicted_entities()): assert (pred_part == pred_whole)
def _test_LocText(corpus_percentage, model, EXPECTED_F=None, predict_entities=None, EXPECTED_F_SE=0.001): # Note: EXPECTED_F=None will make the test fail for non-yet verified evaluations # Note: the real StdErr's are around ~0.0027-0.0095. Decrease them by default to be more strict with tests assert corpus_percentage in [ TEST_MIN_CORPUS_PERCENTAGE, 1.0 ], "corpus_percentage must == {} or 1.0. You gave: {}".format( str(TEST_MIN_CORPUS_PERCENTAGE), str(corpus_percentage)) corpus = read_corpus("LocText", corpus_percentage, predict_entities) args = [ '--model', model, '--corpus_percentage', str(corpus_percentage), '--evaluation_level', str(EVALUATION_LEVEL) ] if predict_entities: args += ['--predict_entities', predict_entities] rel_evaluation = evaluate_with_argv(args) print("LocText " + model, rel_evaluation) assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=EXPECTED_F_SE * 1.1) return rel_evaluation
def test_baseline_D1(corpus_percentage): corpus = read_corpus("LocText", corpus_percentage) if corpus_percentage == 1.0: EXPECTED_F = 0.6421 else: EXPECTED_F = None edge_generator = SentenceDistanceEdgeGenerator(PRO_ID, LOC_ID, REL_PRO_LOC_ID, distance=1) annotator_gen_fun = ( lambda _: StubRelationExtractor(edge_generator).annotate) evaluations = Evaluations.cross_validate(annotator_gen_fun, corpus, EVALUATOR, k_num_folds=5, use_validation_set=True) rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact") print(rel_evaluation) print(evaluations) assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=0.001 * 1.1), rel_evaluation.f_measure return evaluations
def test_baseline_full(corpus_percentage): if (corpus_percentage == 1.0): EXPECTED_F = 0.5050 else: EXPECTED_F = None corpus = read_corpus("LocText", corpus_percentage, predict_entities="9606,3702,4932") annotator_gen_fun = (lambda _: StubSameSentenceRelationExtractor( PRO_ID, LOC_ID, REL_PRO_LOC_ID, use_gold=False, use_pred=True).annotate ) evaluations = Evaluations.cross_validate(annotator_gen_fun, corpus, EVALUATOR, k_num_folds=5, use_validation_set=True) rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact") print(evaluations) assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=0.001 * 1.1), rel_evaluation.f_measure return evaluations
def test_get_evaluation_result_of_corpus(evaluation_level): """ Evaluates the performance of corpus entities [e_1 (Protein), e_2 (Localization) and e_3 (Organism)] [precision, recall and f-measure] :param corpus: :return: """ # Gets both annotation and pred_annotation entities. corpus = read_corpus("LocText", corpus_percentage=1.0, predict_entities="9606,3702,4932") (mention_evaluator, entity_evaluator) = _get_entity_evaluator(evaluation_level) print() print("EVALUATION LEVEL:", evaluation_level) print() # print("-----------------------------------------------------------------------------------") # print("MentionLevelEvaluator") # print(mention_evaluator.evaluate(corpus)) # print("-----------------------------------------------------------------------------------") # print() print() print( "-----------------------------------------------------------------------------------" ) print("EntityEvaluator") print(entity_evaluator.evaluate(corpus)) print( "-----------------------------------------------------------------------------------" )
def test_num_of_normalization_in_new_file(): old_dataset = read_corpus("LocText_v1", corpus_percentage=1.0) new_dataset = read_corpus("LocText_v2", corpus_percentage=1.0) # First of all, count of entities & relations remains equal (and ofc there are 0 predictions) assert (len(list(old_dataset.entities())) == len( list(new_dataset.entities()))) assert (len(list(old_dataset.predicted_entities())) == len( list(new_dataset.predicted_entities())) == 0) assert (len(list(old_dataset.relations())) == len( list(new_dataset.relations()))) assert (len(list(old_dataset.predicted_relations())) == len( list(new_dataset.predicted_relations())) == 0) # Test now new number of normalizations print("Actually obtained number of newly normalized ID's: ", num_normalizations(new_dataset) - num_normalizations(old_dataset)) # 8 is the number of newly normalized records from Tanya, [Greens] in new file. assert num_normalizations( new_dataset) == num_normalizations(old_dataset) + 8
def test_baseline_D0_D1(corpus_percentage): corpus = read_corpus("LocText", corpus_percentage) if corpus_percentage == 1.0: EXPECTED_F = 0.7060 else: EXPECTED_F = None edge_generator = CombinatorEdgeGenerator( SentenceDistanceEdgeGenerator(PRO_ID, LOC_ID, REL_PRO_LOC_ID, distance=0, rewrite_edges=False), SentenceDistanceEdgeGenerator(PRO_ID, LOC_ID, REL_PRO_LOC_ID, distance=1, rewrite_edges=False), # Recall: 88.52 # SentenceDistanceEdgeGenerator(PRO_ID, LOC_ID, REL_PRO_LOC_ID, distance=2, rewrite_edges=False), # SentenceDistanceEdgeGenerator(PRO_ID, LOC_ID, REL_PRO_LOC_ID, distance=3, rewrite_edges=False), # # SentenceDistanceEdgeGenerator(PRO_ID, LOC_ID, REL_PRO_LOC_ID, distance=4, rewrite_edges=False), # SentenceDistanceEdgeGenerator(PRO_ID, LOC_ID, REL_PRO_LOC_ID, distance=5, rewrite_edges=False), # # SentenceDistanceEdgeGenerator(PRO_ID, LOC_ID, REL_PRO_LOC_ID, distance=6, rewrite_edges=False), # Recall: 99.70 ) annotator_gen_fun = ( lambda _: StubRelationExtractor(edge_generator).annotate) evaluations = Evaluations.cross_validate(annotator_gen_fun, corpus, EVALUATOR, k_num_folds=5, use_validation_set=True) rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact") print(rel_evaluation) print(evaluations) assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=0.001 * 1.1), rel_evaluation.f_measure return rel_evaluation
def get_model_and_data(sentence_distance, predict_entities): corpus = read_corpus("LocText", predict_entities=predict_entities) # TODO the specific parameters like C=1 or even `linear` are controversial -- Maybe I should I change that annotator = LocTextDXModelRelationExtractor( PRO_ID, LOC_ID, REL_PRO_LOC_ID, sentence_distance, use_predicted_entities=len(predict_entities) > 0, preprocess=True, kernel='linear', C=1) annotator.pipeline.execute(corpus) X, y, groups = annotator.model.write_vector_instances( corpus, annotator.pipeline.feature_set) X = annotator.model.preprocess.fit_transform(X) return (annotator, X, y, groups)
def _test(corpus_percentage, entity_map_fun, relation_accept_fun, expected_sum_perct_d0_d1, expected_nums, expected_percts): corpus = read_corpus("LocText", corpus_percentage) # Note: the predictor will already split & tokenize the corpus. See the implementation for details StubSamePartRelationExtractor(PRO_ID, LOC_ID, REL_PRO_LOC_ID).annotate(corpus) (counter_nums, counter_percts) = corpus.compute_stats_relations_distances( REL_PRO_LOC_ID, entity_map_fun, relation_accept_fun) print() print("# Documents", len(corpus)) print("# Uniq Rels", sum(counter_nums.values())) print(" ", counter_nums) print(" ", counter_percts) assert expected_nums == counter_nums assert math.isclose(expected_sum_perct_d0_d1, (counter_percts['D0'] + counter_percts['D1']), abs_tol=0.01)
def test_same_stats(): original = read_corpus("LocText_v0", corpus_percentage=1.0) newone = read_corpus("LocText", corpus_percentage=1.0) # Verification original.validate_entity_offsets() newone.validate_entity_offsets() # Basic assert 100 == len(original) == len(newone) assert len(list(original.entities())) == len(list(newone.entities())) and len(list(original.entities())) > 0 assert 0 == len(list(original.predicted_entities())) == len(list(newone.predicted_entities())) assert len(list(original.relations())) == len(list(newone.relations())) and len(list(original.relations())) > 0 assert 0 == len(list(original.predicted_relations())) == len(list(newone.predicted_relations())) # Elaborated edge_generator_d0 = SentenceDistanceEdgeGenerator(PRO_ID, LOC_ID, REL_PRO_LOC_ID, distance=0) annotator = StubRelationExtractor(edge_generator_d0) annotator.annotate(original) annotator.annotate(newone) assert len(list(original.edges())) > 0 and (len(list(original.edges())) == len(list(newone.edges())) == len(list(newone.predicted_relations()))) num_d0 = len(list(newone.predicted_relations())) edge_generator_d1 = SentenceDistanceEdgeGenerator(PRO_ID, LOC_ID, REL_PRO_LOC_ID, distance=1) annotator = StubRelationExtractor(edge_generator_d1) annotator.annotate(original) annotator.annotate(newone) assert len(list(original.edges())) > 0 and (len(list(original.edges())) == len(list(newone.edges())) == (- num_d0 + len(list(newone.predicted_relations())))) # Normalizations assert all(len(e.norms) == 0 for e in original.entities()) count_normalizations = 0 for e in newone.entities(): if str(e.class_id) != "e_4": print(e.norms) assert len(e.norms) == 1, e norm_id = next(iter(e.norms.values())) assert type(norm_id) is str or e.class_id == "e_1" and norm_id is None, e # do not write arrays, only comma-separated strings assert norm_id is None or ' ' not in norm_id, e # We cannot have stuff like 'GO:0005811 lipid droplet' -- let's have only the GO id if e.class_id in ['e_2', 'e_3']: assert norm_id != '', e assert ',' not in norm_id, e if e.class_id == 'e_2': assert norm_id.startswith("GO:") count_normalizations += 1 assert count_normalizations == len(list(newone.entities())) == len(list(original.entities())) # Document based for docid, original_document in original.documents.items(): newone_document = newone.documents[docid] original_count_entities = sum(1 for _ in original_document.entities()) newone_count_entities = sum(1 for _ in newone_document.entities()) assert original_count_entities == newone_count_entities, docid original_count_relations = sum(1 for _ in original_document.relations()) newone_count_relations = sum(1 for _ in newone_document.relations()) assert original_count_relations == newone_count_relations, docid
from sklearn.metrics import classification_report from sklearn.svm import SVC from sklearn.decomposition import PCA from nalaf.learning.lib.sklsvm import SklSVM from nalaf.structures.data import Dataset from loctext.learning.train import read_corpus from loctext.util import PRO_ID, LOC_ID, ORG_ID, REL_PRO_LOC_ID, repo_path from loctext.learning.annotators import LocTextDXModelRelationExtractor import matplotlib.pyplot as plt print(__doc__) corpus = read_corpus("LocText") locTextModel = LocTextDXModelRelationExtractor(PRO_ID, LOC_ID, REL_PRO_LOC_ID) locTextModel.pipeline.execute(corpus) X, y = SklSVM._convert_edges_to_SVC_instances( corpus, locTextModel.pipeline.feature_set) def pca_plot(): X_copy = X.toarray() pca_2d = PCA(n_components=2).fit_transform(X_copy) for instance_i in range(0, pca_2d.shape[0]): if y[instance_i] < 0: neg = plt.scatter(pca_2d[instance_i, 0], pca_2d[instance_i, 1], c='r')