def process_list(data_path, list_name):
"""
:param data_path:
:param list_name:
"""
new_list = open(data_path + list_name + '.txt', 'r', encoding='utf-8')
elements_list = new_list.read().split('\n')
if len(elements_list) == 1:
elements_list = ''.join(elements_list)
elements_list = elements_list.split(',')
merpy.create_lexicon(elements_list, list_name)
merpy.process_lexicon(list_name)
return
def annotations(corpus_path, data_path):
"""
:param corpus_path:
:param data_path:
"""
merpy.download_lexicons()
merpy.process_lexicon("hp")
merpy.process_lexicon("doid")
merpy.process_lexicon("radlex")
process_list(data_path, 'chebi')
process_list(data_path, 'medical_devices')
process_list(data_path, 'temporal_list')
process_list(data_path, 'population_vocabulary')
for f in os.listdir(corpus_path):
file_to_annotate = open(corpus_path + f, 'r', encoding = 'utf-8')
file_content = file_to_annotate.read()
file_to_annotate.close()
entities_hp = merpy.get_entities(file_content, "hp")
clean_hp_list = []
for hp in entities_hp:
if 'HP' not in hp[-1]:
pass
else:
clean_hp_list.append(hp)
entities_doid = merpy.get_entities(file_content, "doid")
entities_radlex = merpy.get_entities(file_content, "radlex")
entities_devices = merpy.get_entities(file_content, "medical_devices")
entities_chebi = merpy.get_entities(file_content, "chebi")
entities_temporal = merpy.get_entities(file_content, "temporal")
entities_population = merpy.get_entities(file_content, "population")
entities = clean_hp_list + entities_doid + entities_devices + entities_chebi + entities_radlex + entities_temporal + entities_population
entities_clean = [x for x in entities if x != ['']]
entities_sorted = sorted(entities_clean, key = lambda position: int(position[0]))
print('\n' + f + '\n')
for entity in entities_sorted:
print(entity)
return
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 23 15:57:57 2020
@author: André
"""
import merpy
import pandas as pd
decs_data = pd.read_csv('../mesinesp_data/DeCS_data.tsv', sep='\t')
conv_dict = {}
for index, row in decs_data.iterrows():
l_terms = str(row['Synonyms']).split('|')
if row['Term_Spanish'] not in l_terms:
l_terms.append(row['Term_Spanish'])
for i in l_terms:
conv_dict[i] = str(row['#DeCS_code'])
merpy.create_lexicon(conv_dict.keys(), "decslex")
merpy.create_mappings(conv_dict, "decslex")
merpy.show_lexicons()
merpy.process_lexicon("decslex")
#DEBUG
merpy.get_entities("lo nervio abducens es una aurelia aurita", "decslex")
import merpy
merpy.create_lexicon_from_file("ncbi.txt", "ncbi")
merpy.process_lexicon("ncbi")
import merpy
import sys
whole_lexicons_parts = []
for lexicon in sys.argv[1].split(','):
lexicon_name, lexicon_type = lexicon.split('.')
merpy.process_lexicon(lexicon_name, lexicon_type)
whole_lexicons_parts.append(lexicon_name)
merpy.merge_processed_lexicons(whole_lexicons_parts, sys.argv[2])
with open('../bioasq_data/mesh_terms_synonyms.txt',
encoding='utf-8') as finput_terms:
l_terms_syn = finput_terms.readlines()
dict_terms_synonyms = {}
for i in l_terms_syn:
aux = i.split('\t')
dict_terms_synonyms[aux[0]] = aux[1].replace('\n', '')
conv_dict = {}
for key, values in dict_terms_synonyms.items():
l_synonyms = values.split(',')
if key not in l_synonyms:
l_synonyms.append(key)
for i in l_synonyms:
conv_dict[i.strip()] = dict_terms.get(key)
merpy.create_lexicon(conv_dict.keys(), "meshlex")
merpy.create_mappings(conv_dict, "meshlex")
merpy.show_lexicons()
merpy.process_lexicon("meshlex")
#DEBUG
print(merpy.get_entities("I like abdominal injuries", "meshlex"))
print(merpy.get_entities("I like Calcimycin", "meshlex"))
print(
merpy.get_entities(
"I like Calcimycin it is a good aurelia aurita and Temefos is awesome! abate lowercase",
"meshlex"))
import merpy
import ssmpy
import urllib.request
# Download the Human Disease Ontology OWL file
doid_link = 'http://purl.obolibrary.org/obo/doid.owl'
with urllib.request.urlopen(doid_link) as response, open('doid.owl',
'wb') as out_file:
data = response.read()
out_file.write(data)
ssmpy.create_semantic_base("doid.owl", "doid.db",
"http://purl.obolibrary.org/obo/",
"http://www.w3.org/2000/01/rdf-schema#subClassOf",
'')
ssmpy.semantic_base("doid.db")
merpy.download_lexicon(doid_link, "doid", "owl")
merpy.process_lexicon("doid", "owl")
document = "zoophilia zoophobia zaspopathy"
entities = merpy.get_entities(document, "doid")
print(entities)
print(merpy.get_similarities(entities, 'doid.db'))
def annotate_documents(task, subset, name_to_id, output_dir):
"""Recognise entities (NER) and link them to the respective CIE-O-3 code (Normalisation), if available, using MERPY"""
lexicon_name = "cieo3"
merpy.create_lexicon(name_to_id.keys(), lexicon_name)
merpy.create_mappings(name_to_id, lexicon_name)
merpy.process_lexicon(lexicon_name)
dataset_dir = str()
if subset == "train":
dataset_dir = "./data/datasets/train-set-to-publish/cantemist-norm/"
elif subset == "dev1":
dataset_dir = "./data/datasets/dev-set1-to-publish/cantemist-norm/"
elif subset == "dev2":
dataset_dir = "./data/datasets/dev-set2-to-publish/cantemist-norm/"
elif subset == "test":
dataset_dir = "./data/datasets/test-background-set-to-publish"
doc_count = int()
doc_annot_ratio = int()
doc_w_annotations_count = int()
total_entity_count = int()
linked_mentions = int()
total_doc_count = int(len(os.listdir(dataset_dir)) / 2)
for doc in os.listdir(dataset_dir):
if doc[-3:] == "txt":
doc_count += 1
output_string = str()
print("Annotating " + str(doc_count) + " of " +
str(total_doc_count) + " documents")
with open(dataset_dir + doc, 'r') as input_file:
text = input_file.read()
input_file.close()
doc_entity_count = int()
entities = merpy.get_entities(text, lexicon_name)
for entity in entities:
if entity != ['']:
total_entity_count += 1
doc_entity_count += 1
if len(entity
) == 4: # linked mentions with CIE-O-3 code
linked_mentions += 1
output_string += "T" + str(
doc_entity_count
) + "\tMORFOLOGIA_NEOPLASIA " + entity[
0] + " " + entity[1] + "\t" + entity[2] + "\n"
if task == "norm":
output_string += "#" + str(
doc_entity_count
) + "\tAnnotatorNotes\tT" + str(
doc_entity_count) + "\t" + entity[3] + "\n"
elif len(entity) == 3: # mentions without CIE-O-3 code
output_string += "T" + str(
doc_entity_count
) + "\tMORFOLOGIA_NEOPLASIA " + entity[
0] + " " + entity[1] + "\t" + entity[2] + "\n"
if task == "norm":
output_string += "#" + str(
doc_entity_count
) + "\tAnnotatorNotes T" + str(
doc_entity_count) + "\tNA\n"
if doc_entity_count > 0:
doc_w_annotations_count += 1
elif doc_entity_count == 0:
output_string = "NA\tNA NA NA\tNA\n"
if task == "norm":
output_string += "#" + str(
doc_entity_count) + "\tAnnotatorNotes T" + str(
doc_entity_count) + "\tNA\n"
# output annotations file
output_filename = output_dir + doc[:-4] + ".ann"
with open(output_filename, 'w') as output_file:
output_file.write(output_string)
output_file.close()
try:
doc_annot_ratio = float(doc_w_annotations_count / total_doc_count)
mentions_ratio = float(total_entity_count / doc_w_annotations_count)
doc_linked_ratio = float(linked_mentions / doc_w_annotations_count)
linked_ratio = float(linked_mentions / total_entity_count)
except:
mentions_ratio = 0.0
doc_linked_ratio = 0.0
linked_ratio = 0.0
output_str = "TOTAL DOCUMENTS: " + str(total_doc_count) + "\n"
output_str += "DOCS WITH ANNOTATIONS: " + str(
doc_w_annotations_count) + "\n"
output_str += "RATIO OF DOCS WITH ANNOTATIONS: " + str(
doc_annot_ratio) + "\n"
output_str += "TOTAL ENTITY MENTIONS: " + str(total_entity_count) + "\n"
output_str += "ENTITY MENTIONS PER DOCUMENT: " + str(mentions_ratio) + "\n"
output_str += "LINKED ENTITY MENTIONS: " + str(linked_mentions) + "\n"
output_str += "LINKED ENTITY MENTIONS PER DOCUMENT: " + str(
doc_linked_ratio) + "\n"
output_str += "RATIO OF LINKED ENTITY MENTIONS: " + str(linked_ratio)
file_name = "./mer_annotations/" + task + "/" + task + "_" + subset + "_stats"
with open(file_name, "w") as output:
output.write(output_str)
output.close()
merpy.download_lexicon(
"ftp://ftp.ebi.ac.uk/pub/databases/chebi/ontology/chebi.owl",
"chebi",
ltype="owl",
)
merpy.download_lexicon("http://purl.obolibrary.org/obo/ncbitaxon.owl",
"taxon",
ltype="owl")
merpy.download_lexicon(
"https://raw.githubusercontent.com/CIDO-ontology/cido/master/src/ontology/cido.owl",
"cido",
"owl",
)
print("process lexicons")
merpy.process_lexicon("do", ltype="owl")
merpy.process_lexicon("go", ltype="owl")
merpy.process_lexicon("hpo", ltype="owl")
merpy.process_lexicon("chebi", ltype="owl")
merpy.process_lexicon("taxon", ltype="owl")
merpy.process_lexicon("cido", "owl")
merpy.delete_obsolete("do")
merpy.delete_obsolete("go")
merpy.delete_obsolete("hpo")
merpy.delete_obsolete("chebi")
merpy.delete_obsolete("taxon")
merpy.delete_obsolete("cido")
merpy.delete_entity("protein", "chebi")
merpy.delete_entity("protein", "cido")
def process_lexicons_4_mer():
print("download latest obo files")
merpy.download_lexicon("http://purl.obolibrary.org/obo/doid.owl",
"do",
ltype="owl")
merpy.download_lexicon("http://purl.obolibrary.org/obo/go.owl",
"go",
ltype="owl")
merpy.download_lexicon("http://purl.obolibrary.org/obo/hp.owl",
"hpo",
ltype="owl")
merpy.download_lexicon(
"ftp://ftp.ebi.ac.uk/pub/databases/chebi/ontology/chebi.owl",
"chebi",
ltype="owl")
merpy.download_lexicon("http://purl.obolibrary.org/obo/ncbitaxon.owl",
"taxon",
ltype="owl")
merpy.download_lexicon(
"https://raw.githubusercontent.com/CIDO-ontology/cido/master/src/ontology/cido.owl",
"cido",
"owl",
)
print("process lexicons")
merpy.process_lexicon("do", ltype="owl")
merpy.process_lexicon("go", ltype="owl")
merpy.process_lexicon("hpo", ltype="owl")
merpy.process_lexicon("chebi", ltype="owl")
merpy.process_lexicon("taxon", ltype="owl")
merpy.process_lexicon("cido", "owl")
#Delete obsolete entities
merpy.delete_obsolete("do")
merpy.delete_obsolete("go")
merpy.delete_obsolete("hpo")
merpy.delete_obsolete("chebi")
merpy.delete_obsolete("taxon")
merpy.delete_obsolete("cido")
merpy.delete_entity("protein", "chebi")
merpy.delete_entity("protein", "cido")
merpy.delete_entity("protein", "hpo")
merpy.delete_entity("polypeptide chain", "chebi")
merpy.delete_entity("data", "taxon")
merpy.delete_entity("one", "chebi")
merpy.delete_entity_by_uri("http://purl.obolibrary.org/obo/PATO_0000070",
"hpo")
#Create and process english vocabularies
#lexicon_name = "medic"
#medic_name_to_id = load_ctd_vocabularies("CTD_diseases.tsv")
#merpy.create_lexicon(medic_name_to_id.keys(), lexicon_name)
#merpy.create_mappings(medic_name_to_id, lexicon_name)
#merpy.process_lexicon(lexicon_name)
#lexicon_name = "ctdChemicals"
#chemicals_name_to_id = load_ctd_vocabularies("CTD_chemicals.tsv")
#merpy.create_lexicon(chemicalsbireme_decs_spa2020.xm_name_to_id.keys(), lexicon_name)
#merpy.create_mappings(chemicals_name_to_id, lexicon_name)
#merpy.process_lexicon(lexicon_name)
#lexicon_name = "ctdAnatomy"
#anatomy_name_to_id = load_ctd_vocabularies("CTD_anatomy.tsv")
#merpy.create_lexicon(anatomy_name_to_id.keys(), lexicon_name)
#merpy.create_mappings(anatomy_name_to_id, lexicon_name)
#merpy.process_lexicon(lexicon_name)
##Create and process english decs
lexicon_name = "decsEN"
name_to_id_spa = load_decs_xml("en")
merpy.create_lexicon(name_to_id_spa.keys(), lexicon_name)
merpy.create_mappings(name_to_id_spa, lexicon_name)
merpy.process_lexicon(lexicon_name)
##Create and process spanish decs
lexicon_name = "decsSPA"
name_to_id_spa = load_decs_xml("spa")
merpy.create_lexicon(name_to_id_spa.keys(), lexicon_name)
merpy.create_mappings(name_to_id_spa, lexicon_name)
merpy.process_lexicon(lexicon_name)
#Create and process portuguese decs
lexicon_name = "decsPT"
name_to_id_spa = load_decs_xml("pt")
merpy.create_lexicon(name_to_id_spa.keys(), lexicon_name)
merpy.create_mappings(name_to_id_spa, lexicon_name)
merpy.process_lexicon(lexicon_name)
def build_relations_dict():
"""Iterates over all sentences in train, dev sets recognizes CIE-O-3, ICD10-CM and DeCS entities and establish a relation
between two entities in a given sentece
Ensures:
dict stored in file './tmp/relations_cieo3_icd10cm.json' ES ICD10-CM <-> CIEO3 relations and in file
'./tmp/relations_cieo3_esdecs.json' with ES DeCS <-> CIEO3 relations
"""
#Create CIE-O-3 lexicon
lexicon_name = "cieo3"
ontology_graph, name_to_id, synonym_to_id = load_cieo3()
merpy.create_lexicon(name_to_id.keys(), lexicon_name)
merpy.create_mappings(name_to_id, lexicon_name)
merpy.process_lexicon(lexicon_name)
#Create ICD10-CM lexicon
lexicon_name = "icd10cmes"
ontology_graph, name_to_id = load_spanish_icd10cm()
merpy.create_lexicon(name_to_id.keys(), lexicon_name)
merpy.create_mappings(name_to_id, lexicon_name)
merpy.process_lexicon(lexicon_name)
#Create DECS lexicon
lexicon_name = "es_decs"
ontology_graph, name_to_id, synonym_to_id = load_es_decs()
merpy.create_lexicon(name_to_id.keys(), lexicon_name)
merpy.create_mappings(name_to_id, lexicon_name)
merpy.process_lexicon(lexicon_name)
filenames_1 = [
"./data/datasets/train-set-to-publish/cantemist-norm/" + input_file
for input_file in os.listdir(
"./data/datasets/train-set-to-publish/cantemist-norm/")
]
filenames_2 = [
"./data/datasets/dev-set1-to-publish/cantemist-norm/" + input_file
for input_file in os.listdir(
"./data/datasets/dev-set1-to-publish/cantemist-norm/")
]
filenames_3 = [
"./data/datasets/dev-set2-to-publish/cantemist-norm/" + input_file
for input_file in os.listdir(
"./data/datasets/dev-set2-to-publish/cantemist-norm/")
]
filenames_4 = [
"./data/datasets/test-background-set-to-publish/" + input_file
for input_file in os.listdir(
"./data/datasets/test-background-set-to-publish/")
]
filenames = filenames_1 + filenames_2 + filenames_3 # + filenames_4
relations_1, relations_2 = dict(), dict()
doc_count = int()
for doc in filenames:
if doc[-3:] == "txt":
#if doc == "cc_onco1016.txt":
doc_count += 1
print("DOC_COUNT:", doc_count)
with open(doc, 'r') as doc_file:
text = doc_file.read()
doc_file.close()
sentences = [Sentence(sent) for sent in split_single(text)]
for sentence in sentences:
sent_text = sentence.to_original_text()
cieo3_entities = merpy.get_entities(sent_text, "cieo3")
icd10cm_entities = merpy.get_entities(sent_text, "icd10cmes")
es_decs_entities = merpy.get_entities(sent_text, "es_decs")
if icd10cm_entities != [['']] and cieo3_entities != [['']]:
icd10cm_codes = [entity[3] for entity in icd10cm_entities]
cieo3_codes = [entity[3] for entity in cieo3_entities]
for code in cieo3_codes:
if code in relations_1:
current_values = relations_1[code]
current_values.extend(icd10cm_codes)
relations_1[code] = current_values
else:
relations_1[code] = icd10cm_codes
if es_decs_entities != [['']] and cieo3_entities != [['']]:
es_decs_codes = [entity[3] for entity in es_decs_entities]
cieo3_codes = [entity[3] for entity in cieo3_entities]
for code in cieo3_codes:
if code in relations_2:
current_values = relations_2[code]
current_values.extend(es_decs_codes)
relations_2[code] = es_decs_codes
else:
relations_2[code] = es_decs_codes
#Output the relations into json files
d = json.dumps(relations_1)
d_file = open("./tmp/relations_cieo3_icd10cm.json", 'w')
d_file.write(d)
d_file.close()
b = json.dumps(relations_2)
b_file = open("./tmp/relations_cieo3_esdecs.json", 'w')
b_file.write(b)
b_file.close()
def annotate_documents(language, name_to_id):
"""
Recognise entities (Named Entity Recognition) and link them to the respective ICD 10 CM code (Named Entity Linking), if available
Requires:
language: str, "pt", "en", "es" for Portuguese, English or Spanish, respectively
Ensures:
for each abstract in 'scielo_abstracts' dir creates an annotation file in 'mer_annotations' dir and an overall statistics file about the annotation process
"""
lexicon_name = "icd10cm_" + language
merpy.create_lexicon(name_to_id.keys(), lexicon_name)
merpy.create_mappings(name_to_id, lexicon_name)
merpy.process_lexicon(lexicon_name)
abstracts_dir = "./scielo_abstracts/"
doc_w_ann_count = int()
entity_count = int()
linked_mentions = int()
for abstract in os.listdir(abstracts_dir):
if abstract[-2:] == language:
output_string = str()
with open(abstracts_dir + abstract, 'r') as input_file:
text = input_file.read()
input_file.close()
document_ent_count = int()
entities = merpy.get_entities(text, lexicon_name)
for entity in entities:
if entity != ['']:
entity_count += 1
document_ent_count += 1
if len(entity) == 4: # linked mentions with ICD code
linked_mentions += 1
output_string += "T" + str(
document_ent_count
) + "\t" + entity[0] + " " + entity[
1] + "\t" + entity[2] + "\t" + entity[3] + "\n"
elif len(entity) == 3: # mentions without ICD code
output_string += "T" + str(
document_ent_count
) + "\t" + entity[0] + " " + entity[
1] + "\t" + entity[2] + "\n"
if document_ent_count > 0:
doc_w_ann_count += 1
output_filename = "./mer_annotations/" + language + "/" + abstract + ".ann"
with open(output_filename, 'w') as output_file:
output_file.write(output_string)
output_file.close()
try:
mentions_ratio = float(entity_count / doc_w_ann_count)
doc_linked_ratio = float(linked_mentions / doc_w_ann_count)
linked_ratio = float(linked_mentions / entity_count)
except:
mentions_ratio = 0.0
doc_linked_ratio = 0.0
linked_ratio = 0.0
output_str = "DOCUMENTS WITH ANNOTATIONS: " + str(doc_w_ann_count) + "\n"
output_str += "TOTAL ENTITY MENTIONS: " + str(entity_count) + "\n"
output_str += "ENTITY MENTIONS PER DOCUMENT: " + str(mentions_ratio) + "\n"
output_str += "LINKED ENTITY MENTIONS: " + str(linked_mentions) + "\n"
output_str += "LINKED ENTITY MENTIONS PER DOCUMENT: " + str(
doc_linked_ratio) + "\n"
output_str += "RATIO OF LINKED ENTITY MENTIONS: " + str(linked_ratio)
file_name = "mer_annotation_stats_" + language
with open(file_name, "w") as output:
output.write(output_str)
output.close()
total_entities = 0
total_sents = 0
total_docs = 0
# can be parallelized
for pmid in document_entities:
if pmid == "":
continue
if pmid not in pmid_to_abst:
print("missing this abstract:", pmid)
#import pdb; pdb.set_trace()
missing_texts += 1
continue
total_docs += 1
doc = nlp(pmid_to_abst[pmid])
merpy.create_lexicon(document_entities[pmid], "biomarker" + pmid)
merpy.process_lexicon("biomarker" + pmid)
doc_entities = merpy.get_entities(pmid_to_abst[pmid], "biomarker" + pmid)
entity_spans = []
for e in doc_entities:
try:
int(e[0]), int(e[1])
except ValueError:
print("ERROR", e)
continue
entity_spans.append(doc.char_span(int(e[0]), int(e[1]), label="GPE"))
entity_spans = [e for e in entity_spans if e is not None]
try:
doc.ents = entity_spans[:]
except:
import pdb
pdb.set_trace()
else:
pmid_to_abst[values[0]] += ". " + values[1].strip().lower()
missing_pmids = []
all_entities = set()
with open("publications_compounds.txt") as compounds_file:
next(compounds_file)
for line in compounds_file:
values = line.strip().split("\t")
pmid = values[1]
all_entities.add(values[3].lower())
if pmid not in pmid_to_abst:
missing_pmids.append(pmid)
merpy.create_lexicon(all_entities, "biomarkers")
merpy.process_lexicon("biomarkers")
#recover missing pmids:
titles, abstracts = query_pubmed.get_titles_abstracts(missing_pmids)
for t in titles:
pmid_to_abst[t[0]] = t[1].lower()
for a in abstracts:
if a[0] not in pmid_to_abst:
pmid_to_abst[a[0]] = a[1].lower()
else:
pmid_to_abst[a[0]] += ". " + a[1].lower()
missing_texts = 0
total_entities = 0
total_sents = 0
total_docs = 0
