def evaluate(documents, dictionary, dictionary_reverse, vsm_model,
neural_model, ensemble_model, d, isMeddra_dict):
if vsm_model is not None:
vsm_model.eval()
if neural_model is not None:
neural_model.eval()
if ensemble_model is not None:
ensemble_model.eval()
ct_predicted = 0
ct_gold = 0
ct_correct = 0
# if opt.norm_rule and opt.norm_vsm and opt.norm_neural:
# ct_correct_rule = 0
# ct_correct_vsm = 0
# ct_correct_neural = 0
# ct_correct_all = 0
# ct_correct_rule_vsm = 0
# ct_correct_rule_neural = 0
# ct_correct_vsm_neural = 0
for document in documents:
# copy entities from gold entities
pred_entities = []
for gold in document.entities:
pred = Entity()
pred.id = gold.id
pred.type = gold.type
pred.spans = gold.spans
pred.section = gold.section
pred.name = gold.name
pred_entities.append(pred)
if opt.norm_rule and opt.norm_vsm and opt.norm_neural:
if opt.ensemble == 'learn':
ensemble_model.process_one_doc(document, pred_entities,
dictionary, dictionary_reverse,
isMeddra_dict)
else:
pred_entities2 = copy.deepcopy(pred_entities)
pred_entities3 = copy.deepcopy(pred_entities)
merge_entities = copy.deepcopy(pred_entities)
multi_sieve.runMultiPassSieve(document, pred_entities,
dictionary, isMeddra_dict)
vsm_model.process_one_doc(document, pred_entities2, dictionary,
dictionary_reverse, isMeddra_dict)
neural_model.process_one_doc(document, pred_entities3,
dictionary, dictionary_reverse,
isMeddra_dict)
elif opt.norm_rule:
multi_sieve.runMultiPassSieve(document, pred_entities, dictionary,
isMeddra_dict)
elif opt.norm_vsm:
vsm_model.process_one_doc(document, pred_entities, dictionary,
dictionary_reverse, isMeddra_dict)
elif opt.norm_neural:
neural_model.process_one_doc(document, pred_entities, dictionary,
dictionary_reverse, isMeddra_dict)
else:
raise RuntimeError("wrong configuration")
if opt.norm_rule and opt.norm_vsm and opt.norm_neural:
# ct_gold += len(document.entities)
# ct_predicted += len(pred_entities)
# up bound of ensemble, if at least one system makes a correct prediction, we count it as correct.
# for idx, gold in enumerate(document.entities):
# if (pred_entities[idx].rule_id is not None and pred_entities[idx].rule_id in gold.norm_ids)\
# and (pred_entities2[idx].vsm_id is not None and pred_entities2[idx].vsm_id in gold.norm_ids) \
# and (pred_entities3[idx].neural_id is not None and pred_entities3[idx].neural_id in gold.norm_ids):
# ct_correct_all += 1
# ct_correct += 1
#
# if (pred_entities[idx].rule_id is not None and pred_entities[idx].rule_id in gold.norm_ids)\
# and (pred_entities2[idx].vsm_id is None or pred_entities2[idx].vsm_id not in gold.norm_ids) \
# and (pred_entities3[idx].neural_id is None or pred_entities3[idx].neural_id not in gold.norm_ids):
# ct_correct_rule += 1
# ct_correct += 1
#
# if (pred_entities[idx].rule_id is None or pred_entities[idx].rule_id not in gold.norm_ids)\
# and (pred_entities2[idx].vsm_id is not None and pred_entities2[idx].vsm_id in gold.norm_ids) \
# and (pred_entities3[idx].neural_id is None or pred_entities3[idx].neural_id not in gold.norm_ids):
# ct_correct_vsm += 1
# ct_correct += 1
#
# if (pred_entities[idx].rule_id is None or pred_entities[idx].rule_id not in gold.norm_ids)\
# and (pred_entities2[idx].vsm_id is None or pred_entities2[idx].vsm_id not in gold.norm_ids) \
# and (pred_entities3[idx].neural_id is not None and pred_entities3[idx].neural_id in gold.norm_ids):
# ct_correct_neural += 1
# ct_correct += 1
#
# if (pred_entities[idx].rule_id is not None and pred_entities[idx].rule_id in gold.norm_ids)\
# and (pred_entities2[idx].vsm_id is not None and pred_entities2[idx].vsm_id in gold.norm_ids) \
# and (pred_entities3[idx].neural_id is None or pred_entities3[idx].neural_id not in gold.norm_ids):
# ct_correct_rule_vsm += 1
# ct_correct += 1
#
# if (pred_entities[idx].rule_id is not None and pred_entities[idx].rule_id in gold.norm_ids)\
# and (pred_entities2[idx].vsm_id is None or pred_entities2[idx].vsm_id not in gold.norm_ids) \
# and (pred_entities3[idx].neural_id is not None and pred_entities3[idx].neural_id in gold.norm_ids):
# ct_correct_rule_neural += 1
# ct_correct += 1
#
# if (pred_entities[idx].rule_id is None or pred_entities[idx].rule_id not in gold.norm_ids)\
# and (pred_entities2[idx].vsm_id is not None and pred_entities2[idx].vsm_id in gold.norm_ids) \
# and (pred_entities3[idx].neural_id is not None and pred_entities3[idx].neural_id in gold.norm_ids):
# ct_correct_vsm_neural += 1
# ct_correct += 1
if opt.ensemble == 'learn':
if isMeddra_dict:
p1, p2, p3 = evaluate_for_fda(document.entities,
pred_entities)
else:
p1, p2, p3 = evaluate_for_ehr(document.entities,
pred_entities, dictionary)
ct_gold += p1
ct_predicted += p2
ct_correct += p3
else:
ensemble.merge_result(pred_entities, pred_entities2,
pred_entities3, merge_entities,
dictionary, isMeddra_dict,
vsm_model.dict_alphabet, d)
if isMeddra_dict:
p1, p2, p3 = evaluate_for_fda(document.entities,
merge_entities)
else:
p1, p2, p3 = evaluate_for_ehr(document.entities,
merge_entities, dictionary)
ct_gold += p1
ct_predicted += p2
ct_correct += p3
else:
if isMeddra_dict:
p1, p2, p3 = evaluate_for_fda(document.entities, pred_entities)
else:
p1, p2, p3 = evaluate_for_ehr(document.entities, pred_entities,
dictionary)
ct_gold += p1
ct_predicted += p2
ct_correct += p3
# if opt.norm_rule and opt.norm_vsm and opt.norm_neural:
# logging.info("ensemble correct. all:{} rule:{} vsm:{} neural:{} rule_vsm:{} rule_neural:{} vsm_neural:{}"
# .format(ct_correct_all, ct_correct_rule, ct_correct_vsm, ct_correct_neural, ct_correct_rule_vsm,
# ct_correct_rule_neural, ct_correct_vsm_neural))
#
# logging.info("gold:{} pred:{} correct:{}".format(ct_gold, ct_predicted, ct_correct))
if ct_gold == 0:
precision = 0
recall = 0
else:
precision = ct_correct * 1.0 / ct_predicted
recall = ct_correct * 1.0 / ct_gold
if precision + recall == 0:
f_measure = 0
else:
f_measure = 2 * precision * recall / (precision + recall)
return precision, recall, f_measure
def test(data, opt):
corpus_dir = opt.test_file
if opt.nlp_tool == "nltk":
nlp_tool = nltk.data.load('tokenizers/punkt/english.pickle')
else:
raise RuntimeError("invalid nlp tool")
corpus_files = [f for f in os.listdir(corpus_dir) if f.find('.xml') != -1]
model = SeqModel(data, opt)
if opt.test_in_cpu:
model.load_state_dict(
torch.load(os.path.join(opt.output, 'model.pkl'), map_location='cpu'))
else:
model.load_state_dict(torch.load(os.path.join(opt.output, 'model.pkl')))
meddra_dict = load_meddra_dict(data)
# initialize norm models
if opt.norm_rule and opt.norm_vsm and opt.norm_neural: # ensemble
logging.info("use ensemble normer")
multi_sieve.init(opt, None, data, meddra_dict, None, True)
if opt.ensemble == 'learn':
if opt.test_in_cpu:
ensemble_model = torch.load(os.path.join(opt.output, 'ensemble.pkl'), map_location='cpu')
else:
ensemble_model = torch.load(os.path.join(opt.output, 'ensemble.pkl'))
ensemble_model.eval()
else:
if opt.test_in_cpu:
vsm_model = torch.load(os.path.join(opt.output, 'vsm.pkl'), map_location='cpu')
neural_model = torch.load(os.path.join(opt.output, 'norm_neural.pkl'), map_location='cpu')
else:
vsm_model = torch.load(os.path.join(opt.output, 'vsm.pkl'))
neural_model = torch.load(os.path.join(opt.output, 'norm_neural.pkl'))
vsm_model.eval()
neural_model.eval()
elif opt.norm_rule:
logging.info("use rule-based normer")
multi_sieve.init(opt, None, data, meddra_dict)
elif opt.norm_vsm:
logging.info("use vsm-based normer")
if opt.test_in_cpu:
vsm_model = torch.load(os.path.join(opt.output, 'vsm.pkl'), map_location='cpu')
else:
vsm_model = torch.load(os.path.join(opt.output, 'vsm.pkl'))
vsm_model.eval()
elif opt.norm_neural:
logging.info("use neural-based normer")
if opt.test_in_cpu:
neural_model = torch.load(os.path.join(opt.output, 'norm_neural.pkl'), map_location='cpu')
else:
neural_model = torch.load(os.path.join(opt.output, 'norm_neural.pkl'))
neural_model.eval()
else:
logging.info("no normalization is performed.")
makedir_and_clear(opt.predict)
ct_success = 0
ct_error = 0
for fileName in corpus_files:
try:
start = time.time()
document, annotation_file = processOneFile_fda(fileName, corpus_dir, nlp_tool, False, opt.types, opt.type_filter, True, False)
pred_entities = []
for section in document:
data.test_texts = []
data.test_Ids = []
read_instance_from_one_document(section, data.word_alphabet, data.char_alphabet, data.label_alphabet,
data.test_texts, data.test_Ids, data)
_, _, _, _, _, pred_results, _ = evaluate(data, opt, model, 'test', False, opt.nbest)
entities = translateResultsintoEntities(section.sentences, pred_results)
# remove the entity in the ignore_region and fill section_id
section_id = section.name[section.name.rfind('_')+1: ]
entities = remove_entity_in_the_ignore_region(annotation_file.ignore_regions, entities, section_id)
if opt.norm_rule and opt.norm_vsm and opt.norm_neural:
if opt.ensemble == 'learn':
ensemble_model.process_one_doc(section, entities, meddra_dict, None, True)
else:
pred_entities1 = copy.deepcopy(entities)
pred_entities2 = copy.deepcopy(entities)
pred_entities3 = copy.deepcopy(entities)
multi_sieve.runMultiPassSieve(section, pred_entities1, meddra_dict, True)
vsm_model.process_one_doc(section, pred_entities2, meddra_dict, None, True)
neural_model.process_one_doc(section, pred_entities3, meddra_dict, None, True)
# merge pred_entities1, pred_entities2, pred_entities3 into entities
ensemble.merge_result(pred_entities1, pred_entities2, pred_entities3, entities, meddra_dict, True, vsm_model.dict_alphabet, data)
elif opt.norm_rule:
multi_sieve.runMultiPassSieve(section, entities, meddra_dict, True)
elif opt.norm_vsm:
vsm_model.process_one_doc(section, entities, meddra_dict, None, True)
elif opt.norm_neural:
neural_model.process_one_doc(section, entities, meddra_dict, None, True)
for entity in entities:
if len(entity.norm_ids)!=0: # if a mention can't be normed, not output it
pred_entities.append(entity)
dump_results(fileName, pred_entities, opt, annotation_file)
end = time.time()
logging.info("process %s complete with %.2fs" % (fileName, end - start))
ct_success += 1
except Exception as e:
logging.error("process file {} error: {}".format(fileName, e))
ct_error += 1
if opt.norm_rule:
multi_sieve.finalize(True)
logging.info("test finished, total {}, error {}".format(ct_success + ct_error, ct_error))
def test(data, opt):
# corpus_dir = join(opt.test_file, 'corpus')
# corpus_dir = join(opt.test_file, 'txt')
corpus_dir = opt.test_file
if opt.nlp_tool == "spacy":
nlp_tool = spacy.load('en')
elif opt.nlp_tool == "nltk":
nlp_tool = nltk.data.load('tokenizers/punkt/english.pickle')
elif opt.nlp_tool == "stanford":
nlp_tool = StanfordCoreNLP('http://localhost:{0}'.format(9000))
else:
raise RuntimeError("invalid nlp tool")
corpus_files = [
f for f in listdir(corpus_dir) if isfile(join(corpus_dir, f))
]
model = SeqModel(data, opt)
if opt.test_in_cpu:
model.load_state_dict(
torch.load(os.path.join(opt.output, 'model.pkl'),
map_location='cpu'))
else:
model.load_state_dict(torch.load(os.path.join(opt.output,
'model.pkl')))
dictionary, dictionary_reverse = umls.load_umls_MRCONSO(
data.config['norm_dict'])
isMeddra_dict = False
# initialize norm models
if opt.norm_rule and opt.norm_vsm and opt.norm_neural: # ensemble
logging.info("use ensemble normer")
multi_sieve.init(opt, None, data, dictionary, dictionary_reverse,
False)
if opt.ensemble == 'learn':
if opt.test_in_cpu:
ensemble_model = torch.load(os.path.join(
opt.output, 'ensemble.pkl'),
map_location='cpu')
else:
ensemble_model = torch.load(
os.path.join(opt.output, 'ensemble.pkl'))
ensemble_model.eval()
else:
if opt.test_in_cpu:
vsm_model = torch.load(os.path.join(opt.output, 'vsm.pkl'),
map_location='cpu')
neural_model = torch.load(os.path.join(opt.output,
'norm_neural.pkl'),
map_location='cpu')
else:
vsm_model = torch.load(os.path.join(opt.output, 'vsm.pkl'))
neural_model = torch.load(
os.path.join(opt.output, 'norm_neural.pkl'))
vsm_model.eval()
neural_model.eval()
elif opt.norm_rule:
logging.info("use rule-based normer")
multi_sieve.init(opt, None, data, dictionary, dictionary_reverse,
False)
elif opt.norm_vsm:
logging.info("use vsm-based normer")
if opt.test_in_cpu:
vsm_model = torch.load(os.path.join(opt.output, 'vsm.pkl'),
map_location='cpu')
else:
vsm_model = torch.load(os.path.join(opt.output, 'vsm.pkl'))
vsm_model.eval()
elif opt.norm_neural:
logging.info("use neural-based normer")
if opt.test_in_cpu:
neural_model = torch.load(os.path.join(opt.output,
'norm_neural.pkl'),
map_location='cpu')
else:
neural_model = torch.load(
os.path.join(opt.output, 'norm_neural.pkl'))
neural_model.eval()
else:
logging.info("no normalization is performed.")
makedir_and_clear(opt.predict)
ct_success = 0
ct_error = 0
for fileName in corpus_files:
try:
start = time.time()
document, _, _, _ = processOneFile(fileName, None, corpus_dir,
nlp_tool, False, opt.types,
opt.type_filter)
data.test_texts = []
data.test_Ids = []
read_instance_from_one_document(document, data.word_alphabet,
data.char_alphabet,
data.label_alphabet,
data.test_texts, data.test_Ids,
data)
_, _, _, _, _, pred_results, _ = evaluate(data, opt, model, 'test',
False, opt.nbest)
entities = translateResultsintoEntities(document.sentences,
pred_results)
if opt.norm_rule and opt.norm_vsm and opt.norm_neural:
if opt.ensemble == 'learn':
ensemble_model.process_one_doc(document, entities,
dictionary,
dictionary_reverse,
isMeddra_dict)
else:
pred_entities1 = copy.deepcopy(entities)
pred_entities2 = copy.deepcopy(entities)
pred_entities3 = copy.deepcopy(entities)
multi_sieve.runMultiPassSieve(document, pred_entities1,
dictionary, isMeddra_dict)
vsm_model.process_one_doc(document, pred_entities2,
dictionary, dictionary_reverse,
isMeddra_dict)
neural_model.process_one_doc(document, pred_entities3,
dictionary,
dictionary_reverse,
isMeddra_dict)
# merge pred_entities1, pred_entities2, pred_entities3 into entities
ensemble.merge_result(pred_entities1, pred_entities2,
pred_entities3, entities, dictionary,
isMeddra_dict,
vsm_model.dict_alphabet, data)
elif opt.norm_rule:
multi_sieve.runMultiPassSieve(document, entities, dictionary,
isMeddra_dict)
elif opt.norm_vsm:
vsm_model.process_one_doc(document, entities, dictionary,
dictionary_reverse, isMeddra_dict)
elif opt.norm_neural:
neural_model.process_one_doc(document, entities, dictionary,
dictionary_reverse, isMeddra_dict)
dump_results(fileName, entities, opt)
end = time.time()
logging.info("process %s complete with %.2fs" %
(fileName, end - start))
ct_success += 1
except Exception as e:
logging.error("process file {} error: {}".format(fileName, e))
ct_error += 1
logging.info("test finished, total {}, error {}".format(
ct_success + ct_error, ct_error))