def assess_label_quality(self, ann_dir, gold_dir, separate_by_label=True, ignore_context=True): if ignore_context: logging.info('doing learning without considering contextual info') # print self.get_top_tfidf_dimensions(self.max_dimensions) cm = self.concept_mapping file_keys = [ f.split('.')[0] for f in listdir(ann_dir) if isfile(join(ann_dir, f)) ] label_type = self.label.replace('neg_', '') query_label_perform = {} for fk in file_keys: cr = CustomisedRecoginiser(join(ann_dir, '%s.json' % fk), cm) if not isfile(join(gold_dir, '%s-ann.xml' % fk)): continue gd = EDIRDoc(join(gold_dir, '%s-ann.xml' % fk)) not_matched_gds = [] for e in gd.get_ess_entities(): if (ignore_context and e.label.replace('neg_', '') == label_type) \ or (not ignore_context and e.label == self.label): not_matched_gds.append(e.id) anns = cr.get_anns_by_label(self.label, no_context=ignore_context) for a in anns: multiple_true_positives = 0 matched = False for g in gd.get_ess_entities(): if g.id in not_matched_gds: gt = g.label.replace('neg_', '') if g.overlap(a) and ( (g.label == self.label and not ignore_context) or (ignore_context and gt == label_type)): if matched: multiple_true_positives += 1 matched = True not_matched_gds.remove(g.id) if separate_by_label: lbl = LabelModel.get_ann_query_label(a) else: lbl = 'united' ql = lbl if ql not in query_label_perform: query_label_perform[ql] = {'c': 0, 'w': 0} if matched: query_label_perform[ql]['c'] += 1 else: query_label_perform[ql]['w'] += 1 lbls = [(l, 1.0 * query_label_perform[l]['c'] / (query_label_perform[l]['c'] + query_label_perform[l]['w']), query_label_perform[l]['c'], query_label_perform[l]['w']) for l in query_label_perform] return sorted(lbls, key=lambda x: x[1])
def populate_semehr_results(label_dir, ann_dir, file_key, label2performances, using_combined=False): label_file = '%s-ann.xml' % file_key ann_file = '%s.json' % file_key print(join(label_dir, label_file)) if not isfile(join(label_dir, label_file)): return ed = EDIRDoc(join(label_dir, label_file)) cm = Concept2Mapping(_concept_mapping) cr = CustomisedRecoginiser(join(ann_dir, ann_file), cm) if using_combined: cr.validate_combined_performance(ed.get_ess_entities(), label2performances) else: cr.validate_mapped_performance(ed.get_ess_entities(), label2performances)
def save_full_text(xml_file, output_dir): """ recover full text from Informatics' xml format :param xml_file: :param output_dir: :return: """ if not isfile(xml_file): return ed = EDIRDoc(xml_file) fn = basename(xml_file) name = fn.replace(r'-ann.xml', '.txt') logging.info('%s processed to be %s' % (fn, name)) utils.save_string(ed.get_full_text, join(output_dir, name))
def get_doc_level_inference(label_dir, ann_dir, file_key, type2insts, type2inst_2, t2missed): """ learn concept to label inference from gold standard - i.e. querying SemEHR annotations to draw conclusions :param label_dir: :param ann_dir: :param file_key: :param type2insts: :param type2inst_2: :return: """ label_file = '%s-ann.xml' % file_key ann_file = '%s.json' % file_key logging.info('working on %s' % join(label_dir, label_file)) ed = EDIRDoc(join(label_dir, label_file)) if not isfile(join(label_dir, label_file)): print('not a file: %s' % join(label_dir, label_file)) return sd = SemEHRAnnDoc(join(ann_dir, ann_file)) sd.learn_mappings_from_labelled(ed, type2insts, t2missed)
def load_data(self, ann_dir, gold_dir, verbose=True, ignore_mappings=[], ignore_context=False, separate_by_label=False, ful_text_dir=None, eHostGD=False, annotated_anns={}): """ :param ann_dir: :param gold_dir: :param verbose: :param ignore_mappings: :param ignore_context: :param separate_by_label: :param ful_text_dir: :param eHostGD: :param annotated_anns: NB: this is for labelling settings where only partial data is annotated on the documents. Therefore, we need to filter out those not assessed by the annotators to avoid kill some true positives (those are correct but not assessed by annotators) :return: """ if ignore_context: logging.info('doing learning without considering contextual info') # print self.get_top_tfidf_dimensions(self.max_dimensions) cm = self.concept_mapping file_keys = [ f[:f.rfind('.')] for f in listdir(ann_dir) if isfile(join(ann_dir, f)) ] lbl2data = {} false_negatives = 0 lbl2tps = {} label_type = self.label.replace('neg_', '') query_label_perform = {} for fk in file_keys: cr = CustomisedRecoginiser(join(ann_dir, '%s.json' % fk), cm) fk = fk.replace('se_ann_', '') if ful_text_dir is not None: cr.full_text_folder = ful_text_dir if eHostGD: if not isfile(join(gold_dir, '%s.txt.knowtator.xml' % fk)): continue # logging.debug('using GD file %s' % join(gold_dir, '%s.txt.knowtator.xml' % fk)) gd = eHostGenedDoc(join(gold_dir, '%s.txt.knowtator.xml' % fk)) else: if not isfile(join(gold_dir, '%s-ann.xml' % fk)): continue logging.debug('using GD file %s' % join(gold_dir, '%s-ann.xml' % fk)) gd = EDIRDoc(join(gold_dir, '%s-ann.xml' % fk)) # re-segement sentences # cr.re_segment_sentences(fk) # cr.relocate_all_anns(fk) # gd.relocate_anns(cr.get_full_text(fk)) not_matched_gds = [] for e in gd.get_ess_entities(): if (ignore_context and e.label.replace('neg_', '') == label_type) \ or (not ignore_context and e.label == self.label): not_matched_gds.append(e.id) anns = cr.get_anns_by_label(self.label, ignore_mappings=ignore_mappings, no_context=ignore_context) if len(annotated_anns) > 0: if '%s.txt' % fk not in annotated_anns: continue kept_anns = [] for a in anns: for aa in annotated_anns['%s.txt' % fk]: if int(aa['s']) == a.start and int(aa['e']) == a.end: kept_anns.append(a) anns = kept_anns for a in anns: logging.debug('%s, %s, %s' % (a.str, a.start, a.end)) multiple_true_positives = 0 t2anns = cr.get_prior_anns(a) # if len(t2anns['umls']) + len(t2anns['phenotype']) == 0: # t2anns = cr.get_prior_anns(a, contenxt_depth=-2) context_anns = [] + t2anns['umls'] + t2anns['phenotype'] + \ cr.get_context_words(a, fk) # context_anns = cr.get_context_words(a, fk) matched = False for g in gd.get_ess_entities(): if g.id in not_matched_gds: gt = g.label.replace('neg_', '') if g.overlap(a) and ( (g.label == self.label and not ignore_context) or (ignore_context and gt == label_type)): if matched: multiple_true_positives += 1 matched = True not_matched_gds.remove(g.id) if verbose: if not matched: logging.debug( '%s %s %s' % ('!', self.get_ann_dim_label(a) + ' // ' + ' | '.join( self.get_ann_dim_label(a, generalise=True) for a in context_anns), fk)) else: logging.debug( '%s %s %s' % ('R', self.get_ann_dim_label(a) + ' // ' + ' | '.join( self.get_ann_dim_label(a, generalise=True) for a in context_anns), fk)) lbl = LabelModel.get_label_specific_data( self, lbl2data, a, context_anns, fk, cr, separate_by_label=separate_by_label) lbl2data[lbl]['multiple_tps'] += multiple_true_positives Y = lbl2data[lbl]['Y'] Y.append([1 if matched else 0]) ql = lbl if ql not in query_label_perform: query_label_perform[ql] = {'c': 0, 'w': 0} if matched: query_label_perform[ql]['c'] += 1 else: query_label_perform[ql]['w'] += 1 false_negatives += len(not_matched_gds) missed = None for g in gd.get_ess_entities(): if g.id in not_matched_gds: missed = g logging.debug('\t'.join([ 'M', g.str, str(g.negated), str(g.start), str(g.end), join(gold_dir, '%s-ann.xml' % fk) ])) # if len(not_matched_gds) > 0: # print not_matched_gds # for a in anns: # logging.debug(a.str, a.start, a.end, missed.overlap(a)) bad_labels = [] for ql in query_label_perform: p = query_label_perform[ql] if p['c'] == 0 or (1.0 * p['w'] / p['c'] < 0.05): bad_labels.append(ql) return { 'lbl2data': lbl2data, 'fns': false_negatives, 'bad_labels': bad_labels, 'files': file_keys }
def collect_tfidf_dimensions(self, ann_dir, gold_dir, ignore_context=False, separate_by_label=False, full_text_dir=None, eHostGD=False): cm = self.concept_mapping file_keys = [ f[:f.rfind('.')] for f in listdir(ann_dir) if isfile(join(ann_dir, f)) ] # collect dimension labels tp_freq = 0 fp_freq = 0 label_type = self.label.replace('neg_', '') fn_freq = 0 for fk in file_keys: cr = CustomisedRecoginiser(join(ann_dir, '%s.json' % fk), cm) fk = fk.replace('se_ann_', '') if full_text_dir is not None: cr.full_text_folder = full_text_dir if eHostGD: if not isfile(join(gold_dir, '%s.txt.knowtator.xml' % fk)): continue gd = eHostGenedDoc(join(gold_dir, '%s.txt.knowtator.xml' % fk)) else: if not isfile(join(gold_dir, '%s-ann.xml' % fk)): continue gd = EDIRDoc(join(gold_dir, '%s-ann.xml' % fk)) t = self.label.replace('neg_', '') anns = cr.get_anns_by_label(t) neg_anns = cr.get_anns_by_label('neg_' + t) # re-segement sentences # cr.re_segment_sentences(fk) # cr.relocate_all_anns(fk) # gd.relocate_anns(cr.get_full_text(fk)) not_matched_gds = [] for e in gd.get_ess_entities(): if (ignore_context and e.label.replace('neg_', '') == label_type) \ or (not ignore_context and e.label == self.label): not_matched_gds.append(e.id) for a in anns + neg_anns: # self.add_context_dimension_by_annotation(a) self.add_label_dimension_by_annotation(a) # if (not ignore_context) and ((a.negation != 'Negated' and self.label.startswith('neg_')) or \ # (a.negation == 'Negated' and not self.label.startswith('neg_'))): # logging.info('skipped because context') # continue matched = False for g in gd.get_ess_entities(): if g.id in not_matched_gds: gt = g.label.replace('neg_', '') if g.overlap(a) and ( (g.label == self.label and not ignore_context) or (ignore_context and gt == label_type)): matched = True tp_freq += 1 not_matched_gds.remove(g.id) if not matched: fp_freq += 1 sanns = cr.get_prior_anns(a, contenxt_depth=-1) context_anns = [] + sanns['umls'] + sanns[ 'phenotype'] + cr.get_context_words(a, fk) # context_anns = cr.get_context_words(a, fk) # collect cui labels for u in sanns['umls']: self._cui2label[u.cui] = u.pref for c in context_anns: self.add_context_dimension_by_annotation( c, tp=True if matched else None, fp=True if not matched else None, lbl='united' if not separate_by_label else LabelModel.get_ann_query_label(a)) fn_freq += len(not_matched_gds) self._tps = tp_freq self._fps = fp_freq logging.debug('tp: %s, fp: %s, fn: %s' % (tp_freq, fp_freq, fn_freq))