def itersplit(collection: bioc.BioCCollection,
num_doc: int) -> Generator[bioc.BioCCollection, None, None]:
subc = bioc.BioCCollection()
subc = subc.copy_infon(collection)
for doc in collection.documents:
subc.add_document(doc)
if len(subc.documents) == num_doc:
yield subc
subc = bioc.BioCCollection()
subc = subc.copy_infon(collection)
if subc.documents:
yield subc
def split(source,
*,
prefix: str,
num_doc: int,
additional_suffix: str = '.xml',
suffix_length: int = 2):
path_format = prefix + '{:0' + str(
suffix_length) + 'x}' + additional_suffix
with open(source, encoding='utf8') as fp:
collection = bioc.load(fp)
newc = bioc.BioCCollection()
newc.infons = collection.infons
newc.source = collection.source
newc.version = collection.version
newc.source = collection.source
newc.standalone = collection.standalone
i = 0
for doc in tqdm.tqdm(collection.documents):
newc.add_document(doc)
if len(newc.documents) == num_doc:
dst = path_format.format(i)
with open(dst, 'w', encoding='utf8') as fp:
bioc.dump(newc, fp)
del newc.documents[:]
i += 1
if newc.documents:
dst = path_format.format(i)
with open(dst, 'w', encoding='utf8') as fp:
bioc.dump(newc, fp)
def dump_results(doc_name, entities, opt):
entity_id = 1
collection = bioc.BioCCollection()
document = bioc.BioCDocument()
collection.add_document(document)
document.id = doc_name
passage = bioc.BioCPassage()
document.add_passage(passage)
passage.offset = 0
for entity in entities:
anno_entity = bioc.BioCAnnotation()
passage.add_annotation(anno_entity)
anno_entity.id = str(entity_id)
entity_id += 1
anno_entity.infons['type'] = entity.type
anno_entity_location = bioc.BioCLocation(
entity.spans[0][0], entity.spans[0][1] - entity.spans[0][0])
anno_entity.add_location(anno_entity_location)
anno_entity.text = entity.name
if len(entity.norm_ids) > 0:
anno_entity.infons['UMLS code'] = entity.norm_ids[0]
anno_entity.infons['UMLS term'] = entity.norm_names[0]
else:
anno_entity.infons['UMLS code'] = 'N/A'
anno_entity.infons['UMLS term'] = 'N/A'
with codecs.open(os.path.join(opt.predict, doc_name + ".bioc.xml"), 'w',
'UTF-8') as fp:
bioc.dump(collection, fp)
def save_as_collection(list_of_pymedext_documents: List[Document]):
"""save a list of pymedext document as a bioc collection .
It will return a bioc collection object.
:param list_of_pymedext_documents: a list of Document
:returns: a bioc collection object
"""
this_bioc_collection = bioc.BioCCollection()
for this_pymedext_doc in list_of_pymedext_documents:
this_bioc_doc = bioc.BioCDocument()
for annot in this_pymedext_doc.annotations:
# print(annot.type)
print(annot.source)
if annot.type == "raw_text":
if this_bioc_collection.source == '':
this_bioc_collection.source = annot.source
if annot.source == "BioCPassage":
print(annot.ngram)
print(annot.value)
this_passage = bioc.BioCPassage()
this_passage.text = annot.ngram
this_passage.offset = annot.span[0]
this_bioc_doc.add_passage(this_passage)
# passageAttributes to add
elif annot.source == "BioCAnnotation":
this_annotation = bioc.BioCAnnotation()
this_annotation.infons = annot.attributes
this_annotation.id = annot.attributes["id"]
this_annotation.text = annot.ngram
thisLocation = bioc.BioCLocation(
annot.span[0], annot.span[1] - annot.span[0])
this_annotation.add_location(thisLocation)
this_bioc_doc.passages[-1].add_annotation(this_annotation)
this_bioc_collection.add_document(this_bioc_doc)
return (this_bioc_collection)
def convertKindredCorpusToBioCCollection(corpus):
assert isinstance(corpus,kindred.Corpus)
collection = bioc.BioCCollection()
for kdoc in corpus.documents:
assert isinstance(kdoc,kindred.Document)
biocDoc = bioc.BioCDocument()
collection.add_document(biocDoc)
if 'id' in kdoc.metadata:
biocDoc.id = kdoc.metadata['id']
biocDoc.infons = kdoc.metadata
passage = bioc.BioCPassage()
passage.text = kdoc.text
passage.offset = 0
biocDoc.add_passage(passage)
seenEntityIDs = set()
kindredID2BiocID = {}
for e in kdoc.entities:
assert isinstance(e,kindred.Entity)
a = bioc.BioCAnnotation()
a.text = e.text
a.infons = {'type':e.entityType}
a.infons.update(e.metadata)
if e.sourceEntityID is None:
a.id = str(e.entityID)
else:
a.id = e.sourceEntityID
assert not a.id in seenEntityIDs, "Multiple entities with the same ID (%s) found" % a.id
seenEntityIDs.add(a.id)
kindredID2BiocID[e.entityID] = a.id
for start,end in e.position:
l = bioc.BioCLocation(offset=start, length=(end-start))
a.locations.append(l)
passage.annotations.append(a)
for r in kdoc.relations:
assert isinstance(r,kindred.Relation)
biocR = bioc.BioCRelation()
biocR.infons = {'type':r.relationType}
entitiesInRelation = r.entities
argNames = r.argNames
if argNames is None:
argNames = [ "arg%d" % i for i,_ in enumerate(entitiesInRelation) ]
for argName,entity in zip(argNames,entitiesInRelation):
node = bioc.BioCNode(role=argName, refid=kindredID2BiocID[entity.entityID])
biocR.nodes.append(node)
passage.relations.append(biocR)
return collection
def load(self):
"""Load and clean the reports."""
collection = bioc.BioCCollection()
reports = pd.read_csv(self.reports_path,
header=None,
names=[REPORTS])[REPORTS].tolist()
for i, report in enumerate(reports):
clean_report = self.clean(report)
document = text2bioc.text2document(str(i), clean_report)
if self.extract_impression:
document = section_split.split_document(document)
self.extract_impression_from_passages(document)
split_document = self.splitter.split_doc(document)
assert len(split_document.passages) == 1,\
('Each document must have a single passage, ' +
'the Impression section.')
collection.add_document(split_document)
self.reports = reports
self.collection = collection
def get_bioc_collection(df):
collection = bioc.BioCCollection()
splitter = NegBioSSplitter()
for i, report in enumerate(df["Report Impression"]):
document = text2bioc.text2document(str(i), report)
document = splitter.split_doc(document)
collection.add_document(document)
return collection
def get_collection(total_doc):
c = bioc.BioCCollection()
c.source = 'source'
for i in range(total_doc):
text = get_random_str()
doc = bioc.BioCDocument.of_text(text)
c.add_document(doc)
return c
def collection(this, json_collection):
collection = bioc.BioCCollection()
collection.source = json_collection['source']
collection.date = json_collection['date']
collection.key = json_collection['key']
collection.infons = json_collection['infons']
collection.documents = [this.document(d)
for d in json_collection['documents']]
return collection
def load(self):
collection = bioc.BioCCollection()
for i, sentence in enumerate(self.report):
clean_report = self.clean(sentence)
document = text2bioc.text2document(str(i), clean_report)
split_document = self.splitter.split_doc(document)
assert len(split_document.passages) == 1,\
('Each document must have a single passage, ' +
'the Impression section.')
collection.add_document(split_document)
self.collection = collection
def export_document(document_id, users):
bcollection = bioc.BioCCollection()
for user_id in users:
document = load_document(document_id, user_id)
bdocument = create_bioc_document_from_document_json(document)
bcollection.add_document(bdocument)
result = bcollection.tobioc()
response = Response(result, mimetype='text/xml')
response.headers[
"Content-Disposition"] = "attachment; filename=" + document_id + ".xml"
return response
def get_figure_text(src, dest, bioc_dir):
df = pd.read_csv(src, dtype=str)
objs = df_to_obj(df)
# add text
objs = add_text(objs, bioc_dir)
collection = bioc.BioCCollection()
for obj in objs:
collection.documents.extend(obj.to_bioc_document())
with open(dest, 'w', encoding='utf8') as fp:
bioc.dump(collection, fp)
def text_to_bioc(list_of_text, type, **kwargs):
if type == 'p/s':
offset = 0
passage = bioc.BioCPassage()
passage.offset = offset
for s in list_of_text:
sentence = bioc.BioCSentence()
sentence.offset = offset
sentence.text = s
offset += len(s) + 1
passage.add_sentence(sentence)
return passage
elif type == 'd/p/s':
document = bioc.BioCDocument()
passage = text_to_bioc(list_of_text, 'p/s')
document.add_passage(passage)
return document
elif type == 'c/d/p/s':
c = bioc.BioCCollection()
d = text_to_bioc(list_of_text, 'd/p/s')
c.add_document(d)
return c
elif type == 'd/p':
document = bioc.BioCDocument()
offset = 0
for s in list_of_text:
passage = bioc.BioCPassage()
passage.offset = offset
offset += len(s) + 1
passage.text = s
document.add_passage(passage)
return document
elif type == 'c/d/p':
c = bioc.BioCCollection()
d = text_to_bioc(list_of_text, 'd/p')
c.add_document(d)
return c
else:
raise KeyError
def translateNCRFPPintoBioc(doc_token, predict_results, file_name):
collection = bioc.BioCCollection()
document = bioc.BioCDocument()
collection.add_document(document)
document.id = file_name
passage = bioc.BioCPassage()
document.add_passage(passage)
passage.offset = 0
entity_id = 1
sent_num = len(predict_results)
for idx in range(sent_num):
sent_length = len(predict_results[idx][0])
sent_token = doc_token[(doc_token['sent_idx'] == idx)]
assert sent_token.shape[0] == sent_length, "file {}, sent {}".format(
file_name, idx)
labelSequence = []
for idy in range(sent_length):
token = sent_token.iloc[idy]
label = predict_results[idx][0][idy]
labelSequence.append(label)
if label[0] == 'S' or label[0] == 'B':
anno_entity = bioc.BioCAnnotation()
passage.add_annotation(anno_entity)
anno_entity.id = str(entity_id)
anno_entity.infons['type'] = label[2:]
anno_entity_location = bioc.BioCLocation(
token['start'], token['end'] - token['start'])
anno_entity.add_location(anno_entity_location)
anno_entity.text = token['text']
entity_id += 1
elif label[0] == 'M' or label[0] == 'E':
if checkWrongState(labelSequence):
anno_entity = passage.annotations[-1]
whitespacetoAdd = token['start'] - anno_entity.locations[
0].end
for _ in range(whitespacetoAdd):
anno_entity.text += " "
anno_entity.text += token['text']
anno_entity.locations[0].length = token[
'end'] - anno_entity.locations[0].offset
bioc_file = open(file_name + ".bioc.xml", 'w')
bioc.dump(collection, bioc_file)
bioc_file.close()
def text2collection(*sources, **kwargs):
"""
Returns a BioCCollection containing documents specified in sources.
Args:
split_document(bool): see text2document
"""
split_document = kwargs.pop('split_document')
collection = bioc.BioCCollection()
for pathname in iter(*sources):
with open(pathname) as fp:
text = fp.read()
id = os.path.splitext(os.path.basename(pathname))[0]
document = text2document(id, text, split_document)
collection.add_document(document)
return collection
def save_predictions(ids, relevant, confidence, output):
collection = bioc.BioCCollection()
collection.source = 'PubMed'
now = datetime.datetime.now()
collection.date = '{}{:02d}{:02d}'.format(now.year, now.month, now.day)
collection.key = 'collection.key'
for i, id in enumerate(ids):
document = bioc.BioCDocument()
document.id = id
document.infons['relevant'] = 'no' if relevant[i] == 0 else 'yes'
if relevant[i] == 1:
document.infons['confidence'] = '{:.2f}'.format(confidence[i][0])
else:
document.infons['confidence'] = '{:.2f}'.format(
1 - confidence[i][0])
collection.add_document(document)
bioc.dump(collection, open(output, 'w'), pretty_print=True)
def prep_collection(self):
"""Apply splitter and create bioc collection"""
collection = bioc.BioCCollection()
for i, report in enumerate(self.reports):
clean_report = self.clean(report)
document = text2bioc.text2document(str(i), clean_report)
if self.extract_impression:
document = section_split.split_document(document)
self.extract_impression_from_passages(document)
split_document = self.splitter.split_doc(document)
assert len(split_document.passages) == 1,\
('Each document must have a single passage, ' +
'the Impression section.')
collection.add_document(split_document)
self.collection = collection
def text2collection(*sources):
"""
Returns a BioCCollection containing documents specified in sources.
Args:
sources: a list of pathname
"""
collection = bioc.BioCCollection()
for pathname in sources:
logging.debug('Process %s', pathname)
try:
with open(pathname) as fp:
text = fp.read()
id = os.path.splitext(os.path.basename(pathname))[0]
document = text2document(id, text)
collection.add_document(document)
except:
logging.exception('Cannot convert %s', pathname)
return collection
def to_bioc_xml(self, outdir):
outpath = os.path.join(outdir, self.file_name + '.bioc.xml')
writer = bioc.BioCWriter()
writer.collection = bioc.BioCCollection()
collection = writer.collection
document = bioc.BioCDocument()
document.id = self.file_name
passage = bioc.BioCPassage()
passage.offset = '0'
document.add_passage(passage)
collection.add_document(document)
# Add annotations that already have bioc annotations
for anno in self.get_annotations():
passage.add_annotation(anno.bioc_anno)
for relat in self.get_relations():
# Create new BioCRelation
relation = bioc.bioc_relation.BioCRelation()
relation.id = relat.id
relation.put_infon('type', relat.type)
# Reference that nodes that contain the annotations
node1 = bioc.bioc_node.BioCNode()
node1.role = 'annotation 1'
node1.refid = relat.annotation_1.id
relation.add_node(node1)
node2 = bioc.bioc_node.BioCNode()
node2.role = 'annotation 2'
node2.refid = relat.annotation_2.id
relation.add_node(node2)
passage.add_relation(relation)
writer.write(outpath)
def evaluate_via_bioc(test_docs,
crf,
extractor,
prediction_dir,
made_base_dir=None):
print('Total documents for evaluation : {}'.format(len(test_docs)))
if not os.path.exists(prediction_dir):
os.makedirs(prediction_dir)
existing_files = glob.glob('{0}/*'.format(prediction_dir))
existing_files_removed = 0
for f in existing_files:
os.remove(f)
existing_files_removed += 1
print('Existing files removed : {}'.format(existing_files_removed))
prediction_documents_written = 0
reference_filenames = []
for test_doc in test_docs:
#print('Working on document : {}'.format(test_doc.filename))
collection = bioc.BioCCollection()
document = bioc.BioCDocument()
document.id = test_doc.filename
collection.add_document(document)
passage = bioc.BioCPassage()
passage.offset = 0
document.add_passage(passage)
next_annotation_id = 1
# now an annotation can be written for each label prediction
for sentence in test_doc.tokenized_doc.sentences:
sentence_tokens = []
# gather tokens in a sentence
for token_offset_pair in sentence:
token = test_doc.text[
token_offset_pair[0]:token_offset_pair[1]]
sentence_tokens.append(token)
if len(sentence_tokens) == 0:
continue
sentence_features = extractor.sent2features(sentence_tokens)
sentence_pred = crf.predict([sentence_features])[0]
if len(sentence_pred) != len(sentence):
print('Sentence Features Length : {}'.format(
len(sentence_features)))
print('Sentence Pred Length : {}'.format(len(sentence_pred)))
print('Sentence Length : {}'.format(len(sentence)))
# walk manually through the predictions and add spans as appropriate
token_idx = 0
while token_idx < len(sentence_pred):
token_pred = sentence_pred[token_idx]
if token_pred != 'O':
base_label = token_pred.replace('B-', '').replace('I-', '')
start_offset = sentence[token_idx][0]
end_offset = sentence[token_idx][1]
# now let's look to the right as long as we see tokens which are part of this same label
while token_idx + 1 < len(sentence_pred) and sentence_pred[
token_idx + 1] == ('I-' + base_label):
# advance the token
token_idx += 1
# update the end of this span
end_offset = sentence[token_idx][1]
# finally we have an annotation that we can add
annotation = bioc.BioCAnnotation()
annotation.infons['type'] = base_label
annotation.text = test_doc.text[start_offset:end_offset]
# current reference replaces newlines with literal '\n'
annotation.text = annotation.text.replace('\n',
'\\n').replace(
'\r', '\\r')
annotation.id = str(next_annotation_id)
location = bioc.BioCLocation(start_offset,
end_offset - start_offset)
next_annotation_id += 1
annotation.add_location(location)
passage.add_annotation(annotation)
# advance the token no matter what happened above
token_idx += 1
prediction_filename = os.path.join(
prediction_dir, '{}.bioc.xml'.format(test_doc.filename))
if made_base_dir is not None:
reference_filename = os.path.join(
os.path.join(made_base_dir, 'annotations'),
'{}.bioc.xml'.format(test_doc.filename))
reference_filenames.append(reference_filename)
with open(prediction_filename, 'w') as fp:
bioc.dump(collection, fp)
prediction_documents_written += 1
print('Total prediction documents written : {}'.format(
prediction_documents_written))
# finally we can invoke some evaluation (if enabled)
if made_base_dir is not None:
annotation_dir = os.path.join(made_base_dir, 'annotations')
text_dir = os.path.join(made_base_dir, 'corpus')
# first param can be an actual directory (string) or a list of filepaths
get_f_scores(reference_filenames, prediction_dir, text_dir)
def predict(opt, data):
seq_model = SeqModel(data)
if opt.test_in_cpu:
seq_model.load_state_dict(torch.load(os.path.join(opt.output, 'ner_model.pkl'), map_location='cpu'))
else:
cuda_src = 'cuda:{}'.format(opt.old_gpu)
cuda_dst = 'cuda:{}'.format(opt.gpu)
seq_model.load_state_dict(torch.load(os.path.join(opt.output, 'ner_model.pkl'), map_location={cuda_src:cuda_dst}))
seq_wordseq = WordSequence(data, False, True, True, True)
if opt.test_in_cpu:
seq_wordseq.load_state_dict(torch.load(os.path.join(opt.output, 'ner_wordseq.pkl'), map_location='cpu'))
else:
cuda_src = 'cuda:{}'.format(opt.old_gpu)
cuda_dst = 'cuda:{}'.format(opt.gpu)
seq_wordseq.load_state_dict(torch.load(os.path.join(opt.output, 'ner_wordseq.pkl'), map_location={cuda_src:cuda_dst}))
classify_model = ClassifyModel(data)
if opt.test_in_cpu:
classify_model.load_state_dict(torch.load(os.path.join(opt.output, 're_model.pkl'), map_location='cpu'))
else:
cuda_src = 'cuda:{}'.format(opt.old_gpu)
cuda_dst = 'cuda:{}'.format(opt.gpu)
classify_model.load_state_dict(torch.load(os.path.join(opt.output, 're_model.pkl'), map_location={cuda_src:cuda_dst}))
classify_wordseq = WordSequence(data, True, False, True, False)
if opt.test_in_cpu:
classify_wordseq.load_state_dict(torch.load(os.path.join(opt.output, 're_wordseq.pkl'), map_location='cpu'))
else:
cuda_src = 'cuda:{}'.format(opt.old_gpu)
cuda_dst = 'cuda:{}'.format(opt.gpu)
classify_wordseq.load_state_dict(torch.load(os.path.join(opt.output, 're_wordseq.pkl'), map_location={cuda_src:cuda_dst}))
input_files = [f for f in listdir(opt.input) if isfile(join(opt.input,f)) and f[0]!='.']
# for idx in tqdm(range(len(input_files))):
for idx in range(len(input_files)):
start = time.time()
fileName = join(opt.input,input_files[idx])
doc_name = input_files[idx]
doc_token = processOneFile(fileName)
doc = generateDataForOneFile(doc_token)
raw_texts, raw_Ids = read_instance(doc, data.word_alphabet, data.char_alphabet,
data.feature_alphabets, data.label_alphabet,
data.number_normalized,
data.MAX_SENTENCE_LENGTH)
decode_results = evaluateWhenTest(data, seq_wordseq, seq_model, raw_Ids)
entities = ner.translateNCRFPPintoEntities(doc_token, decode_results, doc_name)
collection = bioc.BioCCollection()
document = bioc.BioCDocument()
collection.add_document(document)
document.id = doc_name
passage = bioc.BioCPassage()
document.add_passage(passage)
passage.offset = 0
for entity in entities:
anno_entity = bioc.BioCAnnotation()
passage.add_annotation(anno_entity)
anno_entity.id = entity.id
anno_entity.infons['type'] = entity.type
anno_entity_location = bioc.BioCLocation(entity.start, entity.getlength())
anno_entity.add_location(anno_entity_location)
anno_entity.text = entity.text
test_X, test_other = relation_extraction.getRelationInstanceForOneDoc(doc_token, entities, doc_name, data)
relations = relation_extraction.evaluateWhenTest(classify_wordseq, classify_model, test_X, data, test_other, data.re_feature_alphabets[data.re_feature_name2id['[RELATION]']])
for relation in relations:
bioc_relation = bioc.BioCRelation()
passage.add_relation(bioc_relation)
bioc_relation.id = relation.id
bioc_relation.infons['type'] = relation.type
node1 = bioc.BioCNode(relation.node1.id, 'argument 1')
bioc_relation.add_node(node1)
node2 = bioc.BioCNode(relation.node2.id, 'argument 2')
bioc_relation.add_node(node2)
with open(os.path.join(opt.predict, doc_name + ".bioc.xml"), 'w') as fp:
bioc.dump(collection, fp)
end = time.time()
logging.info("process %s complete with %.2fs" % (input_files[idx], end-start))
logging.info("test finished")
def test(data, opt, predict_dir):
test_token, test_entity, test_relation, test_name = preprocess.loadPreprocessData(
data.test_dir)
# evaluate on test data and output results in bioc format, one doc one file
data.load(opt.data_file)
data.MAX_SENTENCE_LENGTH = -1
data.show_data_summary()
data.fix_alphabet()
seq_model = SeqModel(data)
seq_model.load_state_dict(
torch.load(os.path.join(opt.ner_dir, 'model.pkl')))
ner_hiddenlist = []
for i in range(opt.hidden_num):
if i == 0:
input_size = data.word_emb_dim+data.HP_char_hidden_dim+data.feature_emb_dims[data.feature_name2id['[Cap]']]+ \
data.feature_emb_dims[data.feature_name2id['[POS]']]
output_size = data.HP_hidden_dim
else:
input_size = data.HP_hidden_dim
output_size = data.HP_hidden_dim
temp = HiddenLayer(data, input_size, output_size)
temp.load_state_dict(
torch.load(os.path.join(opt.ner_dir, 'hidden_{}.pkl'.format(i))))
ner_hiddenlist.append(temp)
ner_wordrep = WordRep(data, False, True, True, data.use_char)
ner_wordrep.load_state_dict(
torch.load(os.path.join(opt.ner_dir, 'wordrep.pkl')))
classify_model = ClassifyModel(data)
classify_model.load_state_dict(
torch.load(os.path.join(opt.re_dir, 'model.pkl')))
re_hiddenlist = []
for i in range(opt.hidden_num):
if i == 0:
input_size = data.word_emb_dim + data.feature_emb_dims[data.feature_name2id['[POS]']]+\
2*data.re_feature_emb_dims[data.re_feature_name2id['[POSITION]']]
output_size = data.HP_hidden_dim
else:
input_size = data.HP_hidden_dim
output_size = data.HP_hidden_dim
temp = HiddenLayer(data, input_size, output_size)
temp.load_state_dict(
torch.load(os.path.join(opt.re_dir, 'hidden_{}.pkl'.format(i))))
re_hiddenlist.append(temp)
re_wordrep = WordRep(data, True, False, True, False)
re_wordrep.load_state_dict(
torch.load(os.path.join(opt.re_dir, 'wordrep.pkl')))
for i in tqdm(range(len(test_name))):
doc_name = test_name[i]
doc_token = test_token[i]
doc_entity = test_entity[i]
if opt.use_gold_ner:
entities = []
for _, e in doc_entity.iterrows():
entity = Entity()
entity.create(e['id'], e['type'], e['start'], e['end'],
e['text'], e['sent_idx'], e['tf_start'],
e['tf_end'])
entities.append(entity)
else:
ncrf_data = ner.generateDataForOneDoc(doc_token, doc_entity)
data.raw_texts, data.raw_Ids = ner.read_instanceFromBuffer(
ncrf_data, data.word_alphabet, data.char_alphabet,
data.feature_alphabets, data.label_alphabet,
data.number_normalized, data.MAX_SENTENCE_LENGTH)
decode_results = ner_evaluateWhenTest(data, ner_wordrep,
ner_hiddenlist, seq_model)
entities = ner.translateNCRFPPintoEntities(doc_token,
decode_results,
doc_name)
collection = bioc.BioCCollection()
document = bioc.BioCDocument()
collection.add_document(document)
document.id = doc_name
passage = bioc.BioCPassage()
document.add_passage(passage)
passage.offset = 0
for entity in entities:
anno_entity = bioc.BioCAnnotation()
passage.add_annotation(anno_entity)
anno_entity.id = entity.id
anno_entity.infons['type'] = entity.type
anno_entity_location = bioc.BioCLocation(entity.start,
entity.getlength())
anno_entity.add_location(anno_entity_location)
anno_entity.text = entity.text
test_X, test_other = relation_extraction.getRelationInstanceForOneDoc(
doc_token, entities, doc_name, data)
relations = re_evaluateWhenTest(
re_wordrep, re_hiddenlist, classify_model, test_X, data,
test_other,
data.re_feature_alphabets[data.re_feature_name2id['[RELATION]']])
for relation in relations:
bioc_relation = bioc.BioCRelation()
passage.add_relation(bioc_relation)
bioc_relation.id = relation.id
bioc_relation.infons['type'] = relation.type
node1 = bioc.BioCNode(relation.node1.id, 'annotation 1')
bioc_relation.add_node(node1)
node2 = bioc.BioCNode(relation.node2.id, 'annotation 2')
bioc_relation.add_node(node2)
with open(os.path.join(predict_dir, doc_name + ".bioc.xml"),
'w') as fp:
bioc.dump(collection, fp)
def test2(test_token, test_entity, test_relation, test_name, result_dumpdir):
logging.info("loading ... vocab")
relation_vocab = pickle.load(
open(os.path.join(opt.pretrain, 'relation_vocab.pkl'), 'rb'))
logging.info("loading ... result")
results = pickle.load(open(os.path.join(opt.output, 'results.pkl'), "rb"))
for i in tqdm(range(len(test_relation))):
doc_entity = test_entity[i]
doc_name = test_name[i]
collection = bioc.BioCCollection()
document = bioc.BioCDocument()
collection.add_document(document)
document.id = doc_name
passage = bioc.BioCPassage()
document.add_passage(passage)
passage.offset = 0
for _, entity in doc_entity.iterrows():
anno_entity = bioc.BioCAnnotation()
passage.add_annotation(anno_entity)
anno_entity.id = entity['id']
anno_entity.infons['type'] = entity['type']
anno_entity_location = bioc.BioCLocation(
entity['start'], entity['end'] - entity['start'])
anno_entity.add_location(anno_entity_location)
anno_entity.text = entity['text']
relation_id = 1
for result in results:
if doc_name == result['doc_name']:
former = doc_entity[(
doc_entity['id'] == result['former_id'])].iloc[0]
latter = doc_entity[(
doc_entity['id'] == result['latter_id'])].iloc[0]
relation_type = relation_vocab.lookup_id2str(result['type'])
if relation_type == '<unk>':
continue
elif my_utils.relationConstraint1(relation_type,
former['type'],
latter['type']) == False:
continue
else:
bioc_relation = bioc.BioCRelation()
passage.add_relation(bioc_relation)
bioc_relation.id = str(relation_id)
relation_id += 1
bioc_relation.infons['type'] = relation_type
node1 = bioc.BioCNode(former['id'], 'annotation 1')
bioc_relation.add_node(node1)
node2 = bioc.BioCNode(latter['id'], 'annotation 2')
bioc_relation.add_node(node2)
with open(os.path.join(result_dumpdir, doc_name + ".bioc.xml"),
'w') as fp:
bioc.dump(collection, fp)
