def annotate_text(self, text, textname, sentence_filter):
sentences = []
sent_id = 1
doc = self.annotator(text)
for sent in doc.sents:
if sentence_filter([token.text for token in sent]):
continue
sentence = []
for tok_id, token in enumerate(sent):
if not token.is_space:
sentence.append(
collections.OrderedDict([
('id', tok_id + 1), ('form', token.text),
('lemma', token.lemma_), ('upos', token.pos_),
('xpos', token.tag_), ('feats', None),
('head', token.head.i - sent[0].i +
1 if token.dep_ != "ROOT" else 0),
('deprel', token.dep_), ('deps', None),
('misc', None)
]))
sentences.append(
conllu.TokenList(sentence,
metadata=collections.OrderedDict([
('sent_id', textname + '.' + str(sent_id))
])))
sent_id += 1
return sentences
def _tacred_example_to_token_list(
self, example: Dict[str, Any]) -> conllu.TokenList:
id_ = example["id"]
tokens = example["token"]
ner = example["stanford_ner"]
subj_start = example["subj_start"]
subj_end = example["subj_end"]
obj_start = example["obj_start"]
obj_end = example["obj_end"]
subj_tag = example["subj_type"]
obj_tag = example["obj_type"]
label = example["relation"]
metadata = {
"text":
" ".join(tokens),
"sentence_id":
str(id_),
"relations":
";".join([
str(subj_start + 1),
str(subj_end + 1),
str(obj_start + 1),
str(obj_end + 1),
label,
]),
}
prev_tag = None
token_dicts = []
for idx, (token, tag) in enumerate(zip(tokens, ner)):
if subj_start <= idx <= subj_end:
tag = subj_tag
if obj_start <= idx <= obj_end:
tag = obj_tag
prefix = ""
if tag != "O":
if tag != prev_tag:
prefix = "B-"
else:
prefix = "I-"
prev_tag = tag
token_dicts.append(
Token({
"id": str(idx + 1),
"form": convert_ptb_token(token),
"ner": prefix + tag,
}))
return conllu.TokenList(tokens=token_dicts,
metadata=Metadata(metadata))
def tokens2conllu(tokens):
tokens = [
OrderedDict((k, v) for k, v in zip(
conllu.parser.DEFAULT_FIELDS,
[i + 1, unescape(token)] +
["_" for i in range(len(conllu.parser.DEFAULT_FIELDS) - 1)],
)) for i, token in enumerate(tokens)
]
tl = conllu.TokenList(tokens)
return tl
def _src_token_list_to_token_list(self, src_token_list):
tokens = []
token_dicts = []
ner_tags = []
for index, token in enumerate(src_token_list, start=1):
text = token["form"]
ner_tag = token["ner"]
tokens.append(text)
ner_tags.append(ner_tag)
token_dicts.append({
"id": str(index),
"form": text,
"ner": ner_tag,
})
span_end_to_span = {
end: (start, end)
for start, end in self._bio_tags_to_spans(ner_tags)
}
relations = []
for index, token in enumerate(src_token_list):
for relation, head in zip(token["relations"],
token["relation_heads"]):
if relation == "N":
continue
subj_start, subj_end = span_end_to_span[index]
obj_start, obj_end = span_end_to_span[head]
relations.append(
(subj_start, subj_end, obj_start, obj_end, relation))
doc_id = src_token_list.metadata["doc"]
metadata = {
"text":
" ".join(tokens),
"sentence_id":
doc_id,
"relations":
"|".join([
";".join([
str(subj_start + 1),
str(subj_end + 1),
str(obj_start + 1),
str(obj_end + 1),
relation,
]) for subj_start, subj_end, obj_start, obj_end, relation in
relations
]),
}
return conllu.TokenList(tokens=token_dicts, metadata=metadata)
def finalize_sentence(sentence):
encode_entities(sentence)
sent_meta = {
"sent_id":
meta['document_cts_urn'][18:] + "-" + str(len(sentences) + 1),
"doc_id":
meta['document_cts_urn'],
"segmentation":
meta['segmentation'],
"tagging":
meta['tagging'],
"parsing":
meta.get('parsing', 'none'),
"entities":
meta.get('entities', 'none'),
}
tl = conllu.TokenList(sentence, sent_meta)
sentences.append(tl.serialize())
def drugprot_document_to_tokenlists(
self,
pmid: str,
title_sentences: List[Sentence],
abstract_sentences: List[Sentence],
abstract_offset: int,
entities: Dict[str, Tuple[str, int, int, str]],
relations: Set[Tuple[str, str, str]],
) -> List[conllu.TokenList]:
tokenlists: List[conllu.TokenList] = []
sentence_id = 1
for offset, sents in [
(0, title_sentences),
(abstract_offset, abstract_sentences),
]:
for sent in sents:
assert sent.start_pos is not None
assert sent.end_pos is not None
sent_char_start = sent.start_pos + offset
sent_char_end = sent.end_pos + offset
entities_in_sent = set()
for entity_id, (_, char_start, char_end,
_) in entities.items():
if sent_char_start <= char_start and char_end <= sent_char_end:
entities_in_sent.add(entity_id)
entity_char_spans = [(entities[entity_id][1],
entities[entity_id][2])
for entity_id in entities_in_sent]
token_offsets = [(
sent.start_pos + (token.start_pos or 0) + offset,
sent.start_pos + (token.end_pos or 0) + offset,
) for token in sent.tokens]
entity_token_spans = self.char_spans_to_token_spans(
entity_char_spans, token_offsets)
tags_1 = ["O"] * len(sent)
tags_2 = ["O"] * len(sent)
entity_id_to_token_idx = {}
ordered_entities = sorted(
zip(entities_in_sent, entity_token_spans),
key=lambda x: x[1][1] - x[1][0],
reverse=True,
)
for entity_id, entity_span in ordered_entities:
entity_id_to_token_idx[entity_id] = entity_span
# check if first tag row is already occupied
token_start, token_end = entity_span
tag_1_occupied = False
for i in range(token_start, token_end):
if tags_1[i] != "O":
tag_1_occupied = True
# if first tag row is occupied, use second tag row
tags = tags_2 if tag_1_occupied else tags_1
tag = entities[entity_id][0]
token_start, token_end = entity_span
for i in range(token_start, token_end):
if i == token_start:
prefix = "B-"
else:
prefix = "I-"
tags[i] = prefix + tag
token_dicts = []
for i, (token, tag_1,
tag_2) in enumerate(zip(sent, tags_1, tags_2)):
# hardcoded mapping TODO: perhaps find nicer solution
tag_1 = tag_1.replace("GENE-N", "GENE")
tag_1 = tag_1.replace("GENE-Y", "GENE")
tag_2 = tag_2.replace("GENE-N", "GENE")
tag_2 = tag_2.replace("GENE-Y", "GENE")
token_dicts.append(
Token({
"id": str(i + 1),
"form": token.text,
"ner": tag_1,
"ner-2": tag_2,
}))
relations_in_sent = []
for relation, ent1, ent2 in [
r for r in relations
if {r[1], r[2]} <= entities_in_sent
]:
subj_start = entity_id_to_token_idx[ent1][0]
subj_end = entity_id_to_token_idx[ent1][1]
obj_start = entity_id_to_token_idx[ent2][0]
obj_end = entity_id_to_token_idx[ent2][1]
relations_in_sent.append(
(subj_start, subj_end, obj_start, obj_end, relation))
metadata = {
"text":
sent.to_original_text(),
"doc_id":
pmid,
"sentence_id":
str(sentence_id),
"relations":
"|".join([
";".join([
str(subj_start + 1),
str(subj_end),
str(obj_start + 1),
str(obj_end),
relation,
]) for subj_start, subj_end, obj_start, obj_end,
relation in relations_in_sent
]),
}
tokenlists.append(
conllu.TokenList(tokens=token_dicts,
metadata=Metadata(metadata)))
sentence_id += 1
return tokenlists
def _semeval_lines_to_token_list(self, raw_lines, augment_relations):
raw_id, raw_text = raw_lines[0].split("\t")
label = raw_lines[1]
id_ = int(raw_id)
raw_text = raw_text.strip('"')
# Some special cases (e.g., missing spaces before entity marker)
if id_ in [213, 4612, 6373, 8411, 9867]:
raw_text = raw_text.replace("<e2>", " <e2>")
if id_ in [2740, 4219, 4784]:
raw_text = raw_text.replace("<e1>", " <e1>")
if id_ == 9256:
raw_text = raw_text.replace("log- jam", "log-jam")
# necessary if text should be whitespace tokenizeable
if id_ in [2609, 7589]:
raw_text = raw_text.replace("1 1/2", "1-1/2")
if id_ == 10591:
raw_text = raw_text.replace("1 1/4", "1-1/4")
if id_ == 10665:
raw_text = raw_text.replace("6 1/2", "6-1/2")
raw_text = re.sub(r"([.,!?()])$", r" \1", raw_text)
raw_text = re.sub(r"(e[12]>)([',;:\"\(\)])", r"\1 \2", raw_text)
raw_text = re.sub(r"([',;:\"\(\)])(</?e[12])", r"\1 \2", raw_text)
raw_text = raw_text.replace("<e1>", "<e1> ")
raw_text = raw_text.replace("<e2>", "<e2> ")
raw_text = raw_text.replace("</e1>", " </e1>")
raw_text = raw_text.replace("</e2>", " </e2>")
tokens = raw_text.split(" ")
# Handle case where tail may occur before the head
subj_start = tokens.index("<e1>")
obj_start = tokens.index("<e2>")
if subj_start < obj_start:
tokens.pop(subj_start)
subj_end = tokens.index("</e1>")
tokens.pop(subj_end)
obj_start = tokens.index("<e2>")
tokens.pop(obj_start)
obj_end = tokens.index("</e2>")
tokens.pop(obj_end)
else:
tokens.pop(obj_start)
obj_end = tokens.index("</e2>")
tokens.pop(obj_end)
subj_start = tokens.index("<e1>")
tokens.pop(subj_start)
subj_end = tokens.index("</e1>")
tokens.pop(subj_end)
relation = ";".join([
str(subj_start + 1),
str(subj_end),
str(obj_start + 1),
str(obj_end),
label,
])
if augment_relations:
label_inverted = label.replace("e1", "e3")
label_inverted = label_inverted.replace("e2", "e1")
label_inverted = label_inverted.replace("e3", "e2")
relation_inverted = ";".join([
str(obj_start + 1),
str(obj_end),
str(subj_start + 1),
str(subj_end),
label_inverted,
])
metadata = {
"text":
" ".join(tokens),
"sentence_id":
str(id_),
"relations":
relation + "|" +
relation_inverted if augment_relations else relation,
}
token_dicts = []
for idx, token in enumerate(tokens):
tag = "O"
prefix = ""
if subj_start <= idx < subj_end:
prefix = "B-" if idx == subj_start else "I-"
tag = "E1"
elif obj_start <= idx < obj_end:
prefix = "B-" if idx == obj_start else "I-"
tag = "E2"
token_dicts.append({
"id": str(idx + 1),
"form": token,
"ner": prefix + tag,
})
return conllu.TokenList(tokens=token_dicts, metadata=metadata)
def to_conllu(self):
return conllu.TokenList([token.serialize() for token in self[1:]],
metadata=self.meta)
def predict(self,
input: str,
output: Optional[str] = None,
pos: bool = True,
ner: bool = False,
srl: bool = False,
dep: bool = True,
sdp: bool = False):
"""
预测文本并输出为 conllu 格式
:param input: 要预测的文件，每行一句话
:param output: 输出的结果文件，默认是输入文件添加 .conll 后缀
:param pos: 是否输出 词性标注 结果 ['True','False']
:param ner: 是否输出 命名实体识别 结果 ['True','False'], 占用 conllu feats 列
:param srl: 是否输出 语义角色标注 结果 ['True','False'], 占用 conllu misc 列
:param dep: 是否输出 依存句法分析 结果 ['True','False']
:param sdp: 是否输出 语义依存分析 结果 ['True','False']
"""
if output is None:
output = f"{input}.conllu"
with open(output, mode='w', encoding='utf-8') as f:
sentences = sum([sent for idx, sent in iter_lines(input)], [])
results = self._predict(sentences, pos, ner, srl, dep, sdp)
for text, (seg_s, pos_s, ner_s, dep_s, sdp_s,
srl_s) in zip(sentences, results):
tokens = conllu.TokenList([
conllu.models.Token(id=idx + 1,
form=token,
lemma=token,
upos=pos if pos else '_',
xpos=pos if pos else '_',
feats='O' if ner else '_',
head=idx,
deprel='_',
deps='' if sdp else '_',
misc='SpaceAfter=No')
for idx, (token,
pos) in enumerate(zip_longest(seg_s, pos_s))
], conllu.models.Metadata(text=text))
if ner:
for tag, start, end in ner_s:
tokens[start]['feats'] = f'B-{tag}'
for i in range(start + 1, end):
tokens[start]['feats'] = f'I-{tag}'
if dep:
for id, head, tag in dep_s:
tokens[id - 1]['head'] = head
tokens[id - 1]['deprel'] = tag
if sdp:
for id, head, tag in sdp_s:
if tokens[id - 1]['deps']:
tokens[id - 1]['deps'] = tokens[
id - 1]['deps'] + f"|{head}:{tag}"
else:
tokens[id - 1]['deps'] = f"{head}:{tag}"
if srl:
srl_predicate, srl_roles = list(zip(*srl_s))
srl_predicate_num = len(srl_predicate)
if srl_predicate_num > 0:
srl_misc = [[
f'Predicate={"Y" if i in srl_predicate else "_"}',
['O'] * srl_predicate_num
] for i in range(len(tokens))]
for idx, srl_role in enumerate(srl_roles):
for tag, start, end in srl_role:
srl_misc[start][-1][idx] = f'B-{tag}'
for i in range(start + 1, end):
srl_misc[start][-1][idx] = f'I-{tag}'
srl_misc = [
"|".join([s[0], "Role=" + ",".join(s[-1])])
for s in srl_misc
]
for token, misc in zip(tokens, srl_misc):
token['misc'] = f"{token['misc']}|{misc}"
f.write(tokens.serialize())
def drugprot_document_to_tokenlists(
self, pmid: str, title_sentences: List[Sentence],
abstract_sentences: List[Sentence], abstract_offset: int,
entities: Dict[str, Tuple[str, int, int, str]],
relations: Set[Tuple[str, str, str]]) -> List[conllu.TokenList]:
tokenlists: List[conllu.TokenList] = []
sentence_id = 1
for offset, sents in [(0, title_sentences),
(abstract_offset, abstract_sentences)]:
for sent in sents:
sent_char_start = sent.start_pos + offset
sent_char_end = sent.end_pos + offset
entities_in_sent = set()
for entity_id, (_, char_start, char_end,
_) in entities.items():
if sent_char_start <= char_start and char_end <= sent_char_end:
entities_in_sent.add(entity_id)
entity_char_spans = [(entities[entity_id][1],
entities[entity_id][2])
for entity_id in entities_in_sent]
token_offsets = [(sent.start_pos + token.start_pos + offset,
sent.start_pos + token.end_pos + offset)
for token in sent.tokens]
entity_token_spans = self.char_spans_to_token_spans(
entity_char_spans, token_offsets)
tags_1 = ["O"] * len(sent)
tags_2 = ["O"] * len(sent)
entity_id_to_token_idx = {}
prev_entity_span = None
for entity_id, entity_span in sorted(zip(
entities_in_sent, entity_token_spans),
key=lambda x: x[1][0]):
entity_id_to_token_idx[entity_id] = entity_span
overlap = self.has_overlap(prev_entity_span, entity_span)
tags = tags_2 if overlap else tags_1
tag = entities[entity_id][0]
token_start, token_end = entity_span
for i in range(token_start, token_end):
if i == token_start:
prefix = "B-"
else:
prefix = "I-"
tags[i] = prefix + tag
prev_entity_span = entity_span
token_dicts = []
for i, (token, tag_1,
tag_2) in enumerate(zip(sent, tags_1, tags_2)):
# hardcoded mapping TODO: perhaps find nicer solution
tag_1 = tag_1.replace("GENE-N", "GENE")
tag_1 = tag_1.replace("GENE-Y", "GENE")
tag_2 = tag_2.replace("GENE-N", "GENE")
tag_2 = tag_2.replace("GENE-Y", "GENE")
token_dicts.append({
"id": str(i + 1),
"form": token.text,
"ner": tag_1,
"ner-2": tag_2
})
relations_in_sent = []
for relation, ent1, ent2 in [
r for r in relations
if {r[1], r[2]} <= entities_in_sent
]:
subj_start = entity_id_to_token_idx[ent1][0]
subj_end = entity_id_to_token_idx[ent1][1]
obj_start = entity_id_to_token_idx[ent2][0]
obj_end = entity_id_to_token_idx[ent2][1]
relations_in_sent.append(
(subj_start, subj_end, obj_start, obj_end, relation))
metadata = {
"text":
sent.to_original_text(),
"doc_id":
pmid,
"sentence_id":
str(sentence_id),
"relations":
"|".join([
";".join([
str(subj_start + 1),
str(subj_end),
str(obj_start + 1),
str(obj_end), relation
]) for subj_start, subj_end, obj_start, obj_end,
relation in relations_in_sent
]),
}
tokenlists.append(
conllu.TokenList(tokens=token_dicts, metadata=metadata))
sentence_id += 1
return tokenlists