def _bio_to_standoff(bio_tags: List[str], spacy_doc: spacy.tokens.Doc) -> List[Annotation]:
"""Convert BIO tagged document to annotations in standoff format.
The original spaCy document is used to recreate correct entity offsets.
Parameters
----------
bio_tags : List[str]
A BIO tagged sentence. `len(bio_tags) == len(spacy_doc)` has to hold.
spacy_doc : spacy.tokens.Doc
The spaCy doc corresponding to the BIO tags.
Returns
-------
List[Annotation]
The standoff annotations.
"""
bio_tags = fix_dangling_entities(bio_tags)
biluo_tags = _bio_to_biluo(bio_tags)
offsets = offsets_from_biluo_tags(spacy_doc, biluo_tags)
annotations = []
for i, offset in enumerate(offsets):
annotations.append(Annotation(
text=spacy_doc.char_span(offset[0], offset[1]).text,
start=offset[0],
end=offset[1],
tag=offset[2],
ann_id='T{}'.format(i),
))
return annotations
def parse_sp(ner: List[str], sentence: str) -> List[NERMarkerIdx]:
"""Parse a NER tagged sentence based on Spacy (used for longer sentences)."""
# U for unique, i.e. no B or I if single etc (Spacy)
doc = nlp(sentence)
return [
NERMarkerIdx(*offset)
for offset in offsets_from_biluo_tags(doc, ner)
]
def test_roundtrip_offsets_biluo_conversion(en_tokenizer):
text = "I flew to Silicon Valley via London."
biluo_tags = ["O", "O", "O", "B-LOC", "L-LOC", "O", "U-GPE", "O"]
offsets = [(10, 24, "LOC"), (29, 35, "GPE")]
doc = en_tokenizer(text)
biluo_tags_converted = biluo_tags_from_offsets(doc, offsets)
assert biluo_tags_converted == biluo_tags
offsets_converted = offsets_from_biluo_tags(doc, biluo_tags)
assert offsets_converted == offsets
def test_roundtrip_offsets_biluo_conversion(en_tokenizer):
text = "I flew to Silicon Valley via London."
biluo_tags = ["O", "O", "O", "B-LOC", "L-LOC", "O", "U-GPE", "O"]
offsets = [(10, 24, "LOC"), (29, 35, "GPE")]
doc = en_tokenizer(text)
biluo_tags_converted = biluo_tags_from_offsets(doc, offsets)
assert biluo_tags_converted == biluo_tags
offsets_converted = offsets_from_biluo_tags(doc, biluo_tags)
assert offsets_converted == offsets
def _sentence_to_spacy_annotations(self, tokens,
tags) -> Tuple[str, Tuple]:
sentence = " ".join(tokens)
tags = iob_to_biluo(tags)
doc = self.nlp(sentence)
annotations = offsets_from_biluo_tags(doc, tags)
annotations = [(begin, end, tag) for begin, end, tag in annotations
if len(tag) > 0]
return sentence, annotations
def process(self, line, intent_treshold_score=0.5):
doc = self.nlp.make_doc(line)
words_true = [w.text for w in doc]
length = len(words_true)
words_true += ['<EOS>']
words = words_true + ['<PAD>'] * (50 - len(words_true))
words = np.array(words)
batch = [{'words': words, 'length': length}]
decoder_prediction, intent, intent_score = self.model.test(batch)
# batch only contains one element
intent = intent[0]
intent_score = intent_score[0]
# get the part that corresponds to words (truncate PAD and EOS)
decoder_prediction = decoder_prediction[:length, 0]
#print(decoder_prediction, intent[0], intent_score)
# clean up <EOS> and <PAD>
decoder_prediction = [
t if (t != '<EOS>' and t != '<PAD>') else 'O'
for t in decoder_prediction
]
biluo_tags = iob_to_biluo(decoder_prediction)
entities_offsets = offsets_from_biluo_tags(doc, biluo_tags)
entities = []
for ent in entities_offsets:
e_parts = ent[2].split('.')
if len(e_parts) > 1:
# role.type
entity = {'role': e_parts[0], 'type': e_parts[1]}
else:
entity = {'role': None, 'type': e_parts[0]}
value = line[ent[0]:ent[1]]
entities.append({
'_entity': entity['type'],
'role': entity['role'],
'value': value,
'_body': value,
'_start': ent[0],
'_end': ent[1]
})
# now convert to the same format as wit.ai, applying the treshold
if intent_score < intent_treshold_score:
intent_result = None
else:
intent_result = {'confidence': str(intent_score), 'value': intent}
entities_result = {}
for ent in entities:
if ent['role']:
entities_result[ent['role']] = ent
else:
entities_result[ent['_entity']] = ent
return intent_result, entities_result
def _sentence_to_spacy_annotations(self, tokens,
tags) -> Tuple[str, Tuple]:
sentence = " ".join(tokens)
tags = iob_to_biluo(tags)
doc = self.nlp(sentence)
annotations = offsets_from_biluo_tags(doc, tags)
# print(sentence)
# print(tags)
# print(annotations)
return sentence, annotations
def format_predictions_to_display(doc,
predictions,
probability_maps,
pos=False):
"""Format predictions into spacy display formar."""
bert_predictions = []
iob_tags = []
tags_formatted = []
for prediction, probability_map in zip(predictions[0],
probability_maps[0]):
word = list(prediction.keys())[0]
probas = probability_map[word]
normalized_probas = list(softmax(np.mean(probas, axis=0)))
bert_predictions.append(
(word, prediction[word], np.max(normalized_probas)))
if pos:
iob_tags.append("I-" + prediction[word])
else:
iob_tags.append(prediction[word])
biluo_tags = iob_to_biluo(iob_tags)
tags = offsets_from_biluo_tags(doc, biluo_tags)
for tag in tags:
start_token = get_token_for_char(doc, tag[0])
word_span = doc.text[tag[0]:tag[1]]
length_of_span = len(word_span.split())
if length_of_span == 1:
probs = [bert_predictions[start_token][2]]
else:
probs = [
item[2] for item in bert_predictions[start_token:start_token +
length_of_span]
]
tags_formatted.append({
"start": tag[0],
"end": tag[1],
"label": tag[2],
"score": np.prod(probs)
})
return bert_predictions, tags_formatted
def predictions_to_doccano(input_file: str,
output_file: str,
language='en_core_web_sm'):
"""
Convert AllenNLP output json to Doccano style.
:param input_file str: Input AllenNLP json file.
:param output_file str: Doccano style output json.
:param language SpaCy language: For splitting by sentences.
"""
nlp = spacy.blank('en')
json_lines = []
with jsonlines.open(input_file) as reader:
for obj in reader:
text = obj['sentence']
doc = nlp(text)
offsets = offsets_from_biluo_tags(doc, obj['tags'])
json_line = {'text': text, 'labels': offsets}
json_lines.append(json_line)
with jsonlines.open(output_file, mode='w') as writer:
for line in json_lines:
writer.write(line)
def brat2spacy(tokenizer, ann, text):
doc = tokenizer(text)
words = [i.text for i in doc]
entity_ids = defaultdict(tuple)
relation_ids = defaultdict(tuple)
entities = []
for line in ann.strip().split('\n'):
annotation = line.strip().rsplit('\t')
id_ = annotation[0]
if id_ == '*':
ann_type = id_[0]
else:
ann_type = annotation_ids[id_[0]]
if ann_type == 'entity':
if len(annotation[1:]) == 2:
span, surface_form = annotation[1:]
entity_type, start, end = span.split(' ')
entity_ids[id_] = (int(start), int(end))
entities.append((int(start), int(end), entity_type))
if ann_type == 'relation':
if len(annotation[1:]) == 1:
rel_type, head, dep = annotation[1].split(' ')
relation_ids[id_] = (rel_type, head, dep)
entities.sort(key=lambda x: x[0])
tags = biluo_tags_from_offsets(doc, entities)
if relation_ids:
# mapping from brat ids to doc's id
brat_doc_ids_map = {}
for entity in entity_ids:
span = doc.char_span(*entity_ids[entity])
if span.end - span.start == 1:
brat_doc_ids_map[entity] = span.start
else:
# raise Warning("Tokenization mismatch, more than 1 spaCy token in ann token span")
brat_doc_ids_map[entity] = span.start
ids = range(len(doc))
heads = defaultdict(int)
deps = defaultdict(int)
for rel_id, rel in relation_ids.items():
dep, token, head = rel
token, head = brat_doc_ids_map[token.split(
':')[1]], brat_doc_ids_map[head.split(':')[1]]
heads[head] = token
deps[head] = dep
heads = [
i[1] if i[1] > 0 else i[0] for i in [(i, heads[i]) for i in ids]
]
deps = [
i[1] if i[1] != 0 else 'ROOT' for i in [(i, deps[i]) for i in ids]
]
assert len(words) == len(heads) == len(deps) == len(tags)
return GoldParse(doc,
words=words,
heads=heads,
tags=tags,
deps=deps,
entities=entities), text
else:
assert len(words) == len(tags)
return GoldParse(doc,
words=words,
tags=tags,
entities=offsets_from_biluo_tags(doc, tags)), text
class Cached(Provider):
cache: Dict[str, Any]
name = 'cached'
known_schemas = {
# these assume same tokenisation
"bio":
lambda doc, annotation: offsets_from_biluo_tags(
iob_to_biluo(doc, annotation)),
"bilou":
offsets_from_biluo_tags,
"offsets":
OFFSETS,
"list_of_clusters":
convert_clusters_to_offsets,
# these provide their own tokenisation
# annotation: List[Tuple[str,str]]
"list_of_tuples_bio_flat":
lambda doc, annotation: get_offsets(doc.text, annotation),
# annotation: List[List[Tuple[str,str]]]
"list_of_tuples_bio_stacked":
lambda doc, annotation: get_offsets_from_sentences(
doc.text, annotation),
# annotation: Tuple[List[str],List[str]]
"tuple_of_lists_flat":
lambda doc, annotation: get_offsets(doc.text, zip(*annotation[:2])),
# annotation: List[Tuple[List[str]], Tuple[List[str]]]
"list_of_tuples_of_lists":
lambda doc, annotation: get_offsets_from_sentences(
doc.text, ((w, l) for t in annotation for w, l in zip(*t[:2]))),
# annotation: Tuple[List[List[str]], Tuple[List[List[str]]
"tuple_of_lists_of_lists":
lambda doc, annotation: get_offsets_from_sentences(
doc.text, ((w, l) for ws, ls in zip(*annotation[:2])
for w, l in zip(ws, ls)))
# TODO: BRAT
# TODO: Pubmed
}
def __init__(self,
schema: Union[str, Callable[[Doc, Any],
OffsetAnnotation]] = None,
getter=None,
path: str = None):
self.cache = {}
self.loaded = False
if not schema:
self.schema = OFFSETS
elif schema in self.known_schemas:
self.schema = Cached.known_schemas[schema]
elif isinstance(schema, Callable):
self.schema = schema
else:
self.schema = None
self.getter = getter
if path:
self.load(path)
@overrides
def save(self, path: str):
util.save_file(self.cache, path)
# TODO: guess schema
@overrides
def load(self, path):
self.cache = util.load_file(path)
self.loaded = True
@overrides
def annotate_document(self, doc: Doc) -> OffsetAnnotation:
if not self.loaded:
raise ValueError("You forgot to load the cache!")
annotations = self.cache.get(doc._.id, None)
if annotations:
if self.schema:
if self.schema == OFFSETS:
return self.getter(
annotations) if self.getter else annotations
else:
return self.schema(
doc,
self.getter(annotations)
if self.getter else annotations)
else:
logger.info(
f"no schema loaded for {self.__class__.__name__}, good luck!"
)
return annotations
# doc_toks rövidebb általában
j = 0
k = 0
new_tags = []
while j < len(sentences[i]):
if sentences[i][j] == doc_toks[k]:
new_tags.append(iobs[i][j])
j += 1
k += 1
else:
new_tags.append(iobs[i][j])
k += 1
j += 2
tags = iob_to_biluo(new_tags)
else:
tags = iob_to_biluo(iobs[i])
try:
entities = offsets_from_biluo_tags(doc, tags)
e = (detokenized_sent, entities)
corpus.append(e)
except Exception as err:
print(err, detokenized_sent)
continue
print(len(corpus))
corpus = [e for e in corpus if len(e[0]) > 0]
print(len(corpus))
with open("data/interim/corpus.p", "wb") as of:
pickle.dump(corpus, of)
