def complete_case_annotations(spacy_docs: List[Doc],
entity_typename: Dict[str, str]) -> List[Doc]:
"""
Complete/Normalize annotations from the spacy tagger.
:param spacy_docs: the spacy annotations
:param entity_typename: the dictionary with each occurence type
:returns: the updated spacy_annotions (for convenience only, as the update is inplace)
"""
if len(spacy_docs) > 0:
matcher = AcoraMatcher(content=list(entity_typename.keys()),
ignore_case=True)
doc_text, empty_offsets = zip(*[(spacy_doc.text, [])
for spacy_doc in spacy_docs])
document_addresses_offsets = find_address_in_block_of_paragraphs(
texts=list(doc_text), offsets=list(empty_offsets))
for spacy_doc, doc_address_offset in zip(spacy_docs,
document_addresses_offsets):
matches = matcher.get_matches(text=spacy_doc.text, tag="UNKNOWN")
matcher_offsets = list()
for offset in matches:
span_text = spacy_doc.text[offset.start:offset.end]
logger.debug(span_text)
offset.type = entity_typename[span_text.lower()]
matcher_offsets.append(offset)
matcher_offsets_normalized = normalize_offsets(
offsets=matcher_offsets + doc_address_offset)
spacy_matcher_offset: List[Span] = list()
for offset in matcher_offsets_normalized:
# https://spacy.io/usage/linguistic-features#section-named-entities
span_doc: Span = spacy_doc.char_span(offset.start,
offset.end,
label=offset.type)
if span_doc is not None:
# span will be none if the word is incomplete
spacy_matcher_offset.append(span_doc)
else:
logger.error(
f"ERROR char offset [{spacy_doc.text[offset.start:offset.end]}] "
f"from [{spacy_doc.text}]")
spacy_doc.ents = spacy_matcher_offset # all_offsets
return spacy_docs
def convert_to_flair_format(spacy_model: Language, data: List[Tuple[str, List[Offset]]]) -> List[str]:
result: List[str] = list()
for text, offsets in data:
doc: Doc = spacy_model(text)
# remove duplicated offsets
offsets = normalize_offsets(offsets=offsets)
offset_tuples = list(set([offset.to_tuple() for offset in offsets]))
gold_annotations = GoldParse(doc, entities=offset_tuples)
annotations: List[str] = gold_annotations.ner
assert len(annotations) == len(doc)
# Flair uses BIOES and Spacy BILUO
# BILUO for Begin, Inside, Last, Unit, Out
# BIOES for Begin, Inside, Outside, End, Single
annotations = [a.replace("L-", "E-") for a in annotations]
annotations = [a.replace("U-", "S-") for a in annotations]
annotations = ["O" if a == "-" else a for a in annotations] # replace unknown
result += [f"{word} {tag}\n" for word, tag in zip(doc, annotations)]
result.append("\n")
return result
# clerk_names +
# lawyer_names +
# partie_pp +
# postal_code_matches +
# # frequent_entities +
# court_name +
# court_names_matches +
case_dates) #+
# bar +
# rg_from_regex +
# licence_plate_number +
# phone_numbers +
# addresses)
if len(all_matches) > 0:
normalized_offsets = normalize_offsets(all_matches)
last_document_texts.append(current_paragraph)
last_document_offsets.append(normalized_offsets)
elif current_paragraph.isupper(
) and len(current_paragraph) > 10:
# add empty title paragraph to avoid fake solution
last_document_texts.append(current_paragraph)
last_document_offsets.append([])
assert len(last_document_texts) == len(last_document_offsets)
last_document_offsets_with_addresses = find_address_in_block_of_paragraphs(
texts=last_document_texts, offsets=last_document_offsets)
last_document_offsets_with_cleaned_addresses = clean_address_offsets(
def test_normalize_offsets():
data1 = [Offset(5, 10, "LAWYER"), Offset(18, 24, "ORGANIZATION"), Offset(22, 24, "ORGANIZATION"), Offset(120, 133, "LAWYER")]
assert normalize_offsets(data1) == [Offset(5, 10, "LAWYER"), Offset(18, 24, "ORGANIZATION"), Offset(120, 133, "LAWYER")]
data2 = [Offset(71, 75, "PERS"), Offset(76, 85, "PERS")]
assert normalize_offsets(data2) == [Offset(71, 85, "PERS")]
data3 = [Offset(5, 10, "LAWYER"), Offset(71, 75, "PERS"), Offset(76, 85, "PERS"), Offset(120, 133, "LAWYER")]
assert normalize_offsets(data3) == [Offset(5, 10, "LAWYER"), Offset(71, 85, "PERS"), Offset(120, 133, "LAWYER")]
data4 = [Offset(5, 10, "LAWYER"), Offset(77, 85, "PERS"), Offset(77, 85, "PERS"), Offset(120, 133, "LAWYER")]
assert normalize_offsets(data4) == [Offset(5, 10, "LAWYER"), Offset(77, 85, "PERS"), Offset(120, 133, "LAWYER")]
data5 = [Offset(16, 20, "PERS"), Offset(21, 30, "PERS")]
assert normalize_offsets(data5) == [Offset(16, 30, "PERS")]
data6 = [Offset(10, 21, "PERS"), Offset(22, 30, "PERS")]
assert normalize_offsets(data6) == [Offset(10, 30, "PERS")]
data7 = []
assert normalize_offsets(data7) == []
data8 = [Offset(1, 1, "PERS")]
assert normalize_offsets(data8) == []
data9 = [Offset(1, 3, "PERS")]
assert normalize_offsets(data9) == []
data10 = [Offset(1, 10, "PERS"), Offset(1, 10, "PERS"), Offset(3, 10, "PERS")]
assert normalize_offsets(data10) == [Offset(1, 10, "PERS")]
data11 = [Offset(0, 34, "ORGANIZATION"), Offset(0, 8, "ORGANIZATION")]
assert normalize_offsets(data11) == [Offset(0, 34, "ORGANIZATION")]
data12 = [Offset(1, 10, "PERS"), Offset(1, 10, "ORGANIZATION_1")]
assert normalize_offsets(data12) == [Offset(1, 10, "ORGANIZATION")]
data13 = [Offset(1, 10, "PERS"), Offset(5, 10, "ORGANIZATION_1")]
assert normalize_offsets(data13) == [Offset(1, 10, "ORGANIZATION")]
data14 = [Offset(21, 33, "DATE"), Offset(35, 55, "PERS")]
assert normalize_offsets(data14) == data14
data15 = [Offset(start=21, end=37, type='DATE'), Offset(start=45, end=47, type='ORGANIZATION')]
assert normalize_offsets(data15) == [Offset(21, 37, "DATE")]
# clerk_names +
# lawyer_names +
# partie_pp +
# postal_code_matches +
# # frequent_entities +
# court_name +
# court_names_matches +
case_dates) #+
# bar +
# rg_from_regex +
# licence_plate_number +
# phone_numbers +
# addresses)
if len(all_matches) > 0:
normalized_offsets = normalize_offsets(all_matches)
last_document_texts.append(current_paragraph)
last_document_offsets.append(normalized_offsets)
elif current_paragraph.isupper(
) and len(current_paragraph) > 10:
# add empty title paragraph to avoid fake solution
last_document_texts.append(current_paragraph)
last_document_offsets.append([])
assert len(last_document_texts) == len(last_document_offsets)
# last_document_offsets_with_addresses = find_address_in_block_of_paragraphs(texts=last_document_texts,
# offsets=last_document_offsets)
#
# last_document_offsets_with_cleaned_addresses = clean_address_offsets(texts=last_document_texts,
def test_normalize_offsets():
data1 = [(5, 10, "LAWYER"), (18, 24, "ORGANIZATION"),
(22, 24, "ORGANIZATION"), (120, 133, "LAWYER")]
assert normalize_offsets(data1) == [(5, 10, "LAWYER"),
(18, 24, "ORGANIZATION"),
(120, 133, "LAWYER")]
data2 = [(71, 75, "PERS"), (76, 85, "PERS")]
assert normalize_offsets(data2) == [(71, 85, "PERS")]
data3 = [(5, 10, "LAWYER"), (71, 75, "PERS"), (76, 85, "PERS"),
(120, 133, "LAWYER")]
assert normalize_offsets(data3) == [(5, 10, "LAWYER"), (71, 85, "PERS"),
(120, 133, "LAWYER")]
data4 = [(5, 10, "LAWYER"), (77, 85, "PERS"), (77, 85, "PERS"),
(120, 133, "LAWYER")]
assert normalize_offsets(data4) == [(5, 10, "LAWYER"), (77, 85, "PERS"),
(120, 133, "LAWYER")]
data5 = [(16, 20, "PERS"), (21, 30, "PERS")]
assert normalize_offsets(data5) == [(16, 30, "PERS")]
data6 = [(10, 21, "PERS"), (22, 30, "PERS")]
assert normalize_offsets(data6) == [(10, 30, "PERS")]
data7 = []
assert normalize_offsets(data7) == []
data8 = [(1, 1, "PERS")]
assert normalize_offsets(data8) == []
data9 = [(1, 3, "PERS")]
assert normalize_offsets(data9) == []
data10 = [(1, 10, "PERS"), (1, 10, "PERS"), (3, 10, "PERS")]
assert normalize_offsets(data10) == [(1, 10, "PERS")]
data11 = [(0, 34, "ORGANIZATION"), (0, 8, "ORGANIZATION")]
assert normalize_offsets(data11) == [(0, 34, "ORGANIZATION")]
data12 = [(1, 10, "PERS"), (1, 10, "ORGANIZATION_1")]
assert normalize_offsets(data12) == [(1, 10, "ORGANIZATION")]
data13 = [(1, 10, "PERS"), (5, 10, "ORGANIZATION_1")]
assert normalize_offsets(data13) == [(1, 10, "ORGANIZATION")]
data14 = [(21, 33, "DATE"), (35, 55, "PERS")]
assert normalize_offsets(data14) == data14
lawyer_names +
partie_pp +
postal_code_matches +
# frequent_entities +
court_name +
court_names_matches +
case_dates +
bar +
rg_from_regex +
licence_plate_number +
phone_numbers +
addresses)
if len(all_matches) > 0:
normalized_offsets = normalize_offsets(all_matches)
last_document_texts.append(current_paragraph)
last_document_offsets.append(normalized_offsets)
elif current_paragraph.isupper() and len(current_paragraph) > 10:
# add empty title paragraph to avoid fake solution
last_document_texts.append(current_paragraph)
last_document_offsets.append([])
assert len(last_document_texts) == len(last_document_offsets)
last_document_offsets_with_addresses = find_address_in_block_of_paragraphs(texts=last_document_texts,
offsets=last_document_offsets)
last_document_offsets_with_cleaned_addresses = clean_address_offsets(texts=last_document_texts,
offsets=last_document_offsets_with_addresses)
def test_normalize_offsets():
data1 = [
Offset(5, 10, "LAWYER"),
Offset(18, 24, "ORGANIZATION"),
Offset(22, 24, "ORGANIZATION"),
Offset(120, 133, "LAWYER"),
]
assert normalize_offsets(data1) == [
Offset(5, 10, "LAWYER"),
Offset(18, 24, "ORGANIZATION"),
Offset(120, 133, "LAWYER"),
]
data2 = [Offset(71, 75, "PERS"), Offset(76, 85, "PERS")]
assert normalize_offsets(data2) == [Offset(71, 85, "PERS")]
data3 = [
Offset(5, 10, "LAWYER"),
Offset(71, 75, "PERS"),
Offset(76, 85, "PERS"),
Offset(120, 133, "LAWYER")
]
assert normalize_offsets(data3) == [
Offset(5, 10, "LAWYER"),
Offset(71, 85, "PERS"),
Offset(120, 133, "LAWYER")
]
data4 = [
Offset(5, 10, "LAWYER"),
Offset(77, 85, "PERS"),
Offset(77, 85, "PERS"),
Offset(120, 133, "LAWYER")
]
assert normalize_offsets(data4) == [
Offset(5, 10, "LAWYER"),
Offset(77, 85, "PERS"),
Offset(120, 133, "LAWYER")
]
data5 = [Offset(16, 20, "PERS"), Offset(21, 30, "PERS")]
assert normalize_offsets(data5) == [Offset(16, 30, "PERS")]
data6 = [Offset(10, 21, "PERS"), Offset(22, 30, "PERS")]
assert normalize_offsets(data6) == [Offset(10, 30, "PERS")]
data7 = []
assert normalize_offsets(data7) == []
data8 = [Offset(1, 1, "PERS")]
assert normalize_offsets(data8) == []
data9 = [Offset(1, 3, "PERS")]
assert normalize_offsets(data9, min_offset_size=2) == []
data10 = [
Offset(1, 10, "PERS"),
Offset(1, 10, "PERS"),
Offset(3, 10, "PERS")
]
assert normalize_offsets(data10) == [Offset(1, 10, "PERS")]
data11 = [Offset(0, 34, "ORGANIZATION"), Offset(0, 8, "ORGANIZATION")]
assert normalize_offsets(data11) == [Offset(0, 34, "ORGANIZATION")]
data14 = [Offset(21, 33, "DATE"), Offset(35, 55, "PERS")]
assert normalize_offsets(data14) == data14
data15 = [
Offset(start=21, end=37, type="DATE"),
Offset(start=45, end=47, type="ORGANIZATION")
]
assert normalize_offsets(data15,
min_offset_size=2) == [Offset(21, 37, "DATE")]
data16 = [Offset(1, 3, "PERS")]
assert normalize_offsets(data9) == data16
data17 = [Offset(1, 5, "PERS"), Offset(6, 10, "PERS")]
assert normalize_offsets(data17) == [Offset(1, 10, "PERS")]
data18 = [Offset(1, 5, "PERS"), Offset(6, 10, "ORGANIZATION")]
assert normalize_offsets(data18) == data18
data19 = [Offset(1, 5, "PERS"), Offset(7, 10, "ORGANIZATION")]
assert normalize_offsets(data19) == data19