def test_should_find_one_line_with_one_token_at_front_level(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.note(TOKEN_1)]),
container_node_path='text/front')
lines = _get_all_lines(doc)
assert _get_token_texts_for_lines(doc, lines) == [[TOKEN_1]]
assert doc.root.find('./text/front/note').text == TOKEN_1
def test_should_set_empty_token_whitespace(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.note('1.')]))
lines = _get_all_lines(doc)
assert _get_token_texts_for_lines(doc, lines) == [['1', '.']]
tokens = doc.get_all_tokens_of_line(lines[0])
assert tokens[0].whitespace == ''
def test_should_not_include_line_feed_in_tag_if_previous_token_has_different_tag(
self):
original_root = _tei(front_items=[TOKEN_1, '\n ' + TOKEN_2])
LOGGER.debug('original_root: %s', _to_xml(original_root))
doc = GrobidTrainingTeiStructuredDocument(original_root)
lines = _get_all_lines(doc)
line1_tokens = list(doc.get_all_tokens_of_line(lines[0]))
space_tokens = [t for t in line1_tokens if isinstance(t, TeiSpace)]
assert space_tokens
for token in space_tokens:
doc.set_tag(token, TAG_1)
doc.set_tag(line1_tokens[0], TAG_1)
doc.set_tag(line1_tokens[-1], TAG_2)
LOGGER.debug('line1_tokens: %s', line1_tokens)
root = doc.root
front = root.find('./text/front')
LOGGER.debug('xml: %s', _to_xml(front))
assert _to_xml(front) == (
'<front><{tag1}>{token1}</{tag1}>'
'\n <{tag2}>{token2}</{tag2}></front>'.format(tag1=TAG_1,
tag2=TAG_2,
token1=TOKEN_1,
token2=TOKEN_2))
def test_should_be_able_to_set_tag(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.note(TOKEN_1)]))
lines = _get_all_lines(doc)
tokens = list(doc.get_tokens_of_line(lines[0]))
token = tokens[0]
doc.set_tag(token, TAG_1)
assert doc.get_tag(token) == TAG_1
def test_should_be_able_to_set_tag_with_attribute(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.note(TOKEN_1)]),
tag_to_tei_path_mapping={TAG_1: 'div[@tag="tag1"]'})
lines = _get_all_lines(doc)
tokens = list(doc.get_tokens_of_line(lines[0]))
token = tokens[0]
doc.set_tag(token, TAG_1)
assert doc.get_tag(token) == TAG_1
def test_should_find_empty_first_line_and_text_outside_semantic_element(
self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[TOKEN_1,
_tei_lb(), TOKEN_2,
E.note(TOKEN_3)]))
lines = _get_all_lines(doc)
assert _get_token_texts_for_lines(doc, lines) == [[TOKEN_1],
[TOKEN_2, TOKEN_3]]
def test_should_be_able_get_root_with_updated_single_token_tag(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.note(TOKEN_1)]))
lines = _get_all_lines(doc)
tokens = list(doc.get_tokens_of_line(lines[0]))
token = tokens[0]
doc.set_tag(token, TAG_1)
root = doc.root
front = root.find('./text/front')
child_elements = list(front)
assert [c.tag for c in child_elements] == [TAG_1]
assert [c.text for c in child_elements] == [TOKEN_1]
def test_should_reverse_map_tags(self):
tag_to_tei_path_mapping = {TAG_1: 'docTitle/titlePart'}
original_tei_xml = _tei(front_items=[E.docTitle(E.titlePart(TOKEN_1))])
LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml))
doc = GrobidTrainingTeiStructuredDocument(
original_tei_xml,
tag_to_tei_path_mapping=tag_to_tei_path_mapping,
preserve_tags=True)
LOGGER.debug('doc: %s', doc)
assert [[
doc.get_tag_or_preserved_tag_value(t)
for t in doc.get_all_tokens_of_line(line)
] for line in _get_all_lines(doc)] == [[TAG_1]]
def test_should_preserve_existing_tag(self):
original_tei_xml = _tei(
front_items=[E.docTitle(E.titlePart(TOKEN_1)), TOKEN_2])
LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml))
doc = GrobidTrainingTeiStructuredDocument(original_tei_xml,
preserve_tags=True,
tag_to_tei_path_mapping={})
LOGGER.debug('doc: %s', doc)
root = doc.root
front = root.find('./text/front')
LOGGER.debug('xml: %s', _to_xml(front))
assert _to_xml(front) == (
'<front><docTitle><titlePart>{token1}</titlePart></docTitle>{token2}</front>'
.format(token1=TOKEN_1, token2=TOKEN_2))
def test_should_be_able_get_root_with_updated_single_token_tag_with_attribute(
self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.note(TOKEN_1)]),
tag_to_tei_path_mapping={TAG_1: 'div[@tag="tag1"]'})
lines = _get_all_lines(doc)
tokens = list(doc.get_tokens_of_line(lines[0]))
token = tokens[0]
doc.set_tag(token, TAG_1)
root = doc.root
front = root.find('./text/front')
child_elements = list(front)
assert [c.tag for c in child_elements] == ['div']
assert [c.attrib for c in child_elements] == [{'tag': 'tag1'}]
assert [c.text for c in child_elements] == [TOKEN_1]
def test_should_preserve_separation_between_existing_tag(self):
original_tei_xml = _tei(front_items=[
E.byline(E.affiliation(TOKEN_1)),
E.byline(E.affiliation(TOKEN_2))
])
LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml))
doc = GrobidTrainingTeiStructuredDocument(original_tei_xml,
preserve_tags=True,
tag_to_tei_path_mapping={})
LOGGER.debug('doc: %s', doc)
root = doc.root
front = root.find('./text/front')
affiliations = front.xpath('./byline/affiliation')
LOGGER.debug('xml: %s', _to_xml(front))
assert [aff.text for aff in affiliations] == [TOKEN_1, TOKEN_2]
def test_should_preserve_existing_tag_with_attrib(self):
original_tei_xml = _tei(
front_items=[E.div(TOKEN_1, {'tag': TAG_1}), TOKEN_2])
LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml))
doc = GrobidTrainingTeiStructuredDocument(
original_tei_xml,
preserve_tags=True,
tag_to_tei_path_mapping={TAG_1: 'div[@tag="tag1"]'})
LOGGER.debug('doc: %s', doc)
root = doc.root
front = root.find('./text/front')
LOGGER.debug('xml: %s', _to_xml(front))
assert _to_xml(front) == (
'<front><div tag="{TAG_1}">{token1}</div>{token2}</front>'.format(
token1=TOKEN_1, token2=TOKEN_2, TAG_1=TAG_1))
def test_should_preserve_existing_nested_tag(self):
original_tei_xml = _tei(
front_items=[E.div(E.label(TOKEN_1), TOKEN_2), TOKEN_3])
LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml))
doc = GrobidTrainingTeiStructuredDocument(
original_tei_xml,
preserve_tags=True,
tag_to_tei_path_mapping={TAG_1: 'div'})
LOGGER.debug('doc: %s', doc)
root = doc.root
front = root.find('./text/front')
LOGGER.debug('xml: %s', _to_xml(front))
assert _to_xml(front) == (
'<front><div><label>{token1}</label>{token2}</div>{token3}</front>'
.format(token1=TOKEN_1, token2=TOKEN_2, token3=TOKEN_3))
def test_should_preserve_space_after_lb_in_updated_root(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[TOKEN_1,
E(TeiTagNames.LB), ' ' + TOKEN_2]))
lines = _get_all_lines(doc)
line1_tokens = list(doc.get_tokens_of_line(lines[0]))
doc.set_tag(line1_tokens[0], TAG_1)
line2_tokens = list(doc.get_tokens_of_line(lines[1]))
doc.set_tag(line2_tokens[-1], TAG_1)
root = doc.root
front = root.find('./text/front')
child_elements = list(front)
assert [c.tag for c in child_elements] == [TAG_1]
assert _to_xml(child_elements[0]) == (
'<{tag1}>{token1}<{lb}/> {token2}</{tag1}>'.format(
tag1=TAG_1, token1=TOKEN_1, token2=TOKEN_2, lb=TeiTagNames.LB))
def test_should_not_return_preserved_tag_as_tag_and_update_preserved_tag(
self):
original_tei_xml = _tei(front_items=[E.note(TOKEN_1)])
LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml))
doc = GrobidTrainingTeiStructuredDocument(original_tei_xml,
preserve_tags=True,
tag_to_tei_path_mapping={})
LOGGER.debug('doc: %s', doc)
lines = _get_all_lines(doc)
token1 = list(doc.get_tokens_of_line(lines[0]))[0]
assert not doc.get_tag(token1)
doc.set_tag(token1, TAG_1)
root = doc.root
front = root.find('./text/front')
LOGGER.debug('xml: %s', _to_xml(front))
assert _to_xml(front) == (
'<front><{tag1}>{token1}</{tag1}></front>'.format(token1=TOKEN_1,
tag1=TAG_1))
def _simple_document_with_tagged_token_lines(
lines: List[List[Tuple[str,
str]]]) -> GrobidTrainingTeiStructuredDocument:
tei_items = []
for line in lines:
tei_items.append(' '.join(token for _, token in line))
tei_items.append(E.lb())
doc = GrobidTrainingTeiStructuredDocument(
_tei(tei_items), container_node_path=DEFAULT_CONTAINER_NODE_PATH)
doc_lines = [
line for page in doc.get_pages()
for line in doc.get_lines_of_page(page)
]
for line, doc_line in zip(lines, doc_lines):
for (tag, token), doc_token in zip(line,
doc.get_tokens_of_line(doc_line)):
assert token == doc.get_text(doc_token)
if tag:
doc.set_tag(doc_token, tag)
return doc
def test_should_remove_untagged_including_line_feed(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[
TOKEN_1,
E(TeiTagNames.LB), ' ' + TOKEN_2 + ' ' + TOKEN_3
]))
lines = _get_all_lines(doc)
line1_tokens = list(doc.get_tokens_of_line(lines[0]))
doc.set_tag(line1_tokens[0], None)
line2_tokens = list(doc.get_tokens_of_line(lines[1]))
doc.set_tag(line2_tokens[-1], TAG_1)
doc.remove_all_untagged()
root = doc.root
front = root.find('./text/front')
LOGGER.debug('xml: %s', _to_xml(front))
assert _to_xml(front) == (
'<front><{tag1}>{token3}</{tag1}></front>'.format(tag1=TAG_1,
token3=TOKEN_3))
def test_should_not_include_space_in_tag_if_previous_token_has_different_tag(
self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[TOKEN_1,
E(TeiTagNames.LB), ' ' + TOKEN_2]))
lines = _get_all_lines(doc)
line1_tokens = list(doc.get_tokens_of_line(lines[0]))
doc.set_tag(line1_tokens[0], TAG_1)
line2_tokens = list(doc.get_tokens_of_line(lines[1]))
doc.set_tag(line2_tokens[-1], TAG_2)
root = doc.root
front = root.find('./text/front')
LOGGER.debug('xml: %s', _to_xml(front))
assert _to_xml(front) == ('<front><{tag1}>{token1}<{lb}/></{tag1}>'
' <{tag2}>{token2}</{tag2}></front>'.format(
tag1=TAG_1,
tag2=TAG_2,
token1=TOKEN_1,
token2=TOKEN_2,
lb=TeiTagNames.LB))
def test_should_not_fail_on_empty_document(self):
structured_document = GrobidTrainingTeiStructuredDocument(_tei())
TextLineNumberAnnotator().annotate(structured_document)
def test_should_find_two_lines_separated_by_lb_element(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.note(TOKEN_1, _tei_lb(), TOKEN_2)]))
lines = _get_all_lines(doc)
assert _get_token_texts_for_lines(doc, lines) == [[TOKEN_1], [TOKEN_2]]
def test_should_find_text_outside_semantic_element_at_the_end(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.note(TOKEN_1, E(TeiTagNames.LB)), TOKEN_2]))
lines = _get_all_lines(doc)
assert _get_token_texts_for_lines(doc, lines) == [[TOKEN_1], [TOKEN_2]]
def test_should_find_text_in_nested_top_level_element(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.docTitle(E.titlePart(TOKEN_1))]))
lines = _get_all_lines(doc)
assert _get_token_texts_for_lines(doc, lines) == [[TOKEN_1]]
def test_should_split_words_as_separate_tokens(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.note(' '.join([TOKEN_1, TOKEN_2]))]))
lines = _get_all_lines(doc)
assert _get_token_texts_for_lines(doc, lines) == [[TOKEN_1, TOKEN_2]]
def _structured_document_with_title(title=TITLE_1):
return GrobidTrainingTeiStructuredDocument(_tei_with_title(title))
def test_should_return_root_as_pages(self):
root = _tei(front_items=[])
doc = GrobidTrainingTeiStructuredDocument(root)
assert list(doc.get_pages()) == [root]
def _structured_document_with_sub_elements(*args, **kwargs):
return GrobidTrainingTeiStructuredDocument(
_tei_with_sub_elements(*args, **kwargs),
tag_to_tei_path_mapping=TAG_TO_TEI_PATH_MAPPING)
def test_should_not_fail_on_empty_document(self):
structured_document = GrobidTrainingTeiStructuredDocument(_tei())
SegmentationAnnotator(DEFAULT_CONFIG).annotate(structured_document)