def test_should_be_able_to_set_tag(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.note(TOKEN_1)]))
lines = _get_all_lines(doc)
tokens = list(doc.get_tokens_of_line(lines[0]))
token = tokens[0]
doc.set_tag(token, TAG_1)
assert doc.get_tag(token) == TAG_1
def test_should_be_able_to_set_tag_with_attribute(self):
doc = GrobidTrainingTeiStructuredDocument(
_tei(front_items=[E.note(TOKEN_1)]),
tag_to_tei_path_mapping={TAG_1: 'div[@tag="tag1"]'})
lines = _get_all_lines(doc)
tokens = list(doc.get_tokens_of_line(lines[0]))
token = tokens[0]
doc.set_tag(token, TAG_1)
assert doc.get_tag(token) == TAG_1
def _get_document_tagged_token_lines(
doc: GrobidTrainingTeiStructuredDocument
) -> List[List[Tuple[str, str]]]:
document_tagged_token_lines = [[(doc.get_tag(token), doc.get_text(token))
for token in doc.get_tokens_of_line(line)]
for page in doc.get_pages()
for line in doc.get_lines_of_page(page)]
LOGGER.debug('document_tagged_token_lines: %s',
document_tagged_token_lines)
return document_tagged_token_lines
def test_should_not_return_preserved_tag_as_tag_and_update_preserved_tag(
self):
original_tei_xml = _tei(front_items=[E.note(TOKEN_1)])
LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml))
doc = GrobidTrainingTeiStructuredDocument(original_tei_xml,
preserve_tags=True,
tag_to_tei_path_mapping={})
LOGGER.debug('doc: %s', doc)
lines = _get_all_lines(doc)
token1 = list(doc.get_tokens_of_line(lines[0]))[0]
assert not doc.get_tag(token1)
doc.set_tag(token1, TAG_1)
root = doc.root
front = root.find('./text/front')
LOGGER.debug('xml: %s', _to_xml(front))
assert _to_xml(front) == (
'<front><{tag1}>{token1}</{tag1}></front>'.format(token1=TOKEN_1,
tag1=TAG_1))