def iter_find_line_number_tokens(
structured_document: GrobidTrainingTeiStructuredDocument, **kwargs):
for page in structured_document.get_pages():
line_no_tokens = iter_find_line_number_tokens_in_lines(
structured_document,
lines=structured_document.get_lines_of_page(page),
**kwargs)
yield from line_no_tokens
def _get_document_tagged_token_lines(
doc: GrobidTrainingTeiStructuredDocument
) -> List[List[Tuple[str, str]]]:
document_tagged_token_lines = [[(doc.get_tag(token), doc.get_text(token))
for token in doc.get_tokens_of_line(line)]
for page in doc.get_pages()
for line in doc.get_lines_of_page(page)]
LOGGER.debug('document_tagged_token_lines: %s',
document_tagged_token_lines)
return document_tagged_token_lines
def _simple_document_with_tagged_token_lines(
lines: List[List[Tuple[str,
str]]]) -> GrobidTrainingTeiStructuredDocument:
tei_items = []
for line in lines:
tei_items.append(' '.join(token for _, token in line))
tei_items.append(E.lb())
doc = GrobidTrainingTeiStructuredDocument(
_tei(tei_items), container_node_path=DEFAULT_CONTAINER_NODE_PATH)
doc_lines = [
line for page in doc.get_pages()
for line in doc.get_lines_of_page(page)
]
for line, doc_line in zip(lines, doc_lines):
for (tag, token), doc_token in zip(line,
doc.get_tokens_of_line(doc_line)):
assert token == doc.get_text(doc_token)
if tag:
doc.set_tag(doc_token, tag)
return doc