def test_should_filter_by_token_label(self):
tagged_tokens = [(TAG_1, get_layout_tokens_for_text('this is line 1')),
(TAG_2, get_layout_tokens_for_text('this is line 2'))]
line = LayoutLine(
[token for _, tokens in tagged_tokens for token in tokens])
layout_model_labels = [
LayoutModelLabel(label=tag,
label_token_text=token.text,
layout_line=line,
layout_token=token)
for tag, tokens in tagged_tokens for token in tokens
]
layout_document = LayoutDocument(
pages=[LayoutPage(blocks=[LayoutBlock(lines=[line])])])
layout_document_label_result = LayoutDocumentLabelResult(
layout_document, layout_model_labels)
for tag, tokens in tagged_tokens:
assert (join_layout_tokens(
layout_document_label_result.get_filtered_document_by_label(
tag).iter_all_tokens()) == join_layout_tokens(tokens))
def test_should_filter_by_line_without_token(self):
tagged_lines = [(TAG_1, LayoutLine.for_text('this is line 1')),
(TAG_2, LayoutLine.for_text('this is line 2'))]
layout_model_labels = [
LayoutModelLabel(label=tag,
label_token_text=line.text,
layout_line=line,
layout_token=None) for tag, line in tagged_lines
for token in line.tokens
]
layout_document = LayoutDocument(pages=[
LayoutPage(
blocks=[LayoutBlock(lines=[line for _, line in tagged_lines])])
])
layout_document_label_result = LayoutDocumentLabelResult(
layout_document, layout_model_labels)
for tag, line in tagged_lines:
assert (join_layout_tokens(
layout_document_label_result.get_filtered_document_by_label(
tag).iter_all_tokens()) == join_layout_tokens(line.tokens))
def test_should_filter_by_token_multiple_labels(self):
tagged_tokens = [(TAG_1, get_layout_tokens_for_text('tokens tag 1')),
(TAG_2, get_layout_tokens_for_text('tokens tag 2')),
(TAG_3, get_layout_tokens_for_text('tokens tag 3'))]
line = LayoutLine(
[token for _, tokens in tagged_tokens for token in tokens])
layout_model_labels = [
LayoutModelLabel(label=tag,
label_token_text=token.text,
layout_line=line,
layout_token=token)
for tag, tokens in tagged_tokens for token in tokens
]
layout_document = LayoutDocument(
pages=[LayoutPage(blocks=[LayoutBlock(lines=[line])])])
layout_document_label_result = LayoutDocumentLabelResult(
layout_document, layout_model_labels)
assert join_layout_tokens(
layout_document_label_result.get_filtered_document_by_labels([
TAG_1, TAG_3
]).iter_all_tokens()) == join_layout_tokens(tagged_tokens[0][1] +
tagged_tokens[2][1])
def test_should_return_remove_abstract_colon_prefix(self):
layout_block = LayoutBlock.for_text('Abstract: ' + ABSTRACT_1)
cleaned_layout_block = get_cleaned_abstract_layout_block(layout_block)
assert join_layout_tokens(
cleaned_layout_block.lines[0].tokens) == ABSTRACT_1
def test_should_return_abstract_if_it_doesnt_contain_prefix(self):
layout_block = LayoutBlock.for_text(ABSTRACT_1)
cleaned_layout_block = get_cleaned_abstract_layout_block(layout_block)
assert join_layout_tokens(
cleaned_layout_block.lines[0].tokens) == ABSTRACT_1
def write_xml_line_for_layout_tokens(self, xml_writer: XmlTreeWriter,
layout_tokens: Iterable[LayoutToken]):
xml_writer.append_text(join_layout_tokens(layout_tokens))
xml_writer.append(TEI_E('lb'))