def test_should_annotate_other_tags_as_body(self): doc = _simple_document_with_tagged_token_lines(lines=[[(None, TOKEN_1)]]) SegmentationAnnotator(DEFAULT_CONFIG).annotate(doc) assert _get_document_tagged_token_lines(doc) == [[ (SegmentationTagNames.BODY, TOKEN_1) ]]
def test_should_clear_minority_among_untagged_tag(self): doc = _simple_document_with_tagged_token_lines( lines=[[(None, TOKEN_1), (None, TOKEN_2), (OTHER_TAG, TOKEN_3)]]) SegmentationAnnotator(DEFAULT_CONFIG, preserve_tags=True).annotate(doc) assert _get_document_tagged_token_lines(doc) == [[(None, TOKEN_1), (None, TOKEN_2), (None, TOKEN_3)]]
def test_should_annotate_reference_as_reference(self): doc = _simple_document_with_tagged_token_lines( lines=[[(BackTagNames.REFERENCE, TOKEN_1)]]) SegmentationAnnotator(DEFAULT_CONFIG).annotate(doc) assert _get_document_tagged_token_lines(doc) == [[ (SegmentationTagNames.REFERENCE, TOKEN_1) ]]
def test_should_annotate_title_as_front(self): doc = _simple_document_with_tagged_token_lines( lines=[[(FrontTagNames.TITLE, TOKEN_1)]]) SegmentationAnnotator(DEFAULT_CONFIG).annotate(doc) assert _get_document_tagged_token_lines(doc) == [[ (SegmentationTagNames.FRONT, TOKEN_1) ]]
def test_should_annotate_no_tag_as_body_if_preserve_is_disabled(self): doc = _simple_document_with_tagged_token_lines(lines=[[(None, TOKEN_1)]]) SegmentationAnnotator(DEFAULT_CONFIG, preserve_tags=False).annotate(doc) assert _get_document_tagged_token_lines(doc) == [[ (SegmentationTagNames.BODY, TOKEN_1) ]]
def test_should_not_annotate_untagged_lines_after_last_header(self): doc = _simple_document_with_tagged_token_lines(lines=[ [(FrontTagNames.TITLE, TOKEN_1)], [(FrontTagNames.TITLE, TOKEN_2)], [(None, TOKEN_3)], ]) SegmentationAnnotator(DEFAULT_CONFIG, preserve_tags=True).annotate(doc) assert _get_document_tagged_token_lines(doc) == [[ (SegmentationTagNames.FRONT, TOKEN_1) ], [(SegmentationTagNames.FRONT, TOKEN_2)], [(None, TOKEN_3)]]
def test_should_ignore_front_if_start_line_index_beyond_threshold(self): doc = _simple_document_with_tagged_token_lines( lines=[[(None, TOKEN_1)], [(None, TOKEN_2)], [(FrontTagNames.TITLE, TOKEN_3)]]) config = SegmentationConfig(DEFAULT_CONFIG.segmentation_mapping, front_max_start_line_index=1) SegmentationAnnotator(config, preserve_tags=True).annotate(doc) assert _get_document_tagged_token_lines(doc) == [[(None, TOKEN_1)], [(None, TOKEN_2)], [(None, TOKEN_3)]]
def test_should_annotate_line_with_using_common_tag(self): doc = _simple_document_with_tagged_token_lines( lines=[[(FrontTagNames.TITLE, TOKEN_1), (FrontTagNames.TITLE, TOKEN_2), (OTHER_TAG, TOKEN_3)]]) SegmentationAnnotator(DEFAULT_CONFIG).annotate(doc) assert _get_document_tagged_token_lines(doc) == [[ (SegmentationTagNames.FRONT, TOKEN_1), (SegmentationTagNames.FRONT, TOKEN_2), (SegmentationTagNames.FRONT, TOKEN_3) ]]
def test_should_merge_and_fill_remaining_untagged_with_annex(self): doc = _simple_document_with_tagged_token_lines(lines=[[( add_tag_prefix(BackTagNames.APPENDIX, prefix=B_TAG_PREFIX), TOKEN_1 ), (add_tag_prefix(BackTagNames.APPENDIX, prefix=I_TAG_PREFIX), TOKEN_2)], [(None, TOKEN_3), (None, TOKEN_4)]]) SegmentationAnnotator( DEFAULT_CONFIG._replace(no_merge_references=False)).annotate(doc) assert _get_document_tagged_token_lines(doc) == [ [(SegmentationTagNames.ANNEX, TOKEN_1), (SegmentationTagNames.ANNEX, TOKEN_2)], [(SegmentationTagNames.ANNEX, TOKEN_3), (SegmentationTagNames.ANNEX, TOKEN_4)] ]
def test_should_not_annotate_out_of_order_page_number(self): doc = _simple_document_with_tagged_token_lines(lines=[[( None, '2')], [(FrontTagNames.TITLE, TOKEN_1)], [(PageTagNames.PAGE, '2')], [(BodyTagNames.SECTION_TITLE, TOKEN_2)], [(PageTagNames.PAGE, '3')]]) SegmentationAnnotator(DEFAULT_CONFIG, preserve_tags=True).annotate(doc) assert _get_document_tagged_token_lines(doc) == [ [(SegmentationTagNames.FRONT, '2')], [(SegmentationTagNames.FRONT, TOKEN_1)], [(SegmentationTagNames.PAGE, '2')], [(SegmentationTagNames.BODY, TOKEN_2)], [(SegmentationTagNames.PAGE, '3')] ]
def test_should_not_annotate_preserved_page_numbers_as_headnote(self): doc = _simple_document_with_tagged_token_lines(lines=[ [(None, '1')], [(FrontTagNames.TITLE, TOKEN_1)], [(None, '1')], [(BodyTagNames.SECTION_TITLE, TOKEN_2)], ]) all_tokens = list(doc.iter_all_tokens()) doc._set_preserved_tag(all_tokens[0], PageTagNames.PAGE) # pylint: disable=protected-access doc._set_preserved_tag(all_tokens[2], PageTagNames.PAGE) # pylint: disable=protected-access SegmentationAnnotator(DEFAULT_CONFIG, preserve_tags=True).annotate(doc) assert _get_document_tagged_token_lines(doc) == [[ (SegmentationTagNames.PAGE, '1') ], [(SegmentationTagNames.FRONT, TOKEN_1) ], [(SegmentationTagNames.PAGE, '1') ], [(SegmentationTagNames.BODY, TOKEN_2)]]
def test_should_annotate_page_header(self): doc = _simple_document_with_tagged_token_lines(lines=[ [(None, t) for t in LONG_PAGE_HEADER_TEXT_1.split(' ')], [(FrontTagNames.TITLE, TOKEN_1)], [(None, t) for t in LONG_PAGE_HEADER_TEXT_1.split(' ')], [(FrontTagNames.ABSTRACT, TOKEN_2)], ]) SegmentationAnnotator(DEFAULT_CONFIG).annotate(doc) assert _get_document_tagged_token_lines(doc) == [ [(SegmentationTagNames.HEADNOTE, t) for t in LONG_PAGE_HEADER_TEXT_1.split(' ')], [(SegmentationTagNames.FRONT, TOKEN_1)], [(SegmentationTagNames.HEADNOTE, t) for t in LONG_PAGE_HEADER_TEXT_1.split(' ')], [(SegmentationTagNames.FRONT, TOKEN_2)] ]
def test_should_merge_separate_reference_if_enabled(self): doc = _simple_document_with_tagged_token_lines(lines=[ [(add_tag_prefix(BackTagNames.REFERENCE, prefix=B_TAG_PREFIX), TOKEN_1), (add_tag_prefix(BackTagNames.REFERENCE, prefix=I_TAG_PREFIX), TOKEN_2)], [(add_tag_prefix(BackTagNames.REFERENCE, prefix=B_TAG_PREFIX), TOKEN_3), (add_tag_prefix(BackTagNames.REFERENCE, prefix=I_TAG_PREFIX), TOKEN_4)] ]) SegmentationAnnotator( DEFAULT_CONFIG._replace(no_merge_references=False)).annotate(doc) assert _get_document_tagged_token_lines(doc) == [ [(SegmentationTagNames.REFERENCE, TOKEN_1), (SegmentationTagNames.REFERENCE, TOKEN_2)], [(SegmentationTagNames.REFERENCE, TOKEN_3), (SegmentationTagNames.REFERENCE, TOKEN_4)] ]
def test_should_annotate_not_fail_on_empty_line(self): doc = _simple_document_with_tagged_token_lines( lines=[[], [(None, TOKEN_1)]]) SegmentationAnnotator(DEFAULT_CONFIG, preserve_tags=True).annotate(doc) assert _get_document_tagged_token_lines(doc) == [[], [(None, TOKEN_1)]]
def test_should_not_fail_on_empty_document(self): structured_document = GrobidTrainingTeiStructuredDocument(_tei()) SegmentationAnnotator(DEFAULT_CONFIG).annotate(structured_document)