def test_should_extract_single_annotated_line(self): lines = [annotated_line(TEXT_1, TAG_1)] structured_document = SimpleStructuredDocument(lines=lines) result = [(x.tag, x.text) for x in extract_from_annotated_document(structured_document) ] assert result == [(TAG_1, TEXT_1)]
def test_should_extract_sub_tags_from_single_item(self): tokens = [ _token_with_sub_tag(VALUE_1, tag=TAG_1, tag_prefix=B_TAG_PREFIX, sub_tag=TAG_2, sub_tag_prefix=B_TAG_PREFIX), _token_with_sub_tag(VALUE_2, tag=TAG_1, tag_prefix=I_TAG_PREFIX, sub_tag=TAG_2, sub_tag_prefix=I_TAG_PREFIX), _token_with_sub_tag(VALUE_3, tag=TAG_1, tag_prefix=I_TAG_PREFIX, sub_tag=TAG_3, sub_tag_prefix=B_TAG_PREFIX) ] structured_document = SimpleStructuredDocument( lines=[SimpleLine(tokens)]) extracted_items = list( extract_from_annotated_document(structured_document)) result = [(x.tag, x.text, [(sub.tag, sub.text) for sub in x.sub_items]) for x in extracted_items] get_logger().debug('result: %s', result) assert result == [(TAG_1, ' '.join([VALUE_1, VALUE_2, VALUE_3]), [(TAG_2, ' '.join([VALUE_1, VALUE_2])), (TAG_3, VALUE_3)])]
def test_should_extract_from_different_tag_scope(self): lines = [ SimpleLine([SimpleToken(TEXT_1, tag=TAG_1, tag_scope=TAG_SCOPE_1)]) ] structured_document = SimpleStructuredDocument(lines=lines) result = [(x.tag, x.text) for x in extract_from_annotated_document( structured_document, tag_scope=TAG_SCOPE_1)] assert result == [(TAG_1, TEXT_1)]
def test_should_combine_multiple_lines(self): lines = [annotated_line(TEXT_1, TAG_1), annotated_line(TEXT_2, TAG_1)] structured_document = SimpleStructuredDocument(lines=lines) result = [(x.tag, x.text) for x in extract_from_annotated_document(structured_document) ] get_logger().debug('result: %s', result) assert result == [(TAG_1, '\n'.join([TEXT_1, TEXT_2]))]
def test_should_extract_multiple_annotations_on_single_line(self): lines = [ to_line( annotated_tokens(TEXT_1, TAG_1) + to_tokens(TEXT_2) + annotated_tokens(TEXT_3, TAG_3)) ] structured_document = SimpleStructuredDocument(lines=lines) result = [(x.tag, x.text) for x in extract_from_annotated_document(structured_document) ] assert result == [(TAG_1, TEXT_1), (None, TEXT_2), (TAG_3, TEXT_3)]
def test_should_separate_items_based_on_tag_prefix(self): tokens = [ SimpleToken(VALUE_1, tag=TAG_1, tag_prefix=B_TAG_PREFIX), SimpleToken(VALUE_2, tag=TAG_1, tag_prefix=I_TAG_PREFIX), SimpleToken(VALUE_3, tag=TAG_1, tag_prefix=I_TAG_PREFIX), SimpleToken(VALUE_1, tag=TAG_1, tag_prefix=B_TAG_PREFIX), SimpleToken(VALUE_2, tag=TAG_1, tag_prefix=I_TAG_PREFIX), SimpleToken(VALUE_3, tag=TAG_1, tag_prefix=I_TAG_PREFIX) ] structured_document = SimpleStructuredDocument( lines=[SimpleLine(tokens)]) result = [(x.tag, x.text) for x in extract_from_annotated_document(structured_document) ] get_logger().debug('result: %s', result) assert result == [(TAG_1, ' '.join([VALUE_1, VALUE_2, VALUE_3])), (TAG_1, ' '.join([VALUE_1, VALUE_2, VALUE_3]))]
def test_should_not_fail_on_empty_document(self): structured_document = SimpleStructuredDocument() extract_from_annotated_document(structured_document)
def extract_structured_document_to_xml(structured_document, tag_scope=None): return extracted_items_to_xml( extract_from_annotated_document(structured_document, tag_scope=tag_scope))