def annotate(self, structured_document: GrobidTrainingTeiStructuredDocument): # There is currently no support for more than two level tagging, # which would allow the a parent level to be represented. # As a workaround we are adding the a separate parent tag, to the tokens without tag. # That way those tokens will share a common parent element in the output. tag_level = self.config.tag_level all_tokens_iterable = iter_all_tokens_excluding_space( structured_document) unmatched_tags = set() current_group_tag = None for token in all_tokens_iterable: if tag_level: # if we looking at sub tags, then only consider tokens with a tag if not structured_document.get_tag_or_preserved_tag(token): continue tag = structured_document.get_tag_or_preserved_tag_value( token, level=tag_level) if tag: current_group_tag = self.config.get_group_tag_for_tag_fn(tag) if not current_group_tag: unmatched_tags.add(tag) continue if not current_group_tag: continue structured_document.set_tag_only(token, add_tag_prefix( current_group_tag, prefix=I_TAG_PREFIX), level=tag_level) LOGGER.debug('updated group token (%r): %s', current_group_tag, token) LOGGER.debug('ignored unmatched tags: %s', unmatched_tags) return structured_document
def get_line_number_candidates( structured_document: GrobidTrainingTeiStructuredDocument, tokens: list, min_line_number: int, max_line_number_gap: int): line_number_candidates_with_num = [ (token, parse_line_number(structured_document.get_text(token)), 1 + index) for index, token in enumerate(tokens) if is_line_number(structured_document.get_text(token)) ] if not line_number_candidates_with_num: return [] line_number_candidates_with_num = sorted(line_number_candidates_with_num, key=lambda item: (item[1], item[2])) line_number_sequences = [[line_number_candidates_with_num[0]]] for item in line_number_candidates_with_num[1:]: token, num, token_pos = item prev_seq = line_number_sequences[-1] prev_item = prev_seq[-1] _, prev_num, prev_token_pos = prev_item expected_num = prev_num + 1 if token_pos < prev_token_pos or num == prev_num: LOGGER.debug('ignoring out of sequence: %s (prev: %s)', item, prev_item) elif expected_num <= num <= expected_num + max_line_number_gap: prev_seq.append(item) else: line_number_sequences.append([item]) accepted_line_number_sequences = [ seq for seq in line_number_sequences if len(seq) >= min_line_number ] return [ token for seq in accepted_line_number_sequences for token, _, _ in seq ]
def annotate(self, structured_document: GrobidTrainingTeiStructuredDocument): all_tokens_iterable = iter_all_tokens_excluding_space(structured_document) previous_untagged_tokens = [] ignored_token_tag_values = set() for token in all_tokens_iterable: tag_value = structured_document.get_tag_or_preserved_tag_value(token) if not tag_value: previous_untagged_tokens.append(token) continue if not previous_untagged_tokens: continue if tag_value not in self.config.enabled_tags: LOGGER.debug( 'ignoring untagged tokens (before %r): %s', tag_value, previous_untagged_tokens ) ignored_token_tag_values.add(tag_value) previous_untagged_tokens.clear() continue LOGGER.debug( 'updated untagged tokens (before %r): %s', tag_value, previous_untagged_tokens ) for index, previous_untagged_token in enumerate(previous_untagged_tokens): structured_document.set_tag_only( previous_untagged_token, add_tag_prefix(tag_value, B_TAG_PREFIX if index == 0 else I_TAG_PREFIX) ) structured_document.set_tag_only( token, add_tag_prefix(tag_value, I_TAG_PREFIX) ) previous_untagged_tokens.clear() LOGGER.debug('ignore not enabled tag values: %s', ignored_token_tag_values) return structured_document
def test_should_set_empty_token_whitespace(self): doc = GrobidTrainingTeiStructuredDocument( _tei(front_items=[E.note('1.')])) lines = _get_all_lines(doc) assert _get_token_texts_for_lines(doc, lines) == [['1', '.']] tokens = doc.get_all_tokens_of_line(lines[0]) assert tokens[0].whitespace == ''
def annotate(self, structured_document: GrobidTrainingTeiStructuredDocument): all_tokens_iterable = iter_all_tokens_excluding_space(structured_document) previous_enabled_tag_value: Optional[str] = None previous_included_tokens: List[Any] = [] ignored_token_tag_values = set() for token in all_tokens_iterable: tag_value = structured_document.get_tag_or_preserved_tag_value(token) if tag_value: _log_previous_included_tokens( previous_enabled_tag_value, previous_included_tokens ) previous_included_tokens.clear() previous_enabled_tag_value = ( tag_value if tag_value in self.config.enabled_tags else None ) if not previous_enabled_tag_value: ignored_token_tag_values.add(tag_value) continue if not previous_enabled_tag_value: continue structured_document.set_tag_only( token, add_tag_prefix(previous_enabled_tag_value, I_TAG_PREFIX) ) previous_included_tokens.append(token) _log_previous_included_tokens( previous_enabled_tag_value, previous_included_tokens ) LOGGER.debug('ignore not enabled tag values: %s', ignored_token_tag_values) return structured_document
def iter_find_line_number_tokens( structured_document: GrobidTrainingTeiStructuredDocument, **kwargs): for page in structured_document.get_pages(): line_no_tokens = iter_find_line_number_tokens_in_lines( structured_document, lines=structured_document.get_lines_of_page(page), **kwargs) yield from line_no_tokens
def _get_document_tagged_token_lines( doc: GrobidTrainingTeiStructuredDocument ) -> List[List[Tuple[str, str]]]: document_tagged_token_lines = [[(doc.get_tag(token), doc.get_text(token)) for token in doc.get_tokens_of_line(line)] for page in doc.get_pages() for line in doc.get_lines_of_page(page)] LOGGER.debug('document_tagged_token_lines: %s', document_tagged_token_lines) return document_tagged_token_lines
def test_should_not_include_line_feed_in_tag_if_previous_token_has_different_tag( self): original_root = _tei(front_items=[TOKEN_1, '\n ' + TOKEN_2]) LOGGER.debug('original_root: %s', _to_xml(original_root)) doc = GrobidTrainingTeiStructuredDocument(original_root) lines = _get_all_lines(doc) line1_tokens = list(doc.get_all_tokens_of_line(lines[0])) space_tokens = [t for t in line1_tokens if isinstance(t, TeiSpace)] assert space_tokens for token in space_tokens: doc.set_tag(token, TAG_1) doc.set_tag(line1_tokens[0], TAG_1) doc.set_tag(line1_tokens[-1], TAG_2) LOGGER.debug('line1_tokens: %s', line1_tokens) root = doc.root front = root.find('./text/front') LOGGER.debug('xml: %s', _to_xml(front)) assert _to_xml(front) == ( '<front><{tag1}>{token1}</{tag1}>' '\n <{tag2}>{token2}</{tag2}></front>'.format(tag1=TAG_1, tag2=TAG_2, token1=TOKEN_1, token2=TOKEN_2))
def annotate( self, structured_document: GrobidTrainingTeiStructuredDocument ) -> GrobidTrainingTeiStructuredDocument: line_number_tokens = iter_find_line_number_tokens( structured_document, min_line_number=self.config.min_line_number, max_line_number_gap=self.config.max_line_number_gap, line_number_ratio_threshold=self.config.line_number_ratio_threshold ) for t in line_number_tokens: structured_document.set_tag(t, self.config.tag) return structured_document
def test_should_be_able_get_root_with_updated_single_token_tag(self): doc = GrobidTrainingTeiStructuredDocument( _tei(front_items=[E.note(TOKEN_1)])) lines = _get_all_lines(doc) tokens = list(doc.get_tokens_of_line(lines[0])) token = tokens[0] doc.set_tag(token, TAG_1) root = doc.root front = root.find('./text/front') child_elements = list(front) assert [c.tag for c in child_elements] == [TAG_1] assert [c.text for c in child_elements] == [TOKEN_1]
def test_should_reverse_map_tags(self): tag_to_tei_path_mapping = {TAG_1: 'docTitle/titlePart'} original_tei_xml = _tei(front_items=[E.docTitle(E.titlePart(TOKEN_1))]) LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml)) doc = GrobidTrainingTeiStructuredDocument( original_tei_xml, tag_to_tei_path_mapping=tag_to_tei_path_mapping, preserve_tags=True) LOGGER.debug('doc: %s', doc) assert [[ doc.get_tag_or_preserved_tag_value(t) for t in doc.get_all_tokens_of_line(line) ] for line in _get_all_lines(doc)] == [[TAG_1]]
def test_should_be_able_get_root_with_updated_single_token_tag_with_attribute( self): doc = GrobidTrainingTeiStructuredDocument( _tei(front_items=[E.note(TOKEN_1)]), tag_to_tei_path_mapping={TAG_1: 'div[@tag="tag1"]'}) lines = _get_all_lines(doc) tokens = list(doc.get_tokens_of_line(lines[0])) token = tokens[0] doc.set_tag(token, TAG_1) root = doc.root front = root.find('./text/front') child_elements = list(front) assert [c.tag for c in child_elements] == ['div'] assert [c.attrib for c in child_elements] == [{'tag': 'tag1'}] assert [c.text for c in child_elements] == [TOKEN_1]
def test_should_find_one_line_with_one_token_at_front_level(self): doc = GrobidTrainingTeiStructuredDocument( _tei(front_items=[E.note(TOKEN_1)]), container_node_path='text/front') lines = _get_all_lines(doc) assert _get_token_texts_for_lines(doc, lines) == [[TOKEN_1]] assert doc.root.find('./text/front/note').text == TOKEN_1
def test_should_find_empty_first_line_and_text_outside_semantic_element( self): doc = GrobidTrainingTeiStructuredDocument( _tei(front_items=[TOKEN_1, _tei_lb(), TOKEN_2, E.note(TOKEN_3)])) lines = _get_all_lines(doc) assert _get_token_texts_for_lines(doc, lines) == [[TOKEN_1], [TOKEN_2, TOKEN_3]]
def test_should_remove_untagged_including_line_feed(self): doc = GrobidTrainingTeiStructuredDocument( _tei(front_items=[ TOKEN_1, E(TeiTagNames.LB), ' ' + TOKEN_2 + ' ' + TOKEN_3 ])) lines = _get_all_lines(doc) line1_tokens = list(doc.get_tokens_of_line(lines[0])) doc.set_tag(line1_tokens[0], None) line2_tokens = list(doc.get_tokens_of_line(lines[1])) doc.set_tag(line2_tokens[-1], TAG_1) doc.remove_all_untagged() root = doc.root front = root.find('./text/front') LOGGER.debug('xml: %s', _to_xml(front)) assert _to_xml(front) == ( '<front><{tag1}>{token3}</{tag1}></front>'.format(tag1=TAG_1, token3=TOKEN_3))
def test_should_be_able_to_set_tag(self): doc = GrobidTrainingTeiStructuredDocument( _tei(front_items=[E.note(TOKEN_1)])) lines = _get_all_lines(doc) tokens = list(doc.get_tokens_of_line(lines[0])) token = tokens[0] doc.set_tag(token, TAG_1) assert doc.get_tag(token) == TAG_1
def _add_name_text_suffix( structured_document: GrobidTrainingTeiStructuredDocument, tokens: List[Any], config: ReferenceAnnotatorConfig): sub_tags = [structured_document.get_sub_tag(token) for token in tokens] token_texts = [structured_document.get_text(token) for token in tokens] token_whitespaces = [structured_document.get_whitespace(token) for token in tokens] mapped_sub_tags = _map_tags(sub_tags, config.sub_tag_map) transformed_sub_tags = get_suffix_extended_token_tags( mapped_sub_tags, token_texts, token_whitespaces=token_whitespaces, enabled_tags=config.include_suffix_enabled_sub_tags ) LOGGER.debug( 'name suffix sub tokens, transformed: %s -> %s -> %s (tokens: %s)', sub_tags, mapped_sub_tags, transformed_sub_tags, tokens ) for token, token_sub_tag in zip(tokens, transformed_sub_tags): if not token_sub_tag: continue structured_document.set_sub_tag(token, token_sub_tag) return structured_document
def test_should_be_able_to_set_tag_with_attribute(self): doc = GrobidTrainingTeiStructuredDocument( _tei(front_items=[E.note(TOKEN_1)]), tag_to_tei_path_mapping={TAG_1: 'div[@tag="tag1"]'}) lines = _get_all_lines(doc) tokens = list(doc.get_tokens_of_line(lines[0])) token = tokens[0] doc.set_tag(token, TAG_1) assert doc.get_tag(token) == TAG_1
def test_should_preserve_existing_tag(self): original_tei_xml = _tei( front_items=[E.docTitle(E.titlePart(TOKEN_1)), TOKEN_2]) LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml)) doc = GrobidTrainingTeiStructuredDocument(original_tei_xml, preserve_tags=True, tag_to_tei_path_mapping={}) LOGGER.debug('doc: %s', doc) root = doc.root front = root.find('./text/front') LOGGER.debug('xml: %s', _to_xml(front)) assert _to_xml(front) == ( '<front><docTitle><titlePart>{token1}</titlePart></docTitle>{token2}</front>' .format(token1=TOKEN_1, token2=TOKEN_2))
def test_should_preserve_existing_nested_tag(self): original_tei_xml = _tei( front_items=[E.div(E.label(TOKEN_1), TOKEN_2), TOKEN_3]) LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml)) doc = GrobidTrainingTeiStructuredDocument( original_tei_xml, preserve_tags=True, tag_to_tei_path_mapping={TAG_1: 'div'}) LOGGER.debug('doc: %s', doc) root = doc.root front = root.find('./text/front') LOGGER.debug('xml: %s', _to_xml(front)) assert _to_xml(front) == ( '<front><div><label>{token1}</label>{token2}</div>{token3}</front>' .format(token1=TOKEN_1, token2=TOKEN_2, token3=TOKEN_3))
def test_should_preserve_separation_between_existing_tag(self): original_tei_xml = _tei(front_items=[ E.byline(E.affiliation(TOKEN_1)), E.byline(E.affiliation(TOKEN_2)) ]) LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml)) doc = GrobidTrainingTeiStructuredDocument(original_tei_xml, preserve_tags=True, tag_to_tei_path_mapping={}) LOGGER.debug('doc: %s', doc) root = doc.root front = root.find('./text/front') affiliations = front.xpath('./byline/affiliation') LOGGER.debug('xml: %s', _to_xml(front)) assert [aff.text for aff in affiliations] == [TOKEN_1, TOKEN_2]
def test_should_preserve_existing_tag_with_attrib(self): original_tei_xml = _tei( front_items=[E.div(TOKEN_1, {'tag': TAG_1}), TOKEN_2]) LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml)) doc = GrobidTrainingTeiStructuredDocument( original_tei_xml, preserve_tags=True, tag_to_tei_path_mapping={TAG_1: 'div[@tag="tag1"]'}) LOGGER.debug('doc: %s', doc) root = doc.root front = root.find('./text/front') LOGGER.debug('xml: %s', _to_xml(front)) assert _to_xml(front) == ( '<front><div tag="{TAG_1}">{token1}</div>{token2}</front>'.format( token1=TOKEN_1, token2=TOKEN_2, TAG_1=TAG_1))
def test_should_preserve_space_after_lb_in_updated_root(self): doc = GrobidTrainingTeiStructuredDocument( _tei(front_items=[TOKEN_1, E(TeiTagNames.LB), ' ' + TOKEN_2])) lines = _get_all_lines(doc) line1_tokens = list(doc.get_tokens_of_line(lines[0])) doc.set_tag(line1_tokens[0], TAG_1) line2_tokens = list(doc.get_tokens_of_line(lines[1])) doc.set_tag(line2_tokens[-1], TAG_1) root = doc.root front = root.find('./text/front') child_elements = list(front) assert [c.tag for c in child_elements] == [TAG_1] assert _to_xml(child_elements[0]) == ( '<{tag1}>{token1}<{lb}/> {token2}</{tag1}>'.format( tag1=TAG_1, token1=TOKEN_1, token2=TOKEN_2, lb=TeiTagNames.LB))
def _simple_document_with_tagged_token_lines( lines: List[List[Tuple[str, str]]]) -> GrobidTrainingTeiStructuredDocument: tei_items = [] for line in lines: tei_items.append(' '.join(token for _, token in line)) tei_items.append(E.lb()) doc = GrobidTrainingTeiStructuredDocument( _tei(tei_items), container_node_path=DEFAULT_CONTAINER_NODE_PATH) doc_lines = [ line for page in doc.get_pages() for line in doc.get_lines_of_page(page) ] for line, doc_line in zip(lines, doc_lines): for (tag, token), doc_token in zip(line, doc.get_tokens_of_line(doc_line)): assert token == doc.get_text(doc_token) if tag: doc.set_tag(doc_token, tag) return doc
def test_should_not_include_space_in_tag_if_previous_token_has_different_tag( self): doc = GrobidTrainingTeiStructuredDocument( _tei(front_items=[TOKEN_1, E(TeiTagNames.LB), ' ' + TOKEN_2])) lines = _get_all_lines(doc) line1_tokens = list(doc.get_tokens_of_line(lines[0])) doc.set_tag(line1_tokens[0], TAG_1) line2_tokens = list(doc.get_tokens_of_line(lines[1])) doc.set_tag(line2_tokens[-1], TAG_2) root = doc.root front = root.find('./text/front') LOGGER.debug('xml: %s', _to_xml(front)) assert _to_xml(front) == ('<front><{tag1}>{token1}<{lb}/></{tag1}>' ' <{tag2}>{token2}</{tag2}></front>'.format( tag1=TAG_1, tag2=TAG_2, token1=TOKEN_1, token2=TOKEN_2, lb=TeiTagNames.LB))
def test_should_not_return_preserved_tag_as_tag_and_update_preserved_tag( self): original_tei_xml = _tei(front_items=[E.note(TOKEN_1)]) LOGGER.debug('original tei xml: %s', _to_xml(original_tei_xml)) doc = GrobidTrainingTeiStructuredDocument(original_tei_xml, preserve_tags=True, tag_to_tei_path_mapping={}) LOGGER.debug('doc: %s', doc) lines = _get_all_lines(doc) token1 = list(doc.get_tokens_of_line(lines[0]))[0] assert not doc.get_tag(token1) doc.set_tag(token1, TAG_1) root = doc.root front = root.find('./text/front') LOGGER.debug('xml: %s', _to_xml(front)) assert _to_xml(front) == ( '<front><{tag1}>{token1}</{tag1}></front>'.format(token1=TOKEN_1, tag1=TAG_1))
def _structured_document_with_title(title=TITLE_1): return GrobidTrainingTeiStructuredDocument(_tei_with_title(title))
def _structured_document_with_sub_elements(*args, **kwargs): return GrobidTrainingTeiStructuredDocument( _tei_with_sub_elements(*args, **kwargs), tag_to_tei_path_mapping=TAG_TO_TEI_PATH_MAPPING)
def test_should_not_fail_on_empty_document(self): structured_document = GrobidTrainingTeiStructuredDocument(_tei()) TextLineNumberAnnotator().annotate(structured_document)
def iter_first_tokens_of_lines( structured_document: GrobidTrainingTeiStructuredDocument, lines: list): for line in lines: text_tokens = structured_document.get_tokens_of_line(line) if text_tokens: yield text_tokens[0]