def annotate(self, structured_document: GrobidTrainingTeiStructuredDocument): all_tokens_iterable = iter_all_tokens_excluding_space(structured_document) previous_untagged_tokens = [] ignored_token_tag_values = set() for token in all_tokens_iterable: tag_value = structured_document.get_tag_or_preserved_tag_value(token) if not tag_value: previous_untagged_tokens.append(token) continue if not previous_untagged_tokens: continue if tag_value not in self.config.enabled_tags: LOGGER.debug( 'ignoring untagged tokens (before %r): %s', tag_value, previous_untagged_tokens ) ignored_token_tag_values.add(tag_value) previous_untagged_tokens.clear() continue LOGGER.debug( 'updated untagged tokens (before %r): %s', tag_value, previous_untagged_tokens ) for index, previous_untagged_token in enumerate(previous_untagged_tokens): structured_document.set_tag_only( previous_untagged_token, add_tag_prefix(tag_value, B_TAG_PREFIX if index == 0 else I_TAG_PREFIX) ) structured_document.set_tag_only( token, add_tag_prefix(tag_value, I_TAG_PREFIX) ) previous_untagged_tokens.clear() LOGGER.debug('ignore not enabled tag values: %s', ignored_token_tag_values) return structured_document
def annotate(self, structured_document: GrobidTrainingTeiStructuredDocument): all_tokens_iterable = iter_all_tokens_excluding_space(structured_document) previous_enabled_tag_value: Optional[str] = None previous_included_tokens: List[Any] = [] ignored_token_tag_values = set() for token in all_tokens_iterable: tag_value = structured_document.get_tag_or_preserved_tag_value(token) if tag_value: _log_previous_included_tokens( previous_enabled_tag_value, previous_included_tokens ) previous_included_tokens.clear() previous_enabled_tag_value = ( tag_value if tag_value in self.config.enabled_tags else None ) if not previous_enabled_tag_value: ignored_token_tag_values.add(tag_value) continue if not previous_enabled_tag_value: continue structured_document.set_tag_only( token, add_tag_prefix(previous_enabled_tag_value, I_TAG_PREFIX) ) previous_included_tokens.append(token) _log_previous_included_tokens( previous_enabled_tag_value, previous_included_tokens ) LOGGER.debug('ignore not enabled tag values: %s', ignored_token_tag_values) return structured_document
def annotate(self, structured_document: GrobidTrainingTeiStructuredDocument): # There is currently no support for more than two level tagging, # which would allow the a parent level to be represented. # As a workaround we are adding the a separate parent tag, to the tokens without tag. # That way those tokens will share a common parent element in the output. tag_level = self.config.tag_level all_tokens_iterable = iter_all_tokens_excluding_space( structured_document) unmatched_tags = set() current_group_tag = None for token in all_tokens_iterable: if tag_level: # if we looking at sub tags, then only consider tokens with a tag if not structured_document.get_tag_or_preserved_tag(token): continue tag = structured_document.get_tag_or_preserved_tag_value( token, level=tag_level) if tag: current_group_tag = self.config.get_group_tag_for_tag_fn(tag) if not current_group_tag: unmatched_tags.add(tag) continue if not current_group_tag: continue structured_document.set_tag_only(token, add_tag_prefix( current_group_tag, prefix=I_TAG_PREFIX), level=tag_level) LOGGER.debug('updated group token (%r): %s', current_group_tag, token) LOGGER.debug('ignored unmatched tags: %s', unmatched_tags) return structured_document