def _get_wordpiece_detokenized_text(
token_span: _SpanType, raw_prediction: _RawPredictionType,
tokenizer: tokenization.FullTokenizer) -> str:
"""Gets the normalized answer token text given the token span."""
answer_tokens = tokenizer.convert_ids_to_tokens(
raw_prediction["long_token_ids"][token_span[0]:token_span[1] + 1])
return data_utils.wordpiece_tokens_to_normalized_text(answer_tokens)
def _get_sentencepiece_detokenized_text(token_span: _SpanType,
raw_prediction: _RawPredictionType,
tokenizer: tokenization.FullTokenizer):
"""Gets final text using SentencePiece tokens."""
long_token_ids = raw_prediction["long_token_ids"]
answer_tokens = tokenizer.convert_ids_to_tokens(
long_token_ids[token_span[0]:token_span[1] + 1].tolist())
return data_utils.sentencepiece_detokenize(answer_tokens)
def get_sentencepiece_tokenized_text(
text: str, tokenizer: tokenization.FullTokenizer) -> TokenizedText:
"""Gets SentencePiece TokenizedText for a text with indices mapping."""
tokens = [six.ensure_text(tk, "utf-8") for tk in tokenizer.tokenize(text)]
token_ids = tokenizer.convert_tokens_to_ids(tokens)
chars_to_tokens = []
for i, token in enumerate(tokens):
num_chars = len(token)
if i == 0:
num_chars -= 1
chars_to_tokens.extend([i] * num_chars)
token_ids = tokenizer.convert_tokens_to_ids(tokens)
tokenized_text = TokenizedText()
tokenized_text.text = sentencepiece_detokenize(tokens)
tokenized_text.tokens = tokens
tokenized_text.token_ids = token_ids
tokenized_text.chars_to_tokens = chars_to_tokens
return tokenized_text
def _improve_answer_span(
doc_tokens: Sequence[str],
unimproved_span: Tuple[int, int],
orig_answer_text: str,
tokenizer: tokenization.FullTokenizer,
):
"""Returns answer token spans that better match the annotated answer.
This function is branched from the original BERT `run_squad.py` code
Usually question answer span annotations are character based. We first project
them to whitespace-tokenized words (unigrams). But then after WordPiece
tokenization, we can often find a "better match". For example:
Question: What year was John Smith born?
Context: The leader was John Smith (1895-1943).
Answer: 1895
The original whitespace-tokenized answer will be "(1895-1943).". However
after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match
the exact answer, 1895. The purpose of this function is to find such "better
match".
However, this is not always possible. Consider the following:
Question: What country is the top exporter of electornics?
Context: The Japanese electronics industry is the lagest in the world.
Answer: Japan
In this case, the annotator chose "Japan" as a character sub-span of
the word "Japanese". Since our WordPiece tokenizer does not split
"Japanese", we just use "Japanese" as the annotation. This is expected to be
fairly rare.
Args:
doc_tokens: Sequence of Text, the wordpiece tokenized tokens of the doc.
unimproved_span: Tuple of two ints, the unimproved answer token span. In the
first example, it is the token span for "(" and ")".
orig_answer_text: Text, the original answer text. In the first example, it
is "1895".
tokenizer: FullTokenizer, wordpiece tokenizer to tokenize the original
answer text.
Returns:
Tuple of two ints, the improved answer token span. In the first example, it
corresponds to the answer token span for "1895".
"""
tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text))
for new_begin in range(unimproved_span[0], unimproved_span[1] + 1):
for new_end in range(unimproved_span[1], new_begin - 1, -1):
text_span = " ".join(doc_tokens[new_begin:(new_end + 1)])
if text_span == tok_answer_text:
return new_begin, new_end
return unimproved_span
def wordpiece_tokenize_with_indices(
doc_unigrams: Sequence[str], tokenizer: tokenization.FullTokenizer
) -> Tuple[List[str], List[int], List[int]]:
"""Wordpiece tokenizes unigrams to tokens and returns indices mapping."""
token_to_unigram_map = []
unigram_to_token_map = []
doc_tokens = []
for (i, token) in enumerate(doc_unigrams):
unigram_to_token_map.append(len(doc_tokens))
sub_tokens = tokenizer.tokenize(token)
token_to_unigram_map.extend([i] * len(sub_tokens))
doc_tokens.extend(sub_tokens)
return doc_tokens, unigram_to_token_map, token_to_unigram_map
def get_wordpiece_tokenized_text(
text: str, tokenizer: tokenization.FullTokenizer) -> TokenizedText:
"""Gets WordPiece TokenizedText for a text with indices mapping."""
unigrams, _, chars_to_unigrams = whitespace_split_with_indices(text)
tokens, unigrams_to_tokens, tokens_to_unigrams = (
wordpiece_tokenize_with_indices(unigrams, tokenizer))
token_ids = tokenizer.convert_tokens_to_ids(tokens)
tokenized_text = TokenizedText()
tokenized_text.text = text
tokenized_text.tokens = tokens
tokenized_text.token_ids = token_ids
tokenized_text.unigrams = unigrams
tokenized_text.chars_to_unigrams = chars_to_unigrams
tokenized_text.unigrams_to_tokens = unigrams_to_tokens
tokenized_text.tokens_to_unigrams = tokens_to_unigrams
return tokenized_text
def find_candidate_mentions(
input_text: Text,
candidate: Text,
tokenizer: tokenization.FullTokenizer,
offset=0) -> Tuple[List[Text], List[Tuple[int, int]]]:
"""Finds the candidate string mentions in the sentence post tokenization.
Args:
input_text: The input for searching the candidate.
candidate: The candidate to be searched for mentions in the input.
tokenizer: The tokenizer to be used. For BERT tokenzier, we assume an
uncased vocab.
offset: Offset to be added to all the span values.
Returns:
A tuple of (input_tokens_list, list_of_candidate_spans_in_the_list)
Example:
input = "Thisss is Saaan Franciscooo"
candidate = "saan franciscooo"
Let's say we are using the ALBERT tokenizer. Tokenizing the input would give:
['▁This', 'ss', '▁is', '▁Sa', 'aan', '▁Franc', 'isc', 'ooo']
We return the tokens of the sentence.
We also return [(3, 7)] representing the only span where the candidate
occurs in the tokenized sentence. Note that the span is inclusive.
"""
assert isinstance(tokenizer, tokenization.FullTokenizer)
input_lower = input_text.lower()
candidate_lower = candidate.lower()
tokens = tokenizer.tokenize(input_text)
if (not candidate_lower or not input_lower or
candidate_lower not in input_lower):
return (tokens, [])
if isinstance(tokenizer, tokenization.FullTokenizer):
# We assume a tokenizer with lower cased vocab here. We do a simple
# substring match of the candidate tokens to the input text tokens.
if (tokenizer.sp_model is None and
not tokenizer.basic_tokenizer.do_lower_case):
raise ValueError("BERT tokenizer should be lower cased.")
candidate_tokens = tokenizer.tokenize(candidate.lower())
candidate_len = len(candidate_tokens)
candidate_spans = []
for i in range(0, len(tokens)):
if i + candidate_len <= len(tokens):
if tokens[i:i + candidate_len] == candidate_tokens:
candidate_spans.append((offset + i, offset + i + candidate_len - 1))
return (tokens, candidate_spans)
# Now that we know the candidate is present in the input_text, we do a
# best effort matching.
spiece_underline = tokenization.SPIECE_UNDERLINE.decode("utf-8")
char_index_to_token_index = collections.OrderedDict()
i = 0
for (j, token) in enumerate(tokens):
k = 0
if token.startswith(spiece_underline):
k += 1
for c in token[k:len(token)]:
c = c.lower()
# Most chars, in general, other than the special_token in tokens have a
# corresponding mapping in the input_text.
while i < len(input_lower) and _is_whitespace(input_lower[i]):
# To handle cases like a space etc in the input_text. Spaces, tabs etc.
# generally don't appear in the tokens.
char_index_to_token_index[i] = j
i += 1
if _is_whitespace(c):
# This shouldn't generally happen - ALBERT tokenizer collapses
# whitespaces.
continue
if c != input_lower[i]:
# Tokenizer probably has extra characters for this token.
continue
if i < len(input_lower):
assert c == input_lower[i]
char_index_to_token_index[i] = j
i += 1
if i != len(input_text):
# Our best effort matching chars to tokens failed. As a fallback, we will
# just match the given candidate with the entire input_text and return.
# Because we know that the candidate is already present in the input_text,
# it's better to assign the candidate to the entire input_text (which in
# our case is a sentence), rather than dropping it altogether.
return (tokens, [(offset, offset + len(tokens) - 1)])
# We now have matched every char in the input to its corresponding token
# index successfully.
candidate_spans = []
cand_len = len(candidate_lower)
# Using re.finditer for substring match seems to be throwing a weird python
# error -- "nothing to repeat at position 0", in some cases. So doing a
# brute force substring match.
for start in range(0, len(input_lower)):
if (start + cand_len <= len(input_lower) and
input_lower[start:start + cand_len] == candidate_lower):
end = start + cand_len - 1
assert start in char_index_to_token_index, (
"no mapping found for index %d for candidate %s ", start, candidate)
assert end in char_index_to_token_index, (
"no mapping found for index %d for candidate %s ", end, candidate)
token_span_start = char_index_to_token_index[start]
token_span_end = char_index_to_token_index[end]
candidate_spans.append(
(offset + token_span_start, offset + token_span_end))
return (tokens, candidate_spans)