def testWordPieceOpAndVerifyOffsets(self,
tokens,
expected_subwords,
vocab,
expected_start=None,
expected_limit=None,
use_unknown_token=True,
unknown_token="[UNK]",
token_out_type=dtypes.string,
max_bytes_per_word=100):
tokens = ragged_factory_ops.constant(tokens)
vocab_table = _CreateTable(vocab)
self.evaluate(vocab_table.initializer)
tokenizer = WordpieceTokenizer(vocab_table,
unknown_token=unknown_token,
token_out_type=token_out_type,
max_bytes_per_word=max_bytes_per_word)
subwords, begin, end = tokenizer.tokenize_with_offsets(tokens)
self.assertRaggedEqual(subwords, expected_subwords)
# Verify the indices by performing the following:
# - Extract the subwords and join them together to form the original tokens.
# - Then compare the extracted tokens and original tokens.
tokens, begin, end = (self.evaluate((tokens, begin, end)))
# If expected start/limit offsets were provided, check them explicitly.
# Otherwise test the offsets by extracting subwords using token offsets
# from the original 'tokens' input.
if expected_start is None or expected_limit is None:
extracted_tokens = _GetTokensFromWordpieceOffsets(
tokens, begin, end)
self.assertRaggedEqual(extracted_tokens, tokens)
else:
self.assertRaggedEqual(begin, expected_start)
self.assertRaggedEqual(end, expected_limit)
def testWordPieceOpAndVerifyOffsets(self,
tokens,
expected_subwords,
vocab,
expected_start=None,
expected_limit=None,
use_unknown_token=True,
unknown_token="[UNK]",
token_out_type=tf.string):
tokens = tf.ragged.constant(tokens)
vocab_table = _CreateTable(vocab)
self.evaluate(vocab_table.initializer)
tokenizer = WordpieceTokenizer(vocab_table,
unknown_token=unknown_token,
token_out_type=token_out_type)
subwords, begin, end = tokenizer.tokenize_with_offsets(tokens)
self.assertRaggedEqual(subwords, expected_subwords)
# Verify the indices by performing the following:
# - Extract the subwords and join them together to form the original tokens.
# - Then compare the extracted tokens and original tokens.
tokens, begin, end = (self.evaluate((tokens, begin, end)))
extracted_tokens = _GetTokensFromWordpieceOffsets(tokens, begin, end)
self.assertRaggedEqual(extracted_tokens, tokens)
def testWordPieceOpWithIdReturned(self):
"""Let the table determine how to do a lookup on unknown tokens."""
tokens = ragged_factory_ops.constant(
[[b"don't", b"tread", b"cantfindme", b"treadcantfindme"]])
vocab_table = _CreateTable(
_ENGLISH_VOCAB,
100 # OOV values
)
self.evaluate(vocab_table.initializer)
tokenizer = WordpieceTokenizer(
vocab_table, unknown_token=None, token_out_type=dtypes.int64)
subwords, _, _ = tokenizer.tokenize_with_offsets(tokens)
self.assertAllEqual(subwords, [[[0, 1, 2], [3], [96], [46]]])
def testWordPieceOpAndVerifyOffsets(self,
tokens,
expected_subwords,
vocab,
expected_start=None,
expected_limit=None,
use_unknown_token=True,
unknown_token="[UNK]",
token_out_type=dtypes.string,
max_bytes_per_word=100,
split_unknown_characters=False):
for horizon in self._FORWARD_COMPATIBILITY_HORIZONS:
with compat.forward_compatibility_horizon(*horizon):
tokens_t = ragged_factory_ops.constant(tokens)
vocab_table = _CreateTable(vocab)
self.evaluate(vocab_table.initializer)
tokenizer = WordpieceTokenizer(
vocab_table,
unknown_token=unknown_token,
token_out_type=token_out_type,
max_bytes_per_word=max_bytes_per_word,
split_unknown_characters=split_unknown_characters,
)
subwords_t, begin_t, end_t = tokenizer.tokenize_with_offsets(
tokens_t)
self.assertAllEqual(subwords_t, expected_subwords)
# Verify the indices by performing the following:
# - Extract subwords and join them together to form the original tokens.
# - Then compare the extracted tokens and original tokens.
begin, end = (self.evaluate((begin_t, end_t)))
# If expected start/limit offsets were provided, check them explicitly.
# Otherwise test the offsets by extracting subwords using token offsets
# from the original 'tokens' input.
if expected_start is None or expected_limit is None:
extracted_tokens = _GetTokensFromWordpieceOffsets(
tokens, begin, end)
self.assertAllEqual(extracted_tokens, tokens)
else:
self.assertAllEqual(begin, expected_start)
self.assertAllEqual(end, expected_limit)
class BertTokenizer(TokenizerWithOffsets):
"""Tokenizer used for BERT.
This tokenizer applies an end-to-end, text string to wordpiece tokenization.
It first applies basic tokenization, and then follwed by wordpiece
tokenization.
See BasicTokenizer and WordpieceTokenizer for their respective details.
Attributes:
vocab_lookup_table: A lookup table implementing the LookupInterface
containing the vocabulary of subwords or a string which is the file path
to the vocab.txt file.
suffix_indicator: (optional) The characters prepended to a wordpiece to
indicate that it is a suffix to another subword. Default is '##'.
max_bytes_per_word: (optional) Max size of input token. Default is 100.
max_chars_per_token: (optional) Max size of subwords, excluding suffix
indicator. If known, providing this improves the efficiency of decoding
long words.
token_out_type: (optional) The type of the token to return. This can be
`tf.int64` IDs, or `tf.string` subwords. The default is `tf.int64`.
unknown_token: (optional) The value to use when an unknown token is found.
Default is "[UNK]". If this is set to a string, and `token_out_type` is
`tf.int64`, the `vocab_lookup_table` is used to convert the
`unknown_token` to an integer. If this is set to `None`,
out-of-vocabulary tokens are left as is.
split_unknown_characters: (optional) Whether to split out single unknown
characters as subtokens. If False (default), words containing unknown
characters will be treated as single unknown tokens.
lower_case: bool - If true, a preprocessing step is added to lowercase the
text, apply NFD normalization, and strip accents characters.
keep_whitespace: bool - If true, preserves whitespace characters instead of
stripping them away.
normalization_form: If true and lower_case=False, the input text will be
normalized to `normalization_form`. See normalize_utf8() op for a list
of valid values.
"""
def __init__(self,
vocab_lookup_table,
suffix_indicator="##",
max_bytes_per_word=100,
max_chars_per_token=None,
token_out_type=dtypes.int64,
unknown_token="[UNK]",
split_unknown_characters=False,
lower_case=False,
keep_whitespace=False,
normalization_form=None):
if isinstance(vocab_lookup_table, str):
init = lookup_ops.TextFileIdTableInitializer(vocab_lookup_table)
vocab_lookup_table = lookup_ops.StaticVocabularyTableV1(
init, num_oov_buckets=1, lookup_key_dtype=dtypes.string)
self._basic_tokenizer = BasicTokenizer(lower_case, keep_whitespace,
normalization_form)
self._wordpiece_tokenizer = WordpieceTokenizer(
vocab_lookup_table, suffix_indicator, max_bytes_per_word,
max_chars_per_token, token_out_type, unknown_token,
split_unknown_characters)
def tokenize_with_offsets(self, text_input):
tokens, begin, _ = self._basic_tokenizer.tokenize_with_offsets(
text_input)
wordpieces, wp_begin, wp_end = (
self._wordpiece_tokenizer.tokenize_with_offsets(tokens))
begin_expanded = array_ops.expand_dims(begin, axis=2)
final_begin = begin_expanded + wp_begin
final_end = begin_expanded + wp_end
return wordpieces, final_begin, final_end
def tokenize(self, text_input):
"""Performs untokenized text to wordpiece tokenization for BERT.
Args:
text_input: input: A `Tensor` or `RaggedTensor` of untokenized UTF-8
strings.
Returns:
A `RaggedTensor` of tokens where `tokens[i1...iN, j]` is the string
contents (or ID in the vocab_lookup_table representing that string)
of the `jth` token in `input[i1...iN]`
"""
tokens = self._basic_tokenizer.tokenize(text_input)
return self._wordpiece_tokenizer.tokenize(tokens)
class BertTokenizer(TokenizerWithOffsets, Detokenizer):
r"""Tokenizer used for BERT.
This tokenizer applies an end-to-end, text string to wordpiece tokenization.
It first applies basic tokenization, and then followed by wordpiece
tokenization.
See BasicTokenizer and WordpieceTokenizer for their respective details.
Attributes:
vocab_lookup_table: A lookup table implementing the LookupInterface
containing the vocabulary of subwords or a string which is the file path
to the vocab.txt file.
suffix_indicator: (optional) The characters prepended to a wordpiece to
indicate that it is a suffix to another subword. Default is '##'.
max_bytes_per_word: (optional) Max size of input token. Default is 100.
max_chars_per_token: (optional) Max size of subwords, excluding suffix
indicator. If known, providing this improves the efficiency of decoding
long words.
token_out_type: (optional) The type of the token to return. This can be
`tf.int64` IDs, or `tf.string` subwords. The default is `tf.int64`.
unknown_token: (optional) The value to use when an unknown token is found.
Default is "[UNK]". If this is set to a string, and `token_out_type` is
`tf.int64`, the `vocab_lookup_table` is used to convert the
`unknown_token` to an integer. If this is set to `None`, out-of-vocabulary
tokens are left as is.
split_unknown_characters: (optional) Whether to split out single unknown
characters as subtokens. If False (default), words containing unknown
characters will be treated as single unknown tokens.
lower_case: bool - If true, a preprocessing step is added to lowercase the
text, apply NFD normalization, and strip accents characters.
keep_whitespace: bool - If true, preserves whitespace characters instead of
stripping them away.
normalization_form: If true and lower_case=False, the input text will be
normalized to `normalization_form`. See normalize_utf8() op for a list of
valid values.
preserve_unused_token: If true, text in the regex format `\\[unused\\d+\\]`
will be treated as a token and thus remain preserved as is to be looked up
in the vocabulary.
basic_tokenizer_class: If set, the class to use instead of BasicTokenizer
"""
def __init__(self,
vocab_lookup_table,
suffix_indicator="##",
max_bytes_per_word=100,
max_chars_per_token=None,
token_out_type=dtypes.int64,
unknown_token="[UNK]",
split_unknown_characters=False,
lower_case=False,
keep_whitespace=False,
normalization_form=None,
preserve_unused_token=False,
basic_tokenizer_class=BasicTokenizer):
super(BertTokenizer, self).__init__()
_tf_text_bert_tokenizer_op_create_counter.get_cell().increase_by(1)
self._basic_tokenizer = basic_tokenizer_class(lower_case,
keep_whitespace,
normalization_form,
preserve_unused_token)
self._wordpiece_tokenizer = WordpieceTokenizer(
vocab_lookup_table, suffix_indicator, max_bytes_per_word,
max_chars_per_token, token_out_type, unknown_token,
split_unknown_characters)
def tokenize_with_offsets(self, text_input):
tokens, begin, _ = self._basic_tokenizer.tokenize_with_offsets(
text_input)
wordpieces, wp_begin, wp_end = (
self._wordpiece_tokenizer.tokenize_with_offsets(tokens))
begin_expanded = array_ops.expand_dims(begin, axis=2)
final_begin = begin_expanded + wp_begin
final_end = begin_expanded + wp_end
return wordpieces, final_begin, final_end
def tokenize(self, text_input):
"""Performs untokenized text to wordpiece tokenization for BERT.
Args:
text_input: input: A `Tensor` or `RaggedTensor` of untokenized UTF-8
strings.
Returns:
A `RaggedTensor` of tokens where `tokens[i1...iN, j]` is the string
contents (or ID in the vocab_lookup_table representing that string)
of the `jth` token in `input[i1...iN]`
"""
tokens = self._basic_tokenizer.tokenize(text_input)
return self._wordpiece_tokenizer.tokenize(tokens)
def detokenize(self, token_ids):
"""Convert a `Tensor` or `RaggedTensor` of wordpiece IDs to string-words.
See `WordpieceTokenizer.detokenize` for details.
Note: `BertTokenizer.tokenize`/`BertTokenizer.detokenize` does not round
trip losslessly. The result of `detokenize` will not, in general, have the
same content or offsets as the input to `tokenize`. This is because the
"basic tokenization" step, that splits the strings into words before
applying the `WordpieceTokenizer`, includes irreversible
steps like lower-casing and splitting on punctuation. `WordpieceTokenizer`
on the other hand **is** reversible.
Note: This method assumes wordpiece IDs are dense on the interval
`[0, vocab_size)`.
Args:
token_ids: A `RaggedTensor` or `Tensor` with an int dtype.
Returns:
A `RaggedTensor` with dtype `string` and the same rank as the input
`token_ids`.
"""
return self._wordpiece_tokenizer.detokenize(token_ids)