def testDetokenizeIsReversable(self):
table = _CreateTable(_MIXED_LANG_VOCAB + [b""], 2)
self.evaluate(table.initializer)
tokenizer = WordpieceTokenizer(table)
word_lists = [
[b"hello", b"there", b"my", b"name", b"is", b"treadness"],
[b"whatchamacallit?", b"you", b"said"],
[_Utf8(u"大"), _Utf8(u"易")],
]
words = ragged_factory_ops.constant(word_lists)
subwords_ids = tokenizer.tokenize(words)
# detokeinze input shape is (batch, ragged-words, ragged-wordpieces)
words_output = tokenizer.detokenize(subwords_ids)
words_output = array_ops.squeeze(words_output, axis=-1)
self.assertAllEqual(words_output, words)
# detokeinze input shape is (batch, ragged-wordpieces)
subwords_id_seqs = subwords_ids.merge_dims(-2, -1)
words_output = tokenizer.detokenize(subwords_id_seqs)
self.assertAllEqual(words_output, words)
# detokeinze input shape is a dense (batch, padded-wordpieces)
words_output = tokenizer.detokenize(
subwords_ids.merge_dims(-2, -1)
# len(_MIXED_LANG_VOCAB) is ""
.to_tensor(default_value=len(_MIXED_LANG_VOCAB)))
self.assertAllEqual(words_output, words)
def testDetokenizeFailsForSparseVocab(self):
vocab = ["a", "##b", "##c"]
ids = [0, 10, 20]
init = lookup_ops.KeyValueTensorInitializer(vocab,
ids,
key_dtype=dtypes.string,
value_dtype=dtypes.int64)
table = lookup_ops.StaticVocabularyTableV1(
init, num_oov_buckets=1, lookup_key_dtype=dtypes.string)
self.evaluate(table.initializer)
tokenizer = WordpieceTokenizer(table)
words = ragged_factory_ops.constant([["abb", "abc"], ["abcbc"]])
subwords_ids = tokenizer.tokenize(words)
with self.assertRaisesRegex(errors_impl.InvalidArgumentError,
"detokenize.*?dense on the interval"):
result = tokenizer.detokenize(subwords_ids)
self.evaluate(result)
class BertTokenizer(TokenizerWithOffsets, Detokenizer):
r"""Tokenizer used for BERT.
This tokenizer applies an end-to-end, text string to wordpiece tokenization.
It first applies basic tokenization, and then followed by wordpiece
tokenization.
See BasicTokenizer and WordpieceTokenizer for their respective details.
Attributes:
vocab_lookup_table: A lookup table implementing the LookupInterface
containing the vocabulary of subwords or a string which is the file path
to the vocab.txt file.
suffix_indicator: (optional) The characters prepended to a wordpiece to
indicate that it is a suffix to another subword. Default is '##'.
max_bytes_per_word: (optional) Max size of input token. Default is 100.
max_chars_per_token: (optional) Max size of subwords, excluding suffix
indicator. If known, providing this improves the efficiency of decoding
long words.
token_out_type: (optional) The type of the token to return. This can be
`tf.int64` IDs, or `tf.string` subwords. The default is `tf.int64`.
unknown_token: (optional) The value to use when an unknown token is found.
Default is "[UNK]". If this is set to a string, and `token_out_type` is
`tf.int64`, the `vocab_lookup_table` is used to convert the
`unknown_token` to an integer. If this is set to `None`, out-of-vocabulary
tokens are left as is.
split_unknown_characters: (optional) Whether to split out single unknown
characters as subtokens. If False (default), words containing unknown
characters will be treated as single unknown tokens.
lower_case: bool - If true, a preprocessing step is added to lowercase the
text, apply NFD normalization, and strip accents characters.
keep_whitespace: bool - If true, preserves whitespace characters instead of
stripping them away.
normalization_form: If true and lower_case=False, the input text will be
normalized to `normalization_form`. See normalize_utf8() op for a list of
valid values.
preserve_unused_token: If true, text in the regex format `\\[unused\\d+\\]`
will be treated as a token and thus remain preserved as is to be looked up
in the vocabulary.
basic_tokenizer_class: If set, the class to use instead of BasicTokenizer
"""
def __init__(self,
vocab_lookup_table,
suffix_indicator="##",
max_bytes_per_word=100,
max_chars_per_token=None,
token_out_type=dtypes.int64,
unknown_token="[UNK]",
split_unknown_characters=False,
lower_case=False,
keep_whitespace=False,
normalization_form=None,
preserve_unused_token=False,
basic_tokenizer_class=BasicTokenizer):
super(BertTokenizer, self).__init__()
_tf_text_bert_tokenizer_op_create_counter.get_cell().increase_by(1)
self._basic_tokenizer = basic_tokenizer_class(lower_case,
keep_whitespace,
normalization_form,
preserve_unused_token)
self._wordpiece_tokenizer = WordpieceTokenizer(
vocab_lookup_table, suffix_indicator, max_bytes_per_word,
max_chars_per_token, token_out_type, unknown_token,
split_unknown_characters)
def tokenize_with_offsets(self, text_input):
tokens, begin, _ = self._basic_tokenizer.tokenize_with_offsets(
text_input)
wordpieces, wp_begin, wp_end = (
self._wordpiece_tokenizer.tokenize_with_offsets(tokens))
begin_expanded = array_ops.expand_dims(begin, axis=2)
final_begin = begin_expanded + wp_begin
final_end = begin_expanded + wp_end
return wordpieces, final_begin, final_end
def tokenize(self, text_input):
"""Performs untokenized text to wordpiece tokenization for BERT.
Args:
text_input: input: A `Tensor` or `RaggedTensor` of untokenized UTF-8
strings.
Returns:
A `RaggedTensor` of tokens where `tokens[i1...iN, j]` is the string
contents (or ID in the vocab_lookup_table representing that string)
of the `jth` token in `input[i1...iN]`
"""
tokens = self._basic_tokenizer.tokenize(text_input)
return self._wordpiece_tokenizer.tokenize(tokens)
def detokenize(self, token_ids):
"""Convert a `Tensor` or `RaggedTensor` of wordpiece IDs to string-words.
See `WordpieceTokenizer.detokenize` for details.
Note: `BertTokenizer.tokenize`/`BertTokenizer.detokenize` does not round
trip losslessly. The result of `detokenize` will not, in general, have the
same content or offsets as the input to `tokenize`. This is because the
"basic tokenization" step, that splits the strings into words before
applying the `WordpieceTokenizer`, includes irreversible
steps like lower-casing and splitting on punctuation. `WordpieceTokenizer`
on the other hand **is** reversible.
Note: This method assumes wordpiece IDs are dense on the interval
`[0, vocab_size)`.
Args:
token_ids: A `RaggedTensor` or `Tensor` with an int dtype.
Returns:
A `RaggedTensor` with dtype `string` and the same rank as the input
`token_ids`.
"""
return self._wordpiece_tokenizer.detokenize(token_ids)