def testConstruction(self): tensor_values = constant_op.constant( ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']) values = WrappedTensor(tensor_values) row_splits = constant_op.constant([0, 2, 2, 5, 6, 8], dtypes.int64) rt = RaggedTensor.from_row_splits(values, row_splits) self.assertIsInstance(rt.values, WrappedTensor) self.assertAllEqual(rt.values.value, tensor_values) self.assertAllEqual(rt.row_splits, row_splits) row_starts = constant_op.constant([0, 2, 2, 5, 6], dtypes.int64) rt = RaggedTensor.from_row_starts(values, row_starts) self.assertIsInstance(rt.values, WrappedTensor) self.assertAllEqual(rt.values.value, tensor_values) self.assertAllEqual(rt.row_starts(), row_starts) row_limits = constant_op.constant([2, 2, 5, 6, 8], dtypes.int64) rt = RaggedTensor.from_row_limits(values, row_limits) self.assertIsInstance(rt.values, WrappedTensor) self.assertAllEqual(rt.values.value, tensor_values) self.assertAllEqual(rt.row_limits(), row_limits) row_lengths = constant_op.constant([2, 0, 3, 1, 2], dtypes.int64) rt = RaggedTensor.from_row_lengths(values, row_lengths) self.assertIsInstance(rt.values, WrappedTensor) self.assertAllEqual(rt.values.value, tensor_values) self.assertAllEqual(rt.row_lengths(), row_lengths) rt = RaggedTensor.from_uniform_row_length(values, 4) self.assertIsInstance(rt.values, WrappedTensor) self.assertAllEqual(rt.values.value, tensor_values) self.assertAllEqual(rt.uniform_row_length, 4)
def tokenize_with_offsets(self, input): # pylint: disable=redefined-builtin """Tokenizes utf-8 encoded tokens into subword pieces based off of a vocab. ### Example: ```python >>> tokens = [["they're", "the", "greatest"]], >>> tokenizer = WordpieceTokenizer(vocab, token_out_type=tf.string) >>> result = tokenizer.tokenize_with_offsets(tokens) >>> result[0].to_list() # subwords [[['they', "##'", '##re'], ['the'], ['great', '##est']]] >>> result[1].to_list() # offset starts [[[0, 4, 5], [0], [0, 5]]] >>> result[2].to_list() # offset limits [[[4, 5, 7], [3], [5, 8]]] ``` Args: input: An N-dimensional `Tensor` or `RaggedTensor` of UTF-8 strings. Returns: A tuple of `RaggedTensor`s `tokens`, `start_offsets`, and `limit_offsets` where: * `tokens[i1...iN, j]` is the string contents, or ID in the vocab_lookup_table representing that string, of the `j`th token in `input[i1...iN]` * `start_offsets[i1...iN, j]` is the byte offset for the start of the `j`th token in `input[i1...iN]` * `limit_offsets[i1...iN, j]` is the byte offset for the end of the """ name = None if not isinstance(self._vocab_lookup_table, lookup_ops.LookupInterface): raise TypeError('vocab_lookup_table must be a LookupInterface') with ops.name_scope( name, 'WordpieceTokenizeWithOffsets', [input, self._vocab_lookup_table, self._suffix_indicator]): # Check that the types are expected and the ragged rank is appropriate. tokens = ragged_tensor.convert_to_tensor_or_ragged_tensor(input) rank = tokens.shape.ndims if rank is None: raise ValueError('input must have a known rank.') if rank == 0: wordpieces, starts, limits = self.tokenize_with_offsets( array_ops.stack([tokens])) return wordpieces.values, starts.values, limits.values elif rank > 1: if not ragged_tensor.is_ragged(tokens): tokens = ragged_tensor.RaggedTensor.from_tensor( tokens, ragged_rank=rank - 1) wordpieces, starts, limits = self.tokenize_with_offsets( tokens.flat_values) wordpieces = wordpieces.with_row_splits_dtype( tokens.row_splits.dtype) starts = starts.with_row_splits_dtype(tokens.row_splits.dtype) limits = limits.with_row_splits_dtype(tokens.row_splits.dtype) return (tokens.with_flat_values(wordpieces), tokens.with_flat_values(starts), tokens.with_flat_values(limits)) # Tokenize the tokens into subwords values, row_lengths, starts, limits = ( gen_wordpiece_tokenizer.wordpiece_tokenize_with_offsets( input_values=tokens, vocab_lookup_table=self._vocab_lookup_table. resource_handle, suffix_indicator=self._suffix_indicator, use_unknown_token=self._use_unknown_token, max_bytes_per_word=self._max_bytes_per_word, unknown_token=self._unknown_token, )) # If ids are desired, look them up in the vocab table. Otherwise just # return the string values. if self._token_out_type == dtypes.int64: values = self._vocab_lookup_table.lookup(values) wordpieces = RaggedTensor.from_row_lengths(values, row_lengths) starts = RaggedTensor.from_row_lengths(starts, row_lengths) limits = RaggedTensor.from_row_lengths(limits, row_lengths) return wordpieces, starts, limits