def encode(self, s):
"""Encodes text into a list of integers."""
s = tf.compat.as_text(s)
tokens = self._tokenizer.tokenize(s)
tokens = _prepare_tokens_for_encode(tokens)
ids = []
for token in tokens:
ids.extend(self._token_to_ids(token))
return text_encoder.pad_incr(ids)
def test_pad_incr(self): self.assertEqual([4, 3, 2], text_encoder.pad_incr([3, 2, 1])) self.assertEqual([4, 3, 2, 1], text_encoder.pad_incr([3, 2, 1, 0]))
