def test_with_nonalphanum(self): text = 'hi world<<>><<>foo!^* bar && bye (%s hi)' % ZH_HELLO tokens = [ 'hi', ' ', 'world', '<<>><<>', 'foo', '!^* ', 'bar', ' && ', 'bye', ' (', ZH_HELLO.strip(), ' ', 'hi', ')' ] tokenizer = text_encoder.Tokenizer(alphanum_only=False) self.assertEqual(tokens, tokenizer.tokenize(text)) self.assertEqual(text, tokenizer.join(tokenizer.tokenize(text)))
def test_reserved_tokens_with_regex_chars(self): text = r'hello worldba\)r bar foozoo zoo FOO|<EOS>' tokens = [ 'hello', ' ', 'world', r'ba\)r', ' ', 'bar', ' ', 'foozoo', ' ', 'zoo', ' ', 'FOO|', '<EOS>' ] tokenizer = text_encoder.Tokenizer( alphanum_only=False, reserved_tokens=['<EOS>', 'FOO|', r'ba\)r']) self.assertEqual(tokens, tokenizer.tokenize(text)) self.assertEqual(text, tokenizer.join(tokenizer.tokenize(text)))
def _token_counts_from_generator(generator, max_chars, reserved_tokens): """Builds token counts from generator.""" reserved_tokens = list(reserved_tokens) + [_UNDERSCORE_REPLACEMENT] tokenizer = text_encoder.Tokenizer(alphanum_only=False, reserved_tokens=reserved_tokens) num_chars = 0 token_counts = collections.defaultdict(int) for s in generator: s = tf.compat.as_text(s) if max_chars and (num_chars + len(s)) >= max_chars: s = s[:(max_chars - num_chars)] tokens = tokenizer.tokenize(s) tokens = _prepare_tokens_for_encode(tokens) for t in tokens: token_counts[t] += 1 if max_chars: num_chars += len(s) if num_chars > max_chars: break return token_counts
def test_whitespace(self, s, exp): tokenizer = text_encoder.Tokenizer(alphanum_only=False) self.assertEqual(exp, tokenizer.tokenize(s)) self.assertEqual(s, tokenizer.join(tokenizer.tokenize(s)))
def test_default(self): text = 'hi<<>><<>foo!^* bar && bye (%s hi)' % ZH_HELLO self.assertEqual(['hi', 'foo', 'bar', 'bye', ZH_HELLO.strip(), 'hi'], text_encoder.Tokenizer().tokenize(text))