def _testTokensToChars(self, tokens, expected_chars, expected_lengths):
expected_chars = [[tf.compat.as_bytes(c) for c in w] for w in expected_chars]
chars, lengths = text_inputter.tokens_to_chars(tf.constant(tokens))
with self.test_session() as sess:
chars, lengths = sess.run([chars, lengths])
self.assertAllEqual(expected_chars, chars)
self.assertAllEqual(expected_lengths, lengths)
def _testTokensToChars(self, tokens, expected_chars, expected_lengths):
expected_chars = [[tf.compat.as_bytes(c) for c in w] for w in expected_chars]
chars, lengths = text_inputter.tokens_to_chars(tf.constant(tokens))
with self.test_session() as sess:
chars, lengths = sess.run([chars, lengths])
self.assertAllEqual(expected_chars, chars)
self.assertAllEqual(expected_lengths, lengths)
def _testTokensToChars(self, tokens, expected_chars, expected_lengths):
expected_chars = compat.nest.map_structure(tf.compat.as_bytes,
expected_chars)
chars, lengths = text_inputter.tokens_to_chars(
tf.constant(tokens, dtype=tf.string))
chars, lengths = self.evaluate([chars, lengths])
self.assertListEqual(expected_chars, chars.tolist())
self.assertListEqual(expected_lengths, lengths.tolist())
def _testTokensToChars(self, tokens, expected_chars, expected_lengths):
expected_chars = [[tf.compat.as_bytes(c) for c in w] for w in expected_chars]
tokens = tf.placeholder_with_default(tokens, shape=[None])
chars, lengths = text_inputter.tokens_to_chars(tokens)
with self.test_session() as sess:
chars, lengths = sess.run([chars, lengths])
self.assertListEqual(expected_chars, chars.tolist())
self.assertListEqual(expected_lengths, lengths.tolist())