def _testTokensToChars(self, tokens, expected_chars, expected_lengths): expected_chars = [[tf.compat.as_bytes(c) for c in w] for w in expected_chars] chars, lengths = text_inputter.tokens_to_chars(tf.constant(tokens)) with self.test_session() as sess: chars, lengths = sess.run([chars, lengths]) self.assertAllEqual(expected_chars, chars) self.assertAllEqual(expected_lengths, lengths)
def _testTokensToChars(self, tokens, expected_chars, expected_lengths): expected_chars = compat.nest.map_structure(tf.compat.as_bytes, expected_chars) chars, lengths = text_inputter.tokens_to_chars( tf.constant(tokens, dtype=tf.string)) chars, lengths = self.evaluate([chars, lengths]) self.assertListEqual(expected_chars, chars.tolist()) self.assertListEqual(expected_lengths, lengths.tolist())
def _testTokensToChars(self, tokens, expected_chars, expected_lengths): expected_chars = [[tf.compat.as_bytes(c) for c in w] for w in expected_chars] tokens = tf.placeholder_with_default(tokens, shape=[None]) chars, lengths = text_inputter.tokens_to_chars(tokens) with self.test_session() as sess: chars, lengths = sess.run([chars, lengths]) self.assertListEqual(expected_chars, chars.tolist()) self.assertListEqual(expected_lengths, lengths.tolist())