def test_empty(self): expected = tf.constant([], dtype=tf.string) result = split_chars('') self.assertTrue(tf.is_tensor(result)) self.assertNotIsInstance(result, tf.RaggedTensor) expected, result = self.evaluate([expected, result]) self.assertAllEqual(expected, result)
def test_inference_shape(self): source = [ ['1', '2', '3'], ['4', '5', '6'], ] result = split_chars(source) self.assertEqual([2, 3, None], result.shape.as_list())
def test_skip(self): expected = tf.constant([[['x', 'y'], ['zz', '']]], dtype=tf.string) result = split_chars([['xy', 'zz']], skip=['zz']) self.assertIsInstance(result, tf.RaggedTensor) result = result.to_tensor(default_value='') expected, result = self.evaluate([expected, result]) self.assertAllEqual(expected, result)
def test_restore(self): source = tf.constant('Hey\n\tthere\t«word», !!!') splitted = split_chars(source) self.assertTrue(tf.is_tensor(splitted)) self.assertNotIsInstance(splitted, tf.RaggedTensor) restored = tf.strings.reduce_join(splitted) source, restored = self.evaluate(source), self.evaluate(restored) self.assertAllEqual(source, restored)
def test_unicode(self): expected = tf.constant([u'ё', u' ', u'е', u'̈', u'2', u'⁵'], dtype=tf.string) result = split_chars(u'ё ё2⁵') self.assertTrue(tf.is_tensor(result)) self.assertNotIsInstance(result, tf.RaggedTensor) expected, result = self.evaluate([expected, result]) self.assertAllEqual(expected, result)
def test_ragged(self): expected = tf.constant([[['a', 'b', ''], ['c', ' ', 'd']], [['e', '', ''], ['', '', '']]]) result = split_chars(tf.ragged.constant([['ab', 'c d'], ['e']])) self.assertIsInstance(result, tf.RaggedTensor) result = result.to_tensor(default_value='') expected, result = self.evaluate([expected, result]) self.assertAllEqual(expected, result)
def test_actual_shape(self): source = [ ['1', '2', '3'], ['4', '5', '6'], ] result = split_chars(source) self.assertIsInstance(result, tf.RaggedTensor) result = result.to_tensor(default_value='') result = self.evaluate(result) self.assertEqual((2, 3, 1), result.shape)