def test_empty(self):
expected = tf.constant([], dtype=tf.string)
result = split_chars('')
self.assertTrue(tf.is_tensor(result))
self.assertNotIsInstance(result, tf.RaggedTensor)
expected, result = self.evaluate([expected, result])
self.assertAllEqual(expected, result)
def test_inference_shape(self):
source = [
['1', '2', '3'],
['4', '5', '6'],
]
result = split_chars(source)
self.assertEqual([2, 3, None], result.shape.as_list())
def test_skip(self):
expected = tf.constant([[['x', 'y'], ['zz', '']]], dtype=tf.string)
result = split_chars([['xy', 'zz']], skip=['zz'])
self.assertIsInstance(result, tf.RaggedTensor)
result = result.to_tensor(default_value='')
expected, result = self.evaluate([expected, result])
self.assertAllEqual(expected, result)
def test_restore(self):
source = tf.constant('Hey\n\tthere\t«word», !!!')
splitted = split_chars(source)
self.assertTrue(tf.is_tensor(splitted))
self.assertNotIsInstance(splitted, tf.RaggedTensor)
restored = tf.strings.reduce_join(splitted)
source, restored = self.evaluate(source), self.evaluate(restored)
self.assertAllEqual(source, restored)
def test_unicode(self):
expected = tf.constant([u'ё', u' ', u'е', u'̈', u'2', u'⁵'],
dtype=tf.string)
result = split_chars(u'ё ё2⁵')
self.assertTrue(tf.is_tensor(result))
self.assertNotIsInstance(result, tf.RaggedTensor)
expected, result = self.evaluate([expected, result])
self.assertAllEqual(expected, result)
def test_ragged(self):
expected = tf.constant([[['a', 'b', ''], ['c', ' ', 'd']],
[['e', '', ''], ['', '', '']]])
result = split_chars(tf.ragged.constant([['ab', 'c d'], ['e']]))
self.assertIsInstance(result, tf.RaggedTensor)
result = result.to_tensor(default_value='')
expected, result = self.evaluate([expected, result])
self.assertAllEqual(expected, result)
def test_actual_shape(self):
source = [
['1', '2', '3'],
['4', '5', '6'],
]
result = split_chars(source)
self.assertIsInstance(result, tf.RaggedTensor)
result = result.to_tensor(default_value='')
result = self.evaluate(result)
self.assertEqual((2, 3, 1), result.shape)