def test_lowercase_unicode(self):
with self.test_session() as sess:
test_str = ["ŠČŽɬЩЮɦ"]
self.assertEqual(
sess.run(ops.lowercase_op(
tf.convert_to_tensor(test_str))).tolist(),
[test_str[0].lower()])
def test_lowercase(self):
with self.test_session() as sess:
test_str = [["Abc%@||", "DZ dzD", ""]]
self.assertEqual(
sess.run(ops.lowercase_op(
tf.convert_to_tensor(test_str))).tolist(),
[[x.lower() for x in test_str[0]]])
def fn(string_tensor):
"""Builds the output tensor dictionary."""
out = {}
if FLAGS.lowercase:
string_tensor = ops.lowercase_op(string_tensor)
out["wids"] = tf.to_int32(tbl.lookup(string_tensor))
out["cids"] = char_utils.batch_word_to_char_ids(string_tensor, 50)
out["len"] = tf.shape(string_tensor)[-1]
return out
