def fn(string_tensor):
"""Builds the output tensor dictionary."""
out = {}
if FLAGS.lowercase:
string_tensor = ops.lowercase_op(string_tensor)
out["wids"] = tf.to_int32(tbl.lookup(string_tensor))
out["cids"] = char_utils.batch_word_to_char_ids(string_tensor, 50)
out["len"] = tf.shape(string_tensor)[-1]
return out
def _test_words(self, words, expected_char_ids, word_length):
with tf.Graph().as_default():
char_ids = char_utils.batch_word_to_char_ids(
tf.constant(words), word_length)
with tf.Session() as sess:
actual_char_ids = sess.run(char_ids)
for a_cid, e_cid in zip(actual_char_ids, expected_char_ids):
self.assertAllEqual(a_cid, e_cid)
def test_character_cnn(self):
with tf.Graph().as_default():
input_words = [["google", "lumiere"], [u"¯\\_(ツ)_/¯", u"(ᵔᴥᵔ)"],
[u"谷", u"歌"]]
char_ids = char_utils.batch_word_to_char_ids(
tf.constant(input_words), 10)
output_emb = common_layers.character_cnn(char_ids, num_filters=5)
with tf.Session() as sess:
sess.run(tf.compat.v1.global_variables_initializer())
actual_output_emb = sess.run(output_emb)
self.assertAllEqual(actual_output_emb.shape, [3, 2, 5])