def fn(string_tensor): """Builds the output tensor dictionary.""" out = {} if FLAGS.lowercase: string_tensor = ops.lowercase_op(string_tensor) out["wids"] = tf.to_int32(tbl.lookup(string_tensor)) out["cids"] = char_utils.batch_word_to_char_ids(string_tensor, 50) out["len"] = tf.shape(string_tensor)[-1] return out
def _test_words(self, words, expected_char_ids, word_length): with tf.Graph().as_default(): char_ids = char_utils.batch_word_to_char_ids( tf.constant(words), word_length) with tf.Session() as sess: actual_char_ids = sess.run(char_ids) for a_cid, e_cid in zip(actual_char_ids, expected_char_ids): self.assertAllEqual(a_cid, e_cid)
def test_character_cnn(self): with tf.Graph().as_default(): input_words = [["google", "lumiere"], [u"¯\\_(ツ)_/¯", u"(ᵔᴥᵔ)"], [u"谷", u"歌"]] char_ids = char_utils.batch_word_to_char_ids( tf.constant(input_words), 10) output_emb = common_layers.character_cnn(char_ids, num_filters=5) with tf.Session() as sess: sess.run(tf.compat.v1.global_variables_initializer()) actual_output_emb = sess.run(output_emb) self.assertAllEqual(actual_output_emb.shape, [3, 2, 5])