def testEmbeddingRNNDecoder(self):
with self.test_session() as sess:
with variable_scope.variable_scope(
"root", initializer=init_ops.constant_initializer(0.5)):
inp = [constant_op.constant(0.5, shape=[2, 2])] * 2
cell_fn = lambda: core_rnn_cell_impl.BasicLSTMCell(2)
cell = cell_fn()
_, enc_state = core_rnn.static_rnn(cell, inp, dtype=dtypes.float32)
dec_inp = [
constant_op.constant(
i, dtypes.int32, shape=[2]) for i in range(3)
]
# Use a new cell instance since the attention decoder uses a
# different variable scope.
dec, mem = seq2seq_lib.embedding_rnn_decoder(
dec_inp, enc_state, cell_fn(), num_symbols=4, embedding_size=2)
sess.run([variables.global_variables_initializer()])
res = sess.run(dec)
self.assertEqual(3, len(res))
self.assertEqual((2, 2), res[0].shape)
res = sess.run([mem])
self.assertEqual(1, len(res))
self.assertEqual((2, 2), res[0].c.shape)
self.assertEqual((2, 2), res[0].h.shape)
def testEmbeddingRNNDecoder(self):
with self.test_session() as sess:
with variable_scope.variable_scope(
"root", initializer=init_ops.constant_initializer(0.5)):
inp = [constant_op.constant(0.5, shape=[2, 2])] * 2
cell_fn = lambda: core_rnn_cell_impl.BasicLSTMCell(2)
cell = cell_fn()
_, enc_state = core_rnn.static_rnn(cell, inp, dtype=dtypes.float32)
dec_inp = [
constant_op.constant(
i, dtypes.int32, shape=[2]) for i in range(3)
]
# Use a new cell instance since the attention decoder uses a
# different variable scope.
dec, mem = seq2seq_lib.embedding_rnn_decoder(
dec_inp, enc_state, cell_fn(), num_symbols=4, embedding_size=2)
sess.run([variables.global_variables_initializer()])
res = sess.run(dec)
self.assertEqual(3, len(res))
self.assertEqual((2, 2), res[0].shape)
res = sess.run([mem])
self.assertEqual(1, len(res))
self.assertEqual((2, 2), res[0].c.shape)
self.assertEqual((2, 2), res[0].h.shape)
def getDecoding(encoder_state, inputs, cell,
num_symbols, embedding_size,
feed_previous=True, output_prejection=None,
dtype=s2s.dtypes.float32, scope=None):
"""
English:
Model for producing probabilities over x from z
Japanese:
このモデルにおける、z から x へ向かう確率を計算します。
"""
with s2s.variable_scope.variable_scope(scope or 'seq2seq', reuse=True):
if output_prejection is None:
cell = s2s.core_rnn_cell.OutputProjectionWrapper(cell, num_symbols)
decode_probs, _ = s2s.embedding_rnn_decoder(
inputs, encoder_state, cell, num_symbols,
embedding_size, output_projection=output_prejection,
feed_previous=feed_previous)
return decode_probs
def createVariationalVar(inputs, cell, num_symbols, embedding_size,
feed_previous=False, output_projection=None,
dtype=s2s.dtypes.float32, scope=None):
"""
English:
Creates Tensorflow variables which can reused.
Japanese:
再利用可能な Tensorflow の変数を作ります。
"""
with s2s.variable_scope.variable_scope(scope or 'seq2seq'):
encoder_cell = s2s.core_rnn_cell.EmbeddingWrapper(
cell, embedding_classes=num_symbols, embedding_size=embedding_size)
_, encoder_state = s2s.rnn.static_rnn(encoder_cell, inputs, dtype=dtype)
# batch_size x cell.state_size
if output_projection is None:
cell = s2s.core_rnn_cell.OutputProjectionWrapper(cell, num_symbols)
decode_probs, _ = s2s.embedding_rnn_decoder(
inputs, encoder_state, cell, num_symbols,
embedding_size, output_projection=output_projection,
feed_previous=feed_previous)
return None
def build(self):
params = self.params
N, L, Q, F = params.batch_size, params.max_sent_size, params.max_ques_size, params.max_fact_count
V, d, A = params.embed_size, params.hidden_size, self.words.vocab_size
# initialize self
# placeholders
input = tf.placeholder('int32', shape=[N, L], name='x') # [num_batch, sentence_len]
question = tf.placeholder('int32', shape=[N, Q], name='q') # [num_batch, sentence_len]
answer = tf.placeholder('int32', shape=[N], name='y') # [num_batch] - one word answer
input_mask = tf.placeholder(tf.bool, shape=[N, L], name='x_mask') # [num_batch, sentence_len]
is_training = tf.placeholder(tf.bool)
# Prepare parameters
gru = rnn_cell.GRUCell(d)
# Input module
with tf.variable_scope('input') as scope:
input_list = tf.unstack(tf.transpose(input))
input_states, _ = seq2seq.embedding_rnn_decoder(input_list, gru.zero_state(N, tf.float32), gru, A, V)
# Question module
scope.reuse_variables()
ques_list = tf.unstack(tf.transpose(question))
questions, _ = seq2seq.embedding_rnn_decoder(ques_list, gru.zero_state(N, tf.float32), gru, A, V)
question_vec = questions[-1] # use final state
# Masking: to extract fact vectors at end of sentence. (details in paper)
input_states = tf.transpose(tf.stack(input_states), [1, 0, 2]) # [N, L, D]
facts = []
for n in range(N):
filtered = tf.boolean_mask(input_states[n, :, :], input_mask[n, :]) # [?, D]
padding = tf.zeros(tf.stack([F - tf.shape(filtered)[0], d]))
facts.append(tf.concat(0, [filtered, padding])) # [F, D]
facked = tf.stack(facts) # packing for transpose... I hate TF so much
facts = tf.unstack(tf.transpose(facked, [1, 0, 2]), num=F) # F x [N, D]
# Episodic Memory
with tf.variable_scope('episodic') as scope:
episode = EpisodeModule(d, question_vec, facts)
memory = tf.identity(question_vec)
for t in range(params.memory_step):
memory = gru(episode.new(memory), memory)[0]
scope.reuse_variables()
# Regularizations
if params.batch_norm:
memory = batch_norm(memory, is_training=is_training)
memory = dropout(memory, params.keep_prob, is_training)
with tf.name_scope('Answer'):
# Answer module : feed-forward version (for it is one word answer)
w_a = weight('w_a', [d, A])
logits = tf.matmul(memory, w_a) # [N, A]
with tf.name_scope('Loss'):
# Cross-Entropy loss
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, answer)
loss = tf.reduce_mean(cross_entropy)
total_loss = loss + params.weight_decay * tf.add_n(tf.get_collection('l2'))
with tf.variable_scope('Accuracy'):
# Accuracy
predicts = tf.cast(tf.argmax(logits, 1), 'int32')
corrects = tf.equal(predicts, answer)
num_corrects = tf.reduce_sum(tf.cast(corrects, tf.float32))
accuracy = tf.reduce_mean(tf.cast(corrects, tf.float32))
# Training
optimizer = tf.train.AdadeltaOptimizer(params.learning_rate)
opt_op = optimizer.minimize(total_loss, global_step=self.global_step)
# placeholders
self.x = input
self.q = question
self.y = answer
self.mask = input_mask
self.is_training = is_training
# tensors
self.total_loss = total_loss
self.num_corrects = num_corrects
self.accuracy = accuracy
self.opt_op = opt_op
# dec_state_in = tf.placeholder(tf.float32, [None, memory_dim])
encoder_state = tf.nn.tanh(encoder_state)
with tf.variable_scope("decoder1", reuse=None) as scope:
# with tf.device('/cpu:0'):
cell = tf.contrib.rnn.GRUCell(memory_dim)
cell = tf.contrib.rnn.OutputProjectionWrapper(cell, num_decoder_symbols)
# W2 = tf.Variable(tf.constant(0.0, shape=[num_encoder_symbols, embedding_dim]),
# trainable=False, name="embedding_rnn_decoder/embedding")
dec_outputs1, dec_memory1 = embedding_rnn_decoder(decoder_inputs,
encoder_state,
cell,
num_decoder_symbols,
embedding_dim,
feed_previous=True)
with tf.variable_scope("decoder2", reuse=None) as scope:
# with tf.device('/cpu:0'):
cell = tf.contrib.rnn.GRUCell(memory_dim)
cell = tf.contrib.rnn.OutputProjectionWrapper(cell, num_decoder_symbols)
# W2 = tf.Variable(tf.constant(0.0, shape=[num_encoder_symbols, embedding_dim]),
# trainable=False, name="embedding_rnn_decoder/embedding")
dec_outputs2, dec_memory2 = embedding_rnn_decoder(decoder_inputs,
encoder_state,
cell,
num_decoder_symbols,