def test_condition_with_padding(self):
"""Test the DynamicDecoder.condition() method with paddding."""
# pylint: disable=C0103,I0011
T, F = True, False
bt = lambda *args: tf.convert_to_tensor(list(args), dtype=tf.bool)
# pylint: enable=C0103,I0011
pad_to = tf.placeholder(tf.int32, shape=[])
dyndec = layers.DynamicDecoder(decoder=mock.Mock(),
helper=mock.Mock(),
pad_to=pad_to)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# Even if all the sequences in the batch are finished,
# but the decoder is asked to pad, the condition will
# evaluate to True.
time = tf.constant(9, dtype=tf.int32)
finished = bt(T, T, T, T)
feed = {pad_to: 23}
self.assertTrue(
sess.run(dyndec.cond(time, None, None, finished, None), feed))
# Another case to test is when we have to pad to a value
# that is already less than time, but the termination helper
# doesn't say to stop. In this case, condition will evaluate to true.
time = tf.constant(23, dtype=tf.int32)
finished = bt(F, F, F, T)
feed = {pad_to: 9}
self.assertTrue(
sess.run(dyndec.cond(time, None, None, finished, None), feed))
def test_decode_one_step(self):
"""Default test for the DynamicDecoder.decode() method."""
init_value = [[.1, .1], [.2, .2], [.3, .3]]
init_input = tf.constant(init_value)
init_state = 2 * init_input
next_input = 3 * init_input
next_state = 4 * init_input
output = 10 * init_input
finished = tf.constant([False, False, False], dtype=tf.bool)
zero_output = tf.zeros_like(output)
decoder = mock.Mock()
decoder.init_input.side_effect = [init_input]
decoder.init_state.side_effect = [init_state]
decoder.zero_output.side_effect = [zero_output]
decoder.step.side_effect = [(output, next_input, next_state, finished)]
helper = mock.Mock()
helper.finished.side_effect = [tf.logical_not(finished)
] # exit from the loop!
dyndec = layers.DynamicDecoder(decoder, helper)
output_t, state_t = dyndec.decode()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
output_act, state_act = sess.run([output_t, state_t])
# assertions on output.
output_exp = 10 * np.transpose(np.asarray([init_value]), (1, 0, 2))
self.assertAllClose(output_exp, output_act)
state_exp = 4 * np.asarray(init_value)
self.assertAllClose(state_exp, state_act)
# mock assertions.
# we cannot assert more than this since the while
# loop makes all the ops non-fetchable.
decoder.init_input.assert_called_once()
decoder.init_state.assert_called_once()
decoder.zero_output.assert_called_once()
decoder.step.assert_called_once()
helper.finished.assert_called_once()
def test_condition(self):
"""Test the DynamicDecoder.condition() method."""
helper = mock.Mock()
decoder = mock.Mock()
dyndec = layers.DynamicDecoder(decoder, helper)
finished = [(tf.constant([True], dtype=tf.bool), False),
(tf.constant([False], dtype=tf.bool), True),
(tf.constant([True, False], dtype=tf.bool), True),
(tf.constant([True, True], dtype=tf.bool), False)]
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for tensor, expected in finished:
actual = sess.run(dyndec.cond(None, None, None, tensor, None))
self.assertEqual(expected, actual)
helper.assert_not_called()
decoder.assert_not_called()
def test_iterations(self):
"""Test the number of iterations."""
lengths = tf.constant([1, 2, 3], dtype=tf.int32)
def _helper_finished(time, _):
return tf.greater_equal(time + 1, lengths)
helper = mock.Mock()
helper.finished.side_effect = _helper_finished
batch_size = utils.get_dimension(lengths, 0)
inp_size, state_size, output_size = 2, 5, 2
decoder = mock.Mock()
decoder.init_input.side_effect = lambda: tf.zeros(
[batch_size, inp_size])
decoder.init_state.side_effect = lambda: tf.ones(
[batch_size, state_size])
decoder.zero_output.side_effect = lambda: tf.zeros(
[batch_size, output_size])
decoder.step.side_effect = lambda t, i, s:\
((i + 1), 3 * (i + 1), (s + 2), tf.tile([False], [batch_size]))
output_exp = np.asarray(
[[[1, 1], [0, 0], [0, 0]], [[1, 1], [4, 4], [0, 0]],
[[1, 1], [4, 4], [13, 13]]],
dtype=np.float32) # pylint: disable=E1101,I0011
state_exp = np.asarray(
[[7, 7, 7, 7, 7], [7, 7, 7, 7, 7], [7, 7, 7, 7, 7]],
dtype=np.float32) # pylint: disable=E1101,I0011
dyndec = layers.DynamicDecoder(decoder, helper)
output_t, state_t = dyndec.decode()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
output_act, state_act = sess.run([output_t, state_t])
self.assertAllEqual(output_exp, output_act)
self.assertAllEqual(state_exp, state_act)
def test_output(self):
"""Test the DynamicDecoder.output() method."""
helper = mock.Mock()
decoder = mock.Mock()
zero_output = tf.constant([[0, 0, 0], [0, 0, 0]], dtype=tf.float32)
decoder.zero_output.side_effect = [zero_output]
output = tf.constant([[23, 23, 23], [23, 23, 23]], dtype=tf.float32)
finished = tf.constant([True, False], dtype=tf.bool)
dyndec = layers.DynamicDecoder(decoder, helper)
act_output_t = dyndec.output(output, finished)
exp_output = np.asarray([[0, 0, 0], [23, 23, 23]], dtype=np.float32) # pylint: disable=I0011,E1101
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
act_output = sess.run(act_output_t)
helper.finished.assert_not_called()
decoder.zero_output.assert_called_once()
self.assertAllEqual(exp_output, act_output)
def _build_graph(self):
trainable = self.mode == tf.contrib.learn.ModeKeys.TRAIN
words = self.inputs.get(self.inputs.WORDS_KEY)
slengths = self.inputs.get(self.inputs.SENTENCE_LENGTH_KEY)
targets = self.inputs.get(self.inputs.FORMULA_KEY)
flengths = self.inputs.get(self.inputs.FORMULA_LENGTH_KEY)
with self._graph.as_default(): # pylint: disable=E1129
with tf.variable_scope('Embedding'): # pylint: disable=E1129
with tf.device('CPU:0'):
embedding_size = self._params['embedding_size']
vocabulary_size = self._params[self.INPUT_VOC_SIZE_PK]
embeddings = tf.get_variable(
'E', [vocabulary_size, embedding_size])
inputs = tf.nn.embedding_lookup(embeddings, words)
batch_dim = utils.get_dimension(words, 0)
with tf.variable_scope('Encoder'): # pylint: disable=E1129
encoder_params = self._params['encoder']
encoder_cell_type = encoder_params['cell.type']
encoder_cell_params = encoder_params['cell.params']
encoder_cell = configurable.factory(encoder_cell_type,
self._mode,
encoder_cell_params, rnn)
state = encoder_cell.zero_state(batch_dim, tf.float32)
encoder_out, _ = tf.nn.dynamic_rnn(
cell=encoder_cell,
initial_state=state,
inputs=inputs,
sequence_length=slengths,
parallel_iterations=self._params['parallel_iterations'])
with tf.variable_scope('Decoder'): # pylint: disable=E1129
decoder_params = self._params['decoder']
decoder_cell_type = decoder_params['cell.type']
decoder_cell_params = decoder_params['cell.params']
decoder_cell = configurable.factory(decoder_cell_type,
self._mode,
decoder_cell_params, rnn)
attention = layers.BahdanauAttention(
states=encoder_out,
inner_size=self._params['attention_size'],
trainable=trainable)
location = layers.LocationSoftmax(attention=attention,
sequence_length=slengths)
output = layers.PointingSoftmaxOutput(
shortlist_size=self._params[self.OUTPUT_VOC_SIZE_PK],
decoder_out_size=decoder_cell.output_size,
state_size=encoder_out.shape[-1].value,
trainable=trainable)
self._decoder_inputs = None
if trainable:
location_size = utils.get_dimension(words, 1)
output_size = self._params[
self.OUTPUT_VOC_SIZE_PK] + location_size
self._decoder_inputs = tf.one_hot(
targets,
output_size,
dtype=tf.float32,
name='decoder_training_input')
ps_decoder = layers.PointingSoftmaxDecoder(
cell=decoder_cell,
location_softmax=location,
pointing_output=output,
input_size=self._params['feedback_size'],
decoder_inputs=self._decoder_inputs,
trainable=trainable)
eos = None if trainable else self.EOS_IDX
pad_to = None if trainable else utils.get_dimension(targets, 1)
helper = layers.TerminationHelper(lengths=flengths, EOS=eos)
decoder = layers.DynamicDecoder(
decoder=ps_decoder,
helper=helper,
pad_to=pad_to,
parallel_iterations=self._params['parallel_iterations'],
swap_memory=False)
self._predictions, _ = decoder.decode()
def test_iterations_step_by_step(self):
"""Test the number of iterations (step by step)."""
# pylint: disable=C0103,I0011
T, F = True, False
bt = lambda *args: tf.convert_to_tensor(list(args), dtype=tf.bool)
# pylint: enable=C0103,I0011
init_value = [[.1, .1], [.2, .2], [.3, .3]]
inp = tf.placeholder(tf.float32, shape=[None, None])
state = 2 * inp
# next_input = 3 * init_input
# next_state = 4 * init_input
zero_output = tf.zeros_like(inp)
out01 = 10 * inp
out02 = 20 * inp
out03 = 30 * inp
decoder = mock.Mock()
decoder.init_input.side_effect = [inp]
decoder.init_state.side_effect = [state]
decoder.zero_output.return_value = zero_output
decoder_finished_00 = bt(F, F, F)
decoder_finished_01 = bt(F, F, T)
decoder_finished_02 = bt(F, F, T)
decoder.step.side_effect = [
(out01, inp, state, decoder_finished_00), # time=0
(out02, inp, state, decoder_finished_01), # time=1
(out03, inp, state, decoder_finished_02)
] # time=2
helper = mock.Mock()
helper_finished_00 = bt(F, F, F)
helper_finished_01 = bt(F, T, F)
helper_finished_02 = bt(T, F, F)
helper.finished.side_effect = [
helper_finished_00, # time=0
helper_finished_01, # time=1
helper_finished_02
] # time=2
# STEP BY STEP EVALUATION OF `finished` flags.
dyndec = layers.DynamicDecoder(decoder, helper)
time = tf.constant(0, dtype=tf.int32)
finished = tf.tile([F], [utils.get_dimension(inp, 0)])
output_ta = tf.TensorArray(dtype=tf.float32, size=0, dynamic_size=True)
# time=0
next_finished_00_exp = [F, F, F]
results_00 = dyndec.body(time, inp, state, finished, output_ta)
time = results_00[0]
finished = results_00[3]
results_00[-1].stack()
feed = {inp: init_value}
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
next_finished_00_act = sess.run(finished, feed)
self.assertEqual(1, sess.run(time, feed))
self.assertAllEqual(next_finished_00_exp, next_finished_00_act)
# time=1
cond_01_t = dyndec.cond(*results_00)
cond_01_exp = True
feed = {inp: init_value}
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
cond_01_act = sess.run(cond_01_t, feed)
self.assertEqual(cond_01_exp, cond_01_act)
next_finished_01_exp = [F, T, T]
results_01 = dyndec.body(time, inp, state, finished, output_ta)
time = results_01[0]
finished = results_01[3]
results_01[-1].stack()
feed = {inp: init_value}
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
self.assertEqual(2, sess.run(time, feed))
next_finished_01_act = sess.run(finished, feed)
self.assertAllEqual(next_finished_01_exp, next_finished_01_act)
# time=2
cond_02_t = dyndec.cond(*results_01)
cond_02_exp = True
feed = {inp: init_value}
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
cond_02_act = sess.run(cond_02_t, feed)
self.assertEqual(cond_02_exp, cond_02_act)
next_finished_02_exp = [T, T, T]
results_02 = dyndec.body(time, inp, state, finished, output_ta)
time = results_02[0]
finished = results_02[3]
results_02[-1].stack()
feed = {inp: init_value}
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
self.assertEqual(3, sess.run(time, feed))
next_finished_02_act = sess.run(finished, feed)
self.assertAllEqual(next_finished_02_exp, next_finished_02_act)
# time=3
cond_03_t = dyndec.cond(*results_02)
cond_03_exp = False # STOP!
feed = {inp: init_value}
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
cond_03_act = sess.run(cond_03_t, feed)
self.assertEqual(cond_03_exp, cond_03_act)
def test_body(self):
"""Test the DynamicDecoder.body() method."""
time = tf.constant(2, dtype=tf.int32)
inp = tf.constant([[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0]])
state = tf.constant([[5.0, 5.0, 5.0], [6.0, 6.0, 6.0], [7.0, 7.0, 7.0],
[8.0, 8.0, 8.0]])
finished = tf.constant([False, False, False, True], dtype=tf.bool)
output_ta = tf.TensorArray(dtype=tf.float32, size=0, dynamic_size=True)
# Fill the output tensor array with some dummy
# values for items 0 and 1
dummy_out = tf.constant([[-.1, -.1], [-.2, -.2], [-.3, -.3],
[-.4, -.4]])
output_ta = output_ta.write(0, dummy_out).write(1, dummy_out)
dec_out = tf.constant([[10.0, 10.0], [20.0, 20.0], [30.0, 30.0],
[17.0, 17.0]])
next_inp = 7.0 * inp
next_state = 11.0 * state
next_finished = tf.constant([False, False, True, True], dtype=tf.bool)
zero_output = tf.zeros(dtype=tf.float32, shape=dec_out.get_shape())
decoder = mock.Mock()
decoder.step.side_effect = [(dec_out, next_inp, next_state,
next_finished)]
decoder.zero_output.side_effect = [zero_output]
helper_finished = tf.constant([True, False, False, False],
dtype=tf.bool)
helper = mock.Mock()
helper.finished.side_effect = [helper_finished]
# pylint: disable=I0011,E1101
# NOTA BENE: the only masked element is the one that is masked in the beginning.
output_exp = np.asarray([[10.0, 10.0], [20.0, 20.0], [30.0, 30.0],
[0.0, 0.0]])
next_finished_exp = np.asarray([True, False, True, True], np.bool)
# pylint: enable=I0011,E1101
dyndec = layers.DynamicDecoder(decoder, helper)
next_time_t, next_inp_t, next_state_t, next_finished_t, next_output_ta =\
dyndec.body(time, inp, state, finished, output_ta)
outputs_t = next_output_ta.stack(
) # still time major, easier to check.
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
next_inp_exp, next_state_exp, dummy_out_act =\
sess.run([next_inp, next_state, dummy_out])
next_time_act, next_inp_act, next_state_act, next_finished_act, outputs_act =\
sess.run([next_time_t, next_inp_t, next_state_t, next_finished_t, outputs_t])
# assertions on tensors.
self.assertEqual(next_inp, next_inp_t)
self.assertEqual(next_state, next_state_t)
# assertion on mocks.
decoder.step.assert_called_once_with(time, inp, state)
decoder.zero_output.assert_called_once()
helper.finished.assert_called_once_with(time, dec_out)
# assertion on calculated values.
self.assertEqual(3, next_time_act)
self.assertAllEqual(next_inp_exp, next_inp_act)
self.assertAllEqual(next_state_exp, next_state_act)
self.assertAllEqual(next_finished_exp, next_finished_act)
self.assertAllEqual(dummy_out_act, outputs_act[0])
self.assertAllEqual(dummy_out_act, outputs_act[1])
self.assertAllEqual(output_exp, outputs_act[-1])
