def test_init_input_with_decoder_inputs(self): # pylint: disable=C0103
"""Test the .init_input() method when decoder inputs have been provided."""
batch_size = 2
input_size = 4
decoder_inputs_value = np.asarray(
[[[1, 1], [2, 2], [3, 3]], [[10, 10], [20, 20], [30, 30]]],
dtype=np.float32) # pylint: disable=E1101
decoder_inputs = tf.constant(decoder_inputs_value)
states = tf.random_normal([2, 10, 7
]) # nota bene: timesteps can be different!
zero_output = tf.zeros([2, 23])
cell = mock.Mock()
location_softmax = mock.Mock()
location_softmax.attention.states = states
pointing_output = mock.Mock()
pointing_output.zero_output.side_effect = [zero_output]
decoder = layers.PointingSoftmaxDecoder(
cell=cell,
location_softmax=location_softmax,
pointing_output=pointing_output,
input_size=input_size,
decoder_inputs=decoder_inputs)
init_input_act_t = decoder.init_input()
init_input_exp = np.zeros((batch_size, input_size))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
init_input_act = sess.run(init_input_act_t)
self.assertAllEqual(init_input_exp, init_input_act)
pointing_output.zero_output.assert_not_called()
def test_next_inp_without_decoder_inputs(self): # pylint: disable=C0103
"""Test the .next_inp method when decoder inputs are not provided."""
input_size = 4
output_value = [[1, 1, 1], [2, 2, 2], [3, 3, 3]]
states = tf.random_normal([3, 10, 4])
output = tf.constant(output_value, dtype=tf.float32)
time = tf.constant(random.randint(0, 100),
dtype=tf.int32) # irrelevant
cell = mock.Mock()
location_softmax = mock.Mock()
location_softmax.attention.states = states
pointing_output = mock.Mock()
decoder = layers.PointingSoftmaxDecoder(
cell=cell,
location_softmax=location_softmax,
pointing_output=pointing_output,
input_size=input_size)
next_inp_t = decoder.next_inp(time, output)
# pylint: disable=E1101
next_inp_exp = np.asarray([[1, 1, 1, 0], [2, 2, 2, 0], [3, 3, 3, 0]],
dtype=np.float32)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
next_inp_act = sess.run(next_inp_t)
self.assertAllEqual(next_inp_exp, next_inp_act)
def test_init_state(self):
"""Test the .init_state() method."""
batch_size = 2
timesteps = 5
state_size = 9 # useless
cell_output_size = 5
cell_state_size = (3, 4)
input_size = 7
def _cell_zero_state(batch_size, dtype):
self.assertEqual(dtype, tf.float32)
zero_states = []
for state_size in cell_state_size:
zero_state = tf.zeros(tf.stack([batch_size, state_size]))
zero_states.append(zero_state)
return tuple(zero_states)
cell = mock.Mock()
cell.output_size = cell_output_size
cell.zero_state.side_effect = _cell_zero_state
states = tf.placeholder(tf.float32, shape=[None, None, None])
location_softmax = mock.Mock()
location_softmax.attention.states = states
pointing_output = mock.Mock()
decoder = layers.PointingSoftmaxDecoder(
cell=cell,
location_softmax=location_softmax,
pointing_output=pointing_output,
input_size=input_size)
init_state_exp = (np.zeros([batch_size, cell_output_size]),
(np.zeros([batch_size, cell_state_size[0]]),
np.zeros([batch_size, cell_state_size[1]])))
zero_state_act_t = decoder.init_state()
batch_size_act_t, dtype_act = cell.zero_state.call_args[0]
feed = {states: np.random.rand(batch_size, timesteps, state_size)} # pylint: disable=E1101,I0011
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
init_state_act = sess.run(zero_state_act_t, feed)
batch_size_act = sess.run(batch_size_act_t, feed)
cell_out_exp, cell_state_exp = init_state_exp
cell_out_act, cell_state_act = init_state_act
self.assertAllEqual(cell_out_exp, cell_out_act)
self.assertEqual(len(cell_state_exp), len(cell_state_act))
for item_exp, item_act in zip(cell_state_exp, cell_state_act):
self.assertAllEqual(item_exp, item_act)
self.assertEqual(batch_size, batch_size_act)
self.assertEqual(dtype_act, tf.float32)
def test_zero_output(self):
"""Test the .zero_output() method."""
batch_size = 2
timesteps = 5
shortlist_size = 3
output_size = shortlist_size + timesteps
state_size = 9
input_size = 5
cell = mock.Mock()
states = tf.placeholder(tf.float32, shape=[None, None, None])
location_softmax = mock.Mock()
location_softmax.attention.states = states
batch_size_t = utils.get_dimension(states, 0)
timesteps_t = utils.get_dimension(states, 1)
output_size_t = shortlist_size + timesteps_t
zero_output_exp_shape = tf.stack([batch_size_t, output_size_t])
zero_output_exp_t = tf.zeros(zero_output_exp_shape)
pointing_output = mock.Mock()
pointing_output.zero_output.side_effect = [zero_output_exp_t]
decoder = layers.PointingSoftmaxDecoder(
cell=cell,
location_softmax=location_softmax,
pointing_output=pointing_output,
input_size=input_size)
zero_output_act_t = decoder.zero_output()
batch_size_act_t, loc_size_act_t = tuple(
pointing_output.zero_output.call_args[0])
zero_output_exp = np.zeros((batch_size, output_size))
pointing_output.zero_output.assert_called_once()
self.assertEqual(zero_output_exp_t, zero_output_act_t)
feed = {states: np.random.rand(batch_size, timesteps, state_size)} # pylint: disable=E1101,I0011
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
zero_output_act = sess.run(zero_output_act_t, feed)
batch_size_act, loc_size_act = sess.run(
[batch_size_act_t, loc_size_act_t], feed)
self.assertAllEqual(zero_output_exp, zero_output_act)
self.assertEqual(batch_size, batch_size_act)
self.assertEqual(timesteps, loc_size_act)
def test_finished_with_decoder_inputs(self): # pylint: disable=C0103
"""Test the .finished() method when decoder inputs are not provided."""
input_size = 4
decoder_inputs_value = np.asarray(
[[[1, 1], [2, 2], [3, 3]], [[10, 10], [20, 20], [30, 30]]],
dtype=np.float32) # pylint: disable=E1101
decoder_inputs = tf.constant(decoder_inputs_value)
states = tf.random_normal([2, 10, 7
]) # nota bene: timesteps can be different!
cell = mock.Mock()
location_softmax = mock.Mock()
location_softmax.attention.states = states
pointing_output = mock.Mock()
decoder = layers.PointingSoftmaxDecoder(
cell=cell,
location_softmax=location_softmax,
pointing_output=pointing_output,
input_size=input_size,
decoder_inputs=decoder_inputs)
time0 = tf.constant(0, dtype=tf.int32)
time1 = tf.constant(1, dtype=tf.int32)
time2 = tf.constant(2, dtype=tf.int32)
time3 = tf.constant(3, dtype=tf.int32)
time9 = tf.constant(9, dtype=tf.int32)
FF = np.asarray([False, False]) # pylint: disable=C0103,I0011
TT = np.asarray([True, True]) # pylint: disable=C0103,I0011
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
self.assertAllEqual(FF, sess.run(decoder.finished(time0)))
self.assertAllEqual(FF, sess.run(decoder.finished(time1)))
self.assertAllEqual(TT, sess.run(decoder.finished(time2)))
self.assertAllEqual(TT, sess.run(decoder.finished(time3)))
self.assertAllEqual(TT, sess.run(decoder.finished(time9)))
def test_finished_without_decoder_inputs(self): # pylint: disable=C0103
"""Test the .finished() method when decoder inputs are not provided."""
input_size = 4
states = tf.random_normal([3, 10, 4])
time = tf.constant(random.randint(0, 100),
dtype=tf.int32) # irrelevant
cell = mock.Mock()
location_softmax = mock.Mock()
location_softmax.attention.states = states
pointing_output = mock.Mock()
decoder = layers.PointingSoftmaxDecoder(
cell=cell,
location_softmax=location_softmax,
pointing_output=pointing_output,
input_size=input_size)
finished_t = decoder.finished(time)
finished_exp = np.asarray([False, False, False])
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
finished_act = sess.run(finished_t)
self.assertAllEqual(finished_exp, finished_act)
def test_init_input(self):
"""Test the .init_input() method."""
batch_size = 2
timesteps = 5
shortlist_size = 3
state_size = 9 # useless
input_size = 5
states = tf.placeholder(tf.float32, shape=[None, None, None])
cell = mock.Mock()
location_softmax = mock.Mock()
location_softmax.attention.states = states
pointing_output = mock.Mock()
def _zero_output(batch_size, loc_size):
shape = tf.stack([batch_size, shortlist_size + loc_size])
return tf.zeros(shape, dtype=tf.float32)
pointing_output.zero_output.side_effect = _zero_output
decoder = layers.PointingSoftmaxDecoder(
cell=cell,
location_softmax=location_softmax,
pointing_output=pointing_output,
input_size=input_size)
init_input_act_t = decoder.init_input()
init_input_exp = np.zeros((batch_size, input_size))
feed = {states: np.random.rand(batch_size, timesteps, state_size)} # pylint: disable=E1101,I0011
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
init_input_act = sess.run(init_input_act_t, feed)
self.assertAllClose(init_input_exp, init_input_act)
def _build_graph(self):
trainable = self.mode == tf.contrib.learn.ModeKeys.TRAIN
words = self.inputs.get(self.inputs.WORDS_KEY)
slengths = self.inputs.get(self.inputs.SENTENCE_LENGTH_KEY)
targets = self.inputs.get(self.inputs.FORMULA_KEY)
flengths = self.inputs.get(self.inputs.FORMULA_LENGTH_KEY)
with self._graph.as_default(): # pylint: disable=E1129
with tf.variable_scope('Embedding'): # pylint: disable=E1129
with tf.device('CPU:0'):
embedding_size = self._params['embedding_size']
vocabulary_size = self._params[self.INPUT_VOC_SIZE_PK]
embeddings = tf.get_variable(
'E', [vocabulary_size, embedding_size])
inputs = tf.nn.embedding_lookup(embeddings, words)
batch_dim = utils.get_dimension(words, 0)
with tf.variable_scope('Encoder'): # pylint: disable=E1129
encoder_params = self._params['encoder']
encoder_cell_type = encoder_params['cell.type']
encoder_cell_params = encoder_params['cell.params']
encoder_cell = configurable.factory(encoder_cell_type,
self._mode,
encoder_cell_params, rnn)
state = encoder_cell.zero_state(batch_dim, tf.float32)
encoder_out, _ = tf.nn.dynamic_rnn(
cell=encoder_cell,
initial_state=state,
inputs=inputs,
sequence_length=slengths,
parallel_iterations=self._params['parallel_iterations'])
with tf.variable_scope('Decoder'): # pylint: disable=E1129
decoder_params = self._params['decoder']
decoder_cell_type = decoder_params['cell.type']
decoder_cell_params = decoder_params['cell.params']
decoder_cell = configurable.factory(decoder_cell_type,
self._mode,
decoder_cell_params, rnn)
attention = layers.BahdanauAttention(
states=encoder_out,
inner_size=self._params['attention_size'],
trainable=trainable)
location = layers.LocationSoftmax(attention=attention,
sequence_length=slengths)
output = layers.PointingSoftmaxOutput(
shortlist_size=self._params[self.OUTPUT_VOC_SIZE_PK],
decoder_out_size=decoder_cell.output_size,
state_size=encoder_out.shape[-1].value,
trainable=trainable)
self._decoder_inputs = None
if trainable:
location_size = utils.get_dimension(words, 1)
output_size = self._params[
self.OUTPUT_VOC_SIZE_PK] + location_size
self._decoder_inputs = tf.one_hot(
targets,
output_size,
dtype=tf.float32,
name='decoder_training_input')
ps_decoder = layers.PointingSoftmaxDecoder(
cell=decoder_cell,
location_softmax=location,
pointing_output=output,
input_size=self._params['feedback_size'],
decoder_inputs=self._decoder_inputs,
trainable=trainable)
eos = None if trainable else self.EOS_IDX
pad_to = None if trainable else utils.get_dimension(targets, 1)
helper = layers.TerminationHelper(lengths=flengths, EOS=eos)
decoder = layers.DynamicDecoder(
decoder=ps_decoder,
helper=helper,
pad_to=pad_to,
parallel_iterations=self._params['parallel_iterations'],
swap_memory=False)
self._predictions, _ = decoder.decode()
def test_next_inp_with_decoder_inputs(self): # pylint: disable=C0103
"""Test the .next_inp method when decoder inputs are provided.
*NOTE* that at time `t+1` the desired decoder input is the output
from the previous step, `t`, it means that at timestep `t` the next
input is the desired output for the very same timestep.
"""
input_size = 4
decoder_inputs_value = np.asarray(
[[[0, 0], [1, 1], [2, 2], [3, 3]],
[[0, 0], [10, 10], [20, 20], [30, 30]]],
dtype=np.float32) # pylint: disable=E1101
decoder_inputs_padding = np.zeros(decoder_inputs_value.shape)
decoder_inputs = tf.constant(decoder_inputs_value)
states = tf.random_normal([2, 10, 7
]) # nota bene: timesteps can be different!
output = tf.random_normal([2, 5])
zero_output = tf.zeros([2, 23])
cell = mock.Mock()
location_softmax = mock.Mock()
location_softmax.attention.states = states
pointing_output = mock.Mock()
pointing_output.zero_output.return_value = zero_output
decoder = layers.PointingSoftmaxDecoder(
cell=cell,
location_softmax=location_softmax,
pointing_output=pointing_output,
input_size=input_size,
decoder_inputs=decoder_inputs)
# The decoder inputs will be fit (in this case, padded) to the
# input_size` paramter along their last axis. The expected input
# tensor is shaped in a time-major fashion to ease the iteration.
# if the next input is queried 'after' the length of the actual
# input, a zero-vector is returned.
decoder_inputs_fit = np.concatenate(
(decoder_inputs_value, decoder_inputs_padding), axis=-1)
decoder_inputs_time_major = np.transpose(decoder_inputs_fit,
axes=[1, 0, 2])
decoder_inputs_over = np.zeros_like(decoder_inputs_time_major[0])
act_timesteps = 4
max_timesteps = 10
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(act_timesteps):
time = tf.constant(i, dtype=tf.int32, shape=[])
next_input_exp = decoder_inputs_time_major[i]
next_input_act = sess.run(decoder.next_inp(time, output))
self.assertAllEqual(next_input_exp, next_input_act)
for i in range(act_timesteps, max_timesteps):
time = tf.constant(i, dtype=tf.int32, shape=[])
next_input_exp = decoder_inputs_over
next_input_act = sess.run(decoder.next_inp(time, output))
self.assertAllEqual(next_input_exp, next_input_act)
def test_step_without_decoder_inputs(self): # pylint: disable=C0103
"""Test the .step() method when decoder inputs are not available (inference)."""
batch_size = 2
timesteps = 10
shortlist_size = 3
output_size = shortlist_size + timesteps # 13
state_size = 9
input_size = 11
cell_out_size = 4
cell_state_size = 7
# DEFINE ATTENTION STATES AND (variable) DIMENSIONS.
# The `states` variable, even if not used, is the reference
# tensor for the dimensionality of the problem and represents
# the attention states of the model.
states = tf.placeholder(dtype=tf.float32, shape=[None, None, None])
batch_dim = utils.get_dimension(states, 0)
timesteps_dim = utils.get_dimension(states, 1)
state_dim = utils.get_dimension(states, 2)
output_dim = shortlist_size + timesteps_dim
# RECURRENT CELL.
out_cell_out = 8 * tf.ones(shape=tf.stack([batch_dim, cell_out_size]))
out_cell_state = 14 * tf.ones(
shape=tf.stack([batch_dim, cell_state_size]))
cell = mock.Mock()
cell.side_effect = [(out_cell_out, out_cell_state)]
# LOCATION SOFTMAX (and attention).
location = 12 * tf.ones(dtype=tf.float32,
shape=[batch_dim, timesteps_dim])
attention = 13 * tf.ones(dtype=tf.float32,
shape=[batch_dim, state_dim])
location_softmax = mock.Mock()
location_softmax.attention.states = states
location_softmax.side_effect = [(location, attention)]
# OUTPUT.
out_output = 9 * tf.ones(shape=tf.stack([batch_dim, output_dim]))
pointing_output = mock.Mock()
pointing_output.side_effect = [out_output]
# INPUT TENSORS: time, inp, (cell_out, cell_state)
in_time = tf.constant(0, dtype=tf.int32)
in_inp = tf.ones(shape=tf.stack([batch_dim, input_size]))
in_cell_out = 4 * tf.ones(shape=tf.stack([batch_dim, cell_out_size]))
in_cell_state = 7 * tf.ones(
shape=tf.stack([batch_dim, cell_state_size]))
in_state = (in_cell_out, in_cell_state)
in_step_args = (in_time, in_inp, in_state)
# ACTUAL OUT TENSORS.
decoder = layers.PointingSoftmaxDecoder(
cell=cell,
location_softmax=location_softmax,
pointing_output=pointing_output,
input_size=input_size)
output_t, next_inp_t, next_state_t, finished_t = decoder.step(
*in_step_args)
next_cell_out_t, next_cell_state_t = next_state_t
# TENSOR IDENTITY ASSERTIONS.
# 1. Assert that the location and attention are calculated
# with the previous step cell output tensor (in_cell_out).
location_softmax.assert_called_once_with(in_cell_out)
# 2. Assert that the cell state that has been passed to the inner
# recurrent cell is the one coming from the previous step (in_cell_state).
# (apparently pylint doesn't recognize `cell` as a callable mock?)
# pylint: disable=E1136
cell_input_t, in_cell_state_t = tuple(cell.call_args[0])
self.assertEqual(in_cell_state, in_cell_state_t)
# 3. Assert that the pointing output has been invoked with the
# output of the recurrent cell (out_cell_out), the location tensor
# (location) and the attention context tensor (attention).
pointing_output.assert_called_once_with(out_cell_out, location,
attention)
# Actualize the state.
# (actually pylint doesn't recognize np.random.rand())
# pylint: disable=E1101
states_np = np.random.rand(batch_size, timesteps, state_size)
# EXPECTED OUTPUT VALUES for the .step() method.
output_exp = 9 * np.ones((batch_size, output_size))
next_inp_exp = 9 * np.ones((batch_size, input_size))
next_cell_out_exp = 8 * np.ones((batch_size, cell_out_size))
next_cell_state_exp = 14 * np.ones((batch_size, cell_state_size))
finished_exp = np.asarray([False] * batch_size, np.bool)
# Re-built the recurrent cell input as the concatenation
# of the cell output, the attention context vector and the
# current input.
cell_input_rebuilt_t = tf.concat([in_cell_out, attention, in_inp],
axis=1)
feed = {states: states_np}
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# .step() outputs.
self.assertAllEqual(output_exp, sess.run(output_t, feed))
self.assertAllEqual(next_inp_exp, sess.run(next_inp_t, feed))
self.assertAllEqual(next_cell_out_exp,
sess.run(next_cell_out_t, feed))
self.assertAllEqual(next_cell_state_exp,
sess.run(next_cell_state_t, feed))
self.assertAllEqual(finished_exp, sess.run(finished_t, feed))
# recurrent cell input.
cell_input_exp = sess.run(cell_input_rebuilt_t, feed)
cell_input_act = sess.run(cell_input_t, feed)
self.assertAllEqual(cell_input_exp, cell_input_act)
