def testComputation(self):
inputs = tf.placeholder(tf.float32, shape=[self.batch_size, self.in_size])
prev_state = tf.placeholder(tf.float32,
shape=[self.batch_size, self.in_size])
vanilla_rnn = snt.VanillaRNN(name="rnn", hidden_size=self.in_size)
residual = snt.SkipConnectionCore(vanilla_rnn, name="skip")
output, new_state = residual(inputs, prev_state)
in_to_hid = vanilla_rnn.in_to_hidden_variables
hid_to_hid = vanilla_rnn.hidden_to_hidden_variables
with self.test_session() as sess:
# With random data, check the TF calculation matches the Numpy version.
input_data = np.random.randn(self.batch_size, self.in_size)
prev_state_data = np.random.randn(self.batch_size, self.in_size)
tf.global_variables_initializer().run()
fetches = [output, new_state, in_to_hid[0], in_to_hid[1],
hid_to_hid[0], hid_to_hid[1]]
output = sess.run(fetches,
{inputs: input_data, prev_state: prev_state_data})
output_v, new_state_v, in_to_hid_w, in_to_hid_b = output[:4]
hid_to_hid_w, hid_to_hid_b = output[4:]
real_in_to_hid = np.dot(input_data, in_to_hid_w) + in_to_hid_b
real_hid_to_hid = np.dot(prev_state_data, hid_to_hid_w) + hid_to_hid_b
vanilla_output = np.tanh(real_in_to_hid + real_hid_to_hid)
skip_output = np.concatenate((input_data, vanilla_output), -1)
self.assertAllClose(skip_output, output_v)
self.assertAllClose(vanilla_output, new_state_v)
def __init__(self,
num_embedding,
num_hidden,
lstm_depth,
output_size,
use_dynamic_rnn=True,
use_skip_connections=True,
name="text_model"):
"""Constructs a `TextModel`.
Args:
num_embedding: Size of embedding representation, used directly after the
one-hot encoded input.
num_hidden: Number of hidden units in each LSTM layer.
lstm_depth: Number of LSTM layers.
output_size: Size of the output layer on top of the DeepRNN.
use_dynamic_rnn: Whether to use dynamic RNN unrolling. If `False`, it uses
static unrolling. Default is `True`.
use_skip_connections: Whether to use skip connections in the
`snt.DeepRNN`. Default is `True`.
name: Name of the module.
"""
super(TextModel, self).__init__(name=name)
self._num_embedding = num_embedding
self._num_hidden = num_hidden
self._lstm_depth = lstm_depth
self._output_size = output_size
self._use_dynamic_rnn = use_dynamic_rnn
self._use_skip_connections = use_skip_connections
with self._enter_variable_scope():
self._embed_module = snt.Linear(self._num_embedding,
name="linear_embed")
self._output_module = snt.Linear(self._output_size,
name="linear_output")
self._subcores = [
snt.LSTM(self._num_hidden, name="lstm_{}".format(i))
for i in range(self._lstm_depth)
]
if self._use_skip_connections:
skips = []
current_input_shape = self._num_embedding
for lstm in self._subcores:
input_shape = tf.TensorShape([current_input_shape])
skip = snt.SkipConnectionCore(lstm,
input_shape=input_shape,
name="skip_{}".format(
lstm.module_name))
skips.append(skip)
# SkipConnectionCore concatenates the input with the output, so the
# dimensionality increases with depth.
current_input_shape += self._num_hidden
self._subcores = skips
self._core = snt.DeepRNN(self._subcores,
skip_connections=False,
name="deep_lstm")
def testOutputSize(self):
inputs = tf.placeholder(tf.float32, shape=[self.batch_size, self.in_size])
prev_state = tf.placeholder(
tf.float32, shape=[self.batch_size, self.hidden_size])
vanilla_rnn = snt.VanillaRNN(self.hidden_size)
skip_wrapper = snt.SkipConnectionCore(vanilla_rnn, name="skip")
with self.assertRaises(ValueError):
_ = skip_wrapper.output_size
skip_wrapper(inputs, prev_state)
self.assertAllEqual([self.in_size + self.hidden_size],
skip_wrapper.output_size.as_list())
skip_wrapper = snt.SkipConnectionCore(
vanilla_rnn, input_shape=(self.in_size,), name="skip")
self.assertAllEqual([self.in_size + self.hidden_size],
skip_wrapper.output_size.as_list())
def testShape(self):
inputs = tf.placeholder(tf.float32, shape=[self.batch_size, self.in_size])
prev_state = tf.placeholder(
tf.float32, shape=[self.batch_size, self.hidden_size])
vanilla_rnn = snt.VanillaRNN(self.hidden_size)
skip_wrapper = snt.SkipConnectionCore(vanilla_rnn, name="skip")
output, next_state = skip_wrapper(inputs, prev_state)
output_shape = np.ndarray((self.batch_size,
self.in_size + self.hidden_size))
state_shape = np.ndarray((self.batch_size, self.hidden_size))
self.assertShapeEqual(output_shape, output)
self.assertShapeEqual(state_shape, next_state)
def testHeterogeneousState(self):
"""Checks that the shape and type of the initial state are preserved."""
core = HeterogeneousStateCore(name="rnn", hidden_size=self.hidden_size)
skip_wrapper = snt.SkipConnectionCore(core, name="skip")
core_state = core.initial_state(self.batch_size)
skip_state = skip_wrapper.initial_state(self.batch_size)
self.assertEqual(core_state[0].shape.as_list(),
skip_state[0].shape.as_list())
self.assertEqual(core_state[1].shape.as_list(),
skip_state[1].shape.as_list())
self.assertEqual(core_state[0].dtype, skip_state[0].dtype)
self.assertEqual(core_state[1].dtype, skip_state[1].dtype)