def testAttentionCellWrapperFailures(self):
with self.assertRaisesRegexp(TypeError,
"The parameter cell is not RNNCell."):
rnn_cell.AttentionCellWrapper(None, 0)
num_units = 8
for state_is_tuple in [False, True]:
with ops.Graph().as_default():
lstm_cell = core_rnn_cell_impl.BasicLSTMCell(
num_units, state_is_tuple=state_is_tuple)
with self.assertRaisesRegexp(
ValueError, "attn_length should be greater than zero, got 0"):
rnn_cell.AttentionCellWrapper(
lstm_cell, 0, state_is_tuple=state_is_tuple)
with self.assertRaisesRegexp(
ValueError, "attn_length should be greater than zero, got -1"):
rnn_cell.AttentionCellWrapper(
lstm_cell, -1, state_is_tuple=state_is_tuple)
with ops.Graph().as_default():
lstm_cell = core_rnn_cell_impl.BasicLSTMCell(
num_units, state_is_tuple=True)
with self.assertRaisesRegexp(
ValueError, "Cell returns tuple of states, but the flag "
"state_is_tuple is not set. State size is: *"):
rnn_cell.AttentionCellWrapper(lstm_cell, 4, state_is_tuple=False)
def testAttentionCellWrapperZeros(self):
num_units = 8
attn_length = 16
batch_size = 3
input_size = 4
for state_is_tuple in [False, True]:
with ops.Graph().as_default():
with self.test_session() as sess:
with variable_scope.variable_scope("state_is_tuple_" +
str(state_is_tuple)):
lstm_cell = core_rnn_cell_impl.BasicLSTMCell(
num_units, state_is_tuple=state_is_tuple)
cell = rnn_cell.AttentionCellWrapper(
lstm_cell,
attn_length,
state_is_tuple=state_is_tuple)
if state_is_tuple:
zeros = array_ops.zeros([batch_size, num_units],
dtype=np.float32)
attn_state_zeros = array_ops.zeros(
[batch_size, attn_length * num_units],
dtype=np.float32)
zero_state = ((zeros, zeros), zeros,
attn_state_zeros)
else:
zero_state = array_ops.zeros([
batch_size, num_units * 2 +
attn_length * num_units + num_units
],
dtype=np.float32)
inputs = array_ops.zeros([batch_size, input_size],
dtype=dtypes.float32)
output, state = cell(inputs, zero_state)
self.assertEquals(output.get_shape(),
[batch_size, num_units])
if state_is_tuple:
self.assertEquals(len(state), 3)
self.assertEquals(len(state[0]), 2)
self.assertEquals(state[0][0].get_shape(),
[batch_size, num_units])
self.assertEquals(state[0][1].get_shape(),
[batch_size, num_units])
self.assertEquals(state[1].get_shape(),
[batch_size, num_units])
self.assertEquals(
state[2].get_shape(),
[batch_size, attn_length * num_units])
tensors = [output] + list(state)
else:
self.assertEquals(state.get_shape(), [
batch_size, num_units * 2 + num_units +
attn_length * num_units
])
tensors = [output, state]
zero_result = sum([
math_ops.reduce_sum(math_ops.abs(x))
for x in tensors
])
sess.run(variables.global_variables_initializer())
self.assertTrue(sess.run(zero_result) < 1e-6)
def testAttentionCellWrapperCorrectResult(self):
num_units = 4
attn_length = 6
batch_size = 2
expected_output = np.array(
[[1.068372, 0.45496, -0.678277, 0.340538],
[1.018088, 0.378983, -0.572179, 0.268591]],
dtype=np.float32)
expected_state = np.array(
[[0.74946702, 0.34681597, 0.26474735, 1.06485605, 0.38465962,
0.11420801, 0.10272158, 0.30925757, 0.63899988, 0.7181077,
0.47534478, 0.33715725, 0.58086717, 0.49446869, 0.7641536,
0.12814975, 0.92231739, 0.89857256, 0.21889746, 0.38442063,
0.53481543, 0.8876909, 0.45823169, 0.5905602, 0.78038228,
0.56501579, 0.03971386, 0.09870267, 0.8074435, 0.66821432,
0.99211812, 0.12295902, 1.14606023, 0.34370938, -0.79251152,
0.51843399],
[0.5179342, 0.48682183, -0.25426468, 0.96810579, 0.28809637,
0.13607743, -0.11446252, 0.26792109, 0.78047138, 0.63460857,
0.49122369, 0.52007174, 0.73000264, 0.66986895, 0.73576689,
0.86301267, 0.87887371, 0.35185754, 0.93417215, 0.64732957,
0.63173044, 0.66627824, 0.53644657, 0.20477486, 0.98458421,
0.38277245, 0.03746676, 0.92510188, 0.57714164, 0.84932971,
0.36127412, 0.12125921, 1.1362772, 0.34361625, -0.78150457,
0.70582712]],
dtype=np.float32)
seed = 12345
random_seed.set_random_seed(seed)
for state_is_tuple in [False, True]:
with session.Session() as sess:
with variable_scope.variable_scope(
"state_is_tuple", reuse=state_is_tuple,
initializer=init_ops.glorot_uniform_initializer()):
lstm_cell = core_rnn_cell_impl.BasicLSTMCell(
num_units, state_is_tuple=state_is_tuple)
cell = rnn_cell.AttentionCellWrapper(
lstm_cell, attn_length, state_is_tuple=state_is_tuple)
zeros1 = random_ops.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 1)
zeros2 = random_ops.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 2)
zeros3 = random_ops.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 3)
attn_state_zeros = random_ops.random_uniform(
(batch_size, attn_length * num_units), 0.0, 1.0, seed=seed + 4)
zero_state = ((zeros1, zeros2), zeros3, attn_state_zeros)
if not state_is_tuple:
zero_state = array_ops.concat([
zero_state[0][0], zero_state[0][1], zero_state[1], zero_state[2]
], 1)
inputs = random_ops.random_uniform(
(batch_size, num_units), 0.0, 1.0, seed=seed + 5)
output, state = cell(inputs, zero_state)
if state_is_tuple:
state = array_ops.concat(
[state[0][0], state[0][1], state[1], state[2]], 1)
sess.run(variables.global_variables_initializer())
self.assertAllClose(sess.run(output), expected_output)
self.assertAllClose(sess.run(state), expected_state)
def testAttentionCellWrapperValues(self):
num_units = 8
attn_length = 16
batch_size = 3
for state_is_tuple in [False, True]:
with ops.Graph().as_default():
with self.test_session() as sess:
with variable_scope.variable_scope("state_is_tuple_" +
str(state_is_tuple)):
lstm_cell = core_rnn_cell_impl.BasicLSTMCell(
num_units, state_is_tuple=state_is_tuple)
cell = rnn_cell.AttentionCellWrapper(
lstm_cell,
attn_length,
state_is_tuple=state_is_tuple)
if state_is_tuple:
zeros = constant_op.constant(0.1 * np.ones(
[batch_size, num_units], dtype=np.float32),
dtype=dtypes.float32)
attn_state_zeros = constant_op.constant(
0.1 *
np.ones([batch_size, attn_length * num_units],
dtype=np.float32),
dtype=dtypes.float32)
zero_state = ((zeros, zeros), zeros,
attn_state_zeros)
else:
zero_state = constant_op.constant(
0.1 * np.ones([
batch_size, num_units * 2 + num_units +
attn_length * num_units
],
dtype=np.float32),
dtype=dtypes.float32)
inputs = constant_op.constant(np.array(
[[1., 1., 1., 1.], [2., 2., 2., 2.],
[3., 3., 3., 3.]],
dtype=np.float32),
dtype=dtypes.float32)
output, state = cell(inputs, zero_state)
if state_is_tuple:
concat_state = array_ops.concat(
[state[0][0], state[0][1], state[1], state[2]],
1)
else:
concat_state = state
sess.run(variables.global_variables_initializer())
output, state = sess.run([output, concat_state])
# Different inputs so different outputs and states
for i in range(1, batch_size):
self.assertTrue(
float(
np.linalg.norm((output[0, :] -
output[i, :]))) > 1e-6)
self.assertTrue(
float(
np.linalg.norm((state[0, :] -
state[i, :]))) > 1e-6)
def testAttentionCellWrapperCorrectResult(self):
num_units = 4
attn_length = 6
batch_size = 2
expected_output = np.array([[0.955392, 0.408507, -0.60122, 0.270718],
[0.903681, 0.331165, -0.500238, 0.224052]],
dtype=np.float32)
expected_state = np.array(
[[
0.81331915, 0.32036272, 0.28079176, 1.08888793, 0.41264394,
0.1062041, 0.10444493, 0.32050529, 0.64655536, 0.70794445,
0.51896095, 0.31809306, 0.58086717, 0.49446869, 0.7641536,
0.12814975, 0.92231739, 0.89857256, 0.21889746, 0.38442063,
0.53481543, 0.8876909, 0.45823169, 0.5905602, 0.78038228,
0.56501579, 0.03971386, 0.09870267, 0.8074435, 0.66821432,
0.99211812, 0.12295902, 1.01412082, 0.33123279, -0.71114945,
0.40583119
],
[
0.59962207, 0.42597458, -0.22491696, 0.98063421, 0.32548007,
0.11623692, -0.10100613, 0.27708149, 0.76956916, 0.6360054,
0.51719815, 0.50458527, 0.73000264, 0.66986895, 0.73576689,
0.86301267, 0.87887371, 0.35185754, 0.93417215, 0.64732957,
0.63173044, 0.66627824, 0.53644657, 0.20477486, 0.98458421,
0.38277245, 0.03746676, 0.92510188, 0.57714164, 0.84932971,
0.36127412, 0.12125921, 0.99780077, 0.31886846, -0.67595094,
0.56531656
]],
dtype=np.float32)
seed = 12345
random_seed.set_random_seed(seed)
for state_is_tuple in [False, True]:
with session.Session() as sess:
with variable_scope.variable_scope("state_is_tuple",
reuse=state_is_tuple):
lstm_cell = core_rnn_cell_impl.BasicLSTMCell(
num_units, state_is_tuple=state_is_tuple)
cell = rnn_cell.AttentionCellWrapper(
lstm_cell, attn_length, state_is_tuple=state_is_tuple)
zeros1 = random_ops.random_uniform((batch_size, num_units),
0.0,
1.0,
seed=seed + 1)
zeros2 = random_ops.random_uniform((batch_size, num_units),
0.0,
1.0,
seed=seed + 2)
zeros3 = random_ops.random_uniform((batch_size, num_units),
0.0,
1.0,
seed=seed + 3)
attn_state_zeros = random_ops.random_uniform(
(batch_size, attn_length * num_units),
0.0,
1.0,
seed=seed + 4)
zero_state = ((zeros1, zeros2), zeros3, attn_state_zeros)
if not state_is_tuple:
zero_state = array_ops.concat_v2([
zero_state[0][0], zero_state[0][1], zero_state[1],
zero_state[2]
], 1)
inputs = random_ops.random_uniform((batch_size, num_units),
0.0,
1.0,
seed=seed + 5)
output, state = cell(inputs, zero_state)
if state_is_tuple:
state = array_ops.concat_v2(
[state[0][0], state[0][1], state[1], state[2]], 1)
sess.run(variables.global_variables_initializer())
self.assertAllClose(sess.run(output), expected_output)
self.assertAllClose(sess.run(state), expected_state)