def testCompatibleNames(self):
with self.test_session(use_gpu=self._use_gpu, graph=ops.Graph()):
cell = core_rnn_cell_impl.LSTMCell(10)
pcell = core_rnn_cell_impl.LSTMCell(10, use_peepholes=True)
inputs = [array_ops.zeros([4, 5])] * 6
core_rnn.static_rnn(cell,
inputs,
dtype=dtypes.float32,
scope="basic")
core_rnn.static_rnn(pcell,
inputs,
dtype=dtypes.float32,
scope="peephole")
basic_names = {
v.name: v.get_shape()
for v in variables.trainable_variables()
}
with self.test_session(use_gpu=self._use_gpu, graph=ops.Graph()):
cell = lstm_ops.LSTMBlockCell(10)
pcell = lstm_ops.LSTMBlockCell(10, use_peephole=True)
inputs = [array_ops.zeros([4, 5])] * 6
core_rnn.static_rnn(cell,
inputs,
dtype=dtypes.float32,
scope="basic")
core_rnn.static_rnn(pcell,
inputs,
dtype=dtypes.float32,
scope="peephole")
block_names = {
v.name: v.get_shape()
for v in variables.trainable_variables()
}
with self.test_session(use_gpu=self._use_gpu, graph=ops.Graph()):
cell = lstm_ops.LSTMBlockFusedCell(10)
pcell = lstm_ops.LSTMBlockFusedCell(10, use_peephole=True)
inputs = [array_ops.zeros([4, 5])] * 6
cell(inputs, dtype=dtypes.float32, scope="basic/lstm_cell")
pcell(inputs, dtype=dtypes.float32, scope="peephole/lstm_cell")
fused_names = {
v.name: v.get_shape()
for v in variables.trainable_variables()
}
self.assertEqual(basic_names, block_names)
self.assertEqual(basic_names, fused_names)
def __init__(self,
num_features,
input_window_size,
output_window_size,
num_units=128):
"""Construct the LSTM prediction model.
Args:
num_features: number of input features per time step.
input_window_size: Number of past time steps of data to look at when doing
the regression.
output_window_size: Number of future time steps to predict. Note that
setting it to > 1 empirically seems to give a better fit.
num_units: The number of units in the encoder and decoder LSTM cells.
"""
super(LSTMPredictionModel, self).__init__()
self._encoder = lstm_ops.LSTMBlockFusedCell(
num_units=num_units, name="encoder")
self._decoder = lstm_ops.LSTMBlockFusedCell(
num_units=num_units, name="decoder")
self._mean_transform = core.Dense(num_features,
name="mean_transform")
self._covariance_transform = core.Dense(num_features,
name="covariance_transform")
def blocks_match(sess, use_peephole):
batch_size = 2
input_size = 3
cell_size = 4
sequence_length = 4
inputs = []
for _ in range(sequence_length):
inp = ops.convert_to_tensor(np.random.randn(batch_size, input_size),
dtype=dtypes.float32)
inputs.append(inp)
stacked_inputs = array_ops.stack(inputs)
initializer = init_ops.random_uniform_initializer(-0.01,
0.01,
seed=19890212)
with variable_scope.variable_scope("test", initializer=initializer):
# magic naming so that the cells pick up these variables and reuse them
if use_peephole:
wci = variable_scope.get_variable("rnn/lstm_cell/w_i_diag",
shape=[cell_size],
dtype=dtypes.float32)
wcf = variable_scope.get_variable("rnn/lstm_cell/w_f_diag",
shape=[cell_size],
dtype=dtypes.float32)
wco = variable_scope.get_variable("rnn/lstm_cell/w_o_diag",
shape=[cell_size],
dtype=dtypes.float32)
w = variable_scope.get_variable(
"rnn/lstm_cell/kernel",
shape=[input_size + cell_size, cell_size * 4],
dtype=dtypes.float32)
b = variable_scope.get_variable(
"rnn/lstm_cell/bias",
shape=[cell_size * 4],
dtype=dtypes.float32,
initializer=init_ops.zeros_initializer())
basic_cell = rnn_cell.LSTMCell(cell_size,
use_peepholes=use_peephole,
state_is_tuple=True,
reuse=True)
basic_outputs_op, basic_state_op = rnn.static_rnn(basic_cell,
inputs,
dtype=dtypes.float32)
if use_peephole:
_, _, _, _, _, _, block_outputs_op = block_lstm(
ops.convert_to_tensor(sequence_length, dtype=dtypes.int64),
inputs,
w,
b,
wci=wci,
wcf=wcf,
wco=wco,
cell_clip=0,
use_peephole=True)
else:
_, _, _, _, _, _, block_outputs_op = block_lstm(
ops.convert_to_tensor(sequence_length, dtype=dtypes.int64),
inputs,
w,
b,
cell_clip=0)
fused_cell = lstm_ops.LSTMBlockFusedCell(cell_size,
cell_clip=0,
use_peephole=use_peephole,
reuse=True,
name="rnn/lstm_cell")
fused_outputs_op, fused_state_op = fused_cell(stacked_inputs,
dtype=dtypes.float32)
sess.run([variables.global_variables_initializer()])
basic_outputs, basic_state = sess.run(
[basic_outputs_op, basic_state_op[0]])
basic_grads = sess.run(
gradients_impl.gradients(basic_outputs_op, inputs))
xs = [w, b]
if use_peephole:
xs += [wci, wcf, wco]
basic_wgrads = sess.run(gradients_impl.gradients(basic_outputs_op, xs))
block_outputs = sess.run(block_outputs_op)
block_grads = sess.run(
gradients_impl.gradients(block_outputs_op, inputs))
block_wgrads = sess.run(gradients_impl.gradients(block_outputs_op, xs))
xs = [w, b]
if use_peephole:
xs += [wci, wcf, wco]
fused_outputs, fused_state = sess.run(
[fused_outputs_op, fused_state_op[0]])
fused_grads = sess.run(
gradients_impl.gradients(fused_outputs_op, inputs))
fused_wgrads = sess.run(gradients_impl.gradients(fused_outputs_op, xs))
return (basic_state, fused_state, basic_outputs, block_outputs,
fused_outputs, basic_grads, block_grads, fused_grads,
basic_wgrads, block_wgrads, fused_wgrads)
def testLSTMFusedSequenceLengths(self):
"""Verify proper support for sequence lengths in LSTMBlockFusedCell."""
with self.session(use_gpu=True) as sess:
batch_size = 3
input_size = 4
cell_size = 5
max_sequence_length = 6
inputs = []
for _ in range(max_sequence_length):
inp = ops.convert_to_tensor(np.random.randn(
batch_size, input_size),
dtype=dtypes.float32)
inputs.append(inp)
seq_lengths = constant_op.constant([3, 4, 5])
cell_inputs = array_ops.stack(inputs)
initializer = init_ops.random_uniform_initializer(-0.01,
0.01,
seed=19890213)
with variable_scope.variable_scope("lstm_cell",
initializer=initializer):
# magic naming so that the cells pick up these variables and reuse them
variable_scope.get_variable(
"kernel",
shape=[input_size + cell_size, cell_size * 4],
dtype=dtypes.float32)
variable_scope.get_variable(
"bias",
shape=[cell_size * 4],
dtype=dtypes.float32,
initializer=init_ops.zeros_initializer())
cell = lstm_ops.LSTMBlockFusedCell(cell_size,
cell_clip=0,
use_peephole=False,
reuse=True,
name="lstm_cell")
fused_outputs_op, fused_state_op = cell(
cell_inputs, dtype=dtypes.float32, sequence_length=seq_lengths)
cell_vars = [
v for v in variables.trainable_variables()
if v.name.endswith("kernel") or v.name.endswith("bias")
]
# Verify that state propagation works if we turn our sequence into
# tiny (single-time) subsequences, i.e. unfuse the cell
unfused_outputs_op = []
state = None
with variable_scope.variable_scope(
variable_scope.get_variable_scope(), reuse=True):
for i, inp in enumerate(inputs):
lengths = [int(i < l) for l in seq_lengths.eval()]
output, state = cell(array_ops.expand_dims(inp, 0),
initial_state=state,
dtype=dtypes.float32,
sequence_length=lengths)
unfused_outputs_op.append(output[0])
unfused_outputs_op = array_ops.stack(unfused_outputs_op)
sess.run([variables.global_variables_initializer()])
unfused_outputs, unfused_state = sess.run(
[unfused_outputs_op, state[0]])
unfused_grads = sess.run(
gradients_impl.gradients(unfused_outputs_op, inputs))
unfused_wgrads = sess.run(
gradients_impl.gradients(unfused_outputs_op, cell_vars))
fused_outputs, fused_state = sess.run(
[fused_outputs_op, fused_state_op[0]])
fused_grads = sess.run(
gradients_impl.gradients(fused_outputs_op, inputs))
fused_wgrads = sess.run(
gradients_impl.gradients(fused_outputs_op, cell_vars))
self.assertAllClose(fused_outputs, unfused_outputs)
self.assertAllClose(fused_state, unfused_state)
self.assertAllClose(fused_grads, unfused_grads)
for fused, unfused in zip(fused_wgrads, unfused_wgrads):
self.assertAllClose(fused, unfused, rtol=1e-6, atol=1e-6)
def testLSTMFusedSequenceLengths(self):
"""Verify proper support for sequence lengths in LSTMBlockFusedCell."""
with self.test_session(use_gpu=True) as sess:
batch_size = 3
input_size = 4
cell_size = 5
max_sequence_length = 6
inputs = []
for _ in range(max_sequence_length):
inp = ops.convert_to_tensor(np.random.randn(
batch_size, input_size),
dtype=dtypes.float32)
inputs.append(inp)
seq_lengths = constant_op.constant([3, 4, 5])
initializer = init_ops.random_uniform_initializer(-0.01,
0.01,
seed=19890213)
with variable_scope.variable_scope("basic",
initializer=initializer):
cell = rnn_cell.BasicLSTMCell(cell_size, state_is_tuple=True)
outputs, state = rnn.static_rnn(cell,
inputs,
dtype=dtypes.float32,
sequence_length=seq_lengths)
sess.run([variables.global_variables_initializer()])
basic_outputs, basic_state = sess.run([outputs, state[0]])
basic_grads = sess.run(
gradients_impl.gradients(outputs, inputs))
basic_wgrads = sess.run(
gradients_impl.gradients(outputs,
variables.trainable_variables()))
with variable_scope.variable_scope("fused",
initializer=initializer):
cell = lstm_ops.LSTMBlockFusedCell(cell_size,
cell_clip=0,
use_peephole=False)
outputs, state = cell(inputs,
dtype=dtypes.float32,
sequence_length=seq_lengths)
sess.run([variables.global_variables_initializer()])
fused_outputs, fused_state = sess.run([outputs, state[0]])
fused_grads = sess.run(
gradients_impl.gradients(outputs, inputs))
fused_vars = [
v for v in variables.trainable_variables()
if v.name.startswith("fused/")
]
fused_wgrads = sess.run(
gradients_impl.gradients(outputs, fused_vars))
self.assertAllClose(basic_outputs, fused_outputs)
self.assertAllClose(basic_state, fused_state)
self.assertAllClose(basic_grads, fused_grads)
for basic, fused in zip(basic_wgrads, fused_wgrads):
self.assertAllClose(basic, fused, rtol=1e-2, atol=1e-2)
# Verify that state propagation works if we turn our sequence into
# tiny (single-time) subsequences, i.e. unfuse the cell
with variable_scope.variable_scope("unfused",
initializer=initializer) as vs:
cell = lstm_ops.LSTMBlockFusedCell(cell_size,
cell_clip=0,
use_peephole=False)
outputs = []
state = None
for i, inp in enumerate(inputs):
lengths = [int(i < l) for l in seq_lengths.eval()]
output, state = cell([inp],
initial_state=state,
dtype=dtypes.float32,
sequence_length=lengths)
vs.reuse_variables()
outputs.append(output[0])
outputs = array_ops.stack(outputs)
sess.run([variables.global_variables_initializer()])
unfused_outputs, unfused_state = sess.run([outputs, state[0]])
unfused_grads = sess.run(
gradients_impl.gradients(outputs, inputs))
unfused_vars = [
v for v in variables.trainable_variables()
if v.name.startswith("unfused/")
]
unfused_wgrads = sess.run(
gradients_impl.gradients(outputs, unfused_vars))
self.assertAllClose(basic_outputs, unfused_outputs)
self.assertAllClose(basic_state, unfused_state)
self.assertAllClose(basic_grads, unfused_grads)
for basic, unfused in zip(basic_wgrads, unfused_wgrads):
self.assertAllClose(basic, unfused, rtol=1e-2, atol=1e-2)
def testLSTMBasicToBlockPeeping(self):
with self.test_session(use_gpu=True) as sess:
batch_size = 2
input_size = 3
cell_size = 4
sequence_length = 5
inputs = []
for _ in range(sequence_length):
inp = ops.convert_to_tensor(np.random.randn(
batch_size, input_size),
dtype=dtypes.float32)
inputs.append(inp)
initializer = init_ops.random_uniform_initializer(-0.01,
0.01,
seed=19890212)
with variable_scope.variable_scope("basic",
initializer=initializer):
cell = rnn_cell.LSTMCell(cell_size,
use_peepholes=True,
state_is_tuple=True)
outputs, state = rnn.static_rnn(cell,
inputs,
dtype=dtypes.float32)
sess.run([variables.global_variables_initializer()])
basic_outputs, basic_state = sess.run([outputs, state[0]])
basic_grads = sess.run(
gradients_impl.gradients(outputs, inputs))
basic_wgrads = sess.run(
gradients_impl.gradients(outputs,
variables.trainable_variables()))
with variable_scope.variable_scope("block",
initializer=initializer):
w = variable_scope.get_variable(
"w",
shape=[input_size + cell_size, cell_size * 4],
dtype=dtypes.float32)
b = variable_scope.get_variable(
"b",
shape=[cell_size * 4],
dtype=dtypes.float32,
initializer=init_ops.zeros_initializer())
wci = variable_scope.get_variable("wci",
shape=[cell_size],
dtype=dtypes.float32)
wcf = variable_scope.get_variable("wcf",
shape=[cell_size],
dtype=dtypes.float32)
wco = variable_scope.get_variable("wco",
shape=[cell_size],
dtype=dtypes.float32)
_, _, _, _, _, _, outputs = block_lstm(ops.convert_to_tensor(
sequence_length, dtype=dtypes.int64),
inputs,
w,
b,
wci=wci,
wcf=wcf,
wco=wco,
cell_clip=0,
use_peephole=True)
sess.run([variables.global_variables_initializer()])
block_outputs = sess.run(outputs)
block_grads = sess.run(
gradients_impl.gradients(outputs, inputs))
block_wgrads = sess.run(
gradients_impl.gradients(outputs, [w, b, wci, wcf, wco]))
self.assertAllClose(basic_outputs, block_outputs)
self.assertAllClose(basic_grads, block_grads)
for basic, block in zip(basic_wgrads, block_wgrads):
self.assertAllClose(basic, block, rtol=1e-2, atol=1e-2)
with variable_scope.variable_scope("fused",
initializer=initializer):
cell = lstm_ops.LSTMBlockFusedCell(cell_size,
cell_clip=0,
use_peephole=True)
outputs, state = cell(inputs, dtype=dtypes.float32)
sess.run([variables.global_variables_initializer()])
fused_outputs, fused_state = sess.run([outputs, state[0]])
fused_grads = sess.run(
gradients_impl.gradients(outputs, inputs))
fused_vars = [
v for v in variables.trainable_variables()
if v.name.startswith("fused/")
]
fused_wgrads = sess.run(
gradients_impl.gradients(outputs, fused_vars))
self.assertAllClose(basic_outputs, fused_outputs)
self.assertAllClose(basic_state, fused_state)
self.assertAllClose(basic_grads, fused_grads)
for basic, fused in zip(basic_wgrads, fused_wgrads):
self.assertAllClose(basic, fused, rtol=1e-2, atol=1e-2)
def tmp():
initializer = init_ops.random_uniform_initializer(-0.01, 0.01)
def lstm_cell():
hidden_size = RNN_UNIT_SIZE
input_size = CONTENT_DIM
cell = tf.contrib.rnn.LSTMCell(hidden_size,
input_size,
initializer=initializer,
state_is_tuple=True)
return cell
if True:
attn_length = 16
cells = [lstm_cell() for _ in range(2)]
cell = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True)
cell = tf.contrib.rnn.AttentionCellWrapper(cell,
attn_length,
state_is_tuple=True)
outputs, states = tf.nn.dynamic_rnn(cell,
content_embeddings,
sequence_length=content_lengths,
dtype=tf.float32)
#last_outputs = states[0][-1].h
last_outputs = tf.concat([states[0][-1].h, states[-1]], 1)
elif True:
content_embeddings = tf.unstack(content_embeddings, 200, 1)
cell = lstm_ops.LSTMBlockFusedCell(RNN_UNIT_SIZE)
content_lengths = tf.cast(content_lengths, tf.int32)
outputs, state = cell(content_embeddings,
sequence_length=content_lengths,
dtype=tf.float32)
last_outputs = state.h
elif True:
layer_sizes = [RNN_UNIT_SIZE, RNN_UNIT_SIZE]
cell = make_rnn_cell(layer_sizes,
dropout_keep_prob=dropout_keep_prob,
base_cell=lstm_ops.LSTMBlockCell,
attn_length=16)
outputs, final_state = tf.nn.dynamic_rnn(
cell,
content_embeddings,
sequence_length=content_lengths,
swap_memory=True,
dtype=tf.float32)
last_outputs = final_state[-1].h
#last_outputs = tf.concat([final_state[-1].h, final_state[0][1]], 1)
elif True:
cell = tf.contrib.rnn.MultiRNNCell([lstm_cell() for _ in range(2)],
state_is_tuple=True)
outputs, states = tf.nn.dynamic_rnn(cell,
content_embeddings,
sequence_length=content_lengths,
dtype=tf.float32)
last_outputs = states[-1].h
elif True:
num_hidden = RNN_UNIT_SIZE
cell_fw = tf.nn.rnn_cell.LSTMCell(num_units=num_hidden,
state_is_tuple=True)
cell_bw = tf.nn.rnn_cell.LSTMCell(num_units=num_hidden,
state_is_tuple=True)
outputs, states = tf.nn.bidirectional_dynamic_rnn(
cell_fw,
cell_bw,
content_embeddings,
sequence_length=content_lengths,
dtype=tf.float32)
output_fw, output_bw = outputs
output_state_fw, output_state_bw = states
#last_outputs = tf.concat([output_fw[:, 0], output_state_bw.h], 1)
last_outputs = tf.concat([output_state_fw.h, output_state_bw.h], 1)
elif True:
num_hidden = RNN_UNIT_SIZE
lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(num_hidden,
forget_bias=1.0)
lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(num_hidden,
forget_bias=1.0)
content_embeddings = tf.unstack(content_embeddings, 200, 1)
outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn(
lstm_fw_cell,
lstm_bw_cell,
content_embeddings,
sequence_length=content_lengths,
dtype=tf.float32)
last_outputs = outputs[-1]
def testLSTMBasicToBlockPeeping(self):
with self.test_session(use_gpu=True) as sess:
batch_size = 2
input_size = 3
cell_size = 4
sequence_length = 4
inputs = []
for _ in range(sequence_length):
inp = ops.convert_to_tensor(
np.random.randn(batch_size, input_size), dtype=dtypes.float32)
inputs.append(inp)
initializer = init_ops.random_uniform_initializer(
-0.01, 0.01, seed=19890212)
with variable_scope.variable_scope("test", initializer=initializer):
# magic naming so that the cells pick up these variables and resuse them
wci = variable_scope.get_variable(
"rnn/lstm_cell/w_i_diag", shape=[cell_size], dtype=dtypes.float32)
wcf = variable_scope.get_variable(
"rnn/lstm_cell/w_f_diag", shape=[cell_size], dtype=dtypes.float32)
wco = variable_scope.get_variable(
"rnn/lstm_cell/w_o_diag", shape=[cell_size], dtype=dtypes.float32)
w = variable_scope.get_variable(
"rnn/lstm_cell/kernel",
shape=[input_size + cell_size, cell_size * 4],
dtype=dtypes.float32)
b = variable_scope.get_variable(
"rnn/lstm_cell/bias",
shape=[cell_size * 4],
dtype=dtypes.float32,
initializer=init_ops.zeros_initializer())
wci_block = variable_scope.get_variable(
"rnn/lstm_cell/lstm_block_wrapper/w_i_diag",
initializer=wci.initialized_value())
wcf_block = variable_scope.get_variable(
"rnn/lstm_cell/lstm_block_wrapper/w_f_diag",
initializer=wcf.initialized_value())
wco_block = variable_scope.get_variable(
"rnn/lstm_cell/lstm_block_wrapper/w_o_diag",
initializer=wco.initialized_value())
w_block = variable_scope.get_variable(
"rnn/lstm_cell/lstm_block_wrapper/kernel",
initializer=w.initialized_value())
b_block = variable_scope.get_variable(
"rnn/lstm_cell/lstm_block_wrapper/bias",
initializer=b.initialized_value())
basic_cell = rnn_cell.LSTMCell(
cell_size, use_peepholes=True, state_is_tuple=True, reuse=True)
basic_outputs_op, basic_state_op = rnn.static_rnn(
basic_cell, inputs, dtype=dtypes.float32)
_, _, _, _, _, _, block_outputs_op = block_lstm(
ops.convert_to_tensor(sequence_length, dtype=dtypes.int64),
inputs,
w,
b,
wci=wci,
wcf=wcf,
wco=wco,
cell_clip=0,
use_peephole=True)
with variable_scope.variable_scope("rnn/lstm_cell", reuse=True):
fused_cell = lstm_ops.LSTMBlockFusedCell(
cell_size, cell_clip=0, use_peephole=True)
fused_outputs_op, fused_state_op = fused_cell(
inputs, dtype=dtypes.float32)
sess.run([variables.global_variables_initializer()])
basic_outputs, basic_state = sess.run(
[basic_outputs_op, basic_state_op[0]])
basic_grads = sess.run(
gradients_impl.gradients(basic_outputs_op, inputs))
basic_wgrads = sess.run(
gradients_impl.gradients(basic_outputs_op, [w, b, wci, wcf, wco]))
block_outputs = sess.run(block_outputs_op)
block_grads = sess.run(
gradients_impl.gradients(block_outputs_op, inputs))
block_wgrads = sess.run(
gradients_impl.gradients(block_outputs_op, [w, b, wci, wcf, wco]))
fused_outputs, fused_state = sess.run(
[fused_outputs_op, fused_state_op[0]])
fused_grads = sess.run(
gradients_impl.gradients(fused_outputs_op, inputs))
fused_wgrads = sess.run(
gradients_impl.gradients(
fused_outputs_op,
[w_block, b_block, wci_block, wcf_block, wco_block]))
self.assertAllClose(basic_outputs, block_outputs)
self.assertAllClose(basic_grads, block_grads)
for basic, block in zip(basic_wgrads, block_wgrads):
self.assertAllClose(basic, block, rtol=1e-6, atol=1e-6)
self.assertAllClose(basic_outputs, fused_outputs)
self.assertAllClose(basic_state, fused_state)
self.assertAllClose(basic_grads, fused_grads)
for basic, fused in zip(block_wgrads, fused_wgrads):
self.assertAllClose(basic, fused, rtol=1e-6, atol=1e-6)
