def test_minimal_rnn_cell_non_layer():
class MinimalRNNCell(object):
def __init__(self, units, input_dim):
self.units = units
self.state_size = units
self.kernel = keras.backend.variable(
np.random.random((input_dim, units)))
def call(self, inputs, states):
prev_output = states[0]
output = keras.backend.dot(inputs, self.kernel) + prev_output
return output, [output]
# Basic test case.
cell = MinimalRNNCell(32, 5)
x = keras.Input((None, 5))
layer = recurrent.RNN(cell)
y = layer(x)
model = keras.models.Model(x, y)
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
# Test stacking.
cells = [
MinimalRNNCell(8, 5),
MinimalRNNCell(32, 8),
MinimalRNNCell(32, 32)
]
layer = recurrent.RNN(cells)
y = layer(x)
model = keras.models.Model(x, y)
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
def test_stacked_rnn_compute_output_shape():
cells = [recurrent.LSTMCell(3), recurrent.LSTMCell(6)]
layer = recurrent.RNN(cells, return_state=True, return_sequences=True)
output_shape = layer.compute_output_shape((None, timesteps, embedding_dim))
expected_output_shape = [(None, timesteps, 6), (None, 6), (None, 6),
(None, 3), (None, 3)]
assert output_shape == expected_output_shape
def test_stacked_rnn_compute_output_shape():
cells = [recurrent.LSTMCell(3), recurrent.LSTMCell(6)]
layer = recurrent.RNN(cells, return_state=True, return_sequences=True)
output_shape = layer.compute_output_shape((None, timesteps, embedding_dim))
expected_output_shape = [(None, timesteps, 6), (None, 3), (None, 3),
(None, 6), (None, 6)]
assert output_shape == expected_output_shape
# Test reverse_state_order = True for stacked cell.
stacked_cell = recurrent.StackedRNNCells(cells, reverse_state_order=True)
layer = recurrent.RNN(stacked_cell,
return_state=True,
return_sequences=True)
output_shape = layer.compute_output_shape((None, timesteps, embedding_dim))
expected_output_shape = [(None, timesteps, 6), (None, 6), (None, 6),
(None, 3), (None, 3)]
assert output_shape == expected_output_shape
def test_builtin_rnn_cell_layer(cell_class):
# Test basic case.
x = keras.Input((None, 5))
cell = cell_class(32)
layer = recurrent.RNN(cell)
y = layer(x)
model = keras.models.Model(x, y)
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
# Test basic case serialization.
x_np = np.random.random((6, 5, 5))
y_np = model.predict(x_np)
weights = model.get_weights()
config = layer.get_config()
layer = recurrent.RNN.from_config(config)
y = layer(x)
model = keras.models.Model(x, y)
model.set_weights(weights)
y_np_2 = model.predict(x_np)
assert_allclose(y_np, y_np_2, atol=1e-4)
# Test stacking.
cells = [cell_class(8),
cell_class(12),
cell_class(32)]
layer = recurrent.RNN(cells)
y = layer(x)
model = keras.models.Model(x, y)
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
# Test stacked RNN serialization.
x_np = np.random.random((6, 5, 5))
y_np = model.predict(x_np)
weights = model.get_weights()
config = layer.get_config()
layer = recurrent.RNN.from_config(config)
y = layer(x)
model = keras.models.Model(x, y)
model.set_weights(weights)
y_np_2 = model.predict(x_np)
assert_allclose(y_np, y_np_2, atol=1e-4)
def test_masking_correctness_output_size_not_equal_to_first_state_size():
class Cell(keras.layers.Layer):
def __init__(self):
self.state_size = None
self.output_size = None
super(Cell, self).__init__()
def build(self, input_shape):
self.state_size = input_shape[-1]
self.output_size = input_shape[-1] * 2
def call(self, inputs, states):
return keras.layers.concatenate([inputs] *
2), [s + 1 for s in states]
num_samples = 5
num_timesteps = 6
input_size = state_size = 7
# random inputs and state values
x_vals = np.random.random((num_samples, num_timesteps, input_size))
# last timestep masked for first sample (all zero inputs masked by Masking layer)
x_vals[0, -1, :] = 0
s_initial_vals = np.random.random((num_samples, state_size))
# final outputs equal to last inputs concatenated
y_vals_expected = np.concatenate([x_vals[:, -1]] * 2, axis=-1)
# except for first sample, where it is equal to second to last value due to mask
y_vals_expected[0] = np.concatenate([x_vals[0, -2]] * 2, axis=-1)
s_final_vals_expected = s_initial_vals.copy()
# states are incremented `num_timesteps - 1` times for first sample
s_final_vals_expected[0] += (num_timesteps - 1)
# and `num_timesteps - 1` times for remaining samples
s_final_vals_expected[1:] += num_timesteps
for unroll in [True, False]:
x = Input((num_timesteps, input_size), name="x")
x_masked = Masking()(x)
s_initial = Input((state_size, ), name="s_initial")
y, s_final = recurrent.RNN(Cell(), return_state=True,
unroll=unroll)(x_masked,
initial_state=s_initial)
model = Model([x, s_initial], [y, s_final])
model.compile(optimizer='sgd', loss='mse')
y_vals, s_final_vals = model.predict([x_vals, s_initial_vals])
assert_allclose(
y_vals,
y_vals_expected,
err_msg="Unexpected output for unroll={}".format(unroll))
assert_allclose(
s_final_vals,
s_final_vals_expected,
err_msg="Unexpected state for unroll={}".format(unroll))
def test_stacked_rnn_dropout():
cells = [recurrent.LSTMCell(3, dropout=0.1, recurrent_dropout=0.1),
recurrent.LSTMCell(3, dropout=0.1, recurrent_dropout=0.1)]
layer = recurrent.RNN(cells)
x = keras.Input((None, 5))
y = layer(x)
model = keras.models.Model(x, y)
model.compile('sgd', 'mse')
x_np = np.random.random((6, 5, 5))
y_np = np.random.random((6, 3))
model.train_on_batch(x_np, y_np)
def test_minimal_rnn_cell_non_layer_multiple_states():
class MinimalRNNCell(object):
def __init__(self, units, input_dim):
self.units = units
self.state_size = (units, units)
self.kernel = keras.backend.variable(
np.random.random((input_dim, units)))
def call(self, inputs, states):
prev_output_1 = states[0]
prev_output_2 = states[1]
output = keras.backend.dot(inputs, self.kernel)
output += prev_output_1
output -= prev_output_2
return output, [output * 2, output * 3]
# Basic test case.
cell = MinimalRNNCell(32, 5)
x = keras.Input((None, 5))
layer = recurrent.RNN(cell)
y = layer(x)
model = keras.models.Model(x, y)
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
# MXNet does not support cell stacking yet
if K.backend() != 'mxnet':
# Test stacking.
cells = [
MinimalRNNCell(8, 5),
MinimalRNNCell(16, 8),
MinimalRNNCell(32, 16)
]
layer = recurrent.RNN(cells)
assert layer.cell.state_size == (8, 8, 16, 16, 32, 32)
y = layer(x)
model = keras.models.Model(x, y)
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
def test_stacked_rnn_attributes():
cells = [recurrent.LSTMCell(3),
recurrent.LSTMCell(3, kernel_regularizer='l2')]
layer = recurrent.RNN(cells)
layer.build((None, None, 5))
# Test regularization losses
assert len(layer.losses) == 1
# Test weights
assert len(layer.trainable_weights) == 6
cells[0].trainable = False
assert len(layer.trainable_weights) == 3
assert len(layer.non_trainable_weights) == 3
x = keras.Input((None, 5))
y = K.sum(x)
cells[0].add_loss(y, inputs=x)
def test_sequence_example_into_input_layer(self):
examples = [_make_sequence_example().SerializeToString()] * 100
ctx_cols, seq_cols = self._build_feature_columns()
def _parse_example(example):
ctx, seq = tf.io.parse_single_sequence_example(
example,
context_features=tf.feature_column.make_parse_example_spec(
ctx_cols),
sequence_features=tf.feature_column.make_parse_example_spec(
seq_cols))
ctx.update(seq)
return ctx
ds = tf.data.Dataset.from_tensor_slices(examples)
ds = ds.map(_parse_example)
ds = ds.batch(20)
# Test on a single batch
features = tf.compat.v1.data.make_one_shot_iterator(ds).get_next()
# Tile the context features across the sequence features
sequence_input_layer = ksfc.SequenceFeatures(seq_cols)
seq_input, _ = sequence_input_layer(features)
dense_input_layer = dense_features.DenseFeatures(ctx_cols)
ctx_input = dense_input_layer(features)
ctx_input = core.RepeatVector(
tf.compat.v1.shape(seq_input)[1])(ctx_input)
concatenated_input = merge.concatenate([seq_input, ctx_input])
rnn_layer = recurrent.RNN(recurrent.SimpleRNNCell(10))
output = rnn_layer(concatenated_input)
with self.cached_session() as sess:
sess.run(tf.compat.v1.global_variables_initializer())
features_r = sess.run(features)
self.assertAllEqual(features_r['int_list'].dense_shape, [20, 3, 6])
output_r = sess.run(output)
self.assertAllEqual(output_r.shape, [20, 10])
def test_inconsistent_output_state_size():
class PlusOneRNNCell(keras.layers.Layer):
"""Add one to the input and state.
This cell is used for testing state_size and output_size."""
def __init__(self, num_unit, **kwargs):
self.state_size = num_unit
super(PlusOneRNNCell, self).__init__(**kwargs)
def build(self, input_shape):
self.output_size = input_shape[-1]
def call(self, inputs, states):
return inputs + 1, [states[0] + 1]
batch = 32
time_step = 4
state_size = 5
input_size = 6
cell = PlusOneRNNCell(state_size)
x = keras.Input((None, input_size))
layer = recurrent.RNN(cell)
y = layer(x)
assert cell.state_size == state_size
init_state = layer.get_initial_state(x)
assert len(init_state) == 1
if K.backend() != 'theano':
# theano does not support static shape inference.
assert K.int_shape(init_state[0]) == (None, state_size)
model = keras.models.Model(x, y)
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(
np.zeros((batch, time_step, input_size)),
np.zeros((batch, input_size)))
assert model.output_shape == (None, input_size)
def test_rnn_cell_with_constants_layer_passing_initial_state():
class RNNCellWithConstants(keras.layers.Layer):
def __init__(self, units, **kwargs):
self.units = units
self.state_size = units
super(RNNCellWithConstants, self).__init__(**kwargs)
def build(self, input_shape):
if not isinstance(input_shape, list):
raise TypeError('expects constants shape')
[input_shape, constant_shape] = input_shape
# will (and should) raise if more than one constant passed
self.input_kernel = self.add_weight(shape=(input_shape[-1],
self.units),
initializer='uniform',
name='kernel')
self.recurrent_kernel = self.add_weight(shape=(self.units,
self.units),
initializer='uniform',
name='recurrent_kernel')
self.constant_kernel = self.add_weight(shape=(constant_shape[-1],
self.units),
initializer='uniform',
name='constant_kernel')
self.built = True
def call(self, inputs, states, constants):
[prev_output] = states
[constant] = constants
h_input = keras.backend.dot(inputs, self.input_kernel)
h_state = keras.backend.dot(prev_output, self.recurrent_kernel)
h_const = keras.backend.dot(constant, self.constant_kernel)
output = h_input + h_state + h_const
return output, [output]
def get_config(self):
config = {'units': self.units}
base_config = super(RNNCellWithConstants, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
# Test basic case.
x = keras.Input((None, 5))
c = keras.Input((3, ))
s = keras.Input((32, ))
cell = RNNCellWithConstants(32)
layer = recurrent.RNN(cell)
y = layer(x, initial_state=s, constants=c)
model = keras.models.Model([x, s, c], y)
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(
[np.zeros((6, 5, 5)),
np.zeros((6, 32)),
np.zeros((6, 3))], np.zeros((6, 32)))
# Test basic case serialization.
x_np = np.random.random((6, 5, 5))
s_np = np.random.random((6, 32))
c_np = np.random.random((6, 3))
y_np = model.predict([x_np, s_np, c_np])
weights = model.get_weights()
config = layer.get_config()
custom_objects = {'RNNCellWithConstants': RNNCellWithConstants}
with keras.utils.CustomObjectScope(custom_objects):
layer = recurrent.RNN.from_config(config.copy())
y = layer(x, initial_state=s, constants=c)
model = keras.models.Model([x, s, c], y)
model.set_weights(weights)
y_np_2 = model.predict([x_np, s_np, c_np])
assert_allclose(y_np, y_np_2, atol=1e-4)
# verify that state is used
y_np_2_different_s = model.predict([x_np, s_np + 10., c_np])
with pytest.raises(AssertionError):
assert_allclose(y_np, y_np_2_different_s, atol=1e-4)
# test flat list inputs
with keras.utils.CustomObjectScope(custom_objects):
layer = recurrent.RNN.from_config(config.copy())
y = layer([x, s, c])
model = keras.models.Model([x, s, c], y)
model.set_weights(weights)
y_np_3 = model.predict([x_np, s_np, c_np])
assert_allclose(y_np, y_np_3, atol=1e-4)
def test_minimal_rnn_cell_layer():
class MinimalRNNCell(keras.layers.Layer):
def __init__(self, units, **kwargs):
self.units = units
self.state_size = units
super(MinimalRNNCell, self).__init__(**kwargs)
def build(self, input_shape):
self.kernel = self.add_weight(shape=(input_shape[-1], self.units),
initializer='uniform',
name='kernel')
self.recurrent_kernel = self.add_weight(shape=(self.units,
self.units),
initializer='uniform',
name='recurrent_kernel')
self.built = True
def call(self, inputs, states):
prev_output = states[0]
h = keras.backend.dot(inputs, self.kernel)
output = h + keras.backend.dot(prev_output, self.recurrent_kernel)
return output, [output]
def get_config(self):
config = {'units': self.units}
base_config = super(MinimalRNNCell, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
# Test basic case.
x = keras.Input((None, 5))
cell = MinimalRNNCell(32)
layer = recurrent.RNN(cell)
y = layer(x)
model = keras.models.Model(x, y)
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
# Test basic case serialization.
x_np = np.random.random((6, 5, 5))
y_np = model.predict(x_np)
weights = model.get_weights()
config = layer.get_config()
with keras.utils.CustomObjectScope({'MinimalRNNCell': MinimalRNNCell}):
layer = recurrent.RNN.from_config(config)
y = layer(x)
model = keras.models.Model(x, y)
model.set_weights(weights)
y_np_2 = model.predict(x_np)
assert_allclose(y_np, y_np_2, atol=1e-4)
# Test stacking.
cells = [MinimalRNNCell(8), MinimalRNNCell(12), MinimalRNNCell(32)]
layer = recurrent.RNN(cells)
y = layer(x)
model = keras.models.Model(x, y)
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
# Test stacked RNN serialization.
x_np = np.random.random((6, 5, 5))
y_np = model.predict(x_np)
weights = model.get_weights()
config = layer.get_config()
with keras.utils.CustomObjectScope({'MinimalRNNCell': MinimalRNNCell}):
layer = recurrent.RNN.from_config(config)
y = layer(x)
model = keras.models.Model(x, y)
model.set_weights(weights)
y_np_2 = model.predict(x_np)
assert_allclose(y_np, y_np_2, atol=1e-4)
