def test_minimal_rnn_cell_non_layer(): class MinimalRNNCell(object): def __init__(self, units, input_dim): self.units = units self.state_size = units self.kernel = keras.backend.variable( np.random.random((input_dim, units))) def call(self, inputs, states): prev_output = states[0] output = keras.backend.dot(inputs, self.kernel) + prev_output return output, [output] # Basic test case. cell = MinimalRNNCell(32, 5) x = keras.Input((None, 5)) layer = recurrent.RNN(cell) y = layer(x) model = keras.models.Model(x, y) model.compile(optimizer='rmsprop', loss='mse') model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32))) # Test stacking. cells = [ MinimalRNNCell(8, 5), MinimalRNNCell(32, 8), MinimalRNNCell(32, 32) ] layer = recurrent.RNN(cells) y = layer(x) model = keras.models.Model(x, y) model.compile(optimizer='rmsprop', loss='mse') model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
def test_stacked_rnn_compute_output_shape(): cells = [recurrent.LSTMCell(3), recurrent.LSTMCell(6)] layer = recurrent.RNN(cells, return_state=True, return_sequences=True) output_shape = layer.compute_output_shape((None, timesteps, embedding_dim)) expected_output_shape = [(None, timesteps, 6), (None, 6), (None, 6), (None, 3), (None, 3)] assert output_shape == expected_output_shape
def test_stacked_rnn_compute_output_shape(): cells = [recurrent.LSTMCell(3), recurrent.LSTMCell(6)] layer = recurrent.RNN(cells, return_state=True, return_sequences=True) output_shape = layer.compute_output_shape((None, timesteps, embedding_dim)) expected_output_shape = [(None, timesteps, 6), (None, 3), (None, 3), (None, 6), (None, 6)] assert output_shape == expected_output_shape # Test reverse_state_order = True for stacked cell. stacked_cell = recurrent.StackedRNNCells(cells, reverse_state_order=True) layer = recurrent.RNN(stacked_cell, return_state=True, return_sequences=True) output_shape = layer.compute_output_shape((None, timesteps, embedding_dim)) expected_output_shape = [(None, timesteps, 6), (None, 6), (None, 6), (None, 3), (None, 3)] assert output_shape == expected_output_shape
def test_builtin_rnn_cell_layer(cell_class): # Test basic case. x = keras.Input((None, 5)) cell = cell_class(32) layer = recurrent.RNN(cell) y = layer(x) model = keras.models.Model(x, y) model.compile(optimizer='rmsprop', loss='mse') model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32))) # Test basic case serialization. x_np = np.random.random((6, 5, 5)) y_np = model.predict(x_np) weights = model.get_weights() config = layer.get_config() layer = recurrent.RNN.from_config(config) y = layer(x) model = keras.models.Model(x, y) model.set_weights(weights) y_np_2 = model.predict(x_np) assert_allclose(y_np, y_np_2, atol=1e-4) # Test stacking. cells = [cell_class(8), cell_class(12), cell_class(32)] layer = recurrent.RNN(cells) y = layer(x) model = keras.models.Model(x, y) model.compile(optimizer='rmsprop', loss='mse') model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32))) # Test stacked RNN serialization. x_np = np.random.random((6, 5, 5)) y_np = model.predict(x_np) weights = model.get_weights() config = layer.get_config() layer = recurrent.RNN.from_config(config) y = layer(x) model = keras.models.Model(x, y) model.set_weights(weights) y_np_2 = model.predict(x_np) assert_allclose(y_np, y_np_2, atol=1e-4)
def test_masking_correctness_output_size_not_equal_to_first_state_size(): class Cell(keras.layers.Layer): def __init__(self): self.state_size = None self.output_size = None super(Cell, self).__init__() def build(self, input_shape): self.state_size = input_shape[-1] self.output_size = input_shape[-1] * 2 def call(self, inputs, states): return keras.layers.concatenate([inputs] * 2), [s + 1 for s in states] num_samples = 5 num_timesteps = 6 input_size = state_size = 7 # random inputs and state values x_vals = np.random.random((num_samples, num_timesteps, input_size)) # last timestep masked for first sample (all zero inputs masked by Masking layer) x_vals[0, -1, :] = 0 s_initial_vals = np.random.random((num_samples, state_size)) # final outputs equal to last inputs concatenated y_vals_expected = np.concatenate([x_vals[:, -1]] * 2, axis=-1) # except for first sample, where it is equal to second to last value due to mask y_vals_expected[0] = np.concatenate([x_vals[0, -2]] * 2, axis=-1) s_final_vals_expected = s_initial_vals.copy() # states are incremented `num_timesteps - 1` times for first sample s_final_vals_expected[0] += (num_timesteps - 1) # and `num_timesteps - 1` times for remaining samples s_final_vals_expected[1:] += num_timesteps for unroll in [True, False]: x = Input((num_timesteps, input_size), name="x") x_masked = Masking()(x) s_initial = Input((state_size, ), name="s_initial") y, s_final = recurrent.RNN(Cell(), return_state=True, unroll=unroll)(x_masked, initial_state=s_initial) model = Model([x, s_initial], [y, s_final]) model.compile(optimizer='sgd', loss='mse') y_vals, s_final_vals = model.predict([x_vals, s_initial_vals]) assert_allclose( y_vals, y_vals_expected, err_msg="Unexpected output for unroll={}".format(unroll)) assert_allclose( s_final_vals, s_final_vals_expected, err_msg="Unexpected state for unroll={}".format(unroll))
def test_stacked_rnn_dropout(): cells = [recurrent.LSTMCell(3, dropout=0.1, recurrent_dropout=0.1), recurrent.LSTMCell(3, dropout=0.1, recurrent_dropout=0.1)] layer = recurrent.RNN(cells) x = keras.Input((None, 5)) y = layer(x) model = keras.models.Model(x, y) model.compile('sgd', 'mse') x_np = np.random.random((6, 5, 5)) y_np = np.random.random((6, 3)) model.train_on_batch(x_np, y_np)
def test_minimal_rnn_cell_non_layer_multiple_states(): class MinimalRNNCell(object): def __init__(self, units, input_dim): self.units = units self.state_size = (units, units) self.kernel = keras.backend.variable( np.random.random((input_dim, units))) def call(self, inputs, states): prev_output_1 = states[0] prev_output_2 = states[1] output = keras.backend.dot(inputs, self.kernel) output += prev_output_1 output -= prev_output_2 return output, [output * 2, output * 3] # Basic test case. cell = MinimalRNNCell(32, 5) x = keras.Input((None, 5)) layer = recurrent.RNN(cell) y = layer(x) model = keras.models.Model(x, y) model.compile(optimizer='rmsprop', loss='mse') model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32))) # MXNet does not support cell stacking yet if K.backend() != 'mxnet': # Test stacking. cells = [ MinimalRNNCell(8, 5), MinimalRNNCell(16, 8), MinimalRNNCell(32, 16) ] layer = recurrent.RNN(cells) assert layer.cell.state_size == (8, 8, 16, 16, 32, 32) y = layer(x) model = keras.models.Model(x, y) model.compile(optimizer='rmsprop', loss='mse') model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
def test_stacked_rnn_attributes(): cells = [recurrent.LSTMCell(3), recurrent.LSTMCell(3, kernel_regularizer='l2')] layer = recurrent.RNN(cells) layer.build((None, None, 5)) # Test regularization losses assert len(layer.losses) == 1 # Test weights assert len(layer.trainable_weights) == 6 cells[0].trainable = False assert len(layer.trainable_weights) == 3 assert len(layer.non_trainable_weights) == 3 x = keras.Input((None, 5)) y = K.sum(x) cells[0].add_loss(y, inputs=x)
def test_sequence_example_into_input_layer(self): examples = [_make_sequence_example().SerializeToString()] * 100 ctx_cols, seq_cols = self._build_feature_columns() def _parse_example(example): ctx, seq = tf.io.parse_single_sequence_example( example, context_features=tf.feature_column.make_parse_example_spec( ctx_cols), sequence_features=tf.feature_column.make_parse_example_spec( seq_cols)) ctx.update(seq) return ctx ds = tf.data.Dataset.from_tensor_slices(examples) ds = ds.map(_parse_example) ds = ds.batch(20) # Test on a single batch features = tf.compat.v1.data.make_one_shot_iterator(ds).get_next() # Tile the context features across the sequence features sequence_input_layer = ksfc.SequenceFeatures(seq_cols) seq_input, _ = sequence_input_layer(features) dense_input_layer = dense_features.DenseFeatures(ctx_cols) ctx_input = dense_input_layer(features) ctx_input = core.RepeatVector( tf.compat.v1.shape(seq_input)[1])(ctx_input) concatenated_input = merge.concatenate([seq_input, ctx_input]) rnn_layer = recurrent.RNN(recurrent.SimpleRNNCell(10)) output = rnn_layer(concatenated_input) with self.cached_session() as sess: sess.run(tf.compat.v1.global_variables_initializer()) features_r = sess.run(features) self.assertAllEqual(features_r['int_list'].dense_shape, [20, 3, 6]) output_r = sess.run(output) self.assertAllEqual(output_r.shape, [20, 10])
def test_inconsistent_output_state_size(): class PlusOneRNNCell(keras.layers.Layer): """Add one to the input and state. This cell is used for testing state_size and output_size.""" def __init__(self, num_unit, **kwargs): self.state_size = num_unit super(PlusOneRNNCell, self).__init__(**kwargs) def build(self, input_shape): self.output_size = input_shape[-1] def call(self, inputs, states): return inputs + 1, [states[0] + 1] batch = 32 time_step = 4 state_size = 5 input_size = 6 cell = PlusOneRNNCell(state_size) x = keras.Input((None, input_size)) layer = recurrent.RNN(cell) y = layer(x) assert cell.state_size == state_size init_state = layer.get_initial_state(x) assert len(init_state) == 1 if K.backend() != 'theano': # theano does not support static shape inference. assert K.int_shape(init_state[0]) == (None, state_size) model = keras.models.Model(x, y) model.compile(optimizer='rmsprop', loss='mse') model.train_on_batch( np.zeros((batch, time_step, input_size)), np.zeros((batch, input_size))) assert model.output_shape == (None, input_size)
def test_rnn_cell_with_constants_layer_passing_initial_state(): class RNNCellWithConstants(keras.layers.Layer): def __init__(self, units, **kwargs): self.units = units self.state_size = units super(RNNCellWithConstants, self).__init__(**kwargs) def build(self, input_shape): if not isinstance(input_shape, list): raise TypeError('expects constants shape') [input_shape, constant_shape] = input_shape # will (and should) raise if more than one constant passed self.input_kernel = self.add_weight(shape=(input_shape[-1], self.units), initializer='uniform', name='kernel') self.recurrent_kernel = self.add_weight(shape=(self.units, self.units), initializer='uniform', name='recurrent_kernel') self.constant_kernel = self.add_weight(shape=(constant_shape[-1], self.units), initializer='uniform', name='constant_kernel') self.built = True def call(self, inputs, states, constants): [prev_output] = states [constant] = constants h_input = keras.backend.dot(inputs, self.input_kernel) h_state = keras.backend.dot(prev_output, self.recurrent_kernel) h_const = keras.backend.dot(constant, self.constant_kernel) output = h_input + h_state + h_const return output, [output] def get_config(self): config = {'units': self.units} base_config = super(RNNCellWithConstants, self).get_config() return dict(list(base_config.items()) + list(config.items())) # Test basic case. x = keras.Input((None, 5)) c = keras.Input((3, )) s = keras.Input((32, )) cell = RNNCellWithConstants(32) layer = recurrent.RNN(cell) y = layer(x, initial_state=s, constants=c) model = keras.models.Model([x, s, c], y) model.compile(optimizer='rmsprop', loss='mse') model.train_on_batch( [np.zeros((6, 5, 5)), np.zeros((6, 32)), np.zeros((6, 3))], np.zeros((6, 32))) # Test basic case serialization. x_np = np.random.random((6, 5, 5)) s_np = np.random.random((6, 32)) c_np = np.random.random((6, 3)) y_np = model.predict([x_np, s_np, c_np]) weights = model.get_weights() config = layer.get_config() custom_objects = {'RNNCellWithConstants': RNNCellWithConstants} with keras.utils.CustomObjectScope(custom_objects): layer = recurrent.RNN.from_config(config.copy()) y = layer(x, initial_state=s, constants=c) model = keras.models.Model([x, s, c], y) model.set_weights(weights) y_np_2 = model.predict([x_np, s_np, c_np]) assert_allclose(y_np, y_np_2, atol=1e-4) # verify that state is used y_np_2_different_s = model.predict([x_np, s_np + 10., c_np]) with pytest.raises(AssertionError): assert_allclose(y_np, y_np_2_different_s, atol=1e-4) # test flat list inputs with keras.utils.CustomObjectScope(custom_objects): layer = recurrent.RNN.from_config(config.copy()) y = layer([x, s, c]) model = keras.models.Model([x, s, c], y) model.set_weights(weights) y_np_3 = model.predict([x_np, s_np, c_np]) assert_allclose(y_np, y_np_3, atol=1e-4)
def test_minimal_rnn_cell_layer(): class MinimalRNNCell(keras.layers.Layer): def __init__(self, units, **kwargs): self.units = units self.state_size = units super(MinimalRNNCell, self).__init__(**kwargs) def build(self, input_shape): self.kernel = self.add_weight(shape=(input_shape[-1], self.units), initializer='uniform', name='kernel') self.recurrent_kernel = self.add_weight(shape=(self.units, self.units), initializer='uniform', name='recurrent_kernel') self.built = True def call(self, inputs, states): prev_output = states[0] h = keras.backend.dot(inputs, self.kernel) output = h + keras.backend.dot(prev_output, self.recurrent_kernel) return output, [output] def get_config(self): config = {'units': self.units} base_config = super(MinimalRNNCell, self).get_config() return dict(list(base_config.items()) + list(config.items())) # Test basic case. x = keras.Input((None, 5)) cell = MinimalRNNCell(32) layer = recurrent.RNN(cell) y = layer(x) model = keras.models.Model(x, y) model.compile(optimizer='rmsprop', loss='mse') model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32))) # Test basic case serialization. x_np = np.random.random((6, 5, 5)) y_np = model.predict(x_np) weights = model.get_weights() config = layer.get_config() with keras.utils.CustomObjectScope({'MinimalRNNCell': MinimalRNNCell}): layer = recurrent.RNN.from_config(config) y = layer(x) model = keras.models.Model(x, y) model.set_weights(weights) y_np_2 = model.predict(x_np) assert_allclose(y_np, y_np_2, atol=1e-4) # Test stacking. cells = [MinimalRNNCell(8), MinimalRNNCell(12), MinimalRNNCell(32)] layer = recurrent.RNN(cells) y = layer(x) model = keras.models.Model(x, y) model.compile(optimizer='rmsprop', loss='mse') model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32))) # Test stacked RNN serialization. x_np = np.random.random((6, 5, 5)) y_np = model.predict(x_np) weights = model.get_weights() config = layer.get_config() with keras.utils.CustomObjectScope({'MinimalRNNCell': MinimalRNNCell}): layer = recurrent.RNN.from_config(config) y = layer(x) model = keras.models.Model(x, y) model.set_weights(weights) y_np_2 = model.predict(x_np) assert_allclose(y_np, y_np_2, atol=1e-4)