def test_masking_rnn_with_output_and_states(self): class Cell(keras.layers.Layer): def __init__(self): self.state_size = None self.output_size = None super(Cell, self).__init__() def build(self, input_shape): self.state_size = input_shape[-1] self.output_size = input_shape[-1] def call(self, inputs, states): return inputs, [s + 1 for s in states] x = keras.Input((3, 1), name='x') x_masked = keras.layers.Masking()(x) s_0 = keras.Input((1,), name='s_0') y, s = keras.layers.RNN( Cell(), return_state=True)(x_masked, initial_state=s_0) model = keras.models.Model([x, s_0], [y, s]) model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='mse') # last time step masked x_np = np.array([[[1.], [2.], [0.]]]) s_0_np = np.array([[10.]]) y_np, s_np = model.predict([x_np, s_0_np]) # 1 is added to initial state two times self.assertAllClose(s_np, s_0_np + 2) # Expect last output to be the same as last output before masking self.assertAllClose(y_np, x_np[:, 1, :])
def test_zero_output_for_masking(self): for unroll in [True, False]: cell = keras.layers.SimpleRNNCell(5) x = keras.Input((5, 5)) mask = keras.layers.Masking() layer = keras.layers.RNN( cell, return_sequences=True, zero_output_for_mask=True, unroll=unroll) masked_input = mask(x) y = layer(masked_input) model = keras.models.Model(x, y) model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='mse') np_x = np.ones((6, 5, 5)) result_1 = model.predict(np_x) # set the time 4 and 5 for last record to be zero (masked). np_x[5, 3:] = 0 result_2 = model.predict(np_x) # expect the result_2 has same output, except the time 4,5 for last # record. result_1[5, 3:] = 0 self.assertAllClose(result_1, result_2)
def prepare_simple_model(input_tensor, loss_name, target): axis = 1 if K.image_data_format() == 'channels_first' else -1 loss = None num_channels = None activation = None if loss_name == 'sparse_categorical_crossentropy': loss = lambda y_true, y_pred: K.sparse_categorical_crossentropy( # pylint: disable=g-long-lambda y_true, y_pred, axis=axis) num_channels = np.amax(target) + 1 activation = 'softmax' elif loss_name == 'categorical_crossentropy': loss = lambda y_true, y_pred: K.categorical_crossentropy( # pylint: disable=g-long-lambda y_true, y_pred, axis=axis) num_channels = target.shape[axis] activation = 'softmax' elif loss_name == 'binary_crossentropy': loss = lambda y_true, y_pred: K.binary_crossentropy( y_true, y_pred) # pylint: disable=unnecessary-lambda num_channels = target.shape[axis] activation = 'sigmoid' predictions = Conv2D(num_channels, 1, activation=activation, kernel_initializer='ones', bias_initializer='ones')(input_tensor) simple_model = keras.models.Model(inputs=input_tensor, outputs=predictions) simple_model.compile(optimizer=rmsprop.RMSPropOptimizer(1e-3), loss=loss) return simple_model
def test_warm_start_from_keras_ckpt(self): keras_model, (x_train, y_train), ( _, _), train_input_fn, eval_input_fn = get_resource_for_simple_model( model_type='functional', is_evaluate=True) keras_model.compile(optimizer=rmsprop.RMSPropOptimizer(1e-3), loss='categorical_crossentropy', metrics=['accuracy']) keras_model.fit(x_train, y_train, epochs=1) warm_start_path = os.path.join(self._config.model_dir, 'keras', 'warm_start.ckpt') keras_model.save_weights(warm_start_path) est_keras = keras_lib.model_to_estimator(keras_model=keras_model, config=self._config, checkpoint_format='saver') self.assertEqual( warm_start_path, est_keras._warm_start_settings.ckpt_to_initialize_from) before_eval_results = est_keras.evaluate(input_fn=eval_input_fn, steps=1) est_keras.train(input_fn=train_input_fn, steps=_TRAIN_SIZE / 16) after_eval_results = est_keras.evaluate(input_fn=eval_input_fn, steps=1) self.assertLess(after_eval_results['loss'], before_eval_results['loss'])
def test_sequential_deferred_build_serialization(self): num_hidden = 5 input_dim = 3 batch_size = 5 num_classes = 2 model = testing_utils.get_small_sequential_mlp(num_hidden, num_classes) model.compile( loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3), metrics=[keras.metrics.CategoricalAccuracy()], run_eagerly=testing_utils.should_run_eagerly()) self.assertFalse(model.built) x = np.random.random((batch_size, input_dim)) y = np.random.random((batch_size, num_classes)) model.train_on_batch(x, y) self.assertTrue(model.built) config = model.get_config() self.assertIn('build_input_shape', config) new_model = keras.models.Sequential.from_config(config) self.assertEqual(len(new_model.layers), 2) self.assertEqual(len(new_model.weights), 4)
def test_sequential_deferred_build_with_dataset_iterators(self): num_hidden = 5 input_dim = 3 num_classes = 2 num_samples = 50 steps_per_epoch = 10 model = testing_utils.get_small_sequential_mlp(num_hidden, num_classes) model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3), metrics=[keras.metrics.CategoricalAccuracy()], run_eagerly=testing_utils.should_run_eagerly()) self.assertEqual(len(model.layers), 2) self.assertEqual(len(model.weights), 0) self.assertFalse(model.built) x = array_ops.ones((num_samples, input_dim)) y = array_ops.zeros((num_samples, num_classes)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) dataset = dataset.batch(10) iterator = dataset_ops.make_one_shot_iterator(dataset) model.fit(iterator, epochs=1, steps_per_epoch=steps_per_epoch) self.assertTrue(model.built) self.assertEqual(len(model.weights), 2 * 2) self.assertFalse(model._is_graph_network)
def test_high_dimension_RNN_with_init_state(self): unit_a = 10 unit_b = 20 input_a = 5 input_b = 10 batch = 32 time_step = 4 # Basic test case. cell = Minimal2DRNNCell(unit_a, unit_b) x = keras.Input((None, input_a, input_b)) s = keras.Input((unit_a, unit_b)) layer = keras.layers.RNN(cell) y = layer(x, initial_state=s) model = keras.models.Model([x, s], y) model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='mse') model.train_on_batch([ np.zeros((batch, time_step, input_a, input_b)), np.zeros((batch, unit_a, unit_b)) ], np.zeros((batch, unit_a, unit_b))) self.assertEqual(model.output_shape, (None, unit_a, unit_b)) # Bad init state shape. bad_shape_a = unit_a * 2 bad_shape_b = unit_b * 2 cell = Minimal2DRNNCell(unit_a, unit_b) x = keras.Input((None, input_a, input_b)) s = keras.Input((bad_shape_a, bad_shape_b)) layer = keras.layers.RNN(cell) with self.assertRaisesWithPredicateMatch(ValueError, 'however `cell.state_size` is'): layer(x, initial_state=s)
def test_train_with_subclassed_model_with_existing_state(self): keras_model, (_, _), ( _, _), train_input_fn, eval_input_fn = get_resource_for_simple_model( model_type='subclass', is_evaluate=True) keras_model.compile( loss='categorical_crossentropy', optimizer=rmsprop.RMSPropOptimizer(1e-3), metrics=['mse', keras.metrics.categorical_accuracy]) with self.test_session(): # Create state keras_model.train_on_batch(np.random.random((10,) + _INPUT_SIZE), np.random.random((10, _NUM_CLASS))) original_preds = keras_model.predict(np.ones((10,) + _INPUT_SIZE)) est_keras = keras_lib.model_to_estimator( keras_model=keras_model, config=self._config) est_keras.train(input_fn=train_input_fn, steps=_TRAIN_SIZE / 16) before_eval_results = est_keras.evaluate( input_fn=eval_input_fn, steps=1) est_keras.train(input_fn=train_input_fn, steps=_TRAIN_SIZE / 16) after_eval_results = est_keras.evaluate(input_fn=eval_input_fn, steps=1) self.assertLess(after_eval_results['loss'], before_eval_results['loss']) # Check that original model state was not altered preds = keras_model.predict(np.ones((10,) + _INPUT_SIZE)) self.assertAllClose(original_preds, preds, atol=1e-5) # Check that the original model compilation did not break keras_model.train_on_batch(np.random.random((10,) + _INPUT_SIZE), np.random.random((10, _NUM_CLASS)))
def test_dataset_input_shape_validation(self): with self.cached_session(): x = keras.layers.Input(shape=(3,), name='input') y = keras.layers.Dense(4, name='dense')(x) model = keras.Model(x, y) optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001) loss = 'mse' strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:1', '/device:GPU:0']) model.compile(optimizer, loss, distribute=strategy) # User forgets to batch the dataset inputs = np.zeros((10, 3), dtype=np.float32) targets = np.zeros((10, 4), dtype=np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) dataset = dataset.repeat(100) with self.assertRaisesRegexp(ValueError, 'expected input to have shape'): model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0) # Wrong input shape inputs = np.zeros((10, 5), dtype=np.float32) targets = np.zeros((10, 4), dtype=np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) dataset = dataset.repeat(100) dataset = dataset.batch(10) with self.assertRaisesRegexp(ValueError, 'expected input to have shape'): model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0)
def test_extract_model_metrics(self): a = keras.layers.Input(shape=(3, ), name='input_a') b = keras.layers.Input(shape=(3, ), name='input_b') dense = keras.layers.Dense(4, name='dense') c = dense(a) d = dense(b) e = keras.layers.Dropout(0.5, name='dropout')(c) model = keras.models.Model([a, b], [d, e]) extract_metrics = saving_utils.extract_model_metrics(model) self.assertEqual(None, extract_metrics) extract_metric_names = [ 'dense_binary_accuracy', 'dropout_binary_accuracy', 'dense_mean_squared_error', 'dropout_mean_squared_error' ] if tf2.enabled(): extract_metric_names.extend(['dense_mae', 'dropout_mae']) else: extract_metric_names.extend( ['dense_mean_absolute_error', 'dropout_mean_absolute_error']) model_metric_names = ['loss', 'dense_loss', 'dropout_loss' ] + extract_metric_names model.compile(loss='mae', metrics=[ keras.metrics.BinaryAccuracy(), 'mae', keras.metrics.mean_squared_error ], optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.01)) extract_metrics = saving_utils.extract_model_metrics(model) self.assertEqual(set(model_metric_names), set(model.metrics_names)) self.assertEqual(set(extract_metric_names), set(extract_metrics.keys()))
def test_train_sequential_with_distribution_strategy(self): dist = mirrored_strategy.MirroredStrategy( devices=['/device:GPU:0', '/device:GPU:1']) keras_model = simple_sequential_model() keras_model.compile( loss='categorical_crossentropy', metrics=[keras.metrics.CategoricalAccuracy()], optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.01)) config = run_config_lib.RunConfig(tf_random_seed=_RANDOM_SEED, model_dir=self._base_dir, train_distribute=dist) with self.cached_session(): est_keras = keras_lib.model_to_estimator(keras_model=keras_model, config=config) before_eval_results = est_keras.evaluate( input_fn=get_ds_test_input_fn, steps=1) est_keras.train(input_fn=get_ds_train_input_fn, steps=_TRAIN_SIZE / 16) after_eval_results = est_keras.evaluate( input_fn=get_ds_test_input_fn, steps=1) self.assertLess(after_eval_results['loss'], before_eval_results['loss']) writer_cache.FileWriterCache.clear() gfile.DeleteRecursively(self._config.model_dir)
def test_sequential_deferred_build_with_dataset_iterators(self): if not context.executing_eagerly(): # TODO(psv/fchollet): Add support for this use case in graph mode. return num_hidden = 5 input_dim = 3 num_classes = 2 num_samples = 50 steps_per_epoch = 10 model = keras.models.Sequential() # We don't specify the input shape. model.add(keras.layers.Dense(num_hidden)) model.add(keras.layers.Dense(num_classes)) model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3)) self.assertEqual(len(model.layers), 2) self.assertEqual(len(model.weights), 0) self.assertFalse(model.built) x = array_ops.ones((num_samples, input_dim)) y = array_ops.zeros((num_samples, num_classes)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) dataset = dataset.batch(10) iterator = dataset.make_one_shot_iterator() model.fit(iterator, epochs=1, steps_per_epoch=steps_per_epoch) self.assertTrue(model.built) self.assertEqual(model.inputs[0].get_shape().as_list(), [None, input_dim]) self.assertEqual(model.outputs[0].get_shape().as_list(), [None, num_classes]) self.assertEqual(len(model.weights), 2 * 2)
def test_eager_dnc_optimization(self): batch_size = 7 input_size = 15 memory_config = { 'memory_size': 27, 'word_size': 9, 'num_read_heads': 10, } output_size = 36 x = tf.keras.Input(shape=( None, input_size, )) dnc_cell = DNC(output_size, controller_units=30, **memory_config) dnc_initial_state = dnc_cell.get_initial_state(batch_size=batch_size) layer = tf.keras.layers.RNN(dnc_cell) y = layer(x, initial_state=dnc_initial_state) model = tf.keras.models.Model(x, y) model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='mse', run_eagerly=True) model.train_on_batch(np.zeros((batch_size, 5, input_size)), np.zeros((batch_size, output_size))) self.assertEqual(model.output_shape[1], output_size)
def test_train_with_tf_optimizer(self): for model_type in ['sequential', 'functional']: keras_model, (_, _), ( _, _ ), train_input_fn, eval_input_fn = get_resource_for_simple_model( model_type=model_type, is_evaluate=True) keras_model.compile( loss='categorical_crossentropy', optimizer=rmsprop.RMSPropOptimizer(1e-3), metrics=['mse', keras.metrics.categorical_accuracy]) with self.test_session(): est_keras = keras_lib.model_to_estimator( keras_model=keras_model, config=self._config) before_eval_results = est_keras.evaluate( input_fn=eval_input_fn, steps=1) est_keras.train(input_fn=train_input_fn, steps=_TRAIN_SIZE / 16) after_eval_results = est_keras.evaluate(input_fn=eval_input_fn, steps=1) self.assertLess(after_eval_results['loss'], before_eval_results['loss']) writer_cache.FileWriterCache.clear() gfile.DeleteRecursively(self._config.model_dir)
def test_builtin_rnn_cell_serialization(self): for cell_class in [ keras.layers.SimpleRNNCell, keras.layers.GRUCell, keras.layers.LSTMCell ]: # Test basic case. x = keras.Input((None, 5)) cell = cell_class(32) layer = keras.layers.RNN(cell) y = layer(x) model = keras.models.Model(x, y) model.compile( optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='mse') # Test basic case serialization. x_np = np.random.random((6, 5, 5)) y_np = model.predict(x_np) weights = model.get_weights() config = layer.get_config() layer = keras.layers.RNN.from_config(config) y = layer(x) model = keras.models.Model(x, y) model.set_weights(weights) y_np_2 = model.predict(x_np) self.assertAllClose(y_np, y_np_2, atol=1e-4) # Test stacking. cells = [cell_class(8), cell_class(12), cell_class(32)] layer = keras.layers.RNN(cells) y = layer(x) model = keras.models.Model(x, y) model.compile( optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='mse') # Test stacked RNN serialization. x_np = np.random.random((6, 5, 5)) y_np = model.predict(x_np) weights = model.get_weights() config = layer.get_config() layer = keras.layers.RNN.from_config(config) y = layer(x) model = keras.models.Model(x, y) model.set_weights(weights) y_np_2 = model.predict(x_np) self.assertAllClose(y_np, y_np_2, atol=1e-4)
def test_nested_input_output(self): batch = 10 t = 5 i1, i2, i3 = 3, 4, 5 o1, o2, o3 = 2, 3, 4 cell = NestedCell(o1, o2, o3) rnn = keras.layers.RNN(cell) input_1 = keras.Input((t, i1)) input_2 = keras.Input((t, i2, i3)) outputs = rnn((input_1, input_2)) self.assertEqual(len(outputs), 2) self.assertEqual(outputs[0].shape.as_list(), [None, o1]) self.assertEqual(outputs[1].shape.as_list(), [None, o2, o3]) model = keras.models.Model((input_1, input_2), outputs) model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='mse') model.train_on_batch( [np.zeros((batch, t, i1)), np.zeros((batch, t, i2, i3))], [np.zeros((batch, o1)), np.zeros((batch, o2, o3))]) self.assertEqual(model.output_shape, [(None, o1), (None, o2, o3)]) cell = NestedCell(o1, o2, o3, use_tuple=True) rnn = keras.layers.RNN(cell) input_1 = keras.Input((t, i1)) input_2 = keras.Input((t, i2, i3)) outputs = rnn(NestedInput(t1=input_1, t2=input_2)) self.assertEqual(len(outputs), 2) self.assertEqual(outputs[0].shape.as_list(), [None, o1]) self.assertEqual(outputs[1].shape.as_list(), [None, o2, o3]) model = keras.models.Model([input_1, input_2], outputs) model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='mse') model.train_on_batch( [np.zeros((batch, t, i1)), np.zeros((batch, t, i2, i3))], [np.zeros((batch, o1)), np.zeros((batch, o2, o3))]) self.assertEqual(model.output_shape, [(None, o1), (None, o2, o3)])
def test_ops_with_var_and_rmsprop(self): var_list = [ deo.get_variable('sp_var', initializer=0.0, dim=2), ] opt_list = [ rmsprop.RMSPropOptimizer(0.1), ] self.common_run_context(var_list, opt_list, name='rmsprop_test')
def get_multiple_optimizers(): return [ adagrad.AdagradOptimizer(0.1), adam.AdamOptimizer(0.1), ftrl.FtrlOptimizer(0.1), momentum.MomentumOptimizer(0.1, 0.1), rmsprop.RMSPropOptimizer(0.1) ]
def test_high_dimension_RNN(self): # Basic test case. unit_a = 10 unit_b = 20 input_a = 5 input_b = 10 batch = 32 time_step = 4 cell = Minimal2DRNNCell(unit_a, unit_b) x = keras.Input((None, input_a, input_b)) layer = keras.layers.RNN(cell) y = layer(x) self.assertEqual(cell.state_size.as_list(), [unit_a, unit_b]) if not context.executing_eagerly(): init_state = layer.get_initial_state(x) self.assertEqual(len(init_state), 1) self.assertEqual(init_state[0].get_shape().as_list(), [None, unit_a, unit_b]) model = keras.models.Model(x, y) model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='mse') model.train_on_batch( np.zeros((batch, time_step, input_a, input_b)), np.zeros((batch, unit_a, unit_b))) self.assertEqual(model.output_shape, (None, unit_a, unit_b)) # Test stacking. cells = [ Minimal2DRNNCell(unit_a, unit_b), Minimal2DRNNCell(unit_a * 2, unit_b * 2), Minimal2DRNNCell(unit_a * 4, unit_b * 4) ] layer = keras.layers.RNN(cells) y = layer(x) model = keras.models.Model(x, y) model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='mse') model.train_on_batch( np.zeros((batch, time_step, input_a, input_b)), np.zeros((batch, unit_a * 4, unit_b * 4))) self.assertEqual(model.output_shape, (None, unit_a * 4, unit_b * 4))
def test_sequential_nesting(self): model = _get_small_mlp(4, 3) inner_model = _get_small_mlp(4, 5) model.add(inner_model) model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3)) x = np.random.random((2, 6)) y = np.random.random((2, 5)) model.fit(x, y, epochs=1)
def test_pretrained_weights(self): keras_model, (_, _), (_, _), _, _ = get_resource_for_simple_model() keras_model.compile( loss='categorical_crossentropy', optimizer=rmsprop.RMSPropOptimizer(1e-3), metrics=['mse', keras.metrics.categorical_accuracy]) keras_model.train_on_batch(np.random.random((10, ) + _INPUT_SIZE), np.random.random((10, _NUM_CLASS))) weights = keras_model.get_weights() keras_model, (_, _), (_, _), _, _ = get_resource_for_simple_model() keras_model.set_weights(weights) keras_model.compile( loss='categorical_crossentropy', optimizer=rmsprop.RMSPropOptimizer(1e-3), metrics=['mse', keras.metrics.categorical_accuracy]) keras.estimator.model_to_estimator(keras_model=keras_model, config=self._config)
def get_model(): if deferred: model = _get_small_mlp(10, 4) else: model = _get_small_mlp(10, 4, input_dim=3) model.compile(optimizer=rmsprop.RMSPropOptimizer(1e-3), loss='categorical_crossentropy', metrics=['accuracy']) return model
def create_model(): model = keras.Sequential() model.add(keras.layers.Dense(10, activation='relu')) model.add(keras.layers.Dense(4, activation='softmax')) model.compile(optimizer=rmsprop.RMSPropOptimizer(1e-3), loss='categorical_crossentropy', metrics=['accuracy']) return model
def test_build_before_fit(self): # Fix for b/112433577 model = testing_utils.get_small_sequential_mlp(4, 5) model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3)) model.build((None, 6)) x = np.random.random((2, 6)) y = np.random.random((2, 5)) model.fit(x, y, epochs=1)
def test_globalpooling_1d_masking_support(self): model = keras.Sequential() model.add(keras.layers.Masking(mask_value=0., input_shape=(3, 4))) model.add(keras.layers.GlobalAveragePooling1D()) model.compile(loss='mae', optimizer=rmsprop.RMSPropOptimizer(0.001)) model_input = np.random.random((2, 3, 4)) model_input[0, 1:, :] = 0 output = model.predict(model_input) self.assertAllClose(output[0], model_input[0, 0, :])
def test_sequential_nesting(self): model = testing_utils.get_small_sequential_mlp(4, 3) inner_model = testing_utils.get_small_sequential_mlp(4, 5) model.add(inner_model) model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3), run_eagerly=testing_utils.should_run_eagerly()) x = np.random.random((2, 6)) y = np.random.random((2, 5)) model.fit(x, y, epochs=1)
def test_numpy_with_sample_weights(self, distribution): model = get_model() optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001) loss = 'mse' model.compile(optimizer, loss, distribute=distribution) inputs = np.zeros((10, 3), np.float32) targets = np.zeros((10, 4), np.float32) sample_weights = np.ones((10), np.float32) model.fit(inputs, targets, sample_weight=sample_weights, epochs=1, steps_per_epoch=2, verbose=1)
def _get_optimizer(self): lr = self.lr = self.setup_lr() # tf.summary.scalar('lr', self.lr) optimizer_type = self.optimizer_type if optimizer_type == "adam": opt = adam.AdamOptimizer(lr) elif optimizer_type == "sgd": opt = gradient_descent.GradientDescentOptimizer(lr) elif optimizer_type == "rmsprop": opt = rmsprop.RMSPropOptimizer(lr) return opt
def test_minimal_rnn_cell_non_layer_multiple_states(self): class MinimalRNNCell(object): def __init__(self, units, input_dim): self.units = units self.state_size = (units, units) self.kernel = keras.backend.variable( np.random.random((input_dim, units))) def call(self, inputs, states): prev_output_1 = states[0] prev_output_2 = states[1] output = keras.backend.dot(inputs, self.kernel) output += prev_output_1 output -= prev_output_2 return output, [output * 2, output * 3] # Basic test case. cell = MinimalRNNCell(32, 5) x = keras.Input((None, 5)) layer = keras.layers.RNN(cell) y = layer(x) model = keras.models.Model(x, y) model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='mse') model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32))) # Test stacking. cells = [ MinimalRNNCell(8, 5), MinimalRNNCell(16, 8), MinimalRNNCell(32, 16) ] layer = keras.layers.RNN(cells) self.assertEqual(layer.cell.state_size, (8, 8, 16, 16, 32, 32)) self.assertEqual(layer.cell.output_size, 32) y = layer(x) model = keras.models.Model(x, y) model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001), loss='mse') model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
def _GetOptimizer(self, opt): if opt == "adagrad": return adagrad.AdagradOptimizer(learning_rate=1e-2) elif opt == "adam": return adam.AdamOptimizer(learning_rate=1e-2) elif opt == "rmsprop": return rmsprop.RMSPropOptimizer(learning_rate=1e-2) elif opt == "momentum": return momentum.MomentumOptimizer(learning_rate=1e-2, momentum=0.9) elif opt == "sgd": return gradient_descent.GradientDescentOptimizer(learning_rate=1e-2) else: raise ValueError("Unsupported optimizer: %s" % opt)