def test_multi_output_model_with_none_masking(self): def func(x): return [x * 0.2, x * 0.3] def output_shape(input_shape): return [input_shape, input_shape] i = keras.layers.Input(shape=(3, 2, 1)) o = keras.layers.Lambda(function=func, output_shape=output_shape)(i) self.assertEqual(keras.backend.int_shape(o[0]), (None, 3, 2, 1)) self.assertEqual(keras.backend.int_shape(o[1]), (None, 3, 2, 1)) o = keras.layers.add(o) model = keras.Model(i, o) model.run_eagerly = testing_utils.should_run_eagerly() i2 = keras.layers.Input(shape=(3, 2, 1)) o2 = model(i2) model2 = keras.Model(i2, o2) model2.run_eagerly = testing_utils.should_run_eagerly() x = np.random.random((4, 3, 2, 1)) out = model2.predict(x) assert out.shape == (4, 3, 2, 1) self.assertAllClose(out, x * 0.2 + x * 0.3, atol=1e-4)
def test_training_methods(self): # test fit, train_on_batch # on different input types: list, dict num_classes = (2, 3) num_samples = 100 input_dim = 50 x1 = np.ones((num_samples, input_dim)) x2 = np.ones((num_samples, input_dim)) y1 = np.zeros((num_samples, num_classes[0])) y2 = np.zeros((num_samples, num_classes[1])) model = MultiIOTestModel(num_classes=num_classes, use_bn=True) model.compile( loss='mse', optimizer='rmsprop', run_eagerly=testing_utils.should_run_eagerly()) model.fit([x1, x2], [y1, y2], epochs=2, batch_size=32, verbose=0) model.fit({'input_1': x1, 'input_2': x2}, {'output_1': y1, 'output_2': y2}, epochs=2, batch_size=32) model.fit([x1, x2], [y1, y2], epochs=2, batch_size=32, verbose=0, validation_data=([x1, x2], [y1, y2])) model = MultiIOTestModel(num_classes=num_classes, use_bn=True) model.compile( loss='mse', optimizer='rmsprop', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch([x1, x2], [y1, y2]) model.train_on_batch({'input_1': x1, 'input_2': x2}, {'output_1': y1, 'output_2': y2})
def test_merge_dot(self): i1 = keras.layers.Input(shape=(4,)) i2 = keras.layers.Input(shape=(4,)) o = keras.layers.dot([i1, i2], axes=1) self.assertListEqual(o.shape.as_list(), [None, 1]) model = keras.models.Model([i1, i2], o) model.run_eagerly = testing_utils.should_run_eagerly() _ = keras.layers.Dot(axes=1).get_config() x1 = np.random.random((2, 4)) x2 = np.random.random((2, 4)) out = model.predict([x1, x2]) self.assertEqual(out.shape, (2, 1)) expected = np.zeros((2, 1)) expected[0, 0] = np.dot(x1[0], x2[0]) expected[1, 0] = np.dot(x1[1], x2[1]) self.assertAllClose(out, expected, atol=1e-4) # Test with negative tuple of axes. o = keras.layers.dot([i1, i2], axes=(-1, -1)) self.assertListEqual(o.shape.as_list(), [None, 1]) model = keras.models.Model([i1, i2], o) model.run_eagerly = testing_utils.should_run_eagerly() out = model.predict([x1, x2]) self.assertEqual(out.shape, (2, 1)) self.assertAllClose(out, expected, atol=1e-4) # test compute_output_shape layer = keras.layers.Dot(axes=-1) self.assertEqual(layer.compute_output_shape([(4, 5), (4, 5)]), (4, 1))
def test_sequential_pop(self): num_hidden = 5 input_dim = 3 batch_size = 5 num_classes = 2 model = testing_utils.get_small_sequential_mlp( num_hidden, num_classes, input_dim) model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3), run_eagerly=testing_utils.should_run_eagerly()) x = np.random.random((batch_size, input_dim)) y = np.random.random((batch_size, num_classes)) model.fit(x, y, epochs=1) model.pop() self.assertEqual(len(model.layers), 1) self.assertEqual(model.output_shape, (None, num_hidden)) model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3), run_eagerly=testing_utils.should_run_eagerly()) y = np.random.random((batch_size, num_hidden)) model.fit(x, y, epochs=1) # Test popping single-layer model model = keras.models.Sequential() model.add(keras.layers.Dense(num_hidden, input_dim=input_dim)) model.pop() self.assertEqual(model.layers, []) self.assertEqual(model.outputs, None) # Invalid use case model = keras.models.Sequential() with self.assertRaises(TypeError): model.pop()
def test_nested_input_output_with_state(self): batch = 10 t = 5 i1, i2, i3 = 3, 4, 5 o1, o2, o3 = 2, 3, 4 cell = NestedCell(o1, o2, o3) rnn = keras.layers.RNN(cell, return_sequences=True, return_state=True) input_1 = keras.Input((t, i1)) input_2 = keras.Input((t, i2, i3)) output1, output2, s1, s2 = rnn((input_1, input_2)) self.assertEqual(output1.shape.as_list(), [None, t, o1]) self.assertEqual(output2.shape.as_list(), [None, t, o2, o3]) self.assertEqual(s1.shape.as_list(), [None, o1]) self.assertEqual(s2.shape.as_list(), [None, o2, o3]) model = keras.models.Model([input_1, input_2], [output1, output2]) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( [np.zeros((batch, t, i1)), np.zeros((batch, t, i2, i3))], [np.zeros((batch, t, o1)), np.zeros((batch, t, o2, o3))]) self.assertEqual(model.output_shape, [(None, t, o1), (None, t, o2, o3)]) cell = NestedCell(o1, o2, o3, use_tuple=True) rnn = keras.layers.RNN(cell, return_sequences=True, return_state=True) input_1 = keras.Input((t, i1)) input_2 = keras.Input((t, i2, i3)) output1, output2, s1, s2 = rnn(NestedInput(t1=input_1, t2=input_2)) self.assertEqual(output1.shape.as_list(), [None, t, o1]) self.assertEqual(output2.shape.as_list(), [None, t, o2, o3]) self.assertEqual(s1.shape.as_list(), [None, o1]) self.assertEqual(s2.shape.as_list(), [None, o2, o3]) model = keras.models.Model([input_1, input_2], [output1, output2]) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( [np.zeros((batch, t, i1)), np.zeros((batch, t, i2, i3))], [np.zeros((batch, t, o1)), np.zeros((batch, t, o2, o3))]) self.assertEqual(model.output_shape, [(None, t, o1), (None, t, o2, o3)])
def test_clone_functional_model(self, share_weights): if share_weights: clone_fn = functools.partial( keras.models._clone_functional_model, share_weights=True) else: clone_fn = keras.models.clone_model val_a = np.random.random((10, 4)) val_b = np.random.random((10, 4)) val_out = np.random.random((10, 4)) input_a = keras.Input(shape=(4,)) input_b = keras.Input(shape=(4,)) dense_1 = keras.layers.Dense(4,) dense_2 = keras.layers.Dense(4,) x_a = dense_1(input_a) x_a = keras.layers.Dropout(0.5)(x_a) x_a = keras.layers.BatchNormalization()(x_a) x_b = dense_1(input_b) x_a = dense_2(x_a) outputs = keras.layers.add([x_a, x_b]) model = keras.models.Model([input_a, input_b], outputs) # With placeholder creation new_model = clone_fn(model) self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2) new_model.compile( testing_utils.get_v2_optimizer('rmsprop'), 'mse', run_eagerly=testing_utils.should_run_eagerly()) new_model.train_on_batch([val_a, val_b], val_out) # On top of new tensors input_a = keras.Input(shape=(4,), name='a') input_b = keras.Input(shape=(4,), name='b') new_model = keras.models.clone_model( model, input_tensors=[input_a, input_b]) self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2) new_model.compile( testing_utils.get_v2_optimizer('rmsprop'), 'mse', run_eagerly=testing_utils.should_run_eagerly()) new_model.train_on_batch([val_a, val_b], val_out) # On top of new, non-Keras tensors if not context.executing_eagerly(): # TODO(b/121277734):Skip Eager contexts, as Input() layers raise an error # saying they should not be used with EagerTensors input_a = keras.backend.variable(val_a) input_b = keras.backend.variable(val_b) new_model = clone_fn(model, input_tensors=[input_a, input_b]) self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2) new_model.compile( testing_utils.get_v2_optimizer('rmsprop'), 'mse', run_eagerly=testing_utils.should_run_eagerly()) new_model.train_on_batch(None, val_out)
def test_minimal_rnn_cell_abstract_rnn_cell(self): class MinimalRNNCell(keras.layers.AbstractRNNCell): def __init__(self, units, **kwargs): self.units = units super(MinimalRNNCell, self).__init__(**kwargs) @property def state_size(self): return self.units def build(self, input_shape): self.kernel = self.add_weight(shape=(input_shape[-1], self.units), initializer='uniform', name='kernel') self.recurrent_kernel = self.add_weight( shape=(self.units, self.units), initializer='uniform', name='recurrent_kernel') self.built = True def call(self, inputs, states): prev_output = states[0] h = keras.backend.dot(inputs, self.kernel) output = h + keras.backend.dot(prev_output, self.recurrent_kernel) return output, output @property def output_size(self): return self.units cell = MinimalRNNCell(32) x = keras.Input((None, 5)) layer = keras.layers.RNN(cell) y = layer(x) model = keras.models.Model(x, y) model.compile( optimizer="rmsprop", loss="mse", run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32))) # Test stacking. cells = [MinimalRNNCell(8), MinimalRNNCell(16), MinimalRNNCell(32)] layer = keras.layers.RNN(cells) y = layer(x) model = keras.models.Model(x, y) model.compile(optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
def test_nested_input_output(self): batch = 10 t = 5 i1, i2, i3 = 3, 4, 5 o1, o2, o3 = 2, 3, 4 cell = NestedCell(o1, o2, o3) rnn = keras.layers.RNN(cell) input_1 = keras.Input((t, i1)) input_2 = keras.Input((t, i2, i3)) outputs = rnn((input_1, input_2)) self.assertEqual(len(outputs), 2) self.assertEqual(outputs[0].shape.as_list(), [None, o1]) self.assertEqual(outputs[1].shape.as_list(), [None, o2, o3]) model = keras.models.Model((input_1, input_2), outputs) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( [np.zeros((batch, t, i1)), np.zeros((batch, t, i2, i3))], [np.zeros((batch, o1)), np.zeros((batch, o2, o3))]) self.assertEqual(model.output_shape, [(None, o1), (None, o2, o3)]) cell = NestedCell(o1, o2, o3, use_tuple=True) rnn = keras.layers.RNN(cell) input_1 = keras.Input((t, i1)) input_2 = keras.Input((t, i2, i3)) outputs = rnn(NestedInput(t1=input_1, t2=input_2)) self.assertEqual(len(outputs), 2) self.assertEqual(outputs[0].shape.as_list(), [None, o1]) self.assertEqual(outputs[1].shape.as_list(), [None, o2, o3]) model = keras.models.Model([input_1, input_2], outputs) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( [np.zeros((batch, t, i1)), np.zeros((batch, t, i2, i3))], [np.zeros((batch, o1)), np.zeros((batch, o2, o3))]) self.assertEqual(model.output_shape, [(None, o1), (None, o2, o3)])
def test_builtin_rnn_cell_serialization(self): for cell_class in [keras.layers.SimpleRNNCell, keras.layers.GRUCell, keras.layers.LSTMCell]: # Test basic case. x = keras.Input((None, 5)) cell = cell_class(32) layer = keras.layers.RNN(cell) y = layer(x) model = keras.models.Model(x, y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) # Test basic case serialization. x_np = np.random.random((6, 5, 5)) y_np = model.predict(x_np) weights = model.get_weights() config = layer.get_config() layer = keras.layers.RNN.from_config(config) y = layer(x) model = keras.models.Model(x, y) model.set_weights(weights) y_np_2 = model.predict(x_np) self.assertAllClose(y_np, y_np_2, atol=1e-4) # Test stacking. cells = [cell_class(8), cell_class(12), cell_class(32)] layer = keras.layers.RNN(cells) y = layer(x) model = keras.models.Model(x, y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) # Test stacked RNN serialization. x_np = np.random.random((6, 5, 5)) y_np = model.predict(x_np) weights = model.get_weights() config = layer.get_config() layer = keras.layers.RNN.from_config(config) y = layer(x) model = keras.models.Model(x, y) model.set_weights(weights) y_np_2 = model.predict(x_np) self.assertAllClose(y_np, y_np_2, atol=1e-4)
def test_high_dimension_RNN(self): # Basic test case. unit_a = 10 unit_b = 20 input_a = 5 input_b = 10 batch = 32 time_step = 4 cell = Minimal2DRNNCell(unit_a, unit_b) x = keras.Input((None, input_a, input_b)) layer = keras.layers.RNN(cell) y = layer(x) self.assertEqual(cell.state_size.as_list(), [unit_a, unit_b]) if not context.executing_eagerly(): init_state = layer.get_initial_state(x) self.assertEqual(len(init_state), 1) self.assertEqual(init_state[0].get_shape().as_list(), [None, unit_a, unit_b]) model = keras.models.Model(x, y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( np.zeros((batch, time_step, input_a, input_b)), np.zeros((batch, unit_a, unit_b))) self.assertEqual(model.output_shape, (None, unit_a, unit_b)) # Test stacking. cells = [ Minimal2DRNNCell(unit_a, unit_b), Minimal2DRNNCell(unit_a * 2, unit_b * 2), Minimal2DRNNCell(unit_a * 4, unit_b * 4) ] layer = keras.layers.RNN(cells) y = layer(x) model = keras.models.Model(x, y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( np.zeros((batch, time_step, input_a, input_b)), np.zeros((batch, unit_a * 4, unit_b * 4))) self.assertEqual(model.output_shape, (None, unit_a * 4, unit_b * 4))
def test_support_for_manual_training_arg(self): # In most cases, the `training` argument is left unspecified, in which # case it defaults to value corresponding to the Model method being used # (fit -> True, predict -> False, etc). # If the user writes their model `call` method to take # an explicit `training` argument, we must check that the correct value # is being passed to the model for each method call. class DPNet(keras.Model): def __init__(self): super(DPNet, self).__init__() self.dp = keras.layers.Dropout(0.5) self.dense = keras.layers.Dense(1, use_bias=False, kernel_initializer='ones') def call(self, inputs, training=False): x = self.dp(inputs, training=training) return self.dense(x) model = DPNet() x = np.ones((10, 10)) y = model.predict(x) self.assertEqual(np.sum(y), np.sum(x)) model.compile( loss='mse', optimizer='rmsprop', run_eagerly=testing_utils.should_run_eagerly()) loss = model.train_on_batch(x, y) self.assertGreater(loss, 0.1)
def test_inference_methods(self): # test predict, evaluate, test_on_batch, predict_on_batch # on different input types: list, dict num_classes = (2, 3) num_samples = 100 input_dim = 50 x1 = np.ones((num_samples, input_dim)) x2 = np.ones((num_samples, input_dim)) y1 = np.zeros((num_samples, num_classes[0])) y2 = np.zeros((num_samples, num_classes[1])) model = MultiIOTestModel(num_classes=num_classes, use_bn=True) model.compile( loss='mse', optimizer='rmsprop', run_eagerly=testing_utils.should_run_eagerly()) model.evaluate([x1, x2], [y1, y2]) model.test_on_batch([x1, x2], [y1, y2]) model = MultiIOTestModel(num_classes=num_classes, use_bn=True) model.predict([x1, x2]) model = MultiIOTestModel(num_classes=num_classes, use_bn=True) model.predict_on_batch([x1, x2])
def test_merge_add(self): i1 = keras.layers.Input(shape=(4, 5)) i2 = keras.layers.Input(shape=(4, 5)) i3 = keras.layers.Input(shape=(4, 5)) add_layer = keras.layers.Add() o = add_layer([i1, i2, i3]) self.assertListEqual(o.shape.as_list(), [None, 4, 5]) model = keras.models.Model([i1, i2, i3], o) model.run_eagerly = testing_utils.should_run_eagerly() x1 = np.random.random((2, 4, 5)) x2 = np.random.random((2, 4, 5)) x3 = np.random.random((2, 4, 5)) out = model.predict([x1, x2, x3]) self.assertEqual(out.shape, (2, 4, 5)) self.assertAllClose(out, x1 + x2 + x3, atol=1e-4) self.assertEqual( add_layer.compute_mask([i1, i2, i3], [None, None, None]), None) self.assertTrue( np.all( K.eval( add_layer.compute_mask( [i1, i2], [K.variable(x1), K.variable(x2)])))) with self.assertRaisesRegexp(ValueError, "`mask` should be a list."): add_layer.compute_mask([i1, i2, i3], x1) with self.assertRaisesRegexp(ValueError, "`inputs` should be a list."): add_layer.compute_mask(i1, [None, None, None]) with self.assertRaisesRegexp(ValueError, " should have the same length."): add_layer.compute_mask([i1, i2, i3], [None, None])
def test_subclass_nested_in_sequential(self): num_classes = 2 num_samples = 100 input_dim = 50 class Inner(keras.Model): def __init__(self): super(Inner, self).__init__() self.dense1 = keras.layers.Dense(32, activation='relu') self.dense2 = keras.layers.Dense(num_classes, activation='relu') self.bn = keras.layers.BatchNormalization() def call(self, inputs): x = self.dense1(inputs) x = self.dense2(x) return self.bn(x) model = keras.Sequential([Inner()]) model.compile( loss='mse', optimizer='rmsprop', metrics=['acc'], run_eagerly=testing_utils.should_run_eagerly()) x = np.ones((num_samples, input_dim)) y = np.zeros((num_samples, num_classes)) model.fit(x, y, epochs=2, batch_size=32, verbose=0) _ = model.evaluate(x, y, verbose=0) self.assertEqual(len(model.weights), 8) self.assertEqual(len(model.non_trainable_weights), 2) self.assertEqual(len(model.trainable_weights), 6)
def test_training_and_inference_behavior(self): # test that dropout is applied in training and not inference num_samples = 100 input_dim = 50 class DPNet(keras.Model): def __init__(self): super(DPNet, self).__init__() self.dp = keras.layers.Dropout(0.5) self.dense = keras.layers.Dense(1, use_bias=False, kernel_initializer='ones') def call(self, inputs): x = self.dp(inputs) return self.dense(x) model = DPNet() x = np.ones((num_samples, input_dim)) y = model.predict(x) self.assertEqual(np.sum(y), np.sum(x)) model.compile( loss='mse', optimizer='rmsprop', run_eagerly=testing_utils.should_run_eagerly()) loss = model.train_on_batch(x, y) self.assertGreater(loss, 0.1)
def test_merge_concatenate(self): i1 = keras.layers.Input(shape=(4, 5)) i2 = keras.layers.Input(shape=(4, 5)) concat_layer = keras.layers.Concatenate(axis=1) o = concat_layer([i1, i2]) self.assertListEqual(o.shape.as_list(), [None, 8, 5]) model = keras.models.Model([i1, i2], o) model.run_eagerly = testing_utils.should_run_eagerly() x1 = np.random.random((2, 4, 5)) x2 = np.random.random((2, 4, 5)) out = model.predict([x1, x2]) self.assertEqual(out.shape, (2, 8, 5)) self.assertAllClose(out, np.concatenate([x1, x2], axis=1), atol=1e-4) self.assertEqual(concat_layer.compute_mask([i1, i2], [None, None]), None) self.assertTrue( np.all( K.eval( concat_layer.compute_mask( [i1, i2], [K.variable(x1), K.variable(x2)])))) with self.assertRaisesRegexp(ValueError, "`mask` should be a list."): concat_layer.compute_mask([i1, i2], x1) with self.assertRaisesRegexp(ValueError, "`inputs` should be a list."): concat_layer.compute_mask(i1, [None, None]) with self.assertRaisesRegexp(ValueError, "should have the same length"): concat_layer.compute_mask([i1, i2], [None]) with self.assertRaisesRegexp(ValueError, "layer should be called on a list of inputs"): concat_layer(i1)
def test_layer_sharing_at_heterogenous_depth_with_concat(self): input_shape = (16, 9, 3) input_layer = input_layer_lib.Input(shape=input_shape) a = keras.layers.Dense(3, name='dense_A') b = keras.layers.Dense(3, name='dense_B') c = keras.layers.Dense(3, name='dense_C') x1 = b(a(input_layer)) x2 = a(c(input_layer)) output = keras.layers.concatenate([x1, x2]) m = keras.models.Model(inputs=input_layer, outputs=output) m.run_eagerly = testing_utils.should_run_eagerly() x_val = np.random.random((10, 16, 9, 3)) output_val = m.predict(x_val) config = m.get_config() weights = m.get_weights() m2 = keras.models.Model.from_config(config) m2.set_weights(weights) output_val_2 = m2.predict(x_val) self.assertAllClose(output_val, output_val_2, atol=1e-6)
def test_updates(self): # test that updates get run during training num_samples = 100 input_dim = 50 class BNNet(keras.Model): def __init__(self): super(BNNet, self).__init__() self.bn = keras.layers.BatchNormalization(beta_initializer='ones', gamma_initializer='ones') def call(self, inputs): return self.bn(inputs) x = np.ones((num_samples, input_dim)) y = np.ones((num_samples, input_dim)) model = BNNet() model.compile( loss='mse', optimizer='rmsprop', run_eagerly=testing_utils.should_run_eagerly()) y_ref = model.predict(x) model.train_on_batch(x, y) y_new = model.predict(x) self.assertGreater(np.sum(np.abs(y_ref - y_new)), 0.1)
def test_attributes(self): # layers, weights, trainable_weights, non_trainable_weights, inputs, outputs num_classes = (2, 3) num_samples = 100 input_dim = 50 model = MultiIOTestModel(num_classes=num_classes, use_bn=True) x1 = np.ones((num_samples, input_dim)) x2 = np.ones((num_samples, input_dim)) y1 = np.zeros((num_samples, num_classes[0])) y2 = np.zeros((num_samples, num_classes[1])) self.assertEqual(model.name, 'test_model') self.assertEqual(model.built, False) self.assertEqual(len(model.weights), 0) model.compile( loss='mse', optimizer='rmsprop', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch([x1, x2], [y1, y2]) self.assertEqual(model.built, True) self.assertEqual(len(model.layers), 4) self.assertEqual(len(model.weights), 10) self.assertEqual(len(model.trainable_weights), 8) self.assertEqual(len(model.non_trainable_weights), 2) self.assertEqual(len(model.inputs), 2) self.assertEqual(len(model.outputs), 2)
def test_sequential_deferred_build_serialization(self): num_hidden = 5 input_dim = 3 batch_size = 5 num_classes = 2 model = testing_utils.get_small_sequential_mlp(num_hidden, num_classes) model.compile( loss='mse', optimizer='rmsprop', metrics=[keras.metrics.CategoricalAccuracy()], run_eagerly=testing_utils.should_run_eagerly()) self.assertFalse(model.built) x = np.random.random((batch_size, input_dim)) y = np.random.random((batch_size, num_classes)) model.train_on_batch(x, y) self.assertTrue(model.built) config = model.get_config() self.assertIn('build_input_shape', config) new_model = keras.models.Sequential.from_config(config) self.assertEqual(len(new_model.layers), 2) self.assertEqual(len(new_model.weights), 4)
def test_serialization_v2_model(self): np.random.seed(1337) (x_train, y_train), _ = testing_utils.get_test_data( train_samples=100, test_samples=0, input_shape=(10,), num_classes=2) y_train = keras.utils.to_categorical(y_train) model = keras.Sequential([ keras.layers.Flatten(input_shape=x_train.shape[1:]), keras.layers.Dense(10, activation=nn.relu), # To mimic 'tf.nn.softmax' used in TF 2.x. keras.layers.Dense(y_train.shape[-1], activation=nn.softmax_v2), ]) # Check if 'softmax' is in model.get_config(). last_layer_activation = model.get_layer(index=2).get_config()['activation'] self.assertEqual(last_layer_activation, 'softmax') model.compile(loss='categorical_crossentropy', optimizer=keras.optimizer_v2.adam.Adam(0.005), metrics=['accuracy'], run_eagerly=testing_utils.should_run_eagerly()) model.fit(x_train, y_train, epochs=2, batch_size=10, validation_data=(x_train, y_train), verbose=2) output_path = os.path.join(self.get_temp_dir(), 'tf_keras_saved_model') keras.saving.saved_model.export_saved_model(model, output_path) loaded_model = keras.saving.saved_model.load_from_saved_model(output_path) self.assertEqual(model.summary(), loaded_model.summary())
def test_finite_dataset_unknown_cardinality_out_of_data(self): model = testing_utils.get_small_mlp(1, 4, input_dim=3) model.compile('rmsprop', 'mse', run_eagerly=testing_utils.should_run_eagerly()) inputs = np.zeros((100, 3), dtype=np.float32) targets = np.random.randint(0, 4, size=100, dtype=np.int32) dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) dataset = dataset.filter(lambda x, y: True).batch(10) self.assertEqual( keras.backend.get_value(cardinality.cardinality(dataset)), cardinality.UNKNOWN) batch_counter = BatchCounterCallback() with test.mock.patch.object(logging, 'warning') as mock_log: # steps_per_epoch (200) is greater than the dataset size (100). As this is # unexpected, training will stop and not make it to the second epoch. history = model.fit( dataset, epochs=2, verbose=1, callbacks=[batch_counter], steps_per_epoch=200) self.assertIn( 'Your dataset ran out of data; interrupting training. ' 'Make sure that your dataset can generate at least ' '`steps_per_epoch * epochs` batches (in this case, 400 batches). ' 'You may need to use the repeat() function when ' 'building your dataset.', str(mock_log.call_args)) self.assertLen(history.history['loss'], 1) self.assertEqual(batch_counter.batch_count, 10) model.evaluate(dataset) out = model.predict(dataset) self.assertEqual(out.shape[0], 100)
def test_metrics_correctness_with_iterator(self): layers = [ keras.layers.Dense(8, activation='relu', input_dim=4, kernel_initializer='ones'), keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones') ] model = testing_utils.get_model_from_layers(layers, (4,)) model.compile( loss='binary_crossentropy', metrics=['accuracy', metrics_module.BinaryAccuracy()], optimizer='rmsprop', run_eagerly=testing_utils.should_run_eagerly()) np.random.seed(123) x = np.random.randint(10, size=(100, 4)).astype(np.float32) y = np.random.randint(2, size=(100, 1)).astype(np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.batch(10) iterator = dataset_ops.make_one_shot_iterator(dataset) outs = model.evaluate(iterator, steps=10) self.assertEqual(np.around(outs[1], decimals=1), 0.5) self.assertEqual(np.around(outs[2], decimals=1), 0.5) y = np.zeros((100, 1), dtype=np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) dataset = dataset.batch(10) iterator = dataset_ops.make_one_shot_iterator(dataset) outs = model.evaluate(iterator, steps=10) self.assertEqual(outs[1], 0.) self.assertEqual(outs[2], 0.)
def test_dataset_fit_correctness(self): class SumLayer(keras.layers.Layer): def build(self, _): self.w = self.add_weight('w', ()) def call(self, inputs): return keras.backend.sum(inputs) + self.w * 0 model = keras.Sequential([SumLayer(input_shape=(2,))]) model.compile( 'rmsprop', loss='mae', run_eagerly=testing_utils.should_run_eagerly()) inputs = np.zeros((40, 2), dtype=np.float32) inputs[10:20, :] = 2 inputs[20:30, :] = 1 inputs[30:, :] = 4 targets = np.zeros((40, 1), dtype=np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) dataset = dataset.batch(10) history = model.fit(dataset, epochs=2, steps_per_epoch=2, verbose=1, shuffle=False) self.assertListEqual(history.history['loss'], [inputs[:20].sum() / 2, inputs[20:].sum() / 2])
def test_sequential_as_downstream_of_masking_layer(self): inputs = keras.layers.Input(shape=(3, 4)) x = keras.layers.Masking(mask_value=0., input_shape=(3, 4))(inputs) s = keras.Sequential() s.add(keras.layers.Dense(5, input_shape=(4,))) x = keras.layers.wrappers.TimeDistributed(s)(x) model = keras.Model(inputs=inputs, outputs=x) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model_input = np.random.randint( low=1, high=5, size=(10, 3, 4)).astype('float32') for i in range(4): model_input[i, i:, :] = 0. model.fit(model_input, np.random.random((10, 3, 5)), epochs=1, batch_size=6) if not context.executing_eagerly(): # Note: this doesn't work in eager due to DeferredTensor/ops compatibility # issue. mask_outputs = [model.layers[1].compute_mask(model.layers[1].input)] mask_outputs += [model.layers[2].compute_mask( model.layers[2].input, mask_outputs[-1])] func = keras.backend.function([model.input], mask_outputs) mask_outputs_val = func([model_input]) self.assertAllClose(mask_outputs_val[0], np.any(model_input, axis=-1)) self.assertAllClose(mask_outputs_val[1], np.any(model_input, axis=-1))
def test_merge_subtract(self): i1 = keras.layers.Input(shape=(4, 5)) i2 = keras.layers.Input(shape=(4, 5)) i3 = keras.layers.Input(shape=(4, 5)) subtract_layer = keras.layers.Subtract() o = subtract_layer([i1, i2]) self.assertListEqual(o.shape.as_list(), [None, 4, 5]) model = keras.models.Model([i1, i2], o) model.run_eagerly = testing_utils.should_run_eagerly() x1 = np.random.random((2, 4, 5)) x2 = np.random.random((2, 4, 5)) out = model.predict([x1, x2]) self.assertEqual(out.shape, (2, 4, 5)) self.assertAllClose(out, x1 - x2, atol=1e-4) self.assertEqual(subtract_layer.compute_mask([i1, i2], [None, None]), None) self.assertTrue( np.all( K.eval( subtract_layer.compute_mask( [i1, i2], [K.variable(x1), K.variable(x2)])))) with self.assertRaisesRegexp(ValueError, "`mask` should be a list."): subtract_layer.compute_mask([i1, i2], x1) with self.assertRaisesRegexp(ValueError, "`inputs` should be a list."): subtract_layer.compute_mask(i1, [None, None]) with self.assertRaisesRegexp(ValueError, "layer should be called on exactly 2 inputs"): subtract_layer([i1, i2, i3]) with self.assertRaisesRegexp(ValueError, "layer should be called on exactly 2 inputs"): subtract_layer([i1])
def test_zero_output_for_masking(self): for unroll in [True, False]: cell = keras.layers.SimpleRNNCell(5) x = keras.Input((5, 5)) mask = keras.layers.Masking() layer = keras.layers.RNN( cell, return_sequences=True, zero_output_for_mask=True, unroll=unroll) masked_input = mask(x) y = layer(masked_input) model = keras.models.Model(x, y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) np_x = np.ones((6, 5, 5)) result_1 = model.predict(np_x) # set the time 4 and 5 for last record to be zero (masked). np_x[5, 3:] = 0 result_2 = model.predict(np_x) # expect the result_2 has same output, except the time 4,5 for last # record. result_1[5, 3:] = 0 self.assertAllClose(result_1, result_2)
def test_vector_classification(self): np.random.seed(1337) (x_train, y_train), _ = testing_utils.get_test_data( train_samples=100, test_samples=0, input_shape=(10,), num_classes=2) y_train = keras.utils.to_categorical(y_train) model = testing_utils.get_model_from_layers( [keras.layers.Dense(16, activation='relu'), keras.layers.Dropout(0.1), keras.layers.Dense(y_train.shape[-1], activation='softmax')], input_shape=x_train.shape[1:]) model.compile( loss='categorical_crossentropy', optimizer=keras.optimizer_v2.adam.Adam(0.005), metrics=['acc'], run_eagerly=testing_utils.should_run_eagerly()) history = model.fit(x_train, y_train, epochs=10, batch_size=10, validation_data=(x_train, y_train), verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) _, val_acc = model.evaluate(x_train, y_train) self.assertAlmostEqual(history.history['val_acc'][-1], val_acc) predictions = model.predict(x_train) self.assertEqual(predictions.shape, (x_train.shape[0], 2))
def test_inconsistent_output_state_size(self): batch = 32 time_step = 4 state_size = 5 input_size = 6 cell = PlusOneRNNCell(state_size) x = keras.Input((None, input_size)) layer = keras.layers.RNN(cell) y = layer(x) self.assertEqual(cell.state_size, state_size) if not context.executing_eagerly(): init_state = layer.get_initial_state(x) self.assertEqual(len(init_state), 1) self.assertEqual(init_state[0].get_shape().as_list(), [None, state_size]) model = keras.models.Model(x, y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( np.zeros((batch, time_step, input_size)), np.zeros((batch, input_size))) self.assertEqual(model.output_shape, (None, input_size))
def test_timeseries_classification_sequential_tf_rnn(self): np.random.seed(1337) (x_train, y_train), _ = testing_utils.get_test_data( train_samples=100, test_samples=0, input_shape=(4, 10), num_classes=2) y_train = keras.utils.to_categorical(y_train) model = keras.models.Sequential() model.add(keras.layers.RNN(rnn_cell.LSTMCell(5), return_sequences=True, input_shape=x_train.shape[1:])) model.add(keras.layers.RNN(rnn_cell.GRUCell(y_train.shape[-1], activation='softmax', dtype=dtypes.float32))) model.compile( loss='categorical_crossentropy', optimizer=keras.optimizer_v2.adam.Adam(0.005), metrics=['acc'], run_eagerly=testing_utils.should_run_eagerly()) history = model.fit(x_train, y_train, epochs=15, batch_size=10, validation_data=(x_train, y_train), verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) _, val_acc = model.evaluate(x_train, y_train) self.assertAlmostEqual(history.history['val_acc'][-1], val_acc) predictions = model.predict(x_train) self.assertEqual(predictions.shape, (x_train.shape[0], 2))
def test_nest_input_output_with_init_state(self): batch = 10 t = 5 i1, i2, i3 = 3, 4, 5 o1, o2, o3 = 2, 3, 4 cell = NestedCell(o1, o2, o3) rnn = keras.layers.RNN(cell, return_sequences=True, return_state=True) input_1 = keras.Input((t, i1)) input_2 = keras.Input((t, i2, i3)) init_s1 = keras.Input((o1,)) init_s2 = keras.Input((o2, o3)) output1, output2, s1, s2 = rnn((input_1, input_2), initial_state=(init_s1, init_s2)) self.assertEqual(output1.shape.as_list(), [None, t, o1]) self.assertEqual(output2.shape.as_list(), [None, t, o2, o3]) self.assertEqual(s1.shape.as_list(), [None, o1]) self.assertEqual(s2.shape.as_list(), [None, o2, o3]) model = keras.models.Model([input_1, input_2, init_s1, init_s2], [output1, output2]) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( [np.zeros((batch, t, i1)), np.zeros((batch, t, i2, i3)), np.zeros((batch, o1)), np.zeros((batch, o2, o3))], [np.zeros((batch, t, o1)), np.zeros((batch, t, o2, o3))]) self.assertEqual(model.output_shape, [(None, t, o1), (None, t, o2, o3)]) cell = NestedCell(o1, o2, o3, use_tuple=True) rnn = keras.layers.RNN(cell, return_sequences=True, return_state=True) input_1 = keras.Input((t, i1)) input_2 = keras.Input((t, i2, i3)) init_s1 = keras.Input((o1,)) init_s2 = keras.Input((o2, o3)) init_state = NestedState(s1=init_s1, s2=init_s2) output1, output2, s1, s2 = rnn(NestedInput(t1=input_1, t2=input_2), initial_state=init_state) self.assertEqual(output1.shape.as_list(), [None, t, o1]) self.assertEqual(output2.shape.as_list(), [None, t, o2, o3]) self.assertEqual(s1.shape.as_list(), [None, o1]) self.assertEqual(s2.shape.as_list(), [None, o2, o3]) model = keras.models.Model([input_1, input_2, init_s1, init_s2], [output1, output2]) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( [np.zeros((batch, t, i1)), np.zeros((batch, t, i2, i3)), np.zeros((batch, o1)), np.zeros((batch, o2, o3))], [np.zeros((batch, t, o1)), np.zeros((batch, t, o2, o3))]) self.assertEqual(model.output_shape, [(None, t, o1), (None, t, o2, o3)])
def test_minimal_rnn_cell_layer(self): class MinimalRNNCell(keras.layers.Layer): def __init__(self, units, **kwargs): self.units = units self.state_size = units super(MinimalRNNCell, self).__init__(**kwargs) def build(self, input_shape): self.kernel = self.add_weight(shape=(input_shape[-1], self.units), initializer='uniform', name='kernel') self.recurrent_kernel = self.add_weight( shape=(self.units, self.units), initializer='uniform', name='recurrent_kernel') self.built = True def call(self, inputs, states): prev_output = states[0] h = keras.backend.dot(inputs, self.kernel) output = h + keras.backend.dot(prev_output, self.recurrent_kernel) return output, [output] def get_config(self): config = {'units': self.units} base_config = super(MinimalRNNCell, self).get_config() return dict(list(base_config.items()) + list(config.items())) # Test basic case. x = keras.Input((None, 5)) cell = MinimalRNNCell(32) layer = keras.layers.RNN(cell) y = layer(x) model = keras.models.Model(x, y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32))) # Test basic case serialization. x_np = np.random.random((6, 5, 5)) y_np = model.predict(x_np) weights = model.get_weights() config = layer.get_config() with keras.utils.CustomObjectScope({'MinimalRNNCell': MinimalRNNCell}): layer = keras.layers.RNN.from_config(config) y = layer(x) model = keras.models.Model(x, y) model.set_weights(weights) y_np_2 = model.predict(x_np) self.assertAllClose(y_np, y_np_2, atol=1e-4) # Test stacking. cells = [MinimalRNNCell(8), MinimalRNNCell(12), MinimalRNNCell(32)] layer = keras.layers.RNN(cells) y = layer(x) model = keras.models.Model(x, y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32))) # Test stacked RNN serialization. x_np = np.random.random((6, 5, 5)) y_np = model.predict(x_np) weights = model.get_weights() config = layer.get_config() with keras.utils.CustomObjectScope({'MinimalRNNCell': MinimalRNNCell}): layer = keras.layers.RNN.from_config(config) y = layer(x) model = keras.models.Model(x, y) model.set_weights(weights) y_np_2 = model.predict(x_np) self.assertAllClose(y_np, y_np_2, atol=1e-4)
def __init__(self): super(TestModel, self).__init__( name='test_model', dynamic=testing_utils.should_run_eagerly()) self.layer = keras.layers.Dense(2, activity_regularizer='l2')
def test_dynamic_loss_scaling(self, strategy_fn, pass_loss_scale_to_policy=False, get_config=False): strategy = strategy_fn() initial_loss_scale = 2. batch_size = 4 loss_scale = loss_scale_module.DynamicLossScale( initial_loss_scale=initial_loss_scale, increment_period=2) expected_gradient = backend.variable([initial_loss_scale / batch_size], dtype=dtypes.float16) # If this variable is set to True, the model below will have NaN gradients have_nan_gradients = backend.variable(False, dtype=dtypes.bool) with strategy.scope(): opt = gradient_descent.SGD(1.) if pass_loss_scale_to_policy: p = policy.Policy('mixed_float16', loss_scale=loss_scale) else: p = policy.Policy('mixed_float16', loss_scale=None) opt = loss_scale_optimizer.LossScaleOptimizer(opt, loss_scale) with policy.policy_scope(p): x = layers.Input( shape=(1,), batch_size=batch_size, dtype=dtypes.float16) layer = mp_test_util.MultiplyLayer(assert_type=dtypes.float16) y = layer(x) identity_with_nan_grads = ( mp_test_util.create_identity_with_nan_gradients_fn( have_nan_gradients)) y = core.Lambda(identity_with_nan_grads)(y) identity_with_grad_check_fn = ( mp_test_util.create_identity_with_grad_check_fn( expected_dtype=dtypes.float16, expected_gradient=expected_gradient)) y = core.Lambda(identity_with_grad_check_fn)(y) model = models.Model(inputs=x, outputs=y) if get_config: config = model.get_config() model = model.__class__.from_config( config, custom_objects={'MultiplyLayer': mp_test_util.MultiplyLayer}) (layer,) = (layer for layer in model.layers if isinstance(layer, mp_test_util.MultiplyLayer)) def loss_fn(y_true, y_pred): del y_true return math_ops.reduce_mean(y_pred) model.compile( opt, loss=loss_fn, run_eagerly=testing_utils.should_run_eagerly()) self.assertEqual(backend.eval(layer.v), 1) x = np.ones((batch_size, 1)) y = np.ones((batch_size, 1)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(batch_size) model.fit(dataset) # The variables starts with 1 and has a gradient of 1, so will go down by 1 # each step. self.assertEqual(backend.eval(layer.v), 0) model.fit(dataset) self.assertEqual(backend.eval(layer.v), -1) # There have been two steps without NaNs, so the loss scale will double backend.set_value(expected_gradient, backend.get_value(expected_gradient * 2)) model.fit(dataset) self.assertEqual(backend.eval(layer.v), -2) # Next test with NaN gradients. backend.set_value(have_nan_gradients, True) model.fit(dataset) # Variable should not be updated self.assertEqual(backend.eval(layer.v), -2) # Test with finite gradients again backend.set_value(have_nan_gradients, False) # The loss scale will be halved due to the NaNs, so the gradient will also # be halved backend.set_value(expected_gradient, backend.get_value(expected_gradient / 2)) model.fit(dataset) self.assertEqual(backend.eval(layer.v), -3)
def test_dynamic_loss_scaling(self, strategy_fn, experimental_run_tf_function=True): if not self._is_strategy_supported(strategy_fn): return strategy = strategy_fn() initial_loss_scale = 2. batch_size = 4 expected_gradient = backend.variable([initial_loss_scale / batch_size], dtype=dtypes.float16) # If this variable is set to True, the model below will have NaN gradients have_nan_gradients = backend.variable(False, dtype=dtypes.bool) with strategy.scope(): with policy.policy_scope(policy.Policy('infer_float32_vars')): x = layers.Input( shape=(1,), batch_size=batch_size, dtype=dtypes.float16) layer = AddLayer(assert_type=dtypes.float16) y = layer(x) identity_with_nan_grads = ( mp_test_util.create_identity_with_nan_gradients_fn( have_nan_gradients)) y = core.Lambda(identity_with_nan_grads)(y) identity_with_grad_check_fn = ( mp_test_util.create_identity_with_grad_check_fn( expected_dtype=dtypes.float16, expected_gradient=expected_gradient)) y = core.Lambda(identity_with_grad_check_fn)(y) y = math_ops.cast(y, dtypes.float32) model = models.Model(inputs=x, outputs=y) def loss_fn(y_true, y_pred): del y_true return math_ops.reduce_mean(y_pred) opt = gradient_descent.SGD(1.) loss_scale = loss_scale_module.DynamicLossScale( initial_loss_scale=initial_loss_scale, increment_period=2) opt = loss_scale_optimizer.LossScaleOptimizer(opt, loss_scale) model.compile( opt, loss=loss_fn, run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils.should_run_tf_function()) self.assertEqual(backend.eval(layer.v), 1) x = np.ones((batch_size, 1)) y = np.ones((batch_size, 1)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(batch_size) model.fit(dataset) # The variables starts with 1 and has a gradient of 1, so will go down by 1 # each step. self.assertEqual(backend.eval(layer.v), 0) model.fit(dataset) self.assertEqual(backend.eval(layer.v), -1) # There have been two steps without NaNs, so the loss scale will double backend.set_value(expected_gradient, backend.get_value(expected_gradient * 2)) model.fit(dataset) self.assertEqual(backend.eval(layer.v), -2) # Next test with NaN gradients. backend.set_value(have_nan_gradients, True) model.fit(dataset) # Variable should not be updated self.assertEqual(backend.eval(layer.v), -2) # Test with finite gradients again backend.set_value(have_nan_gradients, False) # The loss scale will be halved due to the NaNs, so the gradient will also # be halved backend.set_value(expected_gradient, backend.get_value(expected_gradient / 2)) model.fit(dataset) self.assertEqual(backend.eval(layer.v), -3)
def test_training(self): self.model.compile(loss='sparse_categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly()) self.model.fit(self.tensor_input, self.tensor_target, batch_size=5)
def test_save_model_with_dynamic_loss_scaling( self, strategy_fn, h5=False, use_v1_loss_scale_optimizer=False): # TODO(reedwm): Support and test saving model with a mixed_[b]float16 policy # as well. strategy = strategy_fn() if (isinstance(strategy, mirrored_strategy.MirroredStrategy) and not context.executing_eagerly()): # TODO(b/121381184): Enable running the test in this case. return # Create and run model. with strategy.scope(): x = layers.Input(shape=(2, ), batch_size=2, dtype=dtypes.float32) y = mp_test_util.MultiplyLayer()(x) model = models.Model(inputs=x, outputs=y) opt = gradient_descent.SGD(1.) if use_v1_loss_scale_optimizer: loss_scale = loss_scale_module.DynamicLossScale( initial_loss_scale=1., increment_period=2.) opt = loss_scale_optimizer.LossScaleOptimizerV1( opt, loss_scale) else: opt = loss_scale_optimizer.LossScaleOptimizer( opt, initial_scale=1., dynamic_growth_steps=2.) model.compile(optimizer=opt, loss='mse', run_eagerly=testing_utils.should_run_eagerly()) # Run for 3 steps (6 examples with a batch size of 2) model.fit(np.ones((6, 2)), np.zeros((6, 2)), batch_size=2) self.assertEqual(backend.get_value(opt.loss_scale), 2) self.assertEqual(backend.get_value(opt.dynamic_counter), 1) (weight, ) = model.trainable_weights orig_weight = backend.get_value(weight) # Save model weights. save_path = os.path.join(self.get_temp_dir(), 'model') model.save(save_path, save_format='h5' if h5 else 'tf') # Run model again for 1 step (2 examples with a batch size of 2) model.fit(np.ones((2, 2)), np.zeros((2, 2)), batch_size=2) new_weight = backend.get_value(weight) self.assertNotEqual(new_weight, orig_weight) self.assertEqual(backend.get_value(opt.loss_scale), 4) self.assertEqual(backend.get_value(opt.dynamic_counter), 0) # Load model weights and ensure loss scale weights are restored. model = save.load_model( save_path, custom_objects={'MultiplyLayer': mp_test_util.MultiplyLayer}) (weight, ) = model.trainable_weights loaded_weight = backend.get_value(weight) self.assertEqual(loaded_weight, orig_weight) # Currently the loss scale isn't always saved when the model is saved with # Model.save(). So we assert the loss scale either has the value when it was # saved, or the value it was initialized with. # TODO(reedwm): Always save/restore the loss scale with Model.save(). self.assertIn(backend.get_value(model.optimizer.loss_scale), (1, 2)) self.assertIn(backend.get_value(model.optimizer.dynamic_counter), (0, 1)) # Test optimizer attributes and type self.assertEqual(model.optimizer.initial_scale, 1.) self.assertEqual(model.optimizer.dynamic_growth_steps, 2.) self.assertEqual(type(model.optimizer), loss_scale_optimizer.LossScaleOptimizer)
def test_training_and_eval_methods_on_dataset(self): model = testing_utils.get_small_mlp(1, 4, input_dim=3) optimizer = 'rmsprop' loss = 'mse' metrics = ['mae', metrics_module.CategoricalAccuracy()] model.compile(optimizer, loss, metrics=metrics, run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) inputs = np.zeros((10, 3), np.float32) targets = np.zeros((10, 4), np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) dataset = dataset.repeat() # Infinite dataset. dataset = dataset.batch(10) model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1) model.evaluate(dataset, steps=2, verbose=1) model.predict(dataset, steps=2) # Test with validation data model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, validation_data=dataset, validation_steps=2) # Test with validation split with self.assertRaisesRegexp( ValueError, '`validation_split` argument is not supported when '): model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, validation_split=0.5, validation_steps=2) # Test with sample weight. sample_weight = np.random.random((10, )) with self.assertRaisesRegexp( ValueError, '`sample_weight` argument is not supported ' 'when input `x` is a dataset or a dataset iterator'): model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0, sample_weight=sample_weight) # Test invalid usage with self.assertRaisesRegexp( ValueError, 'The `batch_size` argument must not be specified'): model.fit(dataset, batch_size=10, epochs=1, steps_per_epoch=2, verbose=0) with self.assertRaisesRegexp( ValueError, 'The `batch_size` argument must not be specified'): model.predict(dataset, batch_size=10, steps=2, verbose=0) with self.assertRaisesRegexp( ValueError, 'The `batch_size` argument must not be specified'): model.evaluate(dataset, batch_size=10, steps=2, verbose=0) with self.assertRaisesRegexp(ValueError, 'you should not specify a target'): model.fit(dataset, dataset, epochs=1, steps_per_epoch=2, verbose=0) # With an infinite dataset, `steps_per_epoch`/`steps` argument is required. with self.assertRaisesRegexp(ValueError, 'the `steps_per_epoch` argument'): model.fit(dataset, epochs=1, verbose=0) with self.assertRaisesRegexp(ValueError, 'the `steps` argument'): model.evaluate(dataset, verbose=0) with self.assertRaisesRegexp(ValueError, 'the `steps` argument'): model.predict(dataset, verbose=0)
def test_statefulness_SimpleRNN(self): num_samples = 2 timesteps = 3 embedding_dim = 4 units = 2 layer_class = keras.layers.SimpleRNN model = keras.models.Sequential() model.add( keras.layers.Embedding(4, embedding_dim, mask_zero=True, input_length=timesteps, batch_input_shape=(num_samples, timesteps))) layer = layer_class(units, return_sequences=False, stateful=True, weights=None) model.add(layer) model.compile( optimizer=gradient_descent.GradientDescentOptimizer(0.01), loss='mse', run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils.should_run_tf_function( )) out1 = model.predict(np.ones((num_samples, timesteps))) self.assertEqual(out1.shape, (num_samples, units)) # train once so that the states change model.train_on_batch(np.ones((num_samples, timesteps)), np.ones((num_samples, units))) out2 = model.predict(np.ones((num_samples, timesteps))) # if the state is not reset, output should be different self.assertNotEqual(out1.max(), out2.max()) # check that output changes after states are reset # (even though the model itself didn't change) layer.reset_states() out3 = model.predict(np.ones((num_samples, timesteps))) self.assertNotEqual(out2.max(), out3.max()) # check that container-level reset_states() works model.reset_states() out4 = model.predict(np.ones((num_samples, timesteps))) np.testing.assert_allclose(out3, out4, atol=1e-5) # check that the call to `predict` updated the states out5 = model.predict(np.ones((num_samples, timesteps))) self.assertNotEqual(out4.max(), out5.max()) # Check masking layer.reset_states() left_padded_input = np.ones((num_samples, timesteps)) left_padded_input[0, :1] = 0 left_padded_input[1, :2] = 0 out6 = model.predict(left_padded_input) layer.reset_states() right_padded_input = np.ones((num_samples, timesteps)) right_padded_input[0, -1:] = 0 right_padded_input[1, -2:] = 0 out7 = model.predict(right_padded_input) np.testing.assert_allclose(out7, out6, atol=1e-5)
def test_replace_keras_optimizer_iterations_variable(self): if testing_utils.should_run_eagerly(): # This needs to be updated to run with v2 optimizers. self.skipTest('b/120991591') self.assert_optimizer_iterations_increases('adam')
def _train_model(self, dtype, gtype): model = self._make_model(dtype, gtype) model.compile(optimizer='sgd', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch(np.zeros((8, 32)), np.zeros((8, 8)))
def test_model(self, strategy_fn, use_operator=False, use_regularizer=False, policy_name='mixed_float16', get_config=False, save_format=None, use_input_spec=False): self._skip_if_strategy_unsupported(strategy_fn) self._skip_if_save_format_unsupported(save_format) regularizer = (mp_test_util.IdentityRegularizer() if use_regularizer else None) with strategy_fn().scope(): # Pass loss_scale=None, as this test will fail if the DynamicLossScale # skips applying gradients for a step with policy.policy_scope(policy.Policy(policy_name, loss_scale=None)): layer = mp_test_util.MultiplyLayer( assert_type=dtypes.float16, use_operator=use_operator, regularizer=regularizer, input_shape=(1,)) if use_input_spec: layer.input_spec = input_spec.InputSpec(shape=(2, 1)) model = testing_utils.get_model_from_layers([layer], input_shape=(1,), input_dtype=dtypes.float16) if get_config: config = model.get_config() model = model.__class__.from_config( config, custom_objects={'MultiplyLayer': mp_test_util.MultiplyLayer}) (layer,) = (layer for layer in model.layers if isinstance(layer, mp_test_util.MultiplyLayer)) def loss_fn(y_true, y_pred): del y_true return math_ops.reduce_mean(y_pred) # Learning rate is small enough that if applied to a float16 variable, # the variable will not change. So this tests the learning rate not # applied to a float16 value, but instead the float32 variable. opt = gradient_descent.SGD(2**-14) model.compile( opt, loss=loss_fn, run_eagerly=testing_utils.should_run_eagerly()) x = np.ones((2, 1)) y = np.ones((2, 1)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2) model.fit(dataset) # Variable starts at 1, and should have gradient of 2 ** -14 subtracted # from it. expected = 1 - 2**-14 if use_regularizer: # Regularizer adds another 2 ** -14 to the gradient. expected -= 2**-14 self.assertEqual(backend.eval(layer.v), expected) if save_format: with generic_utils.CustomObjectScope( {'MultiplyLayer': mp_test_util.MultiplyLayer, 'loss_fn': loss_fn}): self._test_saving(model, dataset, save_format, use_regularizer)
def test_model_methods_with_eager_tensors_multi_io(self): if not context.executing_eagerly(): # Only test V2 Function and V2 Eager modes, as V1 Graph mode with # symbolic tensors has different requirements. return input_a = keras.layers.Input(shape=(3, ), name='input_a') input_b = keras.layers.Input(shape=(3, ), name='input_b') dense = keras.layers.Dense(4, name='dense') dropout = keras.layers.Dropout(0.5, name='dropout') model = testing_utils.get_multi_io_model([input_a, dense], [input_b, dense, dropout]) optimizer = rmsprop.RMSprop(learning_rate=0.001) loss = 'mse' loss_weights = [1., 0.5] metrics = ['mae', metrics_module.CategoricalAccuracy()] model.compile(optimizer, loss, metrics=metrics, loss_weights=loss_weights, run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function(), sample_weight_mode=None) input_a = array_ops.zeros(shape=(10, 3)) input_b = array_ops.zeros(shape=(10, 3)) target_a = array_ops.zeros(shape=(10, 4)) target_b = array_ops.zeros(shape=(10, 4)) model.fit([input_a, input_b], [target_a, target_b], epochs=1, batch_size=5, verbose=0) # Test: no shuffle. model.fit([input_a, input_b], [target_a, target_b], epochs=1, batch_size=5, verbose=0, shuffle=False) # Test: validation data. model.fit([input_a, input_b], [target_a, target_b], epochs=1, batch_size=2, verbose=0, validation_data=([input_a, input_b], [target_a, target_b])) model.train_on_batch([input_a, input_b], [target_a, target_b]) model.predict([input_a, input_b], batch_size=5) model.evaluate([input_a, input_b], [target_a, target_b], batch_size=2, verbose=0) model.test_on_batch([input_a, input_b], [target_a, target_b]) # Test: mix np and tensors. input_b = np.zeros(shape=(10, 3)).astype('float32') target_b = np.zeros(shape=(10, 4)).astype('float32') model.fit([input_a, input_b], [target_a, target_b], epochs=1, batch_size=5, verbose=0) model.fit([input_a, input_b], [target_a, target_b], epochs=1, batch_size=2, verbose=0, validation_data=([input_a, input_b], [target_a, target_b])) model.fit([input_a, input_b], [target_a, target_b], epochs=1, batch_size=5, verbose=0, shuffle=False) model.train_on_batch([input_a, input_b], [target_a, target_b]) model.predict([input_a, input_b], batch_size=5) model.evaluate([input_a, input_b], [target_a, target_b], batch_size=2, verbose=0) model.test_on_batch([input_a, input_b], [target_a, target_b])
def test_advanced_model(self, strategy_fn, use_loss_scaling=False): # The advanced model tests mixed-precision-related features that would occur # in a resnet50 model. It tests a model that has: # * Multiple layers, some which use auto-cast variables and some which do # not # * Regularization on some variables and not others. # * A fixed loss scale (if use_loss_scaling is True) strategy = strategy_fn() if use_loss_scaling: loss_scale = 8. else: loss_scale = None learning_rate = 2**-14 with strategy.scope(): with policy.policy_scope(policy.Policy('mixed_float16', loss_scale=loss_scale)): x = layers.Input(shape=(1,), batch_size=2) layer1 = mp_test_util.MultiplyLayer( assert_type=dtypes.float16, regularizer=mp_test_util.IdentityRegularizer(), use_operator=True) layer2 = MultiplyLayerWithoutAutoCast( assert_type=dtypes.float16, use_operator=True) layer3 = mp_test_util.MultiplyLayer(assert_type=dtypes.float16, use_operator=False) layer4 = MultiplyLayerWithoutAutoCast( assert_type=dtypes.float16, regularizer=mp_test_util.IdentityRegularizer(), use_operator=False) y = layer1(x) y = layer2(y) y = layer3(y) y = layer4(y) if use_loss_scaling: # The gradient of 'y' at this point is 1. With loss scaling, the # gradient is 'loss_scale'. We divide by the batch size of 2 since the # loss is averaged across batch elements. expected_gradient = loss_scale / 2 identity_with_grad_check_fn = ( mp_test_util.create_identity_with_grad_check_fn( expected_dtype=dtypes.float16, expected_gradient=[expected_gradient])) y = core.Lambda(identity_with_grad_check_fn)(y) model = models.Model(inputs=x, outputs=y) def loss_fn(y_true, y_pred): del y_true return math_ops.reduce_mean(y_pred) opt = gradient_descent.SGD(learning_rate) model.compile( opt, loss=loss_fn, run_eagerly=testing_utils.should_run_eagerly()) x = np.ones((2, 1)) y = np.ones((2, 1)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2) model.fit(dataset) for layer in (layer1, layer2, layer3, layer4): if layer.losses: # Layer has weight regularizer self.assertEqual(backend.eval(layer.v), 1 - 2 * learning_rate) else: # Layer does not have weight regularizer self.assertEqual(backend.eval(layer.v), 1 - learning_rate)
def get_learning_phase_value(): model = keras.models.Sequential( [LearningPhaseLayer(input_shape=(1, ))]) model._run_eagerly = testing_utils.should_run_eagerly() return np.sum(model(np.ones((1, 1))))
def test_clone_functional_model(self, share_weights): if share_weights: clone_fn = functools.partial(keras.models._clone_functional_model, layer_fn=models.share_weights) else: clone_fn = keras.models.clone_model val_a = np.random.random((10, 4)) val_b = np.random.random((10, 4)) val_out = np.random.random((10, 4)) input_a = keras.Input(shape=(4, )) input_b = keras.Input(shape=(4, )) dense_1 = keras.layers.Dense(4, ) dense_2 = keras.layers.Dense(4, ) x_a = dense_1(input_a) x_a = keras.layers.Dropout(0.5)(x_a) x_a = keras.layers.BatchNormalization()(x_a) x_b = dense_1(input_b) x_a = dense_2(x_a) outputs = keras.layers.add([x_a, x_b]) model = keras.models.Model([input_a, input_b], outputs) # With placeholder creation new_model = clone_fn(model) self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2) new_model.compile( testing_utils.get_v2_optimizer('rmsprop'), 'mse', run_eagerly=testing_utils.should_run_eagerly(), run_distributed=testing_utils.should_run_distributed()) new_model.train_on_batch([val_a, val_b], val_out) # On top of new tensors input_a = keras.Input(shape=(4, ), name='a') input_b = keras.Input(shape=(4, ), name='b') new_model = keras.models.clone_model(model, input_tensors=[input_a, input_b]) self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2) new_model.compile( testing_utils.get_v2_optimizer('rmsprop'), 'mse', run_eagerly=testing_utils.should_run_eagerly(), run_distributed=testing_utils.should_run_distributed()) new_model.train_on_batch([val_a, val_b], val_out) # On top of new, non-Keras tensors if not context.executing_eagerly(): # TODO(b/121277734):Skip Eager contexts, as Input() layers raise an error # saying they should not be used with EagerTensors input_a = keras.backend.variable(val_a) input_b = keras.backend.variable(val_b) new_model = clone_fn(model, input_tensors=[input_a, input_b]) self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2) new_model.compile( testing_utils.get_v2_optimizer('rmsprop'), 'mse', run_eagerly=testing_utils.should_run_eagerly(), run_distributed=testing_utils.should_run_distributed()) new_model.train_on_batch(None, val_out)
def testBody(self, arg): mode = "eager" if context.executing_eagerly() else "graph" should_run_eagerly = testing_utils.should_run_eagerly() should_run_tf_function = testing_utils.should_run_tf_function() l.append((mode, should_run_eagerly, should_run_tf_function, testing_utils.get_model_type()))
def testBody(self): mode = "eager" if context.executing_eagerly() else "graph" should_run_eagerly = testing_utils.should_run_eagerly() l.append((mode, should_run_eagerly))
def test_statefulness_LSTM(self): if test.is_built_with_rocm(): self.skipTest('Skipping the test as ROCm MIOpen does not ' 'support padded input yet.') num_samples = 2 timesteps = 3 embedding_dim = 4 units = 2 layer_class = rnn.LSTM model = keras.models.Sequential() model.add( keras.layers.Embedding(4, embedding_dim, mask_zero=True, input_length=timesteps, batch_input_shape=(num_samples, timesteps))) layer = layer_class(units, return_sequences=False, stateful=True, weights=None) model.add(layer) model.compile( optimizer=gradient_descent.GradientDescentOptimizer(0.01), loss='mse', run_eagerly=testing_utils.should_run_eagerly()) out1 = model.predict(np.ones((num_samples, timesteps))) self.assertEqual(out1.shape, (num_samples, units)) # train once so that the states change model.train_on_batch(np.ones((num_samples, timesteps)), np.ones((num_samples, units))) out2 = model.predict(np.ones((num_samples, timesteps))) # if the state is not reset, output should be different self.assertNotEqual(out1.max(), out2.max()) # check that output changes after states are reset # (even though the model itself didn't change) layer.reset_states() out3 = model.predict(np.ones((num_samples, timesteps))) self.assertNotEqual(out2.max(), out3.max()) # check that container-level reset_states() works model.reset_states() out4 = model.predict(np.ones((num_samples, timesteps))) self.assertAllClose(out3, out4, atol=1e-5) # check that the call to `predict` updated the states out5 = model.predict(np.ones((num_samples, timesteps))) self.assertNotEqual(out4.max(), out5.max()) # Check masking layer.reset_states() left_padded_input = np.ones((num_samples, timesteps)) left_padded_input[0, :1] = 0 left_padded_input[1, :2] = 0 out6 = model.predict(left_padded_input) layer.reset_states() right_padded_input = np.ones((num_samples, timesteps)) right_padded_input[0, -1:] = 0 right_padded_input[1, -2:] = 0 out7 = model.predict(right_padded_input) layer.reset_states() mix_padded_input = np.ones((num_samples, timesteps)) mix_padded_input[0, 1] = 0 mix_padded_input[1, 0] = 0 mix_padded_input[1, 2] = 0 out8 = model.predict(mix_padded_input) self.assertAllClose(out7, out6, atol=1e-5) self.assertAllClose(out8, out7, atol=1e-5)
def test_training(self): self.model.compile(loss='sparse_categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly()) self.model.fit(self.generator_input, steps_per_epoch=10)
def test_training(self): self.model.compile(loss='sparse_categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly()) self.model.fit(self.sequence_input)
def __init__(self): super( MyLayer, self).__init__(dynamic=testing_utils.should_run_eagerly())
def testBody(self, with_brackets): mode = "eager" if context.executing_eagerly() else "graph" with_brackets = "with_brackets" if with_brackets else "without_brackets" should_run_eagerly = testing_utils.should_run_eagerly() l.append((with_brackets, mode, should_run_eagerly))
def _get_compiled_multi_io_model(self, loss): model = get_multi_io_model() model.compile(optimizer='rmsprop', loss=loss, run_eagerly=testing_utils.should_run_eagerly()) return model
def test_training_and_eval_methods_on_iterators_single_io(self): model = testing_utils.get_small_mlp(1, 4, input_dim=3) optimizer = 'rmsprop' loss = 'mse' metrics = ['mae', metrics_module.CategoricalAccuracy()] model.compile(optimizer, loss, metrics=metrics, run_eagerly=testing_utils.should_run_eagerly()) inputs = np.zeros((10, 3), np.float32) targets = np.zeros((10, 4), np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets)) dataset = dataset.repeat(100) dataset = dataset.batch(10) iterator = dataset_ops.make_one_shot_iterator(dataset) model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1) model.evaluate(iterator, steps=2, verbose=1) model.predict(iterator, steps=2) # Test with validation data model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=0, validation_data=iterator, validation_steps=2) # Test with validation split with self.assertRaisesRegexp( ValueError, '`validation_split` argument is not supported ' 'when input `x` is a dataset or a dataset iterator'): model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=0, validation_split=0.5, validation_steps=2) # Test with sample weight. sample_weight = np.random.random((10, )) with self.assertRaisesRegexp( ValueError, '`sample_weight` argument is not supported ' 'when input `x` is a dataset or a dataset iterator'): model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=0, sample_weight=sample_weight) # Test invalid usage with self.assertRaisesRegexp(ValueError, 'you should not specify a target'): model.fit(iterator, iterator, epochs=1, steps_per_epoch=2, verbose=0) with self.assertRaisesRegexp(ValueError, 'the `steps_per_epoch` argument'): model.fit(iterator, epochs=1, verbose=0) with self.assertRaisesRegexp(ValueError, 'the `steps` argument'): model.evaluate(iterator, verbose=0) with self.assertRaisesRegexp(ValueError, 'the `steps` argument'): model.predict(iterator, verbose=0)
def test_training(self): dataset = self.adapter_cls(self.dataset_input).get_dataset() self.model.compile(loss='sparse_categorical_crossentropy', optimizer='sgd', run_eagerly=testing_utils.should_run_eagerly()) self.model.fit(dataset)
def test_rnn_with_time_major(self): batch = 10 time_step = 5 embedding_dim = 4 units = 3 # Test basic case. x = keras.Input((time_step, embedding_dim)) time_major_x = keras.layers.Lambda( lambda t: array_ops.transpose(t, [1, 0, 2]))(x) layer = keras.layers.SimpleRNN( units, time_major=True, return_sequences=True) self.assertEqual( layer.compute_output_shape((time_step, None, embedding_dim)).as_list(), [time_step, None, units]) y = layer(time_major_x) self.assertEqual(layer.output_shape, (time_step, None, units)) y = keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2]))(y) model = keras.models.Model(x, y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( np.zeros((batch, time_step, embedding_dim)), np.zeros((batch, time_step, units))) # Test stacking. x = keras.Input((time_step, embedding_dim)) time_major_x = keras.layers.Lambda( lambda t: array_ops.transpose(t, [1, 0, 2]))(x) cell_units = [10, 8, 6] cells = [keras.layers.SimpleRNNCell(cell_units[i]) for i in range(3)] layer = keras.layers.RNN(cells, time_major=True, return_sequences=True) y = layer(time_major_x) self.assertEqual(layer.output_shape, (time_step, None, cell_units[-1])) y = keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2]))(y) model = keras.models.Model(x, y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( np.zeros((batch, time_step, embedding_dim)), np.zeros((batch, time_step, cell_units[-1]))) # Test masking. x = keras.Input((time_step, embedding_dim)) time_major = keras.layers.Lambda( lambda t: array_ops.transpose(t, [1, 0, 2]))(x) mask = keras.layers.Masking()(time_major) rnn = keras.layers.SimpleRNN( units, time_major=True, return_sequences=True)(mask) y = keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2]))(rnn) model = keras.models.Model(x, y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( np.zeros((batch, time_step, embedding_dim)), np.zeros((batch, time_step, units))) # Test layer output x = keras.Input((time_step, embedding_dim)) rnn_1 = keras.layers.SimpleRNN(units, return_sequences=True) y = rnn_1(x) model = keras.models.Model(x, y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( np.zeros((batch, time_step, embedding_dim)), np.zeros((batch, time_step, units))) x_np = np.random.random((batch, time_step, embedding_dim)) y_np_1 = model.predict(x_np) time_major = keras.layers.Lambda( lambda t: array_ops.transpose(t, [1, 0, 2]))(x) rnn_2 = keras.layers.SimpleRNN( units, time_major=True, return_sequences=True) y_2 = rnn_2(time_major) y_2 = keras.layers.Lambda( lambda t: array_ops.transpose(t, [1, 0, 2]))(y_2) model_2 = keras.models.Model(x, y_2) rnn_2.set_weights(rnn_1.get_weights()) y_np_2 = model_2.predict(x_np) self.assertAllClose(y_np_1, y_np_2, atol=1e-4)
def get_test_mode_kwargs(): run_eagerly = testing_utils.should_run_eagerly() return { "run_eagerly": run_eagerly, }
def _test_optimizer(self, optimizer, target=0.75): if context.executing_eagerly(): self.skipTest('v1 optimizer does not run in eager mode') np.random.seed(1337) (x_train, y_train), _ = testing_utils.get_test_data(train_samples=1000, test_samples=200, input_shape=(10, ), num_classes=2) y_train = np_utils.to_categorical(y_train) model = _get_model(x_train.shape[1], 20, y_train.shape[1]) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc'], run_eagerly=testing_utils.should_run_eagerly()) np.testing.assert_equal( keras.backend.get_value(model.optimizer.iterations), 0) history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0) np.testing.assert_equal( keras.backend.get_value(model.optimizer.iterations), 126) # 63 steps per epoch self.assertGreaterEqual(history.history['acc'][-1], target) config = keras.optimizers.serialize(optimizer) optim = keras.optimizers.deserialize(config) new_config = keras.optimizers.serialize(optim) new_config['class_name'] = new_config['class_name'].lower() new_config['config'].pop('name', None) if 'amsgrad' not in config['config']: new_config['config'].pop('amsgrad', None) if 'decay' in new_config['config'] and 'schedule_decay' in config[ 'config']: new_config['config']['schedule_decay'] = new_config['config'].pop( 'decay') if 'momentum' not in config['config']: new_config['config'].pop('momentum', None) if 'centered' not in config['config']: new_config['config'].pop('centered', None) self.assertDictEqual(config, new_config) # Test constraints. model = keras.models.Sequential() dense = keras.layers.Dense(10, input_shape=(x_train.shape[1], ), kernel_constraint=lambda x: 0. * x + 1., bias_constraint=lambda x: 0. * x + 2., activation='relu') model.add(dense) model.add(keras.layers.Dense(y_train.shape[1], activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'], run_eagerly=testing_utils.should_run_eagerly()) np.testing.assert_equal( keras.backend.get_value(model.optimizer.iterations), 126) # Using same optimizer from before model.train_on_batch(x_train[:10], y_train[:10]) np.testing.assert_equal( keras.backend.get_value(model.optimizer.iterations), 127) kernel, bias = dense.get_weights() np.testing.assert_allclose(kernel, 1., atol=1e-3) np.testing.assert_allclose(bias, 2., atol=1e-3)
def test_rnn_cell_with_constants_layer_passing_initial_state(self): class RNNCellWithConstants(keras.layers.Layer): def __init__(self, units, **kwargs): self.units = units self.state_size = units super(RNNCellWithConstants, self).__init__(**kwargs) def build(self, input_shape): if not isinstance(input_shape, list): raise TypeError('expects constants shape') [input_shape, constant_shape] = input_shape # will (and should) raise if more than one constant passed self.input_kernel = self.add_weight( shape=(input_shape[-1], self.units), initializer='uniform', name='kernel') self.recurrent_kernel = self.add_weight( shape=(self.units, self.units), initializer='uniform', name='recurrent_kernel') self.constant_kernel = self.add_weight( shape=(constant_shape[-1], self.units), initializer='uniform', name='constant_kernel') self.built = True def call(self, inputs, states, constants): [prev_output] = states [constant] = constants h_input = keras.backend.dot(inputs, self.input_kernel) h_state = keras.backend.dot(prev_output, self.recurrent_kernel) h_const = keras.backend.dot(constant, self.constant_kernel) output = h_input + h_state + h_const return output, [output] def get_config(self): config = {'units': self.units} base_config = super(RNNCellWithConstants, self).get_config() return dict(list(base_config.items()) + list(config.items())) # Test basic case. x = keras.Input((None, 5)) c = keras.Input((3,)) s = keras.Input((32,)) cell = RNNCellWithConstants(32) layer = keras.layers.RNN(cell) y = layer(x, initial_state=s, constants=c) model = keras.models.Model([x, s, c], y) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model.train_on_batch( [np.zeros((6, 5, 5)), np.zeros((6, 32)), np.zeros((6, 3))], np.zeros((6, 32)) ) # Test basic case serialization. x_np = np.random.random((6, 5, 5)) s_np = np.random.random((6, 32)) c_np = np.random.random((6, 3)) y_np = model.predict([x_np, s_np, c_np]) weights = model.get_weights() config = layer.get_config() custom_objects = {'RNNCellWithConstants': RNNCellWithConstants} with keras.utils.CustomObjectScope(custom_objects): layer = keras.layers.RNN.from_config(config.copy()) y = layer(x, initial_state=s, constants=c) model = keras.models.Model([x, s, c], y) model.set_weights(weights) y_np_2 = model.predict([x_np, s_np, c_np]) self.assertAllClose(y_np, y_np_2, atol=1e-4) # verify that state is used y_np_2_different_s = model.predict([x_np, s_np + 10., c_np]) with self.assertRaises(AssertionError): self.assertAllClose(y_np, y_np_2_different_s, atol=1e-4) # test flat list inputs with keras.utils.CustomObjectScope(custom_objects): layer = keras.layers.RNN.from_config(config.copy()) y = layer([x, s, c]) model = keras.models.Model([x, s, c], y) model.set_weights(weights) y_np_3 = model.predict([x_np, s_np, c_np]) self.assertAllClose(y_np, y_np_3, atol=1e-4)