def test_vector_classification(self): np.random.seed(1337) (x_train, y_train), _ = testing_utils.get_test_data( train_samples=100, test_samples=0, input_shape=(10,), num_classes=2) y_train = keras.utils.to_categorical(y_train) model = testing_utils.get_model_from_layers( [keras.layers.Dense(16, activation='relu'), keras.layers.Dropout(0.1), keras.layers.Dense(y_train.shape[-1], activation='softmax')], input_shape=x_train.shape[1:]) model.compile( loss='categorical_crossentropy', optimizer=keras.optimizer_v2.adam.Adam(0.005), metrics=['acc'], run_eagerly=testing_utils.should_run_eagerly()) history = model.fit(x_train, y_train, epochs=10, batch_size=10, validation_data=(x_train, y_train), verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) _, val_acc = model.evaluate(x_train, y_train) self.assertAlmostEqual(history.history['val_acc'][-1], val_acc) predictions = model.predict(x_train) self.assertEqual(predictions.shape, (x_train.shape[0], 2))
def _get_simple_bias_model(self): model = testing_utils.get_model_from_layers([Bias()], input_shape=(1,)) model.compile( keras.optimizer_v2.gradient_descent.SGD(0.1), 'mae', run_eagerly=testing_utils.should_run_eagerly()) return model
def test_metrics_correctness_with_iterator(self): layers = [ keras.layers.Dense(8, activation='relu', input_dim=4, kernel_initializer='ones'), keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones') ] model = testing_utils.get_model_from_layers(layers, (4,)) model.compile( loss='binary_crossentropy', metrics=['accuracy', metrics_module.BinaryAccuracy()], optimizer='rmsprop', run_eagerly=testing_utils.should_run_eagerly()) np.random.seed(123) x = np.random.randint(10, size=(100, 4)).astype(np.float32) y = np.random.randint(2, size=(100, 1)).astype(np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.batch(10) iterator = dataset_ops.make_one_shot_iterator(dataset) outs = model.evaluate(iterator, steps=10) self.assertEqual(np.around(outs[1], decimals=1), 0.5) self.assertEqual(np.around(outs[2], decimals=1), 0.5) y = np.zeros((100, 1), dtype=np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) dataset = dataset.batch(10) iterator = dataset_ops.make_one_shot_iterator(dataset) outs = model.evaluate(iterator, steps=10) self.assertEqual(outs[1], 0.) self.assertEqual(outs[2], 0.)
def add_metric_step(defun): optimizer = keras.optimizer_v2.rmsprop.RMSprop() model = testing_utils.get_model_from_layers([ LayerWithMetrics(), keras.layers.Dense(1, kernel_initializer='zeros', activation='softmax') ], input_shape=(10,)) def train_step(x, y): with backprop.GradientTape() as tape: y_pred_1 = model(x) y_pred_2 = model(2 * x) y_pred = y_pred_1 + y_pred_2 loss = keras.losses.mean_squared_error(y, y_pred) gradients = tape.gradient(loss, model.trainable_weights) optimizer.apply_gradients(zip(gradients, model.trainable_weights)) assert len(model.metrics) == 2 return [m.result() for m in model.metrics] if defun: train_step = def_function.function(train_step) x, y = array_ops.ones((10, 10)), array_ops.zeros((10, 1)) metrics = train_step(x, y) assert np.allclose(metrics[0], 1.5) assert np.allclose(metrics[1], 1.5) return metrics
def test_build_with_numpy_data(self): model_layers = [ keras.layers.Dense(3, activation='relu', kernel_initializer='ones'), keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones') ] model = testing_utils.get_model_from_layers(model_layers, input_shape=(4,)) model(np.zeros((2, 4), dtype='float32')) self.assertTrue(model.built)
def test_zero_regularization(self): # Verifies that training with zero regularization works. x, y = np.ones((10, 10)), np.ones((10, 3)) model = testing_utils.get_model_from_layers( [keras.layers.Dense(3, kernel_regularizer=keras.regularizers.l2(0))], input_shape=(10,)) model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly()) model.fit(x, y, batch_size=5, epochs=1)
def _get_model(self): layers = [ keras.layers.Conv2D(8, (3, 3)), keras.layers.Flatten(), keras.layers.Dense(1) ] model = testing_utils.get_model_from_layers(layers, input_shape=(10, 10, 1)) model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly()) return model
def _get_model(self): layers = [ keras.layers.Dense(10, activation='relu'), keras.layers.Dense(1, activation='sigmoid') ] model = testing_utils.get_model_from_layers(layers, input_shape=(10,)) model.compile( adam.AdamOptimizer(0.001), 'binary_crossentropy', run_eagerly=testing_utils.should_run_eagerly()) return model
def test_model_with_internal_sparse_tensors(self): # Create a model that accepts an input, converts it to Sparse, and # converts the sparse tensor back to a dense tensor. layers = [ToSparse(), ToDense(default_value=-1)] model = testing_utils.get_model_from_layers(layers, input_shape=(None,)) # Define some input data with additional padding. input_data = np.array([[1, 0, 0], [2, 3, 0]]) expected_output = np.array([[1, -1, -1], [2, 3, -1]]) output = model.predict(input_data) self.assertAllEqual(expected_output, output)
def test_model_with_ragged_tensor_rebatched_outputs(self): # Create a model that accepts an input, converts it to Ragged, and # converts the ragged tensor back to a dense tensor. layers = [ToRagged(padding=0)] model = testing_utils.get_model_from_layers(layers, input_shape=(None,)) # Define some input data with additional padding. input_data = np.array([[1, 0, 0], [2, 3, 0], [4, 0, 0], [5, 6, 0]]) output = model.predict(input_data, batch_size=2) expected_values = [[1], [2, 3], [4], [5, 6]] self.assertRaggedEqual(expected_values, output)
def _get_model(self): x = layers.Dense(3, kernel_initializer='ones', trainable=False) out = layers.Dense( 1, kernel_initializer='ones', name='output', trainable=False) model = testing_utils.get_model_from_layers([x, out], input_shape=(1,)) model.compile( optimizer='rmsprop', loss='mse', metrics=[metrics.MeanSquaredError()], weighted_metrics=[metrics.MeanSquaredError()], run_eagerly=testing_utils.should_run_eagerly()) return model
def _get_model(self, input_shape=None): layers = [ keras.layers.Dense(3, activation='relu'), keras.layers.Dense(2, activation='softmax') ] model = testing_utils.get_model_from_layers(layers, input_shape=input_shape) model.compile( loss='mse', optimizer='rmsprop', metrics=[keras.metrics.CategoricalAccuracy(name='my_acc')], run_eagerly=testing_utils.should_run_eagerly()) return model
def test_loss_correctness(self): # Test that training loss is the same in eager and graph # (by comparing it to a reference value in a deterministic case) layers = [ keras.layers.Dense(3, activation='relu', kernel_initializer='ones'), keras.layers.Dense(2, activation='softmax', kernel_initializer='ones')] model = testing_utils.get_model_from_layers(layers, input_shape=(4,)) model.compile(loss='sparse_categorical_crossentropy', optimizer=rmsprop.RMSprop(learning_rate=0.001), run_eagerly=testing_utils.should_run_eagerly()) x = np.ones((100, 4)) np.random.seed(123) y = np.random.randint(0, 1, size=(100, 1)) history = model.fit(x, y, epochs=1, batch_size=10) self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
def test_model_with_sparse_tensor_outputs(self): # Create a model that accepts an input, converts it to Ragged, and # converts the ragged tensor back to a dense tensor. layers = [ToSparse()] model = testing_utils.get_model_from_layers(layers, input_shape=(None,)) # Define some input data with additional padding. input_data = np.array([[1, 0, 0], [2, 3, 0]]) output = model.predict(input_data) expected_indices = np.array([[0, 0], [1, 0], [1, 1]]) expected_values = np.array([1, 2, 3]) expected_dense_shape = np.array([2, 3]) self.assertAllEqual(output.indices, expected_indices) self.assertAllEqual(output.values, expected_values) self.assertAllEqual(output.dense_shape, expected_dense_shape)
def test_validation_dataset_with_no_step_arg(self): # Create a model that learns y=Mx. layers = [core.Dense(1)] model = testing_utils.get_model_from_layers(layers, input_shape=(1,)) model.compile(loss="mse", optimizer="adam", metrics=["mean_absolute_error"]) train_dataset = self.create_dataset(num_samples=200, batch_size=10) eval_dataset = self.create_dataset(num_samples=50, batch_size=25) history = model.fit(x=train_dataset, validation_data=eval_dataset, epochs=2) evaluation = model.evaluate(x=eval_dataset) # If the fit call used the entire dataset, then the final val MAE error # from the fit history should be equal to the final element in the output # of evaluating the model on the same eval dataset. self.assertAlmostEqual(history.history["val_mean_absolute_error"][-1], evaluation[-1])
def test_lambda_with_variable_in_model(self): def lambda_fn(x): # Variable will only get created once. v = variables.Variable(1., trainable=True) return x * v model = testing_utils.get_model_from_layers( [keras.layers.Lambda(lambda_fn)], input_shape=(10,)) model.compile( keras.optimizer_v2.gradient_descent.SGD(0.1), 'mae', run_eagerly=testing_utils.should_run_eagerly()) x, y = np.ones((10, 10), 'float32'), 2 * np.ones((10, 10), 'float32') model.fit(x, y, batch_size=2, epochs=2, validation_data=(x, y)) self.assertLen(model.trainable_weights, 1) self.assertAllClose(keras.backend.get_value(model.trainable_weights[0]), 2.)
def test_raw_variable_assignment(self): class RawVariableLayer(keras.layers.Layer): def __init__(self, **kwargs): super(RawVariableLayer, self).__init__(**kwargs) # Test variables in nested structure. self.var_list = [variables.Variable(1.), {'a': variables.Variable(2.)}] def call(self, inputs): return inputs * self.var_list[0] * self.var_list[1]['a'] model = testing_utils.get_model_from_layers([RawVariableLayer()], input_shape=(10,)) model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly()) x, y = np.ones((10, 10)), np.ones((10, 10)) # Checks that variables get initialized. model.fit(x, y, batch_size=2, epochs=2)
def test_loss_correctness(self): # Test that training loss is the same in eager and graph # (by comparing it to a reference value in a deterministic case) layers = [ keras.layers.Dense(3, activation='relu', kernel_initializer='ones'), keras.layers.Dense(2, activation='softmax', kernel_initializer='ones') ] model = testing_utils.get_model_from_layers(layers, input_shape=(4, )) model.compile(loss='sparse_categorical_crossentropy', optimizer=rmsprop.RMSprop(learning_rate=0.001), run_eagerly=testing_utils.should_run_eagerly()) x = np.ones((100, 4)) np.random.seed(123) y = np.random.randint(0, 1, size=(100, 1)) history = model.fit(x, y, epochs=1, batch_size=10) self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
def test_training_arg_in_defun(self): layer = self._get_layer_with_training_arg() model = testing_utils.get_model_from_layers([layer], input_shape=(1, )) model.compile(rmsprop.RMSprop(0.), loss='mae') history = model.fit(np.zeros((1, 1)), np.zeros((1, 1))) self.assertEqual(history.history['loss'][0], 1.) loss = model.evaluate(np.zeros((1, 1)), np.zeros((1, 1))) self.assertEqual(loss, 0.) # Test that the argument injection performed in `call` is not active # when the argument is passed explicitly. layer = self._get_layer_with_training_arg() inputs = keras.Input(shape=(1, )) # Pass `training` by name outputs = layer(inputs, training=False) model = keras.Model(inputs, outputs) model.compile(rmsprop.RMSprop(0.), loss='mae') history = model.fit(np.zeros((1, 1)), np.zeros((1, 1))) self.assertEqual(history.history['loss'][0], 0.)
def add_loss_step(defun): optimizer = keras.optimizer_v2.adam.Adam() model = testing_utils.get_model_from_layers([LayerWithLosses()], input_shape=(10, )) def train_step(x): with backprop.GradientTape() as tape: model(x) assert len(model.losses) == 2 loss = math_ops.reduce_sum(model.losses) gradients = tape.gradient(loss, model.trainable_weights) optimizer.apply_gradients(zip(gradients, model.trainable_weights)) return loss if defun: train_step = def_function.function(train_step) x = array_ops.ones((10, 10)) return train_step(x)
def _get_model(self): x = layers.Dense(3, kernel_initializer='ones', trainable=False) out = layers.Dense(1, kernel_initializer='ones', name='output', trainable=False) model = testing_utils.get_model_from_layers([x, out], input_shape=(1, )) model.compile( optimizer='rmsprop', loss='mse', metrics=[metrics.MeanSquaredError(name='mean_squared_error')], weighted_metrics=[ metrics.MeanSquaredError(name='mean_squared_error_2') ], run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils.should_run_tf_function( )) return model
def test_sparse_tensor_outputs(self): # Create a model that accepts an input, converts it to Ragged, and # converts the ragged tensor back to a dense tensor. layers = [ToSparse()] model = testing_utils.get_model_from_layers(layers, input_shape=(None, )) model._run_eagerly = testing_utils.should_run_eagerly() # Define some input data with additional padding. input_data = np.array([[1, 0, 0], [2, 3, 0]]) output = model.predict(input_data) expected_indices = np.array([[0, 0], [1, 0], [1, 1]]) expected_values = np.array([1, 2, 3]) expected_dense_shape = np.array([2, 3]) self.assertAllEqual(output.indices, expected_indices) self.assertAllEqual(output.values, expected_values) self.assertAllEqual(output.dense_shape, expected_dense_shape)
def add_loss_step(defun): optimizer = keras.optimizer_v2.adam.Adam() model = testing_utils.get_model_from_layers([LayerWithLosses()], input_shape=(10,)) def train_step(x): with backprop.GradientTape() as tape: model(x) assert len(model.losses) == 2 loss = math_ops.reduce_sum(model.losses) gradients = tape.gradient(loss, model.trainable_weights) optimizer.apply_gradients(zip(gradients, model.trainable_weights)) return loss if defun: train_step = def_function.function(train_step) x = array_ops.ones((10, 10)) return train_step(x)
def test_transitive_variable_creation(self): dense = keras.layers.Dense(1, use_bias=False, kernel_initializer='ones') def bad_lambda_fn(x): return dense(x + 1) # Dense layer is built on first call expected_error = textwrap.dedent(r''' ( )?The following Variables were created within a Lambda layer \(bias_dense\) ( )?but are not tracked by said layer: ( )? <tf.Variable \'.*bias_dense/dense/kernel:0\'.+ ( )?The layer cannot safely ensure proper Variable reuse.+''') with self.assertRaisesRegexp(ValueError, expected_error): layer = keras.layers.Lambda(bad_lambda_fn, name='bias_dense') model = testing_utils.get_model_from_layers([layer], input_shape=(1, )) model(array_ops.ones((4, 1)))
def SKIP_test_ragged_input_with_padding(self): input_data = get_input_dataset( ragged_factory_ops.constant([[[1, 2, 3, 4, 5]], [[2], [3]]])) expected_output = np.array([[[1., 2., 3., 4., 5.], [-1., -1., -1., -1., -1.]], [[2., -1., -1., -1., -1.], [3., -1., -1., -1., -1.]]]) layers = [ToDense(pad_value=-1), Final()] model = testing_utils.get_model_from_layers(layers, input_shape=(None, None), input_ragged=True, input_dtype=dtypes.int32) model.compile(optimizer="sgd", loss="mse", metrics=["accuracy"], run_eagerly=testing_utils.should_run_eagerly()) output = model.predict(input_data) self.assertAllEqual(output, expected_output)
def test_vector_classification_shared_model(self): # Test that Sequential models that feature internal updates # and internal losses can be shared. np.random.seed(1337) (x_train, y_train), _ = testing_utils.get_test_data(train_samples=100, test_samples=0, input_shape=(10, ), num_classes=2) y_train = keras.utils.to_categorical(y_train) base_model = testing_utils.get_model_from_layers( [ keras.layers.Dense( 16, activation='relu', kernel_regularizer=keras.regularizers.l2(1e-5), bias_regularizer=keras.regularizers.l2(1e-5)), keras.layers.BatchNormalization() ], input_shape=x_train.shape[1:]) x = keras.layers.Input(x_train.shape[1:]) y = base_model(x) y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y) model = keras.models.Model(x, y) model.compile(loss='categorical_crossentropy', optimizer=keras.optimizer_v2.adam.Adam(0.005), metrics=['acc'], run_eagerly=testing_utils.should_run_eagerly(), run_distributed=testing_utils.should_run_distributed()) if not testing_utils.should_run_eagerly(): self.assertEqual(len(model.get_losses_for(None)), 2) self.assertEqual(len(model.get_updates_for(x)), 2) history = model.fit(x_train, y_train, epochs=10, batch_size=10, validation_data=(x_train, y_train), verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) _, val_acc = model.evaluate(x_train, y_train) self.assertAlmostEqual(history.history['val_acc'][-1], val_acc) predictions = model.predict(x_train) self.assertEqual(predictions.shape, (x_train.shape[0], 2))
def test_warns_on_variable_capture(self): v = variables.Variable(1., trainable=True) def lambda_fn(x): return x * v expected_warning = textwrap.dedent(r''' ( )?The following Variables were used a Lambda layer\'s call \(lambda\), but ( )?are not present in its tracked objects: ( )? <tf.Variable \'.*Variable:0\'.+ ( )?It is possible that this is intended behavior.+''') layer = keras.layers.Lambda(lambda_fn) def patched_warn(msg): raise ValueError(msg) layer._warn = patched_warn with self.assertRaisesRegexp(ValueError, expected_warning): model = testing_utils.get_model_from_layers([layer], input_shape=(1,)) model(array_ops.ones((4, 1)))
def test_sparse_input_shape(self): input_data = get_input_dataset( sparse_tensor.SparseTensor(indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4])) expected_output = np.array([[1., 0., 0., 0.], [0., 0., 2., 0.], [0., 0., 0., 0.]]) layers = [ToDense(shape=[3, 4]), Final()] model = testing_utils.get_model_from_layers(layers, input_shape=(None, ), input_sparse=True, input_dtype=dtypes.int32) model.compile(optimizer="sgd", loss="mse", metrics=["accuracy"], run_eagerly=testing_utils.should_run_eagerly()) output = model.predict(input_data) self.assertAllEqual(output, expected_output)
def test_simple_build_with_constant(self): class BuildConstantLayer(keras.layers.Layer): def build(self, input_shape): self.b = ops.convert_to_tensor(2.0) def call(self, inputs): return self.b * inputs layer = BuildConstantLayer() model = testing_utils.get_model_from_layers( [layer, keras.layers.Dense(1)], input_shape=(1,)) x = ops.convert_to_tensor([[3.0]]) self.assertEqual( tf_utils.is_symbolic_tensor(model(x)), not context.executing_eagerly()) self.assertEqual( tf_utils.is_symbolic_tensor(layer(x)), not context.executing_eagerly()) self.assertAllClose(keras.backend.get_value(layer(x)), [[6.0]])
def test_loss_correctness_with_iterator(self): # Test that training loss is the same in eager and graph # (by comparing it to a reference value in a deterministic case) layers = [ keras.layers.Dense(3, activation='relu', kernel_initializer='ones'), keras.layers.Dense(2, activation='softmax', kernel_initializer='ones')] model = testing_utils.get_model_from_layers(layers, input_shape=(4,)) model.compile( loss='sparse_categorical_crossentropy', optimizer=rmsprop.RMSprop(learning_rate=0.001), run_eagerly=testing_utils.should_run_eagerly()) x = np.ones((100, 4), dtype=np.float32) np.random.seed(123) y = np.random.randint(0, 1, size=(100, 1)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) dataset = dataset.batch(10) iterator = dataset_ops.make_one_shot_iterator(dataset) history = model.fit(iterator, epochs=1, steps_per_epoch=10) self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
def test_training_arg_in_defun(self): layer = self._get_layer_with_training_arg() model = testing_utils.get_model_from_layers([layer], input_shape=(1,)) model.compile(rmsprop.RMSprop(0.), loss='mae') history = model.fit(np.zeros((1, 1)), np.zeros((1, 1))) self.assertEqual(history.history['loss'][0], 1.) loss = model.evaluate(np.zeros((1, 1)), np.zeros((1, 1))) self.assertEqual(loss, 0.) # Test that the argument injection performed in `call` is not active # when the argument is passed explicitly. layer = self._get_layer_with_training_arg() inputs = keras.Input(shape=(1,)) # Pass `training` by name outputs = layer(inputs, training=False) model = keras.Model(inputs, outputs) model.compile(rmsprop.RMSprop(0.), loss='mae') history = model.fit(np.zeros((1, 1)), np.zeros((1, 1))) self.assertEqual(history.history['loss'][0], 0.)
def test_training_model_with_internal_ragged_tensors(self): # Create a model that implements y=Mx. This is easy to learn and will # demonstrate appropriate gradient passing. (We have to use RaggedTensors # for this test, as ToSparse() doesn't support gradient propagation through # the layer.) TODO(b/124796939): Investigate this. layers = [core.Dense(2), ToRagged(padding=0), ToDense(default_value=-1)] model = testing_utils.get_model_from_layers(layers, input_shape=(1,)) input_data = np.random.rand(1024, 1) expected_data = np.concatenate((input_data * 3, input_data * .5), axis=-1) model.compile( loss="mse", optimizer="adam", run_eagerly=testing_utils.should_run_eagerly()) history = model.fit(input_data, expected_data, epochs=10, verbose=0) # If the model trained, the loss stored at history[0] should be different # than the one stored at history[-1]. self.assertNotEqual(history.history["loss"][-1], history.history["loss"][0])
def _testAddUpdate(self, scope): with scope: layer_with_update = LayerWithUpdate(dtype=dtypes.int32) model = testing_utils.get_model_from_layers([layer_with_update], input_shape=(3,), input_dtype=dtypes.int32) if testing_utils.get_model_type() == 'subclass': model._set_inputs(constant_op.constant([[1, 2, 3]], dtype=dtypes.int32)) self.evaluate(variables.variables_initializer(model.variables)) saved_model_dir = self._save_model_dir() model.save(saved_model_dir, save_format='tf') loaded = keras_load.load(saved_model_dir) loaded_layer = loaded.layers[-1] self.evaluate(variables.variables_initializer(loaded.variables)) self.assertEqual(self.evaluate(loaded_layer.v), 0) loaded.predict(constant_op.constant([[1, 2, 3]], dtype=dtypes.int32), steps=1) self.assertEqual(self.evaluate(loaded_layer.v), 6)
def test_training_internal_ragged_tensors(self): # Create a model that implements y=Mx. This is easy to learn and will # demonstrate appropriate gradient passing. (We have to use RaggedTensors # for this test, as ToSparse() doesn't support gradient propagation through # the layer.) TODO(b/124796939): Investigate this. layers = [core.Dense(2), ToRagged(padding=0), ToDense(default_value=-1)] model = testing_utils.get_model_from_layers(layers, input_shape=(1,)) input_data = np.random.rand(1024, 1) expected_data = np.concatenate((input_data * 3, input_data * .5), axis=-1) model.compile( loss="mse", optimizer="adam", run_eagerly=testing_utils.should_run_eagerly()) history = model.fit(input_data, expected_data, epochs=10, verbose=0) # If the model trained, the loss stored at history[0] should be different # than the one stored at history[-1]. self.assertNotEqual(history.history["loss"][-1], history.history["loss"][0])
def test_loss_correctness_with_iterator(self): # Test that training loss is the same in eager and graph # (by comparing it to a reference value in a deterministic case) layers = [ keras.layers.Dense(3, activation='relu', kernel_initializer='ones'), keras.layers.Dense(2, activation='softmax', kernel_initializer='ones')] model = testing_utils.get_model_from_layers(layers, input_shape=(4,)) model.compile( loss='sparse_categorical_crossentropy', optimizer=rmsprop.RMSprop(learning_rate=0.001), run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils.should_run_tf_function()) x = np.ones((100, 4), dtype=np.float32) np.random.seed(123) y = np.random.randint(0, 1, size=(100, 1)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) dataset = dataset.batch(10) history = model.fit(dataset, epochs=1, steps_per_epoch=10) self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
def _testAddUpdate(self, scope): with scope: layer_with_update = LayerWithUpdate() model = testing_utils.get_model_from_layers([layer_with_update], input_shape=(3, )) x = np.ones((10, 3)) if testing_utils.get_model_type() == 'subclass': model.predict(x, batch_size=10) self.evaluate(variables.variables_initializer(model.variables)) saved_model_dir = self._save_model_dir() model.save(saved_model_dir, save_format='tf') loaded = keras_load.load(saved_model_dir) loaded_layer = loaded.layers[-1] self.evaluate(variables.variables_initializer(loaded.variables)) self.assertEqual(self.evaluate(loaded_layer.v), 0.) loaded.compile('sgd', 'mse') loaded.fit(x, x, batch_size=10) self.assertEqual(self.evaluate(loaded_layer.v), 1.)
def test_validation_dataset_with_no_step_arg(self): # Create a model that learns y=Mx. layers = [core.Dense(1)] model = testing_utils.get_model_from_layers(layers, input_shape=(1, )) model.compile(loss="mse", optimizer="adam", metrics=["mean_absolute_error"]) train_dataset = self.create_dataset(num_samples=200, batch_size=10) eval_dataset = self.create_dataset(num_samples=50, batch_size=25) history = model.fit(x=train_dataset, validation_data=eval_dataset, epochs=2) evaluation = model.evaluate(x=eval_dataset) # If the fit call used the entire dataset, then the final val MAE error # from the fit history should be equal to the final element in the output # of evaluating the model on the same eval dataset. self.assertAlmostEqual(history.history["val_mean_absolute_error"][-1], evaluation[-1])
def batch_norm_step(defun): optimizer = keras.optimizer_v2.adadelta.Adadelta() model = testing_utils.get_model_from_layers([ keras.layers.BatchNormalization(momentum=0.9), keras.layers.Dense(1, kernel_initializer='zeros', activation='softmax') ], input_shape=(10, )) def train_step(x, y): with backprop.GradientTape() as tape: y_pred = model(x, training=True) loss = keras.losses.binary_crossentropy(y, y_pred) gradients = tape.gradient(loss, model.trainable_weights) optimizer.apply_gradients(zip(gradients, model.trainable_weights)) return loss, model(x, training=False) if defun: train_step = def_function.function(train_step) x, y = array_ops.ones((10, 10)), array_ops.ones((10, 1)) return train_step(x, y)
def batch_norm_step(defun): optimizer = keras.optimizer_v2.adadelta.Adadelta() model = testing_utils.get_model_from_layers([ keras.layers.BatchNormalization(momentum=0.9), keras.layers.Dense(1, kernel_initializer='zeros', activation='softmax') ], input_shape=(10,)) def train_step(x, y): with backprop.GradientTape() as tape: y_pred = model(x, training=True) loss = keras.losses.binary_crossentropy(y, y_pred) gradients = tape.gradient(loss, model.trainable_weights) optimizer.apply_gradients(zip(gradients, model.trainable_weights)) return loss, model(x, training=False) if defun: train_step = def_function.function(train_step) x, y = array_ops.ones((10, 10)), array_ops.ones((10, 1)) return train_step(x, y)
def test_build_with_derived_constant(self): class BuildDerivedConstantLayer(keras.layers.Layer): def build(self, input_shape): a = ops.convert_to_tensor_v2(1.0) b = 2.0 * a self.variable = variables.Variable(b) self.constant = ops.convert_to_tensor_v2(self.variable) def call(self, inputs): return self.variable * self.constant * inputs layer = BuildDerivedConstantLayer() model = testing_utils.get_model_from_layers( [layer, keras.layers.Dense(1)], input_shape=(1, )) x = ops.convert_to_tensor_v2([[3.0]]) self.assertEqual(tf_utils.is_symbolic_tensor(model(x)), not context.executing_eagerly()) self.assertEqual(tf_utils.is_symbolic_tensor(layer(x)), not context.executing_eagerly()) self.assertAllClose(keras.backend.get_value(layer(x)), [[12.0]])
def test_lambda_with_variable_in_model(self): v = variables.Variable(1., trainable=True) def lambda_fn(x, v): return x * v # While it is generally not advised to mix Variables with Lambda layers, if # the variables are explicitly set as attributes then they are still # tracked. This is consistent with the base Layer behavior. layer = keras.layers.Lambda(lambda_fn, arguments={'v': v}) self.assertLen(layer.trainable_weights, 0) layer.v = v self.assertLen(layer.trainable_weights, 1) model = testing_utils.get_model_from_layers([layer], input_shape=(10,)) model.compile( keras.optimizer_v2.gradient_descent.SGD(0.1), 'mae', run_eagerly=testing_utils.should_run_eagerly()) x, y = np.ones((10, 10), 'float32'), 2 * np.ones((10, 10), 'float32') model.fit(x, y, batch_size=2, epochs=2, validation_data=(x, y)) self.assertLen(model.trainable_weights, 1) self.assertAllClose(keras.backend.get_value(model.trainable_weights[0]), 2.)
def SKIP_test_sparse_input_with_padding(self): input_data = get_input_dataset( sparse_tensor.SparseTensor(indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4])) expected_output = np.array([[1., -1., -1., -1.], [-1., -1., 2., -1.], [-1., -1., -1., -1.]]) layers = [ToDense(pad_value=-1, trainable=False), Final()] model = testing_utils.get_model_from_layers(layers, input_shape=(None, ), input_sparse=True, input_dtype=dtypes.int32) model.compile(optimizer="sgd", loss="mse", metrics=["accuracy"], run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) output = model.predict(input_data) self.assertAllEqual(output, expected_output)
def test_raw_variable_assignment(self): class RawVariableLayer(keras.layers.Layer): def __init__(self, **kwargs): super(RawVariableLayer, self).__init__(**kwargs) # Test variables in nested structure. self.var_list = [ variables.Variable(1.), { 'a': variables.Variable(2.) } ] def call(self, inputs): return inputs * self.var_list[0] * self.var_list[1]['a'] model = testing_utils.get_model_from_layers([RawVariableLayer()], input_shape=(10, )) model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly()) x, y = np.ones((10, 10)), np.ones((10, 10)) # Checks that variables get initialized. model.fit(x, y, batch_size=2, epochs=2)
def SKIP_test_ragged_input_RNN_layer(self, layer): input_data = get_input_dataset( ragged_factory_ops.constant([[1, 2, 3, 4, 5], [5, 6]])) layers = [ ToDense(pad_value=7, mask=True), keras.layers.Embedding(8, 16), layer(16), keras.layers.Dense(3, activation="softmax"), keras.layers.Dense(1, activation="sigmoid") ] model = testing_utils.get_model_from_layers(layers, input_shape=(None, ), input_ragged=True, input_dtype=dtypes.int32) model.compile(optimizer="rmsprop", loss="binary_crossentropy", metrics=["accuracy"], run_eagerly=testing_utils.should_run_eagerly()) output = model.predict(input_data) self.assertAllEqual(np.zeros((2, 1)).shape, output.shape)
def test_vector_classification_shared_model(self): # Test that Sequential models that feature internal updates # and internal losses can be shared. np.random.seed(1337) (x_train, y_train), _ = testing_utils.get_test_data( train_samples=100, test_samples=0, input_shape=(10,), num_classes=2) y_train = keras.utils.to_categorical(y_train) base_model = testing_utils.get_model_from_layers( [keras.layers.Dense(16, activation='relu', kernel_regularizer=keras.regularizers.l2(1e-5), bias_regularizer=keras.regularizers.l2(1e-5)), keras.layers.BatchNormalization()], input_shape=x_train.shape[1:]) x = keras.layers.Input(x_train.shape[1:]) y = base_model(x) y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y) model = keras.models.Model(x, y) model.compile( loss='categorical_crossentropy', optimizer=keras.optimizer_v2.adam.Adam(0.005), metrics=['acc'], run_eagerly=testing_utils.should_run_eagerly()) if not testing_utils.should_run_eagerly(): self.assertEqual(len(model.losses), 2) self.assertEqual(len(model.updates), 2) history = model.fit(x_train, y_train, epochs=10, batch_size=10, validation_data=(x_train, y_train), verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) _, val_acc = model.evaluate(x_train, y_train) self.assertAlmostEqual(history.history['val_acc'][-1], val_acc) predictions = model.predict(x_train) self.assertEqual(predictions.shape, (x_train.shape[0], 2))
def test_metrics_correctness_with_iterator(self): layers = [ keras.layers.Dense(8, activation='relu', input_dim=4, kernel_initializer='ones'), keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones') ] model = testing_utils.get_model_from_layers(layers, (4, )) model.compile(loss='binary_crossentropy', metrics=['accuracy', metrics_module.BinaryAccuracy()], optimizer='rmsprop', run_eagerly=testing_utils.should_run_eagerly(), run_distributed=testing_utils.should_run_distributed()) np.random.seed(123) x = np.random.randint(10, size=(100, 4)).astype(np.float32) y = np.random.randint(2, size=(100, 1)).astype(np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.batch(10) iterator = dataset_ops.make_one_shot_iterator(dataset) outs = model.evaluate(iterator, steps=10) self.assertEqual(np.around(outs[1], decimals=1), 0.5) self.assertEqual(np.around(outs[2], decimals=1), 0.5) y = np.zeros((100, 1), dtype=np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) dataset = dataset.batch(10) iterator = dataset_ops.make_one_shot_iterator(dataset) outs = model.evaluate(iterator, steps=10) self.assertEqual(outs[1], 0.) self.assertEqual(outs[2], 0.)
def test_build_with_derived_constant(self): class BuildDerivedConstantLayer(keras.layers.Layer): def build(self, input_shape): a = ops.convert_to_tensor(1.0) b = 2.0 * a self.variable = variables.Variable(b) self.constant = ops.convert_to_tensor(self.variable) def call(self, inputs): return self.variable * self.constant * inputs layer = BuildDerivedConstantLayer() model = testing_utils.get_model_from_layers( [layer, keras.layers.Dense(1)], input_shape=(1,)) x = ops.convert_to_tensor([[3.0]]) self.assertEqual( tf_utils.is_symbolic_tensor(model(x)), not context.executing_eagerly()) self.assertEqual( tf_utils.is_symbolic_tensor(layer(x)), not context.executing_eagerly()) self.assertAllClose(keras.backend.get_value(layer(x)), [[12.0]])
def test_image_classification(self): if testing_utils.should_run_distributed(): self.skipTest('b/137397816') np.random.seed(1337) (x_train, y_train), _ = testing_utils.get_test_data(train_samples=100, test_samples=0, input_shape=(10, 10, 3), num_classes=2) y_train = keras.utils.to_categorical(y_train) layers = [ keras.layers.Conv2D(4, 3, padding='same', activation='relu'), keras.layers.Conv2D(8, 3, padding='same'), keras.layers.BatchNormalization(), keras.layers.Conv2D(8, 3, padding='same'), keras.layers.Flatten(), keras.layers.Dense(y_train.shape[-1], activation='softmax') ] model = testing_utils.get_model_from_layers( layers, input_shape=x_train.shape[1:]) model.compile(loss='categorical_crossentropy', optimizer=keras.optimizer_v2.adam.Adam(0.005), metrics=['acc'], run_eagerly=testing_utils.should_run_eagerly(), run_distributed=testing_utils.should_run_distributed()) history = model.fit(x_train, y_train, epochs=10, batch_size=10, validation_data=(x_train, y_train), verbose=2) self.assertGreater(history.history['val_acc'][-1], 0.7) _, val_acc = model.evaluate(x_train, y_train) self.assertAlmostEqual(history.history['val_acc'][-1], val_acc) predictions = model.predict(x_train) self.assertEqual(predictions.shape, (x_train.shape[0], 2))
def test_ModelCheckpoint(self): if h5py is None: return # Skip test if models cannot be saved. layers = [ keras.layers.Dense(NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'), keras.layers.Dense(NUM_CLASSES, activation='softmax') ] model = testing_utils.get_model_from_layers(layers, input_shape=(10,)) model.compile( loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc']) temp_dir = self.get_temp_dir() self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True) filepath = os.path.join(temp_dir, 'checkpoint') (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( train_samples=TRAIN_SAMPLES, test_samples=TEST_SAMPLES, input_shape=(INPUT_DIM,), num_classes=NUM_CLASSES) y_test = keras.utils.to_categorical(y_test) y_train = keras.utils.to_categorical(y_train) # case 1 monitor = 'val_loss' save_best_only = False mode = 'auto' model = keras.models.Sequential() model.add( keras.layers.Dense( NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu')) model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax')) model.compile( loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc']) cbks = [ keras.callbacks.ModelCheckpoint( filepath, monitor=monitor, save_best_only=save_best_only, mode=mode) ] model.fit( x_train, y_train, batch_size=BATCH_SIZE, validation_data=(x_test, y_test), callbacks=cbks, epochs=1, verbose=0) assert os.path.exists(filepath) os.remove(filepath) # case 2 mode = 'min' cbks = [ keras.callbacks.ModelCheckpoint( filepath, monitor=monitor, save_best_only=save_best_only, mode=mode) ] model.fit( x_train, y_train, batch_size=BATCH_SIZE, validation_data=(x_test, y_test), callbacks=cbks, epochs=1, verbose=0) assert os.path.exists(filepath) os.remove(filepath) # case 3 mode = 'max' monitor = 'val_acc' cbks = [ keras.callbacks.ModelCheckpoint( filepath, monitor=monitor, save_best_only=save_best_only, mode=mode) ] model.fit( x_train, y_train, batch_size=BATCH_SIZE, validation_data=(x_test, y_test), callbacks=cbks, epochs=1, verbose=0) assert os.path.exists(filepath) os.remove(filepath) # case 4 save_best_only = True cbks = [ keras.callbacks.ModelCheckpoint( filepath, monitor=monitor, save_best_only=save_best_only, mode=mode) ] model.fit( x_train, y_train, batch_size=BATCH_SIZE, validation_data=(x_test, y_test), callbacks=cbks, epochs=1, verbose=0) assert os.path.exists(filepath) os.remove(filepath) # Case: metric not available. cbks = [ keras.callbacks.ModelCheckpoint( filepath, monitor='unknown', save_best_only=True) ] model.fit( x_train, y_train, batch_size=BATCH_SIZE, validation_data=(x_test, y_test), callbacks=cbks, epochs=1, verbose=0) # File won't be written. assert not os.path.exists(filepath) # case 5 save_best_only = False period = 2 mode = 'auto' filepath = os.path.join(temp_dir, 'checkpoint.{epoch:02d}.h5') cbks = [ keras.callbacks.ModelCheckpoint( filepath, monitor=monitor, save_best_only=save_best_only, mode=mode, period=period) ] model.fit( x_train, y_train, batch_size=BATCH_SIZE, validation_data=(x_test, y_test), callbacks=cbks, epochs=4, verbose=1) assert os.path.exists(filepath.format(epoch=2)) assert os.path.exists(filepath.format(epoch=4)) os.remove(filepath.format(epoch=2)) os.remove(filepath.format(epoch=4)) assert not os.path.exists(filepath.format(epoch=1)) assert not os.path.exists(filepath.format(epoch=3)) # Invalid use: this will raise a warning but not an Exception. keras.callbacks.ModelCheckpoint( filepath, monitor=monitor, save_best_only=save_best_only, mode='unknown')
def _get_model(input_shape=(4,)): model_layers = _get_layers(input_shape=None, add_input_layer=False) return testing_utils.get_model_from_layers( model_layers, input_shape=input_shape)
def test_revive(self): input_shape = None if testing_utils.get_model_type() == 'functional': input_shape = (2, 3) layer_with_config = CustomLayerWithConfig(1., 2) layer_without_config = CustomLayerNoConfig(3., 4) subclassed_with_config = SubclassedModelWithConfig(4., 6.) subclassed_without_config = SubclassedModelNoConfig(7., 8.) inputs = keras.Input((2, 3)) x = CustomLayerWithConfig(1., 2)(inputs) x = CustomLayerNoConfig(3., 4)(x) x = SubclassedModelWithConfig(4., 6.)(x) x = SubclassedModelNoConfig(7., 8.)(x) inner_model_functional = keras.Model(inputs, x) inner_model_sequential = keras.Sequential( [CustomLayerWithConfig(1., 2), CustomLayerNoConfig(3., 4), SubclassedModelWithConfig(4., 6.), SubclassedModelNoConfig(7., 8.)]) class SubclassedModel(keras.Model): def __init__(self): super(SubclassedModel, self).__init__() self.all_layers = [CustomLayerWithConfig(1., 2), CustomLayerNoConfig(3., 4), SubclassedModelWithConfig(4., 6.), SubclassedModelNoConfig(7., 8.)] def call(self, inputs): x = inputs for layer in self.all_layers: x = layer(x) return x inner_model_subclassed = SubclassedModel() layers = [layer_with_config, layer_without_config, subclassed_with_config, subclassed_without_config, inner_model_functional, inner_model_sequential, inner_model_subclassed] model = testing_utils.get_model_from_layers( layers, input_shape=input_shape) # The inputs attribute must be defined in order to save the model. if not model.inputs: model._set_inputs(tensor_spec.TensorSpec((None, 2, 3))) # Test that the correct checkpointed values are loaded, whether the layer is # created from the config or SavedModel. layer_with_config.c.assign(2 * layer_with_config.c) layer_without_config.c.assign(3 * layer_without_config.c) model.save(self.path, save_format='tf') revived = keras_load.load(self.path) self._assert_revived_correctness(model, revived)
def test_revive(self): input_shape = None if testing_utils.get_model_type() == 'functional': input_shape = (2, 3) layer_with_config = CustomLayerWithConfig(1., 2) layer_without_config = CustomLayerNoConfig(3., 4) subclassed_with_config = SubclassedModelWithConfig(4., 6.) subclassed_without_config = SubclassedModelNoConfig(7., 8.) inputs = keras.Input((2, 3)) x = CustomLayerWithConfig(1., 2)(inputs) x = CustomLayerNoConfig(3., 4)(x) x = SubclassedModelWithConfig(4., 6.)(x) x = SubclassedModelNoConfig(7., 8.)(x) inner_model_functional = keras.Model(inputs, x) inner_model_sequential = keras.Sequential([ CustomLayerWithConfig(1., 2), CustomLayerNoConfig(3., 4), SubclassedModelWithConfig(4., 6.), SubclassedModelNoConfig(7., 8.) ]) class SubclassedModel(keras.Model): def __init__(self): super(SubclassedModel, self).__init__() self.all_layers = [ CustomLayerWithConfig(1., 2), CustomLayerNoConfig(3., 4), SubclassedModelWithConfig(4., 6.), SubclassedModelNoConfig(7., 8.) ] def call(self, inputs): x = inputs for layer in self.all_layers: x = layer(x) return x inner_model_subclassed = SubclassedModel() layers = [ layer_with_config, layer_without_config, subclassed_with_config, subclassed_without_config, inner_model_functional, inner_model_sequential, inner_model_subclassed ] model = testing_utils.get_model_from_layers(layers, input_shape=input_shape) # Run data through the Model to create save spec and weights. model.predict(np.ones((10, 2, 3)), batch_size=10) # Test that the correct checkpointed values are loaded, whether the layer is # created from the config or SavedModel. layer_with_config.c.assign(2 * layer_with_config.c) layer_without_config.c.assign(3 * layer_without_config.c) model.save(self.path, save_format='tf') revived = keras_load.load(self.path) self._assert_revived_correctness(model, revived)
def test_model(self, strategy_fn, use_operator=False, use_regularizer=False, policy_name='mixed_float16', get_config=False, save_format=None, use_input_spec=False): self._skip_if_strategy_unsupported(strategy_fn) self._skip_if_save_format_unsupported(save_format) regularizer = (mp_test_util.IdentityRegularizer() if use_regularizer else None) with strategy_fn().scope(): # Pass loss_scale=None, as this test will fail if the DynamicLossScale # skips applying gradients for a step with policy.policy_scope( policy.Policy(policy_name, loss_scale=None)): layer = mp_test_util.MultiplyLayer(assert_type=dtypes.float16, use_operator=use_operator, regularizer=regularizer, input_shape=(1, )) if use_input_spec: layer.input_spec = input_spec.InputSpec(shape=(2, 1)) model = testing_utils.get_model_from_layers( [layer], input_shape=(1, ), input_dtype=dtypes.float16) if get_config: config = model.get_config() model = model.__class__.from_config( config, custom_objects={ 'MultiplyLayer': mp_test_util.MultiplyLayer }) (layer, ) = ( layer for layer in model.layers if isinstance(layer, mp_test_util.MultiplyLayer)) def loss_fn(y_true, y_pred): del y_true return math_ops.reduce_mean(y_pred) # Learning rate is small enough that if applied to a float16 variable, # the variable will not change. So this tests the learning rate not # applied to a float16 value, but instead the float32 variable. opt = gradient_descent.SGD(2**-14) model.compile(opt, loss=loss_fn, run_eagerly=testing_utils.should_run_eagerly()) x = np.ones((2, 1)) y = np.ones((2, 1)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2) model.fit(dataset) # Variable starts at 1, and should have gradient of 2 ** -14 subtracted # from it. expected = 1 - 2**-14 if use_regularizer: # Regularizer adds another 2 ** -14 to the gradient. expected -= 2**-14 self.assertEqual(backend.eval(layer.v), expected) if save_format: with generic_utils.CustomObjectScope({ 'MultiplyLayer': mp_test_util.MultiplyLayer, 'loss_fn': loss_fn }): self._test_saving(model, dataset, save_format, use_regularizer)
def test_layer_as_activation(self): layer = keras.layers.Dense(1, activation=keras.layers.ReLU()) model = testing_utils.get_model_from_layers([layer], input_shape=(10,)) model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly()) model.fit(np.ones((10, 10)), np.ones((10, 1)), batch_size=2)
def test_control_flow_layer(self, layer_class): model = testing_utils.get_model_from_layers([layer_class()], input_shape=(3, )) model.compile(rmsprop.RMSprop(0.001), loss='mse') model.train_on_batch(np.random.random((2, 3)), np.random.random( (2, 3)))