def test_vector_classification(self):
np.random.seed(1337)
(x_train, y_train), _ = testing_utils.get_test_data(
train_samples=100,
test_samples=0,
input_shape=(10,),
num_classes=2)
y_train = keras.utils.to_categorical(y_train)
model = testing_utils.get_model_from_layers(
[keras.layers.Dense(16, activation='relu'),
keras.layers.Dropout(0.1),
keras.layers.Dense(y_train.shape[-1], activation='softmax')],
input_shape=x_train.shape[1:])
model.compile(
loss='categorical_crossentropy',
optimizer=keras.optimizer_v2.adam.Adam(0.005),
metrics=['acc'],
run_eagerly=testing_utils.should_run_eagerly())
history = model.fit(x_train, y_train, epochs=10, batch_size=10,
validation_data=(x_train, y_train),
verbose=2)
self.assertGreater(history.history['val_acc'][-1], 0.7)
_, val_acc = model.evaluate(x_train, y_train)
self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
predictions = model.predict(x_train)
self.assertEqual(predictions.shape, (x_train.shape[0], 2))
def _get_simple_bias_model(self):
model = testing_utils.get_model_from_layers([Bias()], input_shape=(1,))
model.compile(
keras.optimizer_v2.gradient_descent.SGD(0.1),
'mae',
run_eagerly=testing_utils.should_run_eagerly())
return model
def test_metrics_correctness_with_iterator(self):
layers = [
keras.layers.Dense(8, activation='relu', input_dim=4,
kernel_initializer='ones'),
keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones')
]
model = testing_utils.get_model_from_layers(layers, (4,))
model.compile(
loss='binary_crossentropy',
metrics=['accuracy', metrics_module.BinaryAccuracy()],
optimizer='rmsprop',
run_eagerly=testing_utils.should_run_eagerly())
np.random.seed(123)
x = np.random.randint(10, size=(100, 4)).astype(np.float32)
y = np.random.randint(2, size=(100, 1)).astype(np.float32)
dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
dataset = dataset.batch(10)
iterator = dataset_ops.make_one_shot_iterator(dataset)
outs = model.evaluate(iterator, steps=10)
self.assertEqual(np.around(outs[1], decimals=1), 0.5)
self.assertEqual(np.around(outs[2], decimals=1), 0.5)
y = np.zeros((100, 1), dtype=np.float32)
dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
dataset = dataset.repeat(100)
dataset = dataset.batch(10)
iterator = dataset_ops.make_one_shot_iterator(dataset)
outs = model.evaluate(iterator, steps=10)
self.assertEqual(outs[1], 0.)
self.assertEqual(outs[2], 0.)
def add_metric_step(defun):
optimizer = keras.optimizer_v2.rmsprop.RMSprop()
model = testing_utils.get_model_from_layers([
LayerWithMetrics(),
keras.layers.Dense(1, kernel_initializer='zeros', activation='softmax')
],
input_shape=(10,))
def train_step(x, y):
with backprop.GradientTape() as tape:
y_pred_1 = model(x)
y_pred_2 = model(2 * x)
y_pred = y_pred_1 + y_pred_2
loss = keras.losses.mean_squared_error(y, y_pred)
gradients = tape.gradient(loss, model.trainable_weights)
optimizer.apply_gradients(zip(gradients, model.trainable_weights))
assert len(model.metrics) == 2
return [m.result() for m in model.metrics]
if defun:
train_step = def_function.function(train_step)
x, y = array_ops.ones((10, 10)), array_ops.zeros((10, 1))
metrics = train_step(x, y)
assert np.allclose(metrics[0], 1.5)
assert np.allclose(metrics[1], 1.5)
return metrics
def test_build_with_numpy_data(self):
model_layers = [
keras.layers.Dense(3, activation='relu', kernel_initializer='ones'),
keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones')
]
model = testing_utils.get_model_from_layers(model_layers, input_shape=(4,))
model(np.zeros((2, 4), dtype='float32'))
self.assertTrue(model.built)
def test_zero_regularization(self):
# Verifies that training with zero regularization works.
x, y = np.ones((10, 10)), np.ones((10, 3))
model = testing_utils.get_model_from_layers(
[keras.layers.Dense(3, kernel_regularizer=keras.regularizers.l2(0))],
input_shape=(10,))
model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly())
model.fit(x, y, batch_size=5, epochs=1)
def _get_model(self):
layers = [
keras.layers.Conv2D(8, (3, 3)),
keras.layers.Flatten(),
keras.layers.Dense(1)
]
model = testing_utils.get_model_from_layers(layers, input_shape=(10, 10, 1))
model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly())
return model
def _get_model(self):
layers = [
keras.layers.Dense(10, activation='relu'),
keras.layers.Dense(1, activation='sigmoid')
]
model = testing_utils.get_model_from_layers(layers, input_shape=(10,))
model.compile(
adam.AdamOptimizer(0.001),
'binary_crossentropy',
run_eagerly=testing_utils.should_run_eagerly())
return model
def test_model_with_internal_sparse_tensors(self):
# Create a model that accepts an input, converts it to Sparse, and
# converts the sparse tensor back to a dense tensor.
layers = [ToSparse(), ToDense(default_value=-1)]
model = testing_utils.get_model_from_layers(layers, input_shape=(None,))
# Define some input data with additional padding.
input_data = np.array([[1, 0, 0], [2, 3, 0]])
expected_output = np.array([[1, -1, -1], [2, 3, -1]])
output = model.predict(input_data)
self.assertAllEqual(expected_output, output)
def test_model_with_ragged_tensor_rebatched_outputs(self):
# Create a model that accepts an input, converts it to Ragged, and
# converts the ragged tensor back to a dense tensor.
layers = [ToRagged(padding=0)]
model = testing_utils.get_model_from_layers(layers, input_shape=(None,))
# Define some input data with additional padding.
input_data = np.array([[1, 0, 0], [2, 3, 0], [4, 0, 0], [5, 6, 0]])
output = model.predict(input_data, batch_size=2)
expected_values = [[1], [2, 3], [4], [5, 6]]
self.assertRaggedEqual(expected_values, output)
def _get_model(self):
x = layers.Dense(3, kernel_initializer='ones', trainable=False)
out = layers.Dense(
1, kernel_initializer='ones', name='output', trainable=False)
model = testing_utils.get_model_from_layers([x, out], input_shape=(1,))
model.compile(
optimizer='rmsprop',
loss='mse',
metrics=[metrics.MeanSquaredError()],
weighted_metrics=[metrics.MeanSquaredError()],
run_eagerly=testing_utils.should_run_eagerly())
return model
def _get_model(self, input_shape=None):
layers = [
keras.layers.Dense(3, activation='relu'),
keras.layers.Dense(2, activation='softmax')
]
model = testing_utils.get_model_from_layers(layers, input_shape=input_shape)
model.compile(
loss='mse',
optimizer='rmsprop',
metrics=[keras.metrics.CategoricalAccuracy(name='my_acc')],
run_eagerly=testing_utils.should_run_eagerly())
return model
def test_loss_correctness(self):
# Test that training loss is the same in eager and graph
# (by comparing it to a reference value in a deterministic case)
layers = [
keras.layers.Dense(3, activation='relu',
kernel_initializer='ones'),
keras.layers.Dense(2, activation='softmax', kernel_initializer='ones')]
model = testing_utils.get_model_from_layers(layers, input_shape=(4,))
model.compile(loss='sparse_categorical_crossentropy',
optimizer=rmsprop.RMSprop(learning_rate=0.001),
run_eagerly=testing_utils.should_run_eagerly())
x = np.ones((100, 4))
np.random.seed(123)
y = np.random.randint(0, 1, size=(100, 1))
history = model.fit(x, y, epochs=1, batch_size=10)
self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
def test_model_with_sparse_tensor_outputs(self):
# Create a model that accepts an input, converts it to Ragged, and
# converts the ragged tensor back to a dense tensor.
layers = [ToSparse()]
model = testing_utils.get_model_from_layers(layers, input_shape=(None,))
# Define some input data with additional padding.
input_data = np.array([[1, 0, 0], [2, 3, 0]])
output = model.predict(input_data)
expected_indices = np.array([[0, 0], [1, 0], [1, 1]])
expected_values = np.array([1, 2, 3])
expected_dense_shape = np.array([2, 3])
self.assertAllEqual(output.indices, expected_indices)
self.assertAllEqual(output.values, expected_values)
self.assertAllEqual(output.dense_shape, expected_dense_shape)
def test_validation_dataset_with_no_step_arg(self):
# Create a model that learns y=Mx.
layers = [core.Dense(1)]
model = testing_utils.get_model_from_layers(layers, input_shape=(1,))
model.compile(loss="mse", optimizer="adam", metrics=["mean_absolute_error"])
train_dataset = self.create_dataset(num_samples=200, batch_size=10)
eval_dataset = self.create_dataset(num_samples=50, batch_size=25)
history = model.fit(x=train_dataset, validation_data=eval_dataset, epochs=2)
evaluation = model.evaluate(x=eval_dataset)
# If the fit call used the entire dataset, then the final val MAE error
# from the fit history should be equal to the final element in the output
# of evaluating the model on the same eval dataset.
self.assertAlmostEqual(history.history["val_mean_absolute_error"][-1],
evaluation[-1])
def test_lambda_with_variable_in_model(self):
def lambda_fn(x):
# Variable will only get created once.
v = variables.Variable(1., trainable=True)
return x * v
model = testing_utils.get_model_from_layers(
[keras.layers.Lambda(lambda_fn)], input_shape=(10,))
model.compile(
keras.optimizer_v2.gradient_descent.SGD(0.1),
'mae',
run_eagerly=testing_utils.should_run_eagerly())
x, y = np.ones((10, 10), 'float32'), 2 * np.ones((10, 10), 'float32')
model.fit(x, y, batch_size=2, epochs=2, validation_data=(x, y))
self.assertLen(model.trainable_weights, 1)
self.assertAllClose(keras.backend.get_value(model.trainable_weights[0]), 2.)
def test_raw_variable_assignment(self):
class RawVariableLayer(keras.layers.Layer):
def __init__(self, **kwargs):
super(RawVariableLayer, self).__init__(**kwargs)
# Test variables in nested structure.
self.var_list = [variables.Variable(1.), {'a': variables.Variable(2.)}]
def call(self, inputs):
return inputs * self.var_list[0] * self.var_list[1]['a']
model = testing_utils.get_model_from_layers([RawVariableLayer()],
input_shape=(10,))
model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly())
x, y = np.ones((10, 10)), np.ones((10, 10))
# Checks that variables get initialized.
model.fit(x, y, batch_size=2, epochs=2)
def test_loss_correctness(self):
# Test that training loss is the same in eager and graph
# (by comparing it to a reference value in a deterministic case)
layers = [
keras.layers.Dense(3, activation='relu',
kernel_initializer='ones'),
keras.layers.Dense(2,
activation='softmax',
kernel_initializer='ones')
]
model = testing_utils.get_model_from_layers(layers, input_shape=(4, ))
model.compile(loss='sparse_categorical_crossentropy',
optimizer=rmsprop.RMSprop(learning_rate=0.001),
run_eagerly=testing_utils.should_run_eagerly())
x = np.ones((100, 4))
np.random.seed(123)
y = np.random.randint(0, 1, size=(100, 1))
history = model.fit(x, y, epochs=1, batch_size=10)
self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
def test_training_arg_in_defun(self):
layer = self._get_layer_with_training_arg()
model = testing_utils.get_model_from_layers([layer], input_shape=(1, ))
model.compile(rmsprop.RMSprop(0.), loss='mae')
history = model.fit(np.zeros((1, 1)), np.zeros((1, 1)))
self.assertEqual(history.history['loss'][0], 1.)
loss = model.evaluate(np.zeros((1, 1)), np.zeros((1, 1)))
self.assertEqual(loss, 0.)
# Test that the argument injection performed in `call` is not active
# when the argument is passed explicitly.
layer = self._get_layer_with_training_arg()
inputs = keras.Input(shape=(1, ))
# Pass `training` by name
outputs = layer(inputs, training=False)
model = keras.Model(inputs, outputs)
model.compile(rmsprop.RMSprop(0.), loss='mae')
history = model.fit(np.zeros((1, 1)), np.zeros((1, 1)))
self.assertEqual(history.history['loss'][0], 0.)
def add_loss_step(defun):
optimizer = keras.optimizer_v2.adam.Adam()
model = testing_utils.get_model_from_layers([LayerWithLosses()],
input_shape=(10, ))
def train_step(x):
with backprop.GradientTape() as tape:
model(x)
assert len(model.losses) == 2
loss = math_ops.reduce_sum(model.losses)
gradients = tape.gradient(loss, model.trainable_weights)
optimizer.apply_gradients(zip(gradients, model.trainable_weights))
return loss
if defun:
train_step = def_function.function(train_step)
x = array_ops.ones((10, 10))
return train_step(x)
def _get_model(self):
x = layers.Dense(3, kernel_initializer='ones', trainable=False)
out = layers.Dense(1,
kernel_initializer='ones',
name='output',
trainable=False)
model = testing_utils.get_model_from_layers([x, out],
input_shape=(1, ))
model.compile(
optimizer='rmsprop',
loss='mse',
metrics=[metrics.MeanSquaredError(name='mean_squared_error')],
weighted_metrics=[
metrics.MeanSquaredError(name='mean_squared_error_2')
],
run_eagerly=testing_utils.should_run_eagerly(),
experimental_run_tf_function=testing_utils.should_run_tf_function(
))
return model
def test_sparse_tensor_outputs(self):
# Create a model that accepts an input, converts it to Ragged, and
# converts the ragged tensor back to a dense tensor.
layers = [ToSparse()]
model = testing_utils.get_model_from_layers(layers,
input_shape=(None, ))
model._run_eagerly = testing_utils.should_run_eagerly()
# Define some input data with additional padding.
input_data = np.array([[1, 0, 0], [2, 3, 0]])
output = model.predict(input_data)
expected_indices = np.array([[0, 0], [1, 0], [1, 1]])
expected_values = np.array([1, 2, 3])
expected_dense_shape = np.array([2, 3])
self.assertAllEqual(output.indices, expected_indices)
self.assertAllEqual(output.values, expected_values)
self.assertAllEqual(output.dense_shape, expected_dense_shape)
def add_loss_step(defun):
optimizer = keras.optimizer_v2.adam.Adam()
model = testing_utils.get_model_from_layers([LayerWithLosses()],
input_shape=(10,))
def train_step(x):
with backprop.GradientTape() as tape:
model(x)
assert len(model.losses) == 2
loss = math_ops.reduce_sum(model.losses)
gradients = tape.gradient(loss, model.trainable_weights)
optimizer.apply_gradients(zip(gradients, model.trainable_weights))
return loss
if defun:
train_step = def_function.function(train_step)
x = array_ops.ones((10, 10))
return train_step(x)
def test_transitive_variable_creation(self):
dense = keras.layers.Dense(1,
use_bias=False,
kernel_initializer='ones')
def bad_lambda_fn(x):
return dense(x + 1) # Dense layer is built on first call
expected_error = textwrap.dedent(r'''
( )?The following Variables were created within a Lambda layer \(bias_dense\)
( )?but are not tracked by said layer:
( )? <tf.Variable \'.*bias_dense/dense/kernel:0\'.+
( )?The layer cannot safely ensure proper Variable reuse.+''')
with self.assertRaisesRegexp(ValueError, expected_error):
layer = keras.layers.Lambda(bad_lambda_fn, name='bias_dense')
model = testing_utils.get_model_from_layers([layer],
input_shape=(1, ))
model(array_ops.ones((4, 1)))
def SKIP_test_ragged_input_with_padding(self):
input_data = get_input_dataset(
ragged_factory_ops.constant([[[1, 2, 3, 4, 5]], [[2], [3]]]))
expected_output = np.array([[[1., 2., 3., 4., 5.],
[-1., -1., -1., -1., -1.]],
[[2., -1., -1., -1., -1.],
[3., -1., -1., -1., -1.]]])
layers = [ToDense(pad_value=-1), Final()]
model = testing_utils.get_model_from_layers(layers,
input_shape=(None, None),
input_ragged=True,
input_dtype=dtypes.int32)
model.compile(optimizer="sgd",
loss="mse",
metrics=["accuracy"],
run_eagerly=testing_utils.should_run_eagerly())
output = model.predict(input_data)
self.assertAllEqual(output, expected_output)
def test_vector_classification_shared_model(self):
# Test that Sequential models that feature internal updates
# and internal losses can be shared.
np.random.seed(1337)
(x_train, y_train), _ = testing_utils.get_test_data(train_samples=100,
test_samples=0,
input_shape=(10, ),
num_classes=2)
y_train = keras.utils.to_categorical(y_train)
base_model = testing_utils.get_model_from_layers(
[
keras.layers.Dense(
16,
activation='relu',
kernel_regularizer=keras.regularizers.l2(1e-5),
bias_regularizer=keras.regularizers.l2(1e-5)),
keras.layers.BatchNormalization()
],
input_shape=x_train.shape[1:])
x = keras.layers.Input(x_train.shape[1:])
y = base_model(x)
y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y)
model = keras.models.Model(x, y)
model.compile(loss='categorical_crossentropy',
optimizer=keras.optimizer_v2.adam.Adam(0.005),
metrics=['acc'],
run_eagerly=testing_utils.should_run_eagerly(),
run_distributed=testing_utils.should_run_distributed())
if not testing_utils.should_run_eagerly():
self.assertEqual(len(model.get_losses_for(None)), 2)
self.assertEqual(len(model.get_updates_for(x)), 2)
history = model.fit(x_train,
y_train,
epochs=10,
batch_size=10,
validation_data=(x_train, y_train),
verbose=2)
self.assertGreater(history.history['val_acc'][-1], 0.7)
_, val_acc = model.evaluate(x_train, y_train)
self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
predictions = model.predict(x_train)
self.assertEqual(predictions.shape, (x_train.shape[0], 2))
def test_warns_on_variable_capture(self):
v = variables.Variable(1., trainable=True)
def lambda_fn(x):
return x * v
expected_warning = textwrap.dedent(r'''
( )?The following Variables were used a Lambda layer\'s call \(lambda\), but
( )?are not present in its tracked objects:
( )? <tf.Variable \'.*Variable:0\'.+
( )?It is possible that this is intended behavior.+''')
layer = keras.layers.Lambda(lambda_fn)
def patched_warn(msg):
raise ValueError(msg)
layer._warn = patched_warn
with self.assertRaisesRegexp(ValueError, expected_warning):
model = testing_utils.get_model_from_layers([layer], input_shape=(1,))
model(array_ops.ones((4, 1)))
def test_sparse_input_shape(self):
input_data = get_input_dataset(
sparse_tensor.SparseTensor(indices=[[0, 0], [1, 2]],
values=[1, 2],
dense_shape=[3, 4]))
expected_output = np.array([[1., 0., 0., 0.], [0., 0., 2., 0.],
[0., 0., 0., 0.]])
layers = [ToDense(shape=[3, 4]), Final()]
model = testing_utils.get_model_from_layers(layers,
input_shape=(None, ),
input_sparse=True,
input_dtype=dtypes.int32)
model.compile(optimizer="sgd",
loss="mse",
metrics=["accuracy"],
run_eagerly=testing_utils.should_run_eagerly())
output = model.predict(input_data)
self.assertAllEqual(output, expected_output)
def test_simple_build_with_constant(self):
class BuildConstantLayer(keras.layers.Layer):
def build(self, input_shape):
self.b = ops.convert_to_tensor(2.0)
def call(self, inputs):
return self.b * inputs
layer = BuildConstantLayer()
model = testing_utils.get_model_from_layers(
[layer, keras.layers.Dense(1)], input_shape=(1,))
x = ops.convert_to_tensor([[3.0]])
self.assertEqual(
tf_utils.is_symbolic_tensor(model(x)), not context.executing_eagerly())
self.assertEqual(
tf_utils.is_symbolic_tensor(layer(x)), not context.executing_eagerly())
self.assertAllClose(keras.backend.get_value(layer(x)), [[6.0]])
def test_loss_correctness_with_iterator(self):
# Test that training loss is the same in eager and graph
# (by comparing it to a reference value in a deterministic case)
layers = [
keras.layers.Dense(3, activation='relu',
kernel_initializer='ones'),
keras.layers.Dense(2, activation='softmax', kernel_initializer='ones')]
model = testing_utils.get_model_from_layers(layers, input_shape=(4,))
model.compile(
loss='sparse_categorical_crossentropy',
optimizer=rmsprop.RMSprop(learning_rate=0.001),
run_eagerly=testing_utils.should_run_eagerly())
x = np.ones((100, 4), dtype=np.float32)
np.random.seed(123)
y = np.random.randint(0, 1, size=(100, 1))
dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
dataset = dataset.repeat(100)
dataset = dataset.batch(10)
iterator = dataset_ops.make_one_shot_iterator(dataset)
history = model.fit(iterator, epochs=1, steps_per_epoch=10)
self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
def test_training_arg_in_defun(self):
layer = self._get_layer_with_training_arg()
model = testing_utils.get_model_from_layers([layer], input_shape=(1,))
model.compile(rmsprop.RMSprop(0.),
loss='mae')
history = model.fit(np.zeros((1, 1)), np.zeros((1, 1)))
self.assertEqual(history.history['loss'][0], 1.)
loss = model.evaluate(np.zeros((1, 1)), np.zeros((1, 1)))
self.assertEqual(loss, 0.)
# Test that the argument injection performed in `call` is not active
# when the argument is passed explicitly.
layer = self._get_layer_with_training_arg()
inputs = keras.Input(shape=(1,))
# Pass `training` by name
outputs = layer(inputs, training=False)
model = keras.Model(inputs, outputs)
model.compile(rmsprop.RMSprop(0.),
loss='mae')
history = model.fit(np.zeros((1, 1)), np.zeros((1, 1)))
self.assertEqual(history.history['loss'][0], 0.)
def test_training_model_with_internal_ragged_tensors(self):
# Create a model that implements y=Mx. This is easy to learn and will
# demonstrate appropriate gradient passing. (We have to use RaggedTensors
# for this test, as ToSparse() doesn't support gradient propagation through
# the layer.) TODO(b/124796939): Investigate this.
layers = [core.Dense(2), ToRagged(padding=0), ToDense(default_value=-1)]
model = testing_utils.get_model_from_layers(layers, input_shape=(1,))
input_data = np.random.rand(1024, 1)
expected_data = np.concatenate((input_data * 3, input_data * .5), axis=-1)
model.compile(
loss="mse",
optimizer="adam",
run_eagerly=testing_utils.should_run_eagerly())
history = model.fit(input_data, expected_data, epochs=10, verbose=0)
# If the model trained, the loss stored at history[0] should be different
# than the one stored at history[-1].
self.assertNotEqual(history.history["loss"][-1], history.history["loss"][0])
def _testAddUpdate(self, scope):
with scope:
layer_with_update = LayerWithUpdate(dtype=dtypes.int32)
model = testing_utils.get_model_from_layers([layer_with_update],
input_shape=(3,),
input_dtype=dtypes.int32)
if testing_utils.get_model_type() == 'subclass':
model._set_inputs(constant_op.constant([[1, 2, 3]], dtype=dtypes.int32))
self.evaluate(variables.variables_initializer(model.variables))
saved_model_dir = self._save_model_dir()
model.save(saved_model_dir, save_format='tf')
loaded = keras_load.load(saved_model_dir)
loaded_layer = loaded.layers[-1]
self.evaluate(variables.variables_initializer(loaded.variables))
self.assertEqual(self.evaluate(loaded_layer.v), 0)
loaded.predict(constant_op.constant([[1, 2, 3]], dtype=dtypes.int32),
steps=1)
self.assertEqual(self.evaluate(loaded_layer.v), 6)
def test_training_internal_ragged_tensors(self):
# Create a model that implements y=Mx. This is easy to learn and will
# demonstrate appropriate gradient passing. (We have to use RaggedTensors
# for this test, as ToSparse() doesn't support gradient propagation through
# the layer.) TODO(b/124796939): Investigate this.
layers = [core.Dense(2), ToRagged(padding=0), ToDense(default_value=-1)]
model = testing_utils.get_model_from_layers(layers, input_shape=(1,))
input_data = np.random.rand(1024, 1)
expected_data = np.concatenate((input_data * 3, input_data * .5), axis=-1)
model.compile(
loss="mse",
optimizer="adam",
run_eagerly=testing_utils.should_run_eagerly())
history = model.fit(input_data, expected_data, epochs=10, verbose=0)
# If the model trained, the loss stored at history[0] should be different
# than the one stored at history[-1].
self.assertNotEqual(history.history["loss"][-1], history.history["loss"][0])
def test_loss_correctness_with_iterator(self):
# Test that training loss is the same in eager and graph
# (by comparing it to a reference value in a deterministic case)
layers = [
keras.layers.Dense(3, activation='relu',
kernel_initializer='ones'),
keras.layers.Dense(2, activation='softmax', kernel_initializer='ones')]
model = testing_utils.get_model_from_layers(layers, input_shape=(4,))
model.compile(
loss='sparse_categorical_crossentropy',
optimizer=rmsprop.RMSprop(learning_rate=0.001),
run_eagerly=testing_utils.should_run_eagerly(),
experimental_run_tf_function=testing_utils.should_run_tf_function())
x = np.ones((100, 4), dtype=np.float32)
np.random.seed(123)
y = np.random.randint(0, 1, size=(100, 1))
dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
dataset = dataset.repeat(100)
dataset = dataset.batch(10)
history = model.fit(dataset, epochs=1, steps_per_epoch=10)
self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
def _testAddUpdate(self, scope):
with scope:
layer_with_update = LayerWithUpdate()
model = testing_utils.get_model_from_layers([layer_with_update],
input_shape=(3, ))
x = np.ones((10, 3))
if testing_utils.get_model_type() == 'subclass':
model.predict(x, batch_size=10)
self.evaluate(variables.variables_initializer(model.variables))
saved_model_dir = self._save_model_dir()
model.save(saved_model_dir, save_format='tf')
loaded = keras_load.load(saved_model_dir)
loaded_layer = loaded.layers[-1]
self.evaluate(variables.variables_initializer(loaded.variables))
self.assertEqual(self.evaluate(loaded_layer.v), 0.)
loaded.compile('sgd', 'mse')
loaded.fit(x, x, batch_size=10)
self.assertEqual(self.evaluate(loaded_layer.v), 1.)
def test_validation_dataset_with_no_step_arg(self):
# Create a model that learns y=Mx.
layers = [core.Dense(1)]
model = testing_utils.get_model_from_layers(layers, input_shape=(1, ))
model.compile(loss="mse",
optimizer="adam",
metrics=["mean_absolute_error"])
train_dataset = self.create_dataset(num_samples=200, batch_size=10)
eval_dataset = self.create_dataset(num_samples=50, batch_size=25)
history = model.fit(x=train_dataset,
validation_data=eval_dataset,
epochs=2)
evaluation = model.evaluate(x=eval_dataset)
# If the fit call used the entire dataset, then the final val MAE error
# from the fit history should be equal to the final element in the output
# of evaluating the model on the same eval dataset.
self.assertAlmostEqual(history.history["val_mean_absolute_error"][-1],
evaluation[-1])
def batch_norm_step(defun):
optimizer = keras.optimizer_v2.adadelta.Adadelta()
model = testing_utils.get_model_from_layers([
keras.layers.BatchNormalization(momentum=0.9),
keras.layers.Dense(1, kernel_initializer='zeros', activation='softmax')
],
input_shape=(10, ))
def train_step(x, y):
with backprop.GradientTape() as tape:
y_pred = model(x, training=True)
loss = keras.losses.binary_crossentropy(y, y_pred)
gradients = tape.gradient(loss, model.trainable_weights)
optimizer.apply_gradients(zip(gradients, model.trainable_weights))
return loss, model(x, training=False)
if defun:
train_step = def_function.function(train_step)
x, y = array_ops.ones((10, 10)), array_ops.ones((10, 1))
return train_step(x, y)
def batch_norm_step(defun):
optimizer = keras.optimizer_v2.adadelta.Adadelta()
model = testing_utils.get_model_from_layers([
keras.layers.BatchNormalization(momentum=0.9),
keras.layers.Dense(1, kernel_initializer='zeros', activation='softmax')
],
input_shape=(10,))
def train_step(x, y):
with backprop.GradientTape() as tape:
y_pred = model(x, training=True)
loss = keras.losses.binary_crossentropy(y, y_pred)
gradients = tape.gradient(loss, model.trainable_weights)
optimizer.apply_gradients(zip(gradients, model.trainable_weights))
return loss, model(x, training=False)
if defun:
train_step = def_function.function(train_step)
x, y = array_ops.ones((10, 10)), array_ops.ones((10, 1))
return train_step(x, y)
def test_build_with_derived_constant(self):
class BuildDerivedConstantLayer(keras.layers.Layer):
def build(self, input_shape):
a = ops.convert_to_tensor_v2(1.0)
b = 2.0 * a
self.variable = variables.Variable(b)
self.constant = ops.convert_to_tensor_v2(self.variable)
def call(self, inputs):
return self.variable * self.constant * inputs
layer = BuildDerivedConstantLayer()
model = testing_utils.get_model_from_layers(
[layer, keras.layers.Dense(1)], input_shape=(1, ))
x = ops.convert_to_tensor_v2([[3.0]])
self.assertEqual(tf_utils.is_symbolic_tensor(model(x)),
not context.executing_eagerly())
self.assertEqual(tf_utils.is_symbolic_tensor(layer(x)),
not context.executing_eagerly())
self.assertAllClose(keras.backend.get_value(layer(x)), [[12.0]])
def test_lambda_with_variable_in_model(self):
v = variables.Variable(1., trainable=True)
def lambda_fn(x, v):
return x * v
# While it is generally not advised to mix Variables with Lambda layers, if
# the variables are explicitly set as attributes then they are still
# tracked. This is consistent with the base Layer behavior.
layer = keras.layers.Lambda(lambda_fn, arguments={'v': v})
self.assertLen(layer.trainable_weights, 0)
layer.v = v
self.assertLen(layer.trainable_weights, 1)
model = testing_utils.get_model_from_layers([layer], input_shape=(10,))
model.compile(
keras.optimizer_v2.gradient_descent.SGD(0.1),
'mae',
run_eagerly=testing_utils.should_run_eagerly())
x, y = np.ones((10, 10), 'float32'), 2 * np.ones((10, 10), 'float32')
model.fit(x, y, batch_size=2, epochs=2, validation_data=(x, y))
self.assertLen(model.trainable_weights, 1)
self.assertAllClose(keras.backend.get_value(model.trainable_weights[0]), 2.)
def SKIP_test_sparse_input_with_padding(self):
input_data = get_input_dataset(
sparse_tensor.SparseTensor(indices=[[0, 0], [1, 2]],
values=[1, 2],
dense_shape=[3, 4]))
expected_output = np.array([[1., -1., -1., -1.], [-1., -1., 2., -1.],
[-1., -1., -1., -1.]])
layers = [ToDense(pad_value=-1, trainable=False), Final()]
model = testing_utils.get_model_from_layers(layers,
input_shape=(None, ),
input_sparse=True,
input_dtype=dtypes.int32)
model.compile(optimizer="sgd",
loss="mse",
metrics=["accuracy"],
run_eagerly=testing_utils.should_run_eagerly(),
experimental_run_tf_function=testing_utils.
should_run_tf_function())
output = model.predict(input_data)
self.assertAllEqual(output, expected_output)
def test_raw_variable_assignment(self):
class RawVariableLayer(keras.layers.Layer):
def __init__(self, **kwargs):
super(RawVariableLayer, self).__init__(**kwargs)
# Test variables in nested structure.
self.var_list = [
variables.Variable(1.), {
'a': variables.Variable(2.)
}
]
def call(self, inputs):
return inputs * self.var_list[0] * self.var_list[1]['a']
model = testing_utils.get_model_from_layers([RawVariableLayer()],
input_shape=(10, ))
model.compile('sgd',
'mse',
run_eagerly=testing_utils.should_run_eagerly())
x, y = np.ones((10, 10)), np.ones((10, 10))
# Checks that variables get initialized.
model.fit(x, y, batch_size=2, epochs=2)
def SKIP_test_ragged_input_RNN_layer(self, layer):
input_data = get_input_dataset(
ragged_factory_ops.constant([[1, 2, 3, 4, 5], [5, 6]]))
layers = [
ToDense(pad_value=7, mask=True),
keras.layers.Embedding(8, 16),
layer(16),
keras.layers.Dense(3, activation="softmax"),
keras.layers.Dense(1, activation="sigmoid")
]
model = testing_utils.get_model_from_layers(layers,
input_shape=(None, ),
input_ragged=True,
input_dtype=dtypes.int32)
model.compile(optimizer="rmsprop",
loss="binary_crossentropy",
metrics=["accuracy"],
run_eagerly=testing_utils.should_run_eagerly())
output = model.predict(input_data)
self.assertAllEqual(np.zeros((2, 1)).shape, output.shape)
def test_vector_classification_shared_model(self):
# Test that Sequential models that feature internal updates
# and internal losses can be shared.
np.random.seed(1337)
(x_train, y_train), _ = testing_utils.get_test_data(
train_samples=100,
test_samples=0,
input_shape=(10,),
num_classes=2)
y_train = keras.utils.to_categorical(y_train)
base_model = testing_utils.get_model_from_layers(
[keras.layers.Dense(16,
activation='relu',
kernel_regularizer=keras.regularizers.l2(1e-5),
bias_regularizer=keras.regularizers.l2(1e-5)),
keras.layers.BatchNormalization()],
input_shape=x_train.shape[1:])
x = keras.layers.Input(x_train.shape[1:])
y = base_model(x)
y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y)
model = keras.models.Model(x, y)
model.compile(
loss='categorical_crossentropy',
optimizer=keras.optimizer_v2.adam.Adam(0.005),
metrics=['acc'],
run_eagerly=testing_utils.should_run_eagerly())
if not testing_utils.should_run_eagerly():
self.assertEqual(len(model.losses), 2)
self.assertEqual(len(model.updates), 2)
history = model.fit(x_train, y_train, epochs=10, batch_size=10,
validation_data=(x_train, y_train),
verbose=2)
self.assertGreater(history.history['val_acc'][-1], 0.7)
_, val_acc = model.evaluate(x_train, y_train)
self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
predictions = model.predict(x_train)
self.assertEqual(predictions.shape, (x_train.shape[0], 2))
def test_metrics_correctness_with_iterator(self):
layers = [
keras.layers.Dense(8,
activation='relu',
input_dim=4,
kernel_initializer='ones'),
keras.layers.Dense(1,
activation='sigmoid',
kernel_initializer='ones')
]
model = testing_utils.get_model_from_layers(layers, (4, ))
model.compile(loss='binary_crossentropy',
metrics=['accuracy',
metrics_module.BinaryAccuracy()],
optimizer='rmsprop',
run_eagerly=testing_utils.should_run_eagerly(),
run_distributed=testing_utils.should_run_distributed())
np.random.seed(123)
x = np.random.randint(10, size=(100, 4)).astype(np.float32)
y = np.random.randint(2, size=(100, 1)).astype(np.float32)
dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
dataset = dataset.batch(10)
iterator = dataset_ops.make_one_shot_iterator(dataset)
outs = model.evaluate(iterator, steps=10)
self.assertEqual(np.around(outs[1], decimals=1), 0.5)
self.assertEqual(np.around(outs[2], decimals=1), 0.5)
y = np.zeros((100, 1), dtype=np.float32)
dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
dataset = dataset.repeat(100)
dataset = dataset.batch(10)
iterator = dataset_ops.make_one_shot_iterator(dataset)
outs = model.evaluate(iterator, steps=10)
self.assertEqual(outs[1], 0.)
self.assertEqual(outs[2], 0.)
def test_build_with_derived_constant(self):
class BuildDerivedConstantLayer(keras.layers.Layer):
def build(self, input_shape):
a = ops.convert_to_tensor(1.0)
b = 2.0 * a
self.variable = variables.Variable(b)
self.constant = ops.convert_to_tensor(self.variable)
def call(self, inputs):
return self.variable * self.constant * inputs
layer = BuildDerivedConstantLayer()
model = testing_utils.get_model_from_layers(
[layer, keras.layers.Dense(1)], input_shape=(1,))
x = ops.convert_to_tensor([[3.0]])
self.assertEqual(
tf_utils.is_symbolic_tensor(model(x)), not context.executing_eagerly())
self.assertEqual(
tf_utils.is_symbolic_tensor(layer(x)), not context.executing_eagerly())
self.assertAllClose(keras.backend.get_value(layer(x)), [[12.0]])
def test_image_classification(self):
if testing_utils.should_run_distributed():
self.skipTest('b/137397816')
np.random.seed(1337)
(x_train,
y_train), _ = testing_utils.get_test_data(train_samples=100,
test_samples=0,
input_shape=(10, 10, 3),
num_classes=2)
y_train = keras.utils.to_categorical(y_train)
layers = [
keras.layers.Conv2D(4, 3, padding='same', activation='relu'),
keras.layers.Conv2D(8, 3, padding='same'),
keras.layers.BatchNormalization(),
keras.layers.Conv2D(8, 3, padding='same'),
keras.layers.Flatten(),
keras.layers.Dense(y_train.shape[-1], activation='softmax')
]
model = testing_utils.get_model_from_layers(
layers, input_shape=x_train.shape[1:])
model.compile(loss='categorical_crossentropy',
optimizer=keras.optimizer_v2.adam.Adam(0.005),
metrics=['acc'],
run_eagerly=testing_utils.should_run_eagerly(),
run_distributed=testing_utils.should_run_distributed())
history = model.fit(x_train,
y_train,
epochs=10,
batch_size=10,
validation_data=(x_train, y_train),
verbose=2)
self.assertGreater(history.history['val_acc'][-1], 0.7)
_, val_acc = model.evaluate(x_train, y_train)
self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
predictions = model.predict(x_train)
self.assertEqual(predictions.shape, (x_train.shape[0], 2))
def test_ModelCheckpoint(self):
if h5py is None:
return # Skip test if models cannot be saved.
layers = [
keras.layers.Dense(NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'),
keras.layers.Dense(NUM_CLASSES, activation='softmax')
]
model = testing_utils.get_model_from_layers(layers, input_shape=(10,))
model.compile(
loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc'])
temp_dir = self.get_temp_dir()
self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
filepath = os.path.join(temp_dir, 'checkpoint')
(x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
train_samples=TRAIN_SAMPLES,
test_samples=TEST_SAMPLES,
input_shape=(INPUT_DIM,),
num_classes=NUM_CLASSES)
y_test = keras.utils.to_categorical(y_test)
y_train = keras.utils.to_categorical(y_train)
# case 1
monitor = 'val_loss'
save_best_only = False
mode = 'auto'
model = keras.models.Sequential()
model.add(
keras.layers.Dense(
NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'))
model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax'))
model.compile(
loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc'])
cbks = [
keras.callbacks.ModelCheckpoint(
filepath,
monitor=monitor,
save_best_only=save_best_only,
mode=mode)
]
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=1,
verbose=0)
assert os.path.exists(filepath)
os.remove(filepath)
# case 2
mode = 'min'
cbks = [
keras.callbacks.ModelCheckpoint(
filepath,
monitor=monitor,
save_best_only=save_best_only,
mode=mode)
]
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=1,
verbose=0)
assert os.path.exists(filepath)
os.remove(filepath)
# case 3
mode = 'max'
monitor = 'val_acc'
cbks = [
keras.callbacks.ModelCheckpoint(
filepath,
monitor=monitor,
save_best_only=save_best_only,
mode=mode)
]
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=1,
verbose=0)
assert os.path.exists(filepath)
os.remove(filepath)
# case 4
save_best_only = True
cbks = [
keras.callbacks.ModelCheckpoint(
filepath,
monitor=monitor,
save_best_only=save_best_only,
mode=mode)
]
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=1,
verbose=0)
assert os.path.exists(filepath)
os.remove(filepath)
# Case: metric not available.
cbks = [
keras.callbacks.ModelCheckpoint(
filepath,
monitor='unknown',
save_best_only=True)
]
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=1,
verbose=0)
# File won't be written.
assert not os.path.exists(filepath)
# case 5
save_best_only = False
period = 2
mode = 'auto'
filepath = os.path.join(temp_dir, 'checkpoint.{epoch:02d}.h5')
cbks = [
keras.callbacks.ModelCheckpoint(
filepath,
monitor=monitor,
save_best_only=save_best_only,
mode=mode,
period=period)
]
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=4,
verbose=1)
assert os.path.exists(filepath.format(epoch=2))
assert os.path.exists(filepath.format(epoch=4))
os.remove(filepath.format(epoch=2))
os.remove(filepath.format(epoch=4))
assert not os.path.exists(filepath.format(epoch=1))
assert not os.path.exists(filepath.format(epoch=3))
# Invalid use: this will raise a warning but not an Exception.
keras.callbacks.ModelCheckpoint(
filepath,
monitor=monitor,
save_best_only=save_best_only,
mode='unknown')
def _get_model(input_shape=(4,)):
model_layers = _get_layers(input_shape=None, add_input_layer=False)
return testing_utils.get_model_from_layers(
model_layers, input_shape=input_shape)
def test_revive(self):
input_shape = None
if testing_utils.get_model_type() == 'functional':
input_shape = (2, 3)
layer_with_config = CustomLayerWithConfig(1., 2)
layer_without_config = CustomLayerNoConfig(3., 4)
subclassed_with_config = SubclassedModelWithConfig(4., 6.)
subclassed_without_config = SubclassedModelNoConfig(7., 8.)
inputs = keras.Input((2, 3))
x = CustomLayerWithConfig(1., 2)(inputs)
x = CustomLayerNoConfig(3., 4)(x)
x = SubclassedModelWithConfig(4., 6.)(x)
x = SubclassedModelNoConfig(7., 8.)(x)
inner_model_functional = keras.Model(inputs, x)
inner_model_sequential = keras.Sequential(
[CustomLayerWithConfig(1., 2),
CustomLayerNoConfig(3., 4),
SubclassedModelWithConfig(4., 6.),
SubclassedModelNoConfig(7., 8.)])
class SubclassedModel(keras.Model):
def __init__(self):
super(SubclassedModel, self).__init__()
self.all_layers = [CustomLayerWithConfig(1., 2),
CustomLayerNoConfig(3., 4),
SubclassedModelWithConfig(4., 6.),
SubclassedModelNoConfig(7., 8.)]
def call(self, inputs):
x = inputs
for layer in self.all_layers:
x = layer(x)
return x
inner_model_subclassed = SubclassedModel()
layers = [layer_with_config,
layer_without_config,
subclassed_with_config,
subclassed_without_config,
inner_model_functional,
inner_model_sequential,
inner_model_subclassed]
model = testing_utils.get_model_from_layers(
layers, input_shape=input_shape)
# The inputs attribute must be defined in order to save the model.
if not model.inputs:
model._set_inputs(tensor_spec.TensorSpec((None, 2, 3)))
# Test that the correct checkpointed values are loaded, whether the layer is
# created from the config or SavedModel.
layer_with_config.c.assign(2 * layer_with_config.c)
layer_without_config.c.assign(3 * layer_without_config.c)
model.save(self.path, save_format='tf')
revived = keras_load.load(self.path)
self._assert_revived_correctness(model, revived)
def _get_model(input_shape=(4,)):
model_layers = _get_layers(input_shape=None, add_input_layer=False)
return testing_utils.get_model_from_layers(
model_layers, input_shape=input_shape)
def test_revive(self):
input_shape = None
if testing_utils.get_model_type() == 'functional':
input_shape = (2, 3)
layer_with_config = CustomLayerWithConfig(1., 2)
layer_without_config = CustomLayerNoConfig(3., 4)
subclassed_with_config = SubclassedModelWithConfig(4., 6.)
subclassed_without_config = SubclassedModelNoConfig(7., 8.)
inputs = keras.Input((2, 3))
x = CustomLayerWithConfig(1., 2)(inputs)
x = CustomLayerNoConfig(3., 4)(x)
x = SubclassedModelWithConfig(4., 6.)(x)
x = SubclassedModelNoConfig(7., 8.)(x)
inner_model_functional = keras.Model(inputs, x)
inner_model_sequential = keras.Sequential([
CustomLayerWithConfig(1., 2),
CustomLayerNoConfig(3., 4),
SubclassedModelWithConfig(4., 6.),
SubclassedModelNoConfig(7., 8.)
])
class SubclassedModel(keras.Model):
def __init__(self):
super(SubclassedModel, self).__init__()
self.all_layers = [
CustomLayerWithConfig(1., 2),
CustomLayerNoConfig(3., 4),
SubclassedModelWithConfig(4., 6.),
SubclassedModelNoConfig(7., 8.)
]
def call(self, inputs):
x = inputs
for layer in self.all_layers:
x = layer(x)
return x
inner_model_subclassed = SubclassedModel()
layers = [
layer_with_config, layer_without_config, subclassed_with_config,
subclassed_without_config, inner_model_functional,
inner_model_sequential, inner_model_subclassed
]
model = testing_utils.get_model_from_layers(layers,
input_shape=input_shape)
# Run data through the Model to create save spec and weights.
model.predict(np.ones((10, 2, 3)), batch_size=10)
# Test that the correct checkpointed values are loaded, whether the layer is
# created from the config or SavedModel.
layer_with_config.c.assign(2 * layer_with_config.c)
layer_without_config.c.assign(3 * layer_without_config.c)
model.save(self.path, save_format='tf')
revived = keras_load.load(self.path)
self._assert_revived_correctness(model, revived)
def test_ModelCheckpoint(self):
if h5py is None:
return # Skip test if models cannot be saved.
layers = [
keras.layers.Dense(NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'),
keras.layers.Dense(NUM_CLASSES, activation='softmax')
]
model = testing_utils.get_model_from_layers(layers, input_shape=(10,))
model.compile(
loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc'])
temp_dir = self.get_temp_dir()
self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
filepath = os.path.join(temp_dir, 'checkpoint')
(x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
train_samples=TRAIN_SAMPLES,
test_samples=TEST_SAMPLES,
input_shape=(INPUT_DIM,),
num_classes=NUM_CLASSES)
y_test = keras.utils.to_categorical(y_test)
y_train = keras.utils.to_categorical(y_train)
# case 1
monitor = 'val_loss'
save_best_only = False
mode = 'auto'
model = keras.models.Sequential()
model.add(
keras.layers.Dense(
NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu'))
model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax'))
model.compile(
loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc'])
cbks = [
keras.callbacks.ModelCheckpoint(
filepath,
monitor=monitor,
save_best_only=save_best_only,
mode=mode)
]
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=1,
verbose=0)
assert os.path.exists(filepath)
os.remove(filepath)
# case 2
mode = 'min'
cbks = [
keras.callbacks.ModelCheckpoint(
filepath,
monitor=monitor,
save_best_only=save_best_only,
mode=mode)
]
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=1,
verbose=0)
assert os.path.exists(filepath)
os.remove(filepath)
# case 3
mode = 'max'
monitor = 'val_acc'
cbks = [
keras.callbacks.ModelCheckpoint(
filepath,
monitor=monitor,
save_best_only=save_best_only,
mode=mode)
]
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=1,
verbose=0)
assert os.path.exists(filepath)
os.remove(filepath)
# case 4
save_best_only = True
cbks = [
keras.callbacks.ModelCheckpoint(
filepath,
monitor=monitor,
save_best_only=save_best_only,
mode=mode)
]
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=1,
verbose=0)
assert os.path.exists(filepath)
os.remove(filepath)
# Case: metric not available.
cbks = [
keras.callbacks.ModelCheckpoint(
filepath,
monitor='unknown',
save_best_only=True)
]
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=1,
verbose=0)
# File won't be written.
assert not os.path.exists(filepath)
# case 5
save_best_only = False
period = 2
mode = 'auto'
filepath = os.path.join(temp_dir, 'checkpoint.{epoch:02d}.h5')
cbks = [
keras.callbacks.ModelCheckpoint(
filepath,
monitor=monitor,
save_best_only=save_best_only,
mode=mode,
period=period)
]
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=4,
verbose=1)
assert os.path.exists(filepath.format(epoch=2))
assert os.path.exists(filepath.format(epoch=4))
os.remove(filepath.format(epoch=2))
os.remove(filepath.format(epoch=4))
assert not os.path.exists(filepath.format(epoch=1))
assert not os.path.exists(filepath.format(epoch=3))
# Invalid use: this will raise a warning but not an Exception.
keras.callbacks.ModelCheckpoint(
filepath,
monitor=monitor,
save_best_only=save_best_only,
mode='unknown')
def test_model(self,
strategy_fn,
use_operator=False,
use_regularizer=False,
policy_name='mixed_float16',
get_config=False,
save_format=None,
use_input_spec=False):
self._skip_if_strategy_unsupported(strategy_fn)
self._skip_if_save_format_unsupported(save_format)
regularizer = (mp_test_util.IdentityRegularizer()
if use_regularizer else None)
with strategy_fn().scope():
# Pass loss_scale=None, as this test will fail if the DynamicLossScale
# skips applying gradients for a step
with policy.policy_scope(
policy.Policy(policy_name, loss_scale=None)):
layer = mp_test_util.MultiplyLayer(assert_type=dtypes.float16,
use_operator=use_operator,
regularizer=regularizer,
input_shape=(1, ))
if use_input_spec:
layer.input_spec = input_spec.InputSpec(shape=(2, 1))
model = testing_utils.get_model_from_layers(
[layer], input_shape=(1, ), input_dtype=dtypes.float16)
if get_config:
config = model.get_config()
model = model.__class__.from_config(
config,
custom_objects={
'MultiplyLayer': mp_test_util.MultiplyLayer
})
(layer, ) = (
layer for layer in model.layers
if isinstance(layer, mp_test_util.MultiplyLayer))
def loss_fn(y_true, y_pred):
del y_true
return math_ops.reduce_mean(y_pred)
# Learning rate is small enough that if applied to a float16 variable,
# the variable will not change. So this tests the learning rate not
# applied to a float16 value, but instead the float32 variable.
opt = gradient_descent.SGD(2**-14)
model.compile(opt,
loss=loss_fn,
run_eagerly=testing_utils.should_run_eagerly())
x = np.ones((2, 1))
y = np.ones((2, 1))
dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2)
model.fit(dataset)
# Variable starts at 1, and should have gradient of 2 ** -14 subtracted
# from it.
expected = 1 - 2**-14
if use_regularizer:
# Regularizer adds another 2 ** -14 to the gradient.
expected -= 2**-14
self.assertEqual(backend.eval(layer.v), expected)
if save_format:
with generic_utils.CustomObjectScope({
'MultiplyLayer':
mp_test_util.MultiplyLayer,
'loss_fn':
loss_fn
}):
self._test_saving(model, dataset, save_format, use_regularizer)
def test_layer_as_activation(self):
layer = keras.layers.Dense(1, activation=keras.layers.ReLU())
model = testing_utils.get_model_from_layers([layer], input_shape=(10,))
model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly())
model.fit(np.ones((10, 10)), np.ones((10, 1)), batch_size=2)
def test_control_flow_layer(self, layer_class):
model = testing_utils.get_model_from_layers([layer_class()],
input_shape=(3, ))
model.compile(rmsprop.RMSprop(0.001), loss='mse')
model.train_on_batch(np.random.random((2, 3)), np.random.random(
(2, 3)))
