def testConstructRMSpropWithLR(self):
opt = rmsprop.RMSprop(lr=1.0)
opt_2 = rmsprop.RMSprop(learning_rate=0.1, lr=1.0)
opt_3 = rmsprop.RMSprop(learning_rate=0.1)
self.assertIsInstance(opt.lr, tf.Variable)
self.assertIsInstance(opt_2.lr, tf.Variable)
self.assertIsInstance(opt_3.lr, tf.Variable)
self.evaluate(tf.compat.v1.global_variables_initializer())
self.assertAllClose(self.evaluate(opt.lr), (1.0))
self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
def test_generator_input_to_fit_eval_predict(self):
val_data = np.ones([10, 10], np.float32), np.ones([10, 1], np.float32)
def ones_generator():
while True:
yield np.ones([10, 10], np.float32), np.ones([10, 1],
np.float32)
model = testing_utils.get_small_mlp(num_hidden=10,
num_classes=1,
input_dim=10)
model.compile(rmsprop.RMSprop(0.001),
'binary_crossentropy',
run_eagerly=testing_utils.should_run_eagerly())
model.fit(ones_generator(),
steps_per_epoch=2,
validation_data=val_data,
epochs=2)
model.evaluate(ones_generator(), steps=2)
model.predict(ones_generator(), steps=2)
# Test with a changing batch size
model = testing_utils.get_small_mlp(num_hidden=3,
num_classes=4,
input_dim=2)
model.compile(loss='mse',
optimizer=rmsprop.RMSprop(1e-3),
metrics=['mae',
metrics_module.CategoricalAccuracy()])
model.fit_generator(custom_generator_changing_batch_size(),
steps_per_epoch=5,
epochs=1,
verbose=1,
max_queue_size=10,
use_multiprocessing=False)
model.fit_generator(
custom_generator_changing_batch_size(),
steps_per_epoch=5,
epochs=1,
verbose=1,
max_queue_size=10,
use_multiprocessing=False,
validation_data=custom_generator_changing_batch_size(),
validation_steps=10)
model.fit(custom_generator_changing_batch_size(),
steps_per_epoch=5,
validation_data=custom_generator_changing_batch_size(),
validation_steps=10,
epochs=2)
model.evaluate(custom_generator_changing_batch_size(), steps=5)
model.predict(custom_generator_changing_batch_size(), steps=5)
def test_control_flow_in_deferred_sequential_model(self):
model = keras.Sequential(
[ControlFlowLayer1(),
keras.layers.Dense(3),
ControlFlowLayer2()])
model.compile(rmsprop.RMSprop(0.001), loss='mse')
model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3)))
def test_fit_generator_method(self):
model = testing_utils.get_small_mlp(num_hidden=3,
num_classes=4,
input_dim=2)
model.compile(loss='mse',
optimizer=rmsprop.RMSprop(1e-3),
metrics=['mae',
metrics_module.CategoricalAccuracy()])
model.fit_generator(custom_generator_threads(),
steps_per_epoch=5,
epochs=1,
verbose=1,
max_queue_size=10,
workers=4,
use_multiprocessing=True)
model.fit_generator(custom_generator(),
steps_per_epoch=5,
epochs=1,
verbose=1,
max_queue_size=10,
use_multiprocessing=False)
model.fit_generator(custom_generator(),
steps_per_epoch=5,
epochs=1,
verbose=1,
max_queue_size=10,
use_multiprocessing=False,
validation_data=custom_generator(),
validation_steps=10)
model.fit_generator(custom_generator(),
steps_per_epoch=5,
validation_data=custom_generator(),
validation_steps=1,
workers=0)
def test_training_with_sequences(self):
class DummySequence(data_utils.Sequence):
def __getitem__(self, idx):
return np.zeros([10, 2]), np.ones([10, 4])
def __len__(self):
return 10
model = testing_utils.get_small_mlp(num_hidden=3,
num_classes=4,
input_dim=2)
model.compile(loss='mse', optimizer=rmsprop.RMSprop(1e-3))
model.fit_generator(DummySequence(),
steps_per_epoch=10,
validation_data=custom_generator(),
validation_steps=1,
max_queue_size=10,
workers=0,
use_multiprocessing=True)
model.fit_generator(DummySequence(),
steps_per_epoch=10,
validation_data=custom_generator(),
validation_steps=1,
max_queue_size=10,
workers=0,
use_multiprocessing=False)
def testMinimizeSparseResourceVariableCentered(self):
# TODO(tanzheny, omalleyt): Fix test in eager mode.
with tf.Graph().as_default():
for dtype in _DATA_TYPES:
if test_util.is_xla_enabled() and dtype.is_complex:
self.skipTest("b/143578550")
var0 = tf.Variable([[1.0, 2.0]], dtype=dtype)
x = tf.constant([[4.0], [5.0]], dtype=dtype)
def loss():
pred = tf.matmul(
tf.compat.v1.nn.embedding_lookup([var0], [0]), x) # pylint: disable=cell-var-from-loop
return pred * pred
# loss = lambda: pred * pred # pylint: disable=cell-var-from-loop
sgd_op = rmsprop.RMSprop(learning_rate=1.0,
rho=0.0,
momentum=0.0,
epsilon=1.0,
centered=True).minimize(
loss, var_list=[var0])
self.evaluate(tf.compat.v1.global_variables_initializer())
# Fetch params to validate initial values
self.assertAllCloseAccordingToType([[1.0, 2.0]],
self.evaluate(var0))
# Run 1 step of sgd
self.evaluate(sgd_op)
# Validate updated params
self.assertAllCloseAccordingToType([[-111, -138]],
self.evaluate(var0),
atol=0.01)
def test_generator_methods_with_sample_weights(self):
model = testing_utils.get_small_mlp(num_hidden=3,
num_classes=4,
input_dim=2)
model.compile(loss='mse',
optimizer=rmsprop.RMSprop(1e-3),
metrics=['mae',
metrics_module.CategoricalAccuracy()],
run_eagerly=testing_utils.should_run_eagerly())
model.fit_generator(custom_generator(mode=3),
steps_per_epoch=5,
epochs=1,
verbose=1,
max_queue_size=10,
use_multiprocessing=False)
model.fit_generator(custom_generator(mode=3),
steps_per_epoch=5,
epochs=1,
verbose=1,
max_queue_size=10,
use_multiprocessing=False,
validation_data=custom_generator(mode=3),
validation_steps=10)
model.predict_generator(custom_generator(mode=3),
steps=5,
max_queue_size=10,
use_multiprocessing=False)
model.evaluate_generator(custom_generator(mode=3),
steps=5,
max_queue_size=10,
use_multiprocessing=False)
def test_evaluate_generator_method(self):
model = testing_utils.get_small_mlp(num_hidden=3,
num_classes=4,
input_dim=2)
model.compile(loss='mse',
optimizer=rmsprop.RMSprop(1e-3),
metrics=['mae',
metrics_module.CategoricalAccuracy()],
run_eagerly=testing_utils.should_run_eagerly())
model.evaluate_generator(custom_generator_threads(),
steps=5,
max_queue_size=10,
workers=2,
verbose=1,
use_multiprocessing=True)
model.evaluate_generator(custom_generator(),
steps=5,
max_queue_size=10,
use_multiprocessing=False)
model.evaluate_generator(custom_generator(),
steps=5,
max_queue_size=10,
use_multiprocessing=False,
workers=0)
def test_model_fit_and_validation_with_missing_arg_errors(self):
model = testing_utils.get_small_mlp(10, 4, 3)
model.compile(optimizer=rmsprop.RMSprop(learning_rate=0.001),
loss='mse',
run_eagerly=True)
x = tf.zeros(shape=(10, 3))
y = tf.zeros(shape=(10, 4))
dataset = tf.data.Dataset.from_tensor_slices(
(x, y)).repeat(10).batch(5)
validation_dataset = tf.data.Dataset.from_tensor_slices(
(x, y)).repeat().batch(5) # Infinite dataset.
model.fit(dataset, epochs=1, verbose=0)
# Step argument is required for infinite datasets.
with self.assertRaises(ValueError):
model.fit(dataset,
steps_per_epoch=2,
epochs=1,
verbose=0,
validation_data=validation_dataset)
with self.assertRaises(ValueError):
model.fit(dataset,
steps_per_epoch=2,
epochs=1,
verbose=0,
validation_data=validation_dataset)
def test_generator_methods(self):
model = testing_utils.get_small_mlp(10, 4, 3)
optimizer = rmsprop.RMSprop(learning_rate=0.001)
model.compile(optimizer,
loss='mse',
metrics=['mae',
metrics_module.CategoricalAccuracy()],
run_eagerly=True)
x = np.random.random((10, 3))
y = np.random.random((10, 4))
def numpy_iterator():
while True:
yield x, y
model.fit_generator(numpy_iterator(), steps_per_epoch=3, epochs=1)
model.evaluate_generator(numpy_iterator(), steps=3)
def inference_numpy_iterator():
while True:
yield x
out = model.predict_generator(inference_numpy_iterator(), steps=3)
self.assertEqual(out.shape, (30, 4))
def test_sequence_input_to_fit_eval_predict(self):
val_data = np.ones([10, 10], np.float32), np.ones([10, 1], np.float32)
class CustomSequence(data_utils.Sequence):
def __getitem__(self, idx):
return np.ones([10, 10], np.float32), np.ones([10, 1],
np.float32)
def __len__(self):
return 2
class CustomSequenceChangingBatchSize(data_utils.Sequence):
def __getitem__(self, idx):
batch_size = 10 - idx
return (np.ones([batch_size, 10], np.float32),
np.ones([batch_size, 1], np.float32))
def __len__(self):
return 2
model = testing_utils.get_small_mlp(num_hidden=10,
num_classes=1,
input_dim=10)
model.compile(rmsprop.RMSprop(0.001), 'binary_crossentropy')
model.fit(CustomSequence(), validation_data=val_data, epochs=2)
model.evaluate(CustomSequence())
model.predict(CustomSequence())
with self.assertRaisesRegex(ValueError,
'`y` argument is not supported'):
model.fit(CustomSequence(), y=np.ones([10, 1]))
with self.assertRaisesRegex(
ValueError, '`sample_weight` argument is not supported'):
model.fit(CustomSequence(), sample_weight=np.ones([10, 1]))
model.compile(rmsprop.RMSprop(0.001), 'binary_crossentropy')
model.fit(CustomSequenceChangingBatchSize(),
validation_data=val_data,
epochs=2)
model.evaluate(CustomSequenceChangingBatchSize())
model.predict(CustomSequenceChangingBatchSize())
def testTrainAndServeWithKPL(self, distribution):
use_adapt = False
test_utils_obj = kpl_test_utils.DistributeKplTestUtils()
with distribution.scope():
feature_mapper, label_mapper = test_utils_obj.define_kpls_for_training(
use_adapt)
model = test_utils_obj.define_model()
optimizer = rmsprop.RMSprop(learning_rate=0.1)
accuracy = keras.metrics.Accuracy()
def dataset_fn(_):
return test_utils_obj.dataset_fn(feature_mapper, label_mapper)
@tf.function
def train_step(iterator):
"""The step function for one training step."""
def step_fn(inputs):
"""The computation to run on each replica(GPU)."""
features, labels = inputs
with tf.GradientTape() as tape:
pred = model(features, training=True)
loss = keras.losses.binary_crossentropy(labels, pred)
loss = tf.nn.compute_average_loss(loss)
grads = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(
list(zip(grads, model.trainable_variables)))
actual_pred = tf.cast(tf.greater(pred, 0.5), tf.int64)
accuracy.update_state(labels, actual_pred)
distribution.run(step_fn, args=(next(iterator), ))
distributed_dataset = distribution.distribute_datasets_from_function(
dataset_fn)
distributed_iterator = iter(distributed_dataset)
num_epochs = 4
num_steps = 7
for _ in range(num_epochs):
accuracy.reset_state()
for _ in range(num_steps):
train_step(distributed_iterator)
self.assertGreater(accuracy.result().numpy(), 0.5)
self.assertEqual(optimizer.iterations.numpy(),
num_epochs * num_steps)
# Test save/load/serving the trained model.
test_utils_obj.test_save_load_serving_model(
model, feature_mapper,
test_utils_obj.define_reverse_lookup_layer())
def testSlotsUniqueEager(self):
v1 = tf.Variable(1.)
v2 = tf.Variable(1.)
opt = rmsprop.RMSprop(1., momentum=0., centered=False)
opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
# There should be iteration, and one unique slot variable for v1 and v2.
self.assertLen(set({id(v) for v in opt.variables()}), 3)
self.assertEqual(self.evaluate(opt.variables()[0]),
self.evaluate(opt.iterations))
opt = rmsprop.RMSprop(learning_rate=1., momentum=0.2, centered=False)
opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
# There should be iteration, and two unique slot variables for v1 and v2.
self.assertLen(set({id(v) for v in opt.variables()}), 5)
self.assertEqual(self.evaluate(opt.variables()[0]),
self.evaluate(opt.iterations))
opt = rmsprop.RMSprop(learning_rate=1., momentum=0.2, centered=True)
opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
# There should be iteration, and three unique slot variables for v1 and v2
self.assertLen(set({id(v) for v in opt.variables()}), 7)
self.assertEqual(self.evaluate(opt.variables()[0]),
self.evaluate(opt.iterations))
def testCallableParams(self):
for dtype in _DATA_TYPES:
var0 = tf.Variable([1.0, 2.0], dtype=dtype)
var1 = tf.Variable([3.0, 4.0], dtype=dtype)
grads0 = tf.constant([0.1, 0.1], dtype=dtype)
grads1 = tf.constant([0.01, 0.01], dtype=dtype)
learning_rate = lambda: 2.0
rho = lambda: 0.9
momentum = lambda: 0.0
epsilon = 1.0
opt = rmsprop.RMSprop(learning_rate, rho, momentum, epsilon)
# Fetch params to validate initial values
self.assertAllClose([1.0, 2.0], self.evaluate(var0))
self.assertAllClose([3.0, 4.0], self.evaluate(var1))
# Step 1: the rms accumulators where 1. So we should see a normal
# update: v -= grad * learning_rate
opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
# Check the parameters.
self.assertAllCloseAccordingToType(
np.array([
1.0 - (0.1 * 2.0 / math.sqrt(0.001 + 1.0)),
2.0 - (0.1 * 2.0 / math.sqrt(0.001 + 1.0))
]), self.evaluate(var0))
self.assertAllCloseAccordingToType(
np.array([
3.0 - (0.01 * 2.0 / math.sqrt(0.00001 + 1.0)),
4.0 - (0.01 * 2.0 / math.sqrt(0.00001 + 1.0))
]), self.evaluate(var1))
# Step 2: the root mean square accumulators contain the previous update.
opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
# Check the parameters.
self.assertAllCloseAccordingToType(
np.array([
1.0 - (0.1 * 2.0 / math.sqrt(0.001 + 1.0)) -
(0.1 * 2.0 / math.sqrt(0.001 * 0.9 + 0.001 + 1.0)),
2.0 - (0.1 * 2.0 / math.sqrt(0.001 + 1.0)) -
(0.1 * 2.0 / math.sqrt(0.001 * 0.9 + 0.001 + 1.0))
]), self.evaluate(var0))
self.assertAllCloseAccordingToType(
np.array([
3.0 - (0.01 * 2.0 / math.sqrt(0.00001 + 1.0)) -
(0.01 * 2.0 / math.sqrt(0.00001 * 0.9 + 1e-5 + 1.0)),
4.0 - (0.01 * 2.0 / math.sqrt(0.00001 + 1.0)) -
(0.01 * 2.0 / math.sqrt(0.00001 * 0.9 + 1e-5 + 1.0))
]), self.evaluate(var1))
def test_loss_correctness(self, optimizer_kwargs):
# Test that training loss is the same in eager and graph
# (by comparing it to a reference value in a deterministic case)
layers = [
keras.layers.Dense(3, activation='relu',
kernel_initializer='ones'),
keras.layers.Dense(2,
activation='softmax',
kernel_initializer='ones')
]
model = testing_utils.get_model_from_layers(layers, input_shape=(4, ))
model.compile(loss='sparse_categorical_crossentropy',
optimizer=rmsprop.RMSprop(learning_rate=0.001,
**optimizer_kwargs),
run_eagerly=testing_utils.should_run_eagerly())
x = np.ones((100, 4))
np.random.seed(123)
y = np.random.randint(0, 1, size=(100, 1))
history = model.fit(x, y, epochs=1, batch_size=10)
self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
def test_loss_correctness_with_iterator(self):
# Test that training loss is the same in eager and graph
# (by comparing it to a reference value in a deterministic case)
layers = [
keras.layers.Dense(3, activation='relu',
kernel_initializer='ones'),
keras.layers.Dense(2,
activation='softmax',
kernel_initializer='ones')
]
model = testing_utils.get_model_from_layers(layers, input_shape=(4, ))
model.compile(loss='sparse_categorical_crossentropy',
optimizer=rmsprop.RMSprop(learning_rate=0.001),
run_eagerly=testing_utils.should_run_eagerly())
x = np.ones((100, 4), dtype=np.float32)
np.random.seed(123)
y = np.random.randint(0, 1, size=(100, 1))
dataset = tf.data.Dataset.from_tensor_slices((x, y))
dataset = dataset.repeat(100)
dataset = dataset.batch(10)
history = model.fit(dataset, epochs=1, steps_per_epoch=10)
self.assertAlmostEqual(history.history['loss'][-1], 0.5836, 4)
def testRunMinimizeOnGPUForCPUVariables(self, use_resource):
with tf.compat.v1.device("/device:CPU:0"):
if use_resource:
var0 = tf.Variable([1.0, 2.0], dtype=tf.float32)
var1 = tf.Variable([3.0, 4.0], dtype=tf.float32)
else:
var0 = tf.Variable([1.0, 2.0], dtype=tf.float32)
var1 = tf.Variable([3.0, 4.0], dtype=tf.float32)
def loss():
return 5 * var0 + 3 * var1
opt = rmsprop.RMSprop(learning_rate=1.0,
decay=0.9,
momentum=0.5,
epsilon=1.0)
# Fetch params to validate initial values
self.evaluate(tf.compat.v1.global_variables_initializer())
self.assertAllClose([1.0, 2.0], self.evaluate(var0))
self.assertAllClose([3.0, 4.0], self.evaluate(var1))
# Run 1 step through optimizer on GPU.
# Slot variables are created the first time optimizer is used on some
# variable. This tests that slot variables will be colocated with the base
# variable.
with tf.compat.v1.device("/device:GPU:0"):
# Note that for eager execution, minimize expects a function instead of a
# Tensor.
opt_op = opt.minimize(loss, [var0, var1])
self.evaluate(tf.compat.v1.global_variables_initializer())
self.evaluate(opt_op)
# Validate updated params, All variables should have decreased.
self.assertTrue(all(v < 0.0 for v in self.evaluate(var0)),
msg="updated variables: %s" % self.evaluate(var0))
self.assertTrue(all(v < 2.0 for v in self.evaluate(var1)),
msg="updated variables: %s" % self.evaluate(var1))
def test_generator_methods_invalid_use_case(self):
def invalid_generator():
while 1:
yield (0, 0, 0, 0)
model = testing_utils.get_small_mlp(num_hidden=3,
num_classes=4,
input_dim=2)
model.compile(loss='mse',
optimizer=rmsprop.RMSprop(1e-3),
run_eagerly=testing_utils.should_run_eagerly())
with self.assertRaises(ValueError):
model.fit_generator(invalid_generator(),
steps_per_epoch=5,
epochs=1,
verbose=1,
max_queue_size=10,
use_multiprocessing=False)
with self.assertRaises(ValueError):
model.fit_generator(custom_generator(),
steps_per_epoch=5,
epochs=1,
verbose=1,
max_queue_size=10,
use_multiprocessing=False,
validation_data=invalid_generator(),
validation_steps=10)
with self.assertRaises(ValueError):
model.predict_generator(invalid_generator(),
steps=5,
max_queue_size=10,
use_multiprocessing=False)
with self.assertRaises(ValueError):
model.evaluate_generator(invalid_generator(),
steps=5,
max_queue_size=10,
use_multiprocessing=False)
def test_saving_sequential_model(self):
with self.cached_session():
model = keras.models.Sequential()
model.add(keras.layers.Dense(2, input_shape=(3, )))
model.add(keras.layers.RepeatVector(3))
model.add(keras.layers.TimeDistributed(keras.layers.Dense(3)))
model.compile(loss=keras.losses.MSE,
optimizer=rmsprop.RMSprop(lr=0.0001),
metrics=[keras.metrics.categorical_accuracy],
sample_weight_mode='temporal')
x = np.random.random((1, 3))
y = np.random.random((1, 3, 3))
model.train_on_batch(x, y)
ref_y = model.predict(x)
saved_model_dir = self._save_model_dir()
keras_saved_model.export_saved_model(model, saved_model_dir)
loaded_model = keras_saved_model.load_from_saved_model(
saved_model_dir)
y = loaded_model.predict(x)
self.assertAllClose(ref_y, y, atol=1e-05)
def test_saving_functional_model(self):
with self.cached_session():
inputs = keras.layers.Input(shape=(3, ))
x = keras.layers.Dense(2)(inputs)
output = keras.layers.Dense(3)(x)
model = keras.models.Model(inputs, output)
model.compile(loss=keras.losses.MSE,
optimizer=rmsprop.RMSprop(lr=0.0001),
metrics=[keras.metrics.categorical_accuracy])
x = np.random.random((1, 3))
y = np.random.random((1, 3))
model.train_on_batch(x, y)
ref_y = model.predict(x)
saved_model_dir = self._save_model_dir()
keras_saved_model.export_saved_model(model, saved_model_dir)
loaded_model = keras_saved_model.load_from_saved_model(
saved_model_dir)
y = loaded_model.predict(x)
self.assertAllClose(ref_y, y, atol=1e-05)
def test_model_methods_with_eager_tensors_single_io(self):
if not tf.executing_eagerly():
# Only test V2 Function and V2 Eager modes, as V1 Graph mode with
# symbolic tensors has different requirements.
return
model = testing_utils.get_small_mlp(10, 4, 3)
optimizer = rmsprop.RMSprop(learning_rate=0.001)
loss = 'mse'
metrics = ['mae', metrics_module.CategoricalAccuracy()]
model.compile(optimizer,
loss,
metrics=metrics,
run_eagerly=testing_utils.should_run_eagerly())
inputs = tf.zeros(shape=(10, 3))
targets = tf.zeros(shape=(10, 4))
model.fit(inputs, targets, epochs=1, batch_size=2, verbose=0)
model.fit(inputs,
targets,
epochs=1,
batch_size=3,
verbose=0,
shuffle=False)
model.fit(inputs,
targets,
epochs=1,
batch_size=4,
verbose=0,
validation_data=(inputs, targets))
model.evaluate(inputs, targets, batch_size=2, verbose=0)
model.predict(inputs, batch_size=2)
model.train_on_batch(inputs, targets)
model.test_on_batch(inputs, targets)
def testRMSprop(self): self._compare_numerical(rmsprop_new.RMSprop(), rmsprop_old.RMSprop())
def testDenseWithLearningRateInverseTimeDecay(self):
# TODO(tanzheny, omalleyt): Fix test in eager mode.
with tf.Graph().as_default():
var0_np = np.array([1.0, 2.0])
grads0_np = np.array([0.1, 0.2])
var1_np = np.array([3.0, 4.0])
grads1_np = np.array([0.01, 0.2])
var0 = tf.Variable(var0_np)
var1 = tf.Variable(var1_np)
grads0 = tf.constant(grads0_np)
grads1 = tf.constant(grads1_np)
learning_rate = 0.01
rho = 0.9
momentum = 0.0
epsilon = 1e-7
centered = False
decay = 0.5
lr_schedule = learning_rate_schedule.InverseTimeDecay(
learning_rate, decay_steps=1.0, decay_rate=decay)
opt = rmsprop.RMSprop(learning_rate=lr_schedule,
rho=rho,
momentum=momentum,
epsilon=epsilon,
centered=centered)
update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
self.evaluate(tf.compat.v1.global_variables_initializer())
rms0 = opt.get_slot(var0, "rms")
self.assertIsNotNone(rms0)
rms1 = opt.get_slot(var1, "rms")
self.assertIsNotNone(rms1)
if momentum > 0.:
mom0 = opt.get_slot(var0, "momentum")
mom1 = opt.get_slot(var1, "momentum")
else:
mom0 = None
mom1 = None
mg0_np = np.array([0.0, 0.0])
mg1_np = np.array([0.0, 0.0])
rms0_np = np.array([0.0, 0.0])
rms1_np = np.array([0.0, 0.0])
mom0_np = np.array([0.0, 0.0])
mom1_np = np.array([0.0, 0.0])
# Fetch params to validate initial values
self.assertAllClose([1.0, 2.0], self.evaluate(var0))
self.assertAllClose([3.0, 4.0], self.evaluate(var1))
# Run 4 steps of RMSprop
for t in range(2):
self.evaluate(update)
lr = learning_rate / (1 + decay * t)
var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
var0_np, grads0_np, mg0_np, rms0_np, mom0_np, lr, rho,
momentum, epsilon, centered)
var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy(
var1_np, grads1_np, mg1_np, rms1_np, mom1_np, lr, rho,
momentum, epsilon, centered)
# Validate updated params
self.assertAllCloseAccordingToType(rms0_np,
self.evaluate(rms0))
self.assertAllCloseAccordingToType(rms1_np,
self.evaluate(rms1))
if momentum > 0.:
self.assertAllCloseAccordingToType(mom0_np,
self.evaluate(mom0))
self.assertAllCloseAccordingToType(mom1_np,
self.evaluate(mom1))
self.assertAllCloseAccordingToType(var0_np,
self.evaluate(var0))
self.assertAllCloseAccordingToType(var1_np,
self.evaluate(var1))
def test_control_flow_model(self, model_class):
model = model_class()
model.compile(rmsprop.RMSprop(0.001), loss='mse')
model.train_on_batch(np.random.random((2, 3)), np.random.random(
(2, 3)))
adagrad_optimizer_keras_v2_fn = tf.__internal__.test.combinations.NamedObject(
"AdagradKerasV2", lambda: adagrad_keras_v2.Adagrad(0.001))
adam_optimizer_keras_v2_fn = tf.__internal__.test.combinations.NamedObject(
"AdamKerasV2", lambda: adam_keras_v2.Adam(0.001, epsilon=1.0))
adam_experimental_fn = tf.__internal__.test.combinations.NamedObject(
"AdamExperimental", lambda: adam_experimental.Adam(0.001))
adamax_optimizer_keras_v2_fn = tf.__internal__.test.combinations.NamedObject(
"AdamaxKerasV2", lambda: adamax_keras_v2.Adamax(0.001, epsilon=1.0))
nadam_optimizer_keras_v2_fn = tf.__internal__.test.combinations.NamedObject(
"NadamKerasV2", lambda: nadam_keras_v2.Nadam(0.001, epsilon=1.0))
ftrl_optimizer_keras_v2_fn = tf.__internal__.test.combinations.NamedObject(
"FtrlKerasV2", lambda: ftrl_keras_v2.Ftrl(0.001))
gradient_descent_optimizer_keras_v2_fn = tf.__internal__.test.combinations.NamedObject(
"GradientDescentKerasV2", lambda: gradient_descent_keras_v2.SGD(0.001))
rmsprop_optimizer_keras_v2_fn = tf.__internal__.test.combinations.NamedObject(
"RmsPropKerasV2", lambda: rmsprop_keras_v2.RMSprop(0.001))
# TODO(shiningsun): consider adding the other v2 optimizers
optimizers_v2 = [
gradient_descent_optimizer_keras_v2_fn, adagrad_optimizer_keras_v2_fn
]
optimizers_v1_and_v2 = optimizers_v1 + optimizers_v2
def distributions_and_v1_optimizers():
"""A common set of combination with DistributionStrategies and Optimizers."""
return tf.__internal__.test.combinations.combine(
distribution=[
tf.__internal__.distribute.combinations.one_device_strategy,
tf.__internal__.distribute.combinations.
def test_model_methods_with_eager_tensors_multi_io(self):
if not tf.executing_eagerly():
# Only test V2 Function and V2 Eager modes, as V1 Graph mode with
# symbolic tensors has different requirements.
return
input_a = keras.layers.Input(shape=(3, ), name='input_a')
input_b = keras.layers.Input(shape=(3, ), name='input_b')
dense = keras.layers.Dense(4, name='dense')
dropout = keras.layers.Dropout(0.5, name='dropout')
model = testing_utils.get_multi_io_model([input_a, dense],
[input_b, dense, dropout])
optimizer = rmsprop.RMSprop(learning_rate=0.001)
loss = 'mse'
loss_weights = [1., 0.5]
metrics = ['mae', metrics_module.CategoricalAccuracy()]
model.compile(optimizer,
loss,
metrics=metrics,
loss_weights=loss_weights,
run_eagerly=testing_utils.should_run_eagerly(),
sample_weight_mode=None)
input_a = tf.zeros(shape=(10, 3))
input_b = tf.zeros(shape=(10, 3))
target_a = tf.zeros(shape=(10, 4))
target_b = tf.zeros(shape=(10, 4))
model.fit([input_a, input_b], [target_a, target_b],
epochs=1,
batch_size=5,
verbose=0)
# Test: no shuffle.
model.fit([input_a, input_b], [target_a, target_b],
epochs=1,
batch_size=5,
verbose=0,
shuffle=False)
# Test: validation data.
model.fit([input_a, input_b], [target_a, target_b],
epochs=1,
batch_size=2,
verbose=0,
validation_data=([input_a, input_b], [target_a, target_b]))
model.train_on_batch([input_a, input_b], [target_a, target_b])
model.predict([input_a, input_b], batch_size=5)
model.evaluate([input_a, input_b], [target_a, target_b],
batch_size=2,
verbose=0)
model.test_on_batch([input_a, input_b], [target_a, target_b])
# Test: mix np and tensors.
input_b = np.zeros(shape=(10, 3)).astype('float32')
target_b = np.zeros(shape=(10, 4)).astype('float32')
model.fit([input_a, input_b], [target_a, target_b],
epochs=1,
batch_size=5,
verbose=0)
model.fit([input_a, input_b], [target_a, target_b],
epochs=1,
batch_size=2,
verbose=0,
validation_data=([input_a, input_b], [target_a, target_b]))
model.fit([input_a, input_b], [target_a, target_b],
epochs=1,
batch_size=5,
verbose=0,
shuffle=False)
model.train_on_batch([input_a, input_b], [target_a, target_b])
model.predict([input_a, input_b], batch_size=5)
model.evaluate([input_a, input_b], [target_a, target_b],
batch_size=2,
verbose=0)
model.test_on_batch([input_a, input_b], [target_a, target_b])
def testDense(self):
# TODO(tanzheny, omalleyt): Fix test in eager mode.
for (dtype, learning_rate, rho, momentum, epsilon,
centered) in _TESTPARAMS:
with tf.compat.v1.get_default_graph().as_default(
), testing_utils.use_gpu():
# Initialize variables for numpy implementation.
var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
grads0_np = np.array([0.1, 0.2], dtype=dtype.as_numpy_dtype)
var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
grads1_np = np.array([0.01, 0.2], dtype=dtype.as_numpy_dtype)
var0 = tf.Variable(var0_np, dtype=dtype)
var1 = tf.Variable(var1_np, dtype=dtype)
grads0 = tf.constant(grads0_np, dtype=dtype)
grads1 = tf.constant(grads1_np, dtype=dtype)
opt = rmsprop.RMSprop(learning_rate=learning_rate,
rho=rho,
momentum=momentum,
epsilon=epsilon,
centered=centered)
update = opt.apply_gradients(
zip([grads0, grads1], [var0, var1]))
self.evaluate(tf.compat.v1.global_variables_initializer())
if centered:
mg0 = opt.get_slot(var0, "mg")
mg1 = opt.get_slot(var1, "mg")
else:
mg0 = None
mg1 = None
if momentum > 0.:
mom0 = opt.get_slot(var0, "momentum")
mom1 = opt.get_slot(var1, "momentum")
else:
mom0 = None
mom1 = None
rms0 = opt.get_slot(var0, "rms")
self.assertIsNotNone(rms0)
rms1 = opt.get_slot(var1, "rms")
self.assertIsNotNone(rms1)
mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
rms0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
rms1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
# Fetch params to validate initial values
self.assertAllClose([1.0, 2.0], self.evaluate(var0))
self.assertAllClose([3.0, 4.0], self.evaluate(var1))
# Run 3 steps of RMSprop
for _ in range(1, 4):
self.evaluate(update)
var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
var0_np, grads0_np, mg0_np, rms0_np, mom0_np,
learning_rate, rho, momentum, epsilon, centered)
var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy(
var1_np, grads1_np, mg1_np, rms1_np, mom1_np,
learning_rate, rho, momentum, epsilon, centered)
# Validate updated params
if centered:
self.assertAllCloseAccordingToType(
mg0_np, self.evaluate(mg0))
self.assertAllCloseAccordingToType(
mg1_np, self.evaluate(mg1))
if momentum > 0.:
self.assertAllCloseAccordingToType(
mom0_np, self.evaluate(mom0))
self.assertAllCloseAccordingToType(
mom1_np, self.evaluate(mom1))
self.assertAllCloseAccordingToType(rms0_np,
self.evaluate(rms0))
self.assertAllCloseAccordingToType(rms1_np,
self.evaluate(rms1))
self.assertAllCloseAccordingToType(var0_np,
self.evaluate(var0))
self.assertAllCloseAccordingToType(var1_np,
self.evaluate(var1))
def test_control_flow_layer(self, layer_class):
model = testing_utils.get_model_from_layers([layer_class()],
input_shape=(3, ))
model.compile(rmsprop.RMSprop(0.001), loss='mse')
model.train_on_batch(np.random.random((2, 3)), np.random.random(
(2, 3)))
def testTrainAndServe(self, use_adapt):
with self.coordinator.strategy.scope():
feature_ps, label_ps = self.define_kpls_for_training(use_adapt)
def dataset_fn():
def feature_and_label_gen():
while True:
features = random.sample(FEATURE_VOCAB, 3)
label = ["yes"] if "avenger" in features else ["no"]
yield {"features": features, "label": label}
# The dataset will be created on the coordinator.
raw_dataset = tf.data.Dataset.from_generator(
feature_and_label_gen,
output_signature={
"features": tf.TensorSpec([3], tf.string),
"label": tf.TensorSpec([1], tf.string)
}).shuffle(100).batch(32)
train_dataset = raw_dataset.map(lambda x: ( # pylint: disable=g-long-lambda
{
"features": feature_ps(x["features"])
}, label_ps(x["label"])))
return train_dataset
# Create the model. The input needs to be compatible with KPLs.
model_input = keras.layers.Input(shape=(3, ),
dtype=tf.int64,
name="model_input")
# input_dim includes a mask token and an oov token.
emb_output = keras.layers.Embedding(input_dim=len(FEATURE_VOCAB) +
2,
output_dim=20)(model_input)
emb_output = tf.reduce_mean(emb_output, axis=1)
dense_output = keras.layers.Dense(units=1,
activation="sigmoid")(emb_output)
model = keras.Model({"features": model_input}, dense_output)
optimizer = rmsprop.RMSprop(learning_rate=0.1)
accuracy = keras.metrics.Accuracy()
@tf.function
def worker_fn(iterator):
def replica_fn(iterator):
batch_data, labels = next(iterator)
with tf.GradientTape() as tape:
pred = model(batch_data, training=True)
loss = tf.nn.compute_average_loss(
keras.losses.BinaryCrossentropy(
reduction=losses_utils.ReductionV2.NONE)(labels,
pred))
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(
zip(gradients, model.trainable_variables))
actual_pred = tf.cast(tf.greater(pred, 0.5), tf.int64)
accuracy.update_state(labels, actual_pred)
self.coordinator.strategy.run(replica_fn, args=(iterator, ))
distributed_dataset = self.coordinator.create_per_worker_dataset(
dataset_fn)
distributed_iterator = iter(distributed_dataset)
for _ in range(4):
accuracy.reset_state()
for _ in range(7):
self.coordinator.schedule(worker_fn,
args=(distributed_iterator, ))
self.coordinator.join()
self.assertGreater(accuracy.result().numpy(), 0.5)
# Create a saved model.
model.feature_ps = feature_ps
model.label_ps = label_ps
model.label_inverse_lookup_layer = self.define_reverse_lookup_layer()
def create_serving_signature(model):
@tf.function
def serve_fn(raw_features):
raw_features = tf.compat.v1.expand_dims(raw_features, axis=0)
transformed_features = model.feature_ps(raw_features)
outputs = model(transformed_features)
outputs = tf.compat.v1.squeeze(outputs, axis=0)
outputs = tf.cast(tf.greater(outputs, 0.5), tf.int64)
decoded_outputs = model.label_inverse_lookup_layer(outputs)
return tf.compat.v1.squeeze(decoded_outputs, axis=0)
# serving does NOT have batch dimension
return serve_fn.get_concrete_function(
tf.TensorSpec(shape=(3), dtype=tf.string, name="example"))
serving_fn = create_serving_signature(model)
saved_model_dir = tempfile.mkdtemp(dir=self.get_temp_dir())
model.save(saved_model_dir, signatures={"serving_default": serving_fn})
# Test the saved_model.
loaded_serving_fn = keras.saving.save.load_model(
saved_model_dir).signatures["serving_default"]
# check the result w/ and w/o avenger.
prediction0 = loaded_serving_fn(
tf.constant(["avenger", "ironman", "avenger"]))["output_0"]
self.assertIn(prediction0, ("yes", "no"))
prediction1 = loaded_serving_fn(
tf.constant(["ironman", "ironman", "unkonwn"]))["output_0"]
self.assertIn(prediction1, ("yes", "no"))
def testMomentumProperValue(self):
with self.assertRaisesRegex(
ValueError, r"`momentum` must be between \[0, 1\]. "
r"Received: momentum=2.5 \(of type <class "
r"\'float\'>\)."):
rmsprop.RMSprop(1., momentum=2.5, centered=False)