def testNumericEquivalenceForNesterovMomentum(self):
if tf.executing_eagerly():
self.skipTest(
'v1 optimizer does not run in eager mode')
np.random.seed(1331)
with testing_utils.use_gpu():
train_samples = 20
input_dim = 3
num_classes = 2
(x, y), _ = testing_utils.get_test_data(
train_samples=train_samples,
test_samples=10,
input_shape=(input_dim,),
num_classes=num_classes)
y = np_utils.to_categorical(y)
num_hidden = 5
model_k_v1 = testing_utils.get_small_sequential_mlp(
num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
model_k_v2 = testing_utils.get_small_sequential_mlp(
num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
model_k_v2.set_weights(model_k_v1.get_weights())
model_tf = testing_utils.get_small_sequential_mlp(
num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
model_tf.set_weights(model_k_v2.get_weights())
opt_k_v1 = optimizer_v1.SGD(momentum=0.9, nesterov=True)
opt_k_v2 = gradient_descent.SGD(momentum=0.9, nesterov=True)
opt_tf = tf.compat.v1.train.MomentumOptimizer(
learning_rate=0.01, momentum=0.9, use_nesterov=True)
model_k_v1.compile(
opt_k_v1,
loss='categorical_crossentropy',
metrics=[],
run_eagerly=testing_utils.should_run_eagerly())
model_k_v2.compile(
opt_k_v2,
loss='categorical_crossentropy',
metrics=[],
run_eagerly=testing_utils.should_run_eagerly())
model_tf.compile(
opt_tf,
loss='categorical_crossentropy',
metrics=[],
run_eagerly=testing_utils.should_run_eagerly())
hist_k_v1 = model_k_v1.fit(x, y, batch_size=5, epochs=10, shuffle=False)
hist_k_v2 = model_k_v2.fit(x, y, batch_size=5, epochs=10, shuffle=False)
hist_tf = model_tf.fit(x, y, batch_size=5, epochs=10, shuffle=False)
self.assertAllClose(model_k_v1.get_weights(), model_tf.get_weights())
self.assertAllClose(model_k_v1.get_weights(), model_k_v2.get_weights())
self.assertAllClose(opt_k_v1.get_weights(), opt_k_v2.get_weights())
self.assertAllClose(hist_k_v1.history['loss'], hist_tf.history['loss'])
self.assertAllClose(hist_k_v1.history['loss'], hist_k_v2.history['loss'])
def test_pass_invalid_optimizer_with_loss_scaling(self):
with policy.policy_scope(policy.PolicyV1('float32', loss_scale=10.)):
x = layers.Input(shape=(1,))
y = mp_test_util.MultiplyLayer()(x)
model = models.Model(x, y)
if tf.executing_eagerly():
error_msg = 'Use a `tf.keras` Optimizer instead'
else:
error_msg = 'optimizer" must be an instance of '
with self.assertRaisesRegex(ValueError, error_msg):
model.compile(optimizer_v1.SGD(1.), 'mse')
def test_negative_clipvalue_or_clipnorm(self):
with self.assertRaises(ValueError):
_ = optimizer_v1.SGD(lr=0.01, clipvalue=-0.5)
with self.assertRaises(ValueError):
_ = optimizer_v1.Adam(clipnorm=-2.0)
def test_clipvalue(self):
with self.cached_session():
self._test_optimizer(
optimizer_v1.SGD(lr=0.01, momentum=0.9, clipvalue=0.5))
def test_momentum(self):
with self.cached_session():
self._test_optimizer(
optimizer_v1.SGD(lr=0.01, momentum=0.9, nesterov=True))
def test_sgd(self):
with self.cached_session():
self._test_optimizer(optimizer_v1.SGD())
def testSGDCompatibility(self): opt_v1 = optimizer_v1.SGD(lr=0.01) opt_v2 = gradient_descent.SGD(learning_rate=0.01) self._testOptimizersCompatibility(opt_v1, opt_v2, False)
def testMomentumCompatibility(self): opt_v1 = optimizer_v1.SGD(lr=0.01, momentum=0.9) opt_v2 = gradient_descent.SGD(learning_rate=0.01, momentum=0.9) self._testOptimizersCompatibility(opt_v1, opt_v2)
