def testNumericEquivalenceForNesterovMomentum(self): if context.executing_eagerly(): self.skipTest( 'v1 optimizer does not run in eager mode') np.random.seed(1331) with testing_utils.use_gpu(): train_samples = 20 input_dim = 3 num_classes = 2 (x, y), _ = testing_utils.get_test_data( train_samples=train_samples, test_samples=10, input_shape=(input_dim,), num_classes=num_classes) y = np_utils.to_categorical(y) num_hidden = 5 model_k_v1 = testing_utils.get_small_sequential_mlp( num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim) model_k_v2 = testing_utils.get_small_sequential_mlp( num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim) model_k_v2.set_weights(model_k_v1.get_weights()) model_tf = testing_utils.get_small_sequential_mlp( num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim) model_tf.set_weights(model_k_v2.get_weights()) opt_k_v1 = optimizer_v1.SGD(momentum=0.9, nesterov=True) opt_k_v2 = gradient_descent.SGD(momentum=0.9, nesterov=True) opt_tf = momentum.MomentumOptimizer( learning_rate=0.01, momentum=0.9, use_nesterov=True) model_k_v1.compile( opt_k_v1, loss='categorical_crossentropy', metrics=[], run_eagerly=testing_utils.should_run_eagerly()) model_k_v2.compile( opt_k_v2, loss='categorical_crossentropy', metrics=[], run_eagerly=testing_utils.should_run_eagerly()) model_tf.compile( opt_tf, loss='categorical_crossentropy', metrics=[], run_eagerly=testing_utils.should_run_eagerly()) hist_k_v1 = model_k_v1.fit(x, y, batch_size=5, epochs=10, shuffle=False) hist_k_v2 = model_k_v2.fit(x, y, batch_size=5, epochs=10, shuffle=False) hist_tf = model_tf.fit(x, y, batch_size=5, epochs=10, shuffle=False) self.assertAllClose(model_k_v1.get_weights(), model_tf.get_weights()) self.assertAllClose(model_k_v1.get_weights(), model_k_v2.get_weights()) self.assertAllClose(opt_k_v1.get_weights(), opt_k_v2.get_weights()) self.assertAllClose(hist_k_v1.history['loss'], hist_tf.history['loss']) self.assertAllClose(hist_k_v1.history['loss'], hist_k_v2.history['loss'])
def test_pass_invalid_optimizer_with_loss_scaling(self): with policy.policy_scope(policy.PolicyV1('float32', loss_scale=10.)): x = layers.Input(shape=(1, )) y = mp_test_util.MultiplyLayer()(x) model = models.Model(x, y) if context.executing_eagerly(): error_msg = 'Use a `tf.keras` Optimizer instead' else: error_msg = 'optimizer" must be an instance of ' with self.assertRaisesRegex(ValueError, error_msg): model.compile(optimizer_v1.SGD(1.), 'mse')
def test_negative_clipvalue_or_clipnorm(self): with self.assertRaises(ValueError): _ = optimizer_v1.SGD(lr=0.01, clipvalue=-0.5) with self.assertRaises(ValueError): _ = optimizer_v1.Adam(clipnorm=-2.0)
def test_clipvalue(self): with self.cached_session(): self._test_optimizer( optimizer_v1.SGD(lr=0.01, momentum=0.9, clipvalue=0.5))
def test_momentum(self): with self.cached_session(): self._test_optimizer( optimizer_v1.SGD(lr=0.01, momentum=0.9, nesterov=True))
def test_sgd(self): with self.cached_session(): self._test_optimizer(optimizer_v1.SGD())
def testSGDCompatibility(self): opt_v1 = optimizer_v1.SGD(lr=0.01) opt_v2 = gradient_descent.SGD(learning_rate=0.01) self._testOptimizersCompatibility(opt_v1, opt_v2, False)
def testMomentumCompatibility(self): opt_v1 = optimizer_v1.SGD(lr=0.01, momentum=0.9) opt_v2 = gradient_descent.SGD(learning_rate=0.01, momentum=0.9) self._testOptimizersCompatibility(opt_v1, opt_v2)