def testMovingAverageOptimizer(self):
optimizer = sgd_new.SGD(learning_rate=1,
use_ema=True,
ema_momentum=0.5,
ema_overwrite_frequency=3)
var1, var2 = tf.Variable(2.0), tf.Variable(2.0)
with tf.GradientTape() as tape:
loss = var1 + var2
grads = tape.gradient(loss, [var1, var2])
# First iteration: [var1, var2] = [1.0, 1.0]
optimizer.apply_gradients(zip(grads, [var1, var2]))
self.assertAllEqual([var1.numpy(), var2.numpy()], [1.0, 1.0])
# Second iteration: [var1, var2] = [0.0, 0.0]
optimizer.apply_gradients(zip(grads, [var1, var2]))
self.assertAllEqual([var1.numpy(), var2.numpy()], [0.0, 0.0])
# Third iteration, without EMA, we should see [var1, var2] = [-1.0, -1.0],
# but overwriting results in [var1, var2] = [-0.125, -0.125].
optimizer.apply_gradients(zip(grads, [var1, var2]))
self.assertAllEqual([var1.numpy(), var2.numpy()], [-0.125, -0.125])
def testSgd(self, nesterov):
self._compare_numerical(
sgd_old.SGD(nesterov=True), sgd_new.SGD(nesterov=True))
ds_combinations.central_storage_strategy_with_two_gpus,
]
adadelta_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentaladadelta",
lambda: adadelta_new.Adadelta( # pylint: disable=g-long-lambda
0.002, use_ema=True, ema_overwrite_frequency=None))
adagrad_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentaladagrad", lambda: adagrad_new.Adagrad(0.002))
adam_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentaladam", lambda: adam_new.Adam(0.002))
rmsprop_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentalrmsprop", lambda: rmsprop_new.RMSprop(0.002))
sgd_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentalsgdaverage",
lambda: sgd_new.SGD( # pylint: disable=g-long-lambda
0.002, use_ema=True, ema_overwrite_frequency=1))
OPTIMIZER_FN = [
adadelta_new_fn,
adagrad_new_fn,
adam_new_fn,
rmsprop_new_fn,
sgd_new_fn,
]
class OptimizerFuntionalityTest(tf.test.TestCase, parameterized.TestCase):
"""Test the functionality of optimizer."""
def testAddVariableFromReference(self):
optimizer = adam_new.Adam()
adadelta_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentaladadelta",
lambda: adadelta_new.Adadelta( # pylint: disable=g-long-lambda
0.002,
ema_option=optimizer_lib.EMAOption(use_ema=True,
ema_overwrite_frequency=None)))
adagrad_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentaladagrad", lambda: adagrad_new.Adagrad(0.002))
adam_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentaladam", lambda: adam_new.Adam(0.002))
rmsprop_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentalrmsprop", lambda: rmsprop_new.RMSprop(0.002))
sgd_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentalsgdaverage",
lambda: sgd_new.SGD( # pylint: disable=g-long-lambda
0.002,
ema_option=optimizer_lib.EMAOption(use_ema=True,
ema_overwrite_frequency=1)))
OPTIMIZER_FN = [
adadelta_new_fn,
adagrad_new_fn,
adam_new_fn,
rmsprop_new_fn,
sgd_new_fn,
]
class OptimizerFuntionalityTest(tf.test.TestCase, parameterized.TestCase):
"""Test the functionality of optimizer."""
def testAddVariableFromReference(self):
optimizer = adam_new.Adam()
ds_combinations.mirrored_strategy_with_two_gpus,
ds_combinations.tpu_strategy,
ds_combinations.cloud_tpu_strategy,
ds_combinations.multi_worker_mirrored_2x1_cpu,
ds_combinations.multi_worker_mirrored_2x2_gpu,
ds_combinations.central_storage_strategy_with_two_gpus,
]
adadelta_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentaladadelta", lambda: adadelta_new.Adadelta(0.002))
adagrad_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentaladagrad", lambda: adagrad_new.Adagrad(0.002))
adam_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentaladam", lambda: adam_new.Adam(0.002))
sgd_new_fn = tf.__internal__.test.combinations.NamedObject(
"experimentalsgd", lambda: sgd_new.SGD(0.002))
OPTIMIZER_FN = [
adadelta_new_fn,
adagrad_new_fn,
adam_new_fn,
sgd_new_fn,
]
class OptimizerFuntionalityTest(tf.test.TestCase, parameterized.TestCase):
"""Test the functionality of optimizer."""
def testAddVariableFromReference(self):
optimizer = adam_new.Adam()
variable = optimizer.add_variable_from_reference(