def testContinuous(self):
self.evaluate(variables.global_variables_initializer())
step = 5
decayed_lr = learning_rate_decay.exponential_decay(
0.05, step, 10, 0.96)
expected = .05 * 0.96**(5.0 / 10.0)
self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)
def testVariables(self):
step = variables.VariableV1(1)
assign_1 = step.assign(1)
assign_2 = step.assign(2)
assign_100 = step.assign(100)
decayed_lr = learning_rate_decay.exponential_decay(
.1, step, 3, 0.96, staircase=True)
self.evaluate(variables.global_variables_initializer())
# No change to learning rate
self.evaluate(assign_1.op)
self.assertAllClose(self.evaluate(decayed_lr), .1, 1e-6)
self.evaluate(assign_2.op)
self.assertAllClose(self.evaluate(decayed_lr), .1, 1e-6)
# Decayed learning rate
self.evaluate(assign_100.op)
expected = .1 * 0.96**(100 // 3)
self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)
def testStaircase(self):
if context.executing_eagerly():
step = resource_variable_ops.ResourceVariable(0)
self.evaluate(variables.global_variables_initializer())
decayed_lr = learning_rate_decay.exponential_decay(
.1, step, 3, 0.96, staircase=True)
# No change to learning rate due to staircase
expected = .1
self.evaluate(step.assign(1))
self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)
expected = .1
self.evaluate(step.assign(2))
self.assertAllClose(self.evaluate(decayed_lr), .1, 1e-6)
# Decayed learning rate
expected = .1 * 0.96 ** (100 // 3)
self.evaluate(step.assign(100))
self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)