def testContinuous(self): self.evaluate(variables.global_variables_initializer()) step = 5 decayed_lr = learning_rate_decay.exponential_decay( 0.05, step, 10, 0.96) expected = .05 * 0.96**(5.0 / 10.0) self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)
def testVariables(self): step = variables.VariableV1(1) assign_1 = step.assign(1) assign_2 = step.assign(2) assign_100 = step.assign(100) decayed_lr = learning_rate_decay.exponential_decay( .1, step, 3, 0.96, staircase=True) self.evaluate(variables.global_variables_initializer()) # No change to learning rate self.evaluate(assign_1.op) self.assertAllClose(self.evaluate(decayed_lr), .1, 1e-6) self.evaluate(assign_2.op) self.assertAllClose(self.evaluate(decayed_lr), .1, 1e-6) # Decayed learning rate self.evaluate(assign_100.op) expected = .1 * 0.96**(100 // 3) self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)
def testStaircase(self): if context.executing_eagerly(): step = resource_variable_ops.ResourceVariable(0) self.evaluate(variables.global_variables_initializer()) decayed_lr = learning_rate_decay.exponential_decay( .1, step, 3, 0.96, staircase=True) # No change to learning rate due to staircase expected = .1 self.evaluate(step.assign(1)) self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6) expected = .1 self.evaluate(step.assign(2)) self.assertAllClose(self.evaluate(decayed_lr), .1, 1e-6) # Decayed learning rate expected = .1 * 0.96 ** (100 // 3) self.evaluate(step.assign(100)) self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)