def testExponentialDampingValuesWithDecayRate(self): init_value = 0.01 decay_rate = 0.3 num_decay_steps = 4 num_delay_steps = 3 opt = optimizers.Kfac(learning_rate=0.01, damping=init_value, model=self.model, loss='mse') self.model.compile(opt, 'mse') damping_list = [] cbs = [ callbacks.ExponentialDecay(hyperparameter='damping', init_value=init_value, decay_rate=decay_rate, num_decay_steps=num_decay_steps, num_delay_steps=num_delay_steps, verbose=1), HyperParamTracker('damping', damping_list, HyperParamTracker.BATCH) ] self.model.fit(self.data, self.labels, batch_size=self.batch_size, callbacks=cbs) expected_list = [init_value] * num_delay_steps + [ init_value * decay_rate**min(i, num_decay_steps) for i in range(self.num_steps - num_delay_steps) ] self.assertAllClose(damping_list, expected_list)
def testExponentialDampingFailsWithAllOptionals(self): with self.assertRaisesRegex(ValueError, '.*must specify exactly two of.*'): callbacks.ExponentialDecay(hyperparameter='learning_rate', init_value=0.01, final_value=0.001, decay_rate=0.99, num_decay_steps=50)
def testDampingDecayFailsWithNoDamping(self): with self.assertRaisesRegex(ValueError, '.*must have a "damping".*'): self.model.compile('adam', 'mse') cb = callbacks.ExponentialDecay(hyperparameter='damping', init_value=0.01, decay_rate=0.3, num_decay_steps=4) self.model.fit(self.data, self.data, callbacks=[cb])
class CallbacksTest(parameterized.TestCase, tf.test.TestCase): def __init__(self, *args, **kwargs): super(CallbacksTest, self).__init__(*args, **kwargs) self.batch_size = 16 self.num_steps = 20 self.data = np.random.random((self.batch_size * self.num_steps)) self.labels = np.random.random((self.batch_size * self.num_steps)) def setUp(self): super(CallbacksTest, self).setUp() self.model = tf.keras.Sequential([layers.Dense(1, input_shape=(1, ))]) tf.random.set_random_seed(_SEED) def testPolynomialDecayValues(self): init_value = 0.01 final_value = 0.0002 power = 0.6 num_decay_steps = 11 num_delay_steps = 3 opt = tf.keras.optimizers.Adam(learning_rate=init_value) self.model.compile(opt, 'mse') lr_list = [] cbs = [ callbacks.PolynomialDecay(hyperparameter='learning_rate', init_value=init_value, final_value=final_value, power=power, num_decay_steps=num_decay_steps, num_delay_steps=num_delay_steps, verbose=1), HyperParamTracker('learning_rate', lr_list, HyperParamTracker.BATCH) ] self.model.fit(self.data, self.labels, batch_size=self.batch_size, callbacks=cbs) expected_list = [init_value] * num_delay_steps + [ (init_value - final_value) * (1 - min(i, num_decay_steps) / float(num_decay_steps))**power + final_value for i in range(self.num_steps - num_delay_steps) ] self.assertAllClose(lr_list, expected_list) def testExponentialDampingValuesWithDecayRate(self): init_value = 0.01 decay_rate = 0.3 num_decay_steps = 4 num_delay_steps = 3 opt = optimizers.Kfac(learning_rate=0.01, damping=init_value, model=self.model, loss='mse') self.model.compile(opt, 'mse') damping_list = [] cbs = [ callbacks.ExponentialDecay(hyperparameter='damping', init_value=init_value, decay_rate=decay_rate, num_decay_steps=num_decay_steps, num_delay_steps=num_delay_steps, verbose=1), HyperParamTracker('damping', damping_list, HyperParamTracker.BATCH) ] self.model.fit(self.data, self.labels, batch_size=self.batch_size, callbacks=cbs) expected_list = [init_value] * num_delay_steps + [ init_value * decay_rate**min(i, num_decay_steps) for i in range(self.num_steps - num_delay_steps) ] self.assertAllClose(damping_list, expected_list) def testExponentialDampingValuesWithFinalValue(self): init_value = 0.01 final_value = 0.0001 num_decay_steps = 4 num_delay_steps = 3 opt = optimizers.Kfac(learning_rate=0.01, damping=init_value, model=self.model, loss='mse') self.model.compile(opt, 'mse') damping_list = [] cbs = [ callbacks.ExponentialDecay(hyperparameter='damping', init_value=init_value, final_value=final_value, num_decay_steps=num_decay_steps, num_delay_steps=num_delay_steps, verbose=1), HyperParamTracker('damping', damping_list, HyperParamTracker.BATCH) ] self.model.fit(self.data, self.labels, batch_size=self.batch_size, callbacks=cbs) expected_list = [init_value] * num_delay_steps + [ init_value * (final_value / init_value)** (min(i, num_decay_steps) * 1. / num_decay_steps) for i in range(self.num_steps - num_delay_steps) ] self.assertAllClose(damping_list, expected_list) self.assertNear(damping_list[-1], final_value, err=1e-5) def testExponentialDampingValuesWithFinalValueAndRate(self): init_value = 0.01 final_value = 0.0001 decay_rate = 0.6 num_delay_steps = 3 opt = optimizers.Kfac(learning_rate=0.01, damping=init_value, model=self.model, loss='mse') self.model.compile(opt, 'mse') damping_list = [] cbs = [ callbacks.ExponentialDecay(hyperparameter='damping', init_value=init_value, final_value=final_value, decay_rate=decay_rate, num_delay_steps=num_delay_steps, verbose=1), HyperParamTracker('damping', damping_list, HyperParamTracker.BATCH) ] self.model.fit(self.data, self.labels, batch_size=self.batch_size, callbacks=cbs) expected_list = [init_value] * num_delay_steps + [ max((init_value * decay_rate**i), final_value) for i in range(self.num_steps - num_delay_steps) ] self.assertAllClose(damping_list, expected_list) self.assertNear(damping_list[-1], final_value, err=1e-5) @parameterized.named_parameters( ('_Exponential', 'damping', callbacks.ExponentialDecay(hyperparameter='damping', init_value=0.01, decay_rate=0.3, num_decay_steps=30)), ('_Polynomial', 'learning_rate', callbacks.PolynomialDecay(hyperparameter='learning_rate', init_value=0.001, final_value=0.002, power=0.6, num_decay_steps=30))) def testTrainHistory(self, hyper, callback): opt = optimizers.Kfac(learning_rate=0.001, damping=0.01, model=self.model, loss='mse', num_burnin_steps=5) self.model.compile(opt, 'mse') lst = [] cbs = [ callback, HyperParamTracker(hyper, lst, HyperParamTracker.EPOCH) ] hist = self.model.fit(self.data, self.labels, batch_size=self.batch_size, epochs=3, callbacks=cbs) self.assertAllClose(lst, hist.history[hyper]) def testDampingDecayFailsWithNoDamping(self): with self.assertRaisesRegex(ValueError, '.*must have a "damping".*'): self.model.compile('adam', 'mse') cb = callbacks.ExponentialDecay(hyperparameter='damping', init_value=0.01, decay_rate=0.3, num_decay_steps=4) self.model.fit(self.data, self.data, callbacks=[cb]) def testExponentialDampingFailsNoRateOrFinalValue(self): with self.assertRaisesRegex(ValueError, '.*must specify exactly two of.*'): callbacks.ExponentialDecay(hyperparameter='damping', init_value=0.01) def testExponentialDampingFailsWithAllOptionals(self): with self.assertRaisesRegex(ValueError, '.*must specify exactly two of.*'): callbacks.ExponentialDecay(hyperparameter='learning_rate', init_value=0.01, final_value=0.001, decay_rate=0.99, num_decay_steps=50)
def testExponentialDampingFailsNoRateOrFinalValue(self): with self.assertRaisesRegex(ValueError, '.*must specify exactly two of.*'): callbacks.ExponentialDecay(hyperparameter='damping', init_value=0.01)