def testExponentialDampingValuesWithDecayRate(self):
init_value = 0.01
decay_rate = 0.3
num_decay_steps = 4
num_delay_steps = 3
opt = optimizers.Kfac(learning_rate=0.01,
damping=init_value,
model=self.model,
loss='mse')
self.model.compile(opt, 'mse')
damping_list = []
cbs = [
callbacks.ExponentialDecay(hyperparameter='damping',
init_value=init_value,
decay_rate=decay_rate,
num_decay_steps=num_decay_steps,
num_delay_steps=num_delay_steps,
verbose=1),
HyperParamTracker('damping', damping_list, HyperParamTracker.BATCH)
]
self.model.fit(self.data,
self.labels,
batch_size=self.batch_size,
callbacks=cbs)
expected_list = [init_value] * num_delay_steps + [
init_value * decay_rate**min(i, num_decay_steps)
for i in range(self.num_steps - num_delay_steps)
]
self.assertAllClose(damping_list, expected_list)
def testExponentialDampingFailsWithAllOptionals(self):
with self.assertRaisesRegex(ValueError,
'.*must specify exactly two of.*'):
callbacks.ExponentialDecay(hyperparameter='learning_rate',
init_value=0.01,
final_value=0.001,
decay_rate=0.99,
num_decay_steps=50)
def testDampingDecayFailsWithNoDamping(self):
with self.assertRaisesRegex(ValueError, '.*must have a "damping".*'):
self.model.compile('adam', 'mse')
cb = callbacks.ExponentialDecay(hyperparameter='damping',
init_value=0.01,
decay_rate=0.3,
num_decay_steps=4)
self.model.fit(self.data, self.data, callbacks=[cb])
class CallbacksTest(parameterized.TestCase, tf.test.TestCase):
def __init__(self, *args, **kwargs):
super(CallbacksTest, self).__init__(*args, **kwargs)
self.batch_size = 16
self.num_steps = 20
self.data = np.random.random((self.batch_size * self.num_steps))
self.labels = np.random.random((self.batch_size * self.num_steps))
def setUp(self):
super(CallbacksTest, self).setUp()
self.model = tf.keras.Sequential([layers.Dense(1, input_shape=(1, ))])
tf.random.set_random_seed(_SEED)
def testPolynomialDecayValues(self):
init_value = 0.01
final_value = 0.0002
power = 0.6
num_decay_steps = 11
num_delay_steps = 3
opt = tf.keras.optimizers.Adam(learning_rate=init_value)
self.model.compile(opt, 'mse')
lr_list = []
cbs = [
callbacks.PolynomialDecay(hyperparameter='learning_rate',
init_value=init_value,
final_value=final_value,
power=power,
num_decay_steps=num_decay_steps,
num_delay_steps=num_delay_steps,
verbose=1),
HyperParamTracker('learning_rate', lr_list,
HyperParamTracker.BATCH)
]
self.model.fit(self.data,
self.labels,
batch_size=self.batch_size,
callbacks=cbs)
expected_list = [init_value] * num_delay_steps + [
(init_value - final_value) *
(1 - min(i, num_decay_steps) / float(num_decay_steps))**power +
final_value for i in range(self.num_steps - num_delay_steps)
]
self.assertAllClose(lr_list, expected_list)
def testExponentialDampingValuesWithDecayRate(self):
init_value = 0.01
decay_rate = 0.3
num_decay_steps = 4
num_delay_steps = 3
opt = optimizers.Kfac(learning_rate=0.01,
damping=init_value,
model=self.model,
loss='mse')
self.model.compile(opt, 'mse')
damping_list = []
cbs = [
callbacks.ExponentialDecay(hyperparameter='damping',
init_value=init_value,
decay_rate=decay_rate,
num_decay_steps=num_decay_steps,
num_delay_steps=num_delay_steps,
verbose=1),
HyperParamTracker('damping', damping_list, HyperParamTracker.BATCH)
]
self.model.fit(self.data,
self.labels,
batch_size=self.batch_size,
callbacks=cbs)
expected_list = [init_value] * num_delay_steps + [
init_value * decay_rate**min(i, num_decay_steps)
for i in range(self.num_steps - num_delay_steps)
]
self.assertAllClose(damping_list, expected_list)
def testExponentialDampingValuesWithFinalValue(self):
init_value = 0.01
final_value = 0.0001
num_decay_steps = 4
num_delay_steps = 3
opt = optimizers.Kfac(learning_rate=0.01,
damping=init_value,
model=self.model,
loss='mse')
self.model.compile(opt, 'mse')
damping_list = []
cbs = [
callbacks.ExponentialDecay(hyperparameter='damping',
init_value=init_value,
final_value=final_value,
num_decay_steps=num_decay_steps,
num_delay_steps=num_delay_steps,
verbose=1),
HyperParamTracker('damping', damping_list, HyperParamTracker.BATCH)
]
self.model.fit(self.data,
self.labels,
batch_size=self.batch_size,
callbacks=cbs)
expected_list = [init_value] * num_delay_steps + [
init_value * (final_value / init_value)**
(min(i, num_decay_steps) * 1. / num_decay_steps)
for i in range(self.num_steps - num_delay_steps)
]
self.assertAllClose(damping_list, expected_list)
self.assertNear(damping_list[-1], final_value, err=1e-5)
def testExponentialDampingValuesWithFinalValueAndRate(self):
init_value = 0.01
final_value = 0.0001
decay_rate = 0.6
num_delay_steps = 3
opt = optimizers.Kfac(learning_rate=0.01,
damping=init_value,
model=self.model,
loss='mse')
self.model.compile(opt, 'mse')
damping_list = []
cbs = [
callbacks.ExponentialDecay(hyperparameter='damping',
init_value=init_value,
final_value=final_value,
decay_rate=decay_rate,
num_delay_steps=num_delay_steps,
verbose=1),
HyperParamTracker('damping', damping_list, HyperParamTracker.BATCH)
]
self.model.fit(self.data,
self.labels,
batch_size=self.batch_size,
callbacks=cbs)
expected_list = [init_value] * num_delay_steps + [
max((init_value * decay_rate**i), final_value)
for i in range(self.num_steps - num_delay_steps)
]
self.assertAllClose(damping_list, expected_list)
self.assertNear(damping_list[-1], final_value, err=1e-5)
@parameterized.named_parameters(
('_Exponential', 'damping',
callbacks.ExponentialDecay(hyperparameter='damping',
init_value=0.01,
decay_rate=0.3,
num_decay_steps=30)),
('_Polynomial', 'learning_rate',
callbacks.PolynomialDecay(hyperparameter='learning_rate',
init_value=0.001,
final_value=0.002,
power=0.6,
num_decay_steps=30)))
def testTrainHistory(self, hyper, callback):
opt = optimizers.Kfac(learning_rate=0.001,
damping=0.01,
model=self.model,
loss='mse',
num_burnin_steps=5)
self.model.compile(opt, 'mse')
lst = []
cbs = [
callback,
HyperParamTracker(hyper, lst, HyperParamTracker.EPOCH)
]
hist = self.model.fit(self.data,
self.labels,
batch_size=self.batch_size,
epochs=3,
callbacks=cbs)
self.assertAllClose(lst, hist.history[hyper])
def testDampingDecayFailsWithNoDamping(self):
with self.assertRaisesRegex(ValueError, '.*must have a "damping".*'):
self.model.compile('adam', 'mse')
cb = callbacks.ExponentialDecay(hyperparameter='damping',
init_value=0.01,
decay_rate=0.3,
num_decay_steps=4)
self.model.fit(self.data, self.data, callbacks=[cb])
def testExponentialDampingFailsNoRateOrFinalValue(self):
with self.assertRaisesRegex(ValueError,
'.*must specify exactly two of.*'):
callbacks.ExponentialDecay(hyperparameter='damping',
init_value=0.01)
def testExponentialDampingFailsWithAllOptionals(self):
with self.assertRaisesRegex(ValueError,
'.*must specify exactly two of.*'):
callbacks.ExponentialDecay(hyperparameter='learning_rate',
init_value=0.01,
final_value=0.001,
decay_rate=0.99,
num_decay_steps=50)
def testExponentialDampingFailsNoRateOrFinalValue(self):
with self.assertRaisesRegex(ValueError,
'.*must specify exactly two of.*'):
callbacks.ExponentialDecay(hyperparameter='damping',
init_value=0.01)