def testZeroEpochScheduleWithWarmup(self):
global_step = tf.placeholder_with_default(0, [], 'global_step')
with self.assertRaises(ValueError):
utils.build_learning_rate_schedule(
learning_rate=1.,
decay_type=enums.DecayType.EXPONENTIAL,
warmup_start_epoch=12,
max_learning_rate_epoch=24,
decay_end_epoch=24,
global_step=global_step,
steps_per_epoch=100)
def testPiecewiseLinearDecayWithoutWarmup(self):
global_step = tf.placeholder_with_default(0, [], 'global_step')
steps_per_epoch = 100
learning_rate = utils.build_learning_rate_schedule(
learning_rate=1.,
decay_type=enums.DecayType.PIECEWISE_LINEAR,
warmup_start_epoch=0,
max_learning_rate_epoch=0,
decay_end_epoch=1000,
global_step=global_step,
steps_per_epoch=steps_per_epoch)
with self.cached_session() as sess:
initial_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 1})
self.assertAlmostEqual(1., initial_learning_rate)
for i, epoch in enumerate(self.DEFAULT_BOUNDARY_EPOCHS):
before_decay_learning_rate = sess.run(
learning_rate,
feed_dict={global_step: epoch * steps_per_epoch})
after_decay_learning_rate = sess.run(
learning_rate,
feed_dict={global_step: epoch * steps_per_epoch + 1})
self.assertAlmostEqual(before_decay_learning_rate,
self.DEFAULT_DECAY_RATES[i])
self.assertAlmostEqual(after_decay_learning_rate,
self.DEFAULT_DECAY_RATES[i + 1])
final_learning_rate = sess.run(
learning_rate, feed_dict={global_step: 1000 * steps_per_epoch})
self.assertAlmostEqual(1e-4, final_learning_rate, places=7)
def testCosineDecayWithWarmup(self):
global_step = tf.placeholder_with_default(0, [], 'global_step')
learning_rate = utils.build_learning_rate_schedule(
learning_rate=1.,
decay_type=enums.DecayType.COSINE,
warmup_start_epoch=0,
max_learning_rate_epoch=5,
decay_end_epoch=10,
global_step=global_step,
steps_per_epoch=100)
with self.cached_session() as sess:
initial_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 0})
self.assertAlmostEqual(0., initial_learning_rate)
intermediate_warmup_learning_rate = sess.run(
learning_rate, feed_dict={global_step: 250})
self.assertAlmostEqual(0.5, intermediate_warmup_learning_rate)
max_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 500})
self.assertAlmostEqual(1., max_learning_rate)
final_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 1000})
self.assertAlmostEqual(0., final_learning_rate)
def testExponentialDecayWithDecayRate(self):
global_step = tf.placeholder_with_default(0, [], 'global_step')
learning_rate = utils.build_learning_rate_schedule(
learning_rate=1.,
decay_type=enums.DecayType.EXPONENTIAL,
warmup_start_epoch=0,
max_learning_rate_epoch=5,
decay_end_epoch=10,
global_step=global_step,
steps_per_epoch=100,
decay_rate=0.99)
with self.cached_session() as sess:
initial_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 0})
self.assertAlmostEqual(0., initial_learning_rate)
max_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 500})
self.assertAlmostEqual(1., max_learning_rate)
final_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 1000})
self.assertAlmostEqual((0.99)**np.floor((500) / (2.4 * 100)),
final_learning_rate,
places=5)
def testExponentialDecayWithWarmup(self):
global_step = tf.placeholder_with_default(0, [], 'global_step')
learning_rate = utils.build_learning_rate_schedule(
learning_rate=1.,
decay_type=enums.DecayType.EXPONENTIAL,
warmup_start_epoch=0,
max_learning_rate_epoch=5,
decay_end_epoch=10,
global_step=global_step,
steps_per_epoch=100)
with self.cached_session() as sess:
initial_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 0})
self.assertAlmostEqual(0., initial_learning_rate)
max_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 500})
self.assertAlmostEqual(1., max_learning_rate)
intermediate_warmup_learning_rate = sess.run(
learning_rate, feed_dict={global_step: 250})
self.assertAlmostEqual(0.5, intermediate_warmup_learning_rate)
final_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 1000})
self.assertAlmostEqual(self.DEFAULT_DECAY_FACTOR**np.floor(
500 / (self.DEFAULT_EPOCHS_PER_DECAY * 100)),
final_learning_rate,
places=5)
def stage_learning_rate(stage_training_params, start_epoch,
end_epoch):
schedule_kwargs = {}
if (stage_training_params.learning_rate_decay
in (enums.DecayType.PIECEWISE_LINEAR,
enums.DecayType.EXPONENTIAL)):
schedule_kwargs[
'decay_rate'] = stage_training_params.decay_rate
if (stage_training_params.learning_rate_decay ==
enums.DecayType.PIECEWISE_LINEAR):
schedule_kwargs['boundary_epochs'] = (
stage_training_params.decay_boundary_epochs)
if (stage_training_params.learning_rate_decay ==
enums.DecayType.EXPONENTIAL):
schedule_kwargs['epochs_per_decay'] = (
stage_training_params.epochs_per_decay)
return utils.build_learning_rate_schedule(
learning_rate=(stage_training_params.base_learning_rate *
(batch_size / BASE_BATCH_SIZE)),
decay_type=stage_training_params.learning_rate_decay,
warmup_start_epoch=start_epoch,
max_learning_rate_epoch=(
start_epoch +
stage_training_params.learning_rate_warmup_epochs),
decay_end_epoch=end_epoch,
global_step=global_step,
steps_per_epoch=steps_per_epoch,
**schedule_kwargs)
def testZeroEpochSchedule(self):
global_step = tf.placeholder_with_default(0, [], 'global_step')
learning_rate = utils.build_learning_rate_schedule(
learning_rate=1.,
decay_type=enums.DecayType.EXPONENTIAL,
warmup_start_epoch=24,
max_learning_rate_epoch=24,
decay_end_epoch=24,
global_step=global_step,
steps_per_epoch=100)
self.assertEqual(1., learning_rate)
def testPiecewiseLinearDecayWithBoundaries(self):
global_step = tf.placeholder_with_default(0, [], 'global_step')
boundary_epochs = [1, 2, 3, 4]
decay_rate = 1e-1
max_learning_rate_epoch = 5
steps_per_epoch = 100
learning_rate = utils.build_learning_rate_schedule(
learning_rate=1.,
decay_type=enums.DecayType.PIECEWISE_LINEAR,
warmup_start_epoch=0,
max_learning_rate_epoch=max_learning_rate_epoch,
decay_end_epoch=1000,
global_step=global_step,
steps_per_epoch=steps_per_epoch,
boundary_epochs=boundary_epochs,
decay_rate=decay_rate)
with self.cached_session() as sess:
initial_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 0})
self.assertAlmostEqual(0., initial_learning_rate)
max_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 500})
self.assertAlmostEqual(1., max_learning_rate)
for i, epoch in enumerate(boundary_epochs):
before_decay_learning_rate = sess.run(
learning_rate,
feed_dict={
global_step:
(epoch + max_learning_rate_epoch) * steps_per_epoch
})
after_decay_learning_rate = sess.run(
learning_rate,
feed_dict={
global_step:
((epoch + max_learning_rate_epoch) * steps_per_epoch) +
1
})
self.assertAlmostEqual(before_decay_learning_rate,
decay_rate**i)
self.assertAlmostEqual(after_decay_learning_rate,
decay_rate**(i + 1))
final_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 1000})
self.assertAlmostEqual(1e-4, final_learning_rate, places=5)
def testExponentialDecayWithoutWarmup(self, decay_index):
global_step = tf.placeholder_with_default(0, [], 'global_step')
learning_rate = utils.build_learning_rate_schedule(
learning_rate=1.,
decay_type=enums.DecayType.EXPONENTIAL,
warmup_start_epoch=0,
max_learning_rate_epoch=0,
decay_end_epoch=24,
global_step=global_step,
steps_per_epoch=100)
with self.cached_session() as sess:
initial_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 0})
self.assertAlmostEqual(1., initial_learning_rate)
staircase_drop_step = 100 * decay_index * self.DEFAULT_EPOCHS_PER_DECAY
before_staircase_drop_learning_rate = sess.run(
learning_rate, feed_dict={global_step: staircase_drop_step})
after_staircase_drop_learning_rate = sess.run(
learning_rate,
feed_dict={global_step: staircase_drop_step + 1})
self.assertAlmostEqual(self.DEFAULT_DECAY_FACTOR**(
np.floor(staircase_drop_step /
(self.DEFAULT_EPOCHS_PER_DECAY * 100)) - 1),
before_staircase_drop_learning_rate,
places=5)
self.assertAlmostEqual(self.DEFAULT_DECAY_FACTOR**(np.floor(
staircase_drop_step / (self.DEFAULT_EPOCHS_PER_DECAY * 100))),
after_staircase_drop_learning_rate,
places=5)
final_learning_rate = sess.run(learning_rate,
feed_dict={global_step: 2400})
self.assertAlmostEqual(self.DEFAULT_DECAY_FACTOR**(
np.floor(2400 / (self.DEFAULT_EPOCHS_PER_DECAY * 100)) - 1.),
final_learning_rate,
places=5)
