def test_cyclic_exp_lr(self):
"""Test a exponential cyclic learning rate"""
gold = [0.1, 0.26200002, 0.39159995, 0.49366, 0.5723919, 0.631441, 0.48263744, 0.35828033, 0.25496817, 0.1697357, 0.1,
0.15648592, 0.20167467, 0.23726073, 0.2647129, 0.285302, 0.23341745, 0.19005677, 0.15403408, 0.12431534, 0.1,
0.1196954, 0.13545176, 0.14785986, 0.15743186]
lrs = cyclic_learning_rate.ExponentialCyclicalLearningRate(
initial_learning_rate=0.1,
maximal_learning_rate=1.0,
step_size=5,
gamma=0.9,
)
for i in range(25):
assert round(float(lrs(i).numpy()), 5) == round(gold[i], 5)
def test_cyclic_exp_lr(self):
"""Test a exponential cyclic learning rate"""
gold = [0.1,0.26200002431869507,0.39159995317459106,0.49366000294685364,0.572391927242279,0.6314409971237183,
0.4826374351978302,0.35828033089637756,0.25496816635131836,0.16973569989204407,0.10000000149011612,
0.156485915184021,0.2016746699810028,0.23726072907447815,0.2647128999233246,0.2853020131587982,
0.23341745138168335,0.19005677103996277,0.1540340781211853,0.12431533634662628,0.10000000149011612,
0.11969540268182755,0.1354517638683319,0.1478598564863205,0.15743185579776764]
lrs = cyclic_learning_rate.ExponentialCyclicalLearningRate(
initial_learning_rate=0.1,
maximal_learning_rate=1.0,
step_size=5,
gamma=0.9,
)
scheduler = [lrs(i) for i in range(25)]
self.assertTrue(self.compare_lr_values(scheduler, gold))
def choose_scheduler(model_config):
"""
Define the optimizer used for training the RelevanceModel
Users have the option to define an ExponentialDecay learning rate schedule
Parameters
----------
model_config : dict
model configuration doctionary
Returns
-------
tensorflow learning rate scheduler
Notes
-----
References:
https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer
https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/ExponentialDecay
https://arxiv.org/pdf/1506.01186.pdf
"""
if 'lr_schedule' not in model_config:
# use constant lr schedule
learning_rate_schedule = ExponentialDecay(
initial_learning_rate=OptimizerDefaultValues.CONSTANT_LR,
decay_steps=10000000,
decay_rate=1.0,
)
else:
lr_schedule = model_config['lr_schedule']
lr_schedule_key = lr_schedule['key']
if lr_schedule_key == LearningRateScheduleKey.EXPONENTIAL:
learning_rate_schedule = ExponentialDecay(
initial_learning_rate=lr_schedule.get(
'learning_rate', OptimizerDefaultValues.CONSTANT_LR),
decay_steps=lr_schedule.get(
'learning_rate_decay_steps',
OptimizerDefaultValues.EXP_DECAY_STEPS),
decay_rate=lr_schedule.get(
'learning_rate_decay',
OptimizerDefaultValues.EXP_DECAY_RATE),
staircase=True,
)
elif lr_schedule_key == LearningRateScheduleKey.CONSTANT:
learning_rate_schedule = ExponentialDecay(
initial_learning_rate=lr_schedule.get(
'learning_rate', OptimizerDefaultValues.CONSTANT_LR),
decay_steps=10000000,
decay_rate=1.0,
)
elif lr_schedule_key == LearningRateScheduleKey.REDUCE_LR_ON_PLATEAU:
learning_rate_schedule = lr_schedule.get(
'learning_rate', OptimizerDefaultValues.CONSTANT_LR)
elif lr_schedule_key == LearningRateScheduleKey.CYCLIC:
lr_schedule_type = lr_schedule['type']
if lr_schedule_type == CyclicLearningRateType.TRIANGULAR:
learning_rate_schedule = cyclic_learning_rate.TriangularCyclicalLearningRate(
initial_learning_rate=lr_schedule.get(
'initial_learning_rate',
OptimizerDefaultValues.CYCLIC_INITIAL_LEARNING_RATE),
maximal_learning_rate=lr_schedule.get(
'maximal_learning_rate',
OptimizerDefaultValues.CYCLIC_MAXIMAL_LEARNING_RATE),
step_size=lr_schedule.get(
'step_size', OptimizerDefaultValues.CYCLIC_STEP_SIZE),
)
elif lr_schedule_type == CyclicLearningRateType.TRIANGULAR2:
learning_rate_schedule = cyclic_learning_rate.Triangular2CyclicalLearningRate(
initial_learning_rate=lr_schedule.get(
'initial_learning_rate',
OptimizerDefaultValues.CYCLIC_INITIAL_LEARNING_RATE),
maximal_learning_rate=lr_schedule.get(
'maximal_learning_rate',
OptimizerDefaultValues.CYCLIC_MAXIMAL_LEARNING_RATE),
step_size=lr_schedule.get(
'step_size', OptimizerDefaultValues.CYCLIC_STEP_SIZE),
)
elif lr_schedule_type == CyclicLearningRateType.EXPONENTIAL:
learning_rate_schedule = cyclic_learning_rate.ExponentialCyclicalLearningRate(
initial_learning_rate=lr_schedule.get(
'initial_learning_rate',
OptimizerDefaultValues.CYCLIC_INITIAL_LEARNING_RATE),
maximal_learning_rate=lr_schedule.get(
'maximal_learning_rate',
OptimizerDefaultValues.CYCLIC_MAXIMAL_LEARNING_RATE),
step_size=lr_schedule.get(
'step_size', OptimizerDefaultValues.CYCLIC_STEP_SIZE),
gamma=lr_schedule.get('gamma',
OptimizerDefaultValues.CYCLIC_GAMMA),
)
else:
raise ValueError(
"Unsupported cyclic learning rate schedule type key: " +
lr_schedule_type)
else:
raise ValueError("Unsupported learning rate schedule key: " +
lr_schedule_key)
return learning_rate_schedule
def choose_scheduler(model_config):
"""
Define the optimizer used for training the RelevanceModel
Users have the option to define an ExponentialDecay learning rate schedule
Parameters
----------
model_config : dict
model configuration doctionary
Returns
-------
tensorflow learning rate scheduler
Notes
-----
References:
https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer
https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/ExponentialDecay
https://arxiv.org/pdf/1506.01186.pdf
"""
if 'lr_schedule' not in model_config:
#use constant lr schedule
learning_rate_schedule = ExponentialDecay(
initial_learning_rate=0.01,
decay_steps=10000000,
decay_rate=1.0,
)
else:
lr_schedule = model_config['lr_schedule']
lr_schedule_key = lr_schedule['key']
if lr_schedule_key == LearningRateScheduleKey.EXPONENTIAL:
learning_rate_schedule = ExponentialDecay(
initial_learning_rate=lr_schedule['learning_rate']
if 'learning_rate' in lr_schedule else 0.01,
decay_steps=lr_schedule['learning_rate_decay_steps']
if 'learning_rate_decay_steps' in lr_schedule else 100000,
decay_rate=lr_schedule['learning_rate_decay']
if 'learning_rate_decay' in lr_schedule else 0.96,
staircase=True,
)
elif lr_schedule_key == LearningRateScheduleKey.CONSTANT:
learning_rate_schedule = ExponentialDecay(
initial_learning_rate=lr_schedule['learning_rate']
if 'learning_rate' in lr_schedule else 0.01,
decay_steps=10000000,
decay_rate=1.0,
)
elif lr_schedule_key == LearningRateScheduleKey.CYCLIC:
lr_schedule_type = lr_schedule['type']
if lr_schedule_type == CyclicLearningRateType.TRIANGULAR:
learning_rate_schedule = cyclic_learning_rate.TriangularCyclicalLearningRate(
initial_learning_rate=lr_schedule['initial_learning_rate']
if 'initial_learning_rate' in lr_schedule else 0.001,
maximal_learning_rate=lr_schedule['maximal_learning_rate']
if 'maximal_learning_rate' in lr_schedule else 0.01,
step_size=lr_schedule['step_size']
if 'step_size' in lr_schedule else 10,
)
elif lr_schedule_type == CyclicLearningRateType.TRIANGULAR2:
learning_rate_schedule = cyclic_learning_rate.Triangular2CyclicalLearningRate(
initial_learning_rate=lr_schedule['initial_learning_rate']
if 'initial_learning_rate' in lr_schedule else 0.001,
maximal_learning_rate=lr_schedule['maximal_learning_rate']
if 'maximal_learning_rate' in lr_schedule else 0.01,
step_size=lr_schedule['step_size']
if 'step_size' in lr_schedule else 10,
)
elif lr_schedule_type == CyclicLearningRateType.EXPONENTIAL:
learning_rate_schedule = cyclic_learning_rate.ExponentialCyclicalLearningRate(
initial_learning_rate=lr_schedule['initial_learning_rate']
if 'initial_learning_rate' in lr_schedule else 0.001,
maximal_learning_rate=lr_schedule['maximal_learning_rate']
if 'maximal_learning_rate' in lr_schedule else 0.01,
step_size=lr_schedule['step_size']
if 'step_size' in lr_schedule else 10,
gamma=lr_schedule['gamma']
if 'gamma' in lr_schedule else 1.0,
)
else:
raise ValueError(
"Unsupported cyclic learning rate schedule type key: " +
lr_schedule_type)
else:
raise ValueError("Unsupported learning rate schedule key: " +
lr_schedule_key)
return learning_rate_schedule