def test_cyclic_tri_lr(self):
"""Test a triangular cyclic learning rate"""
gold = [0.1,0.28000003,0.45999995,0.64000005,0.81999993,1.0,0.81999993,0.64000005,0.45999995,0.28000003,
0.1,0.28000003,0.46000007,0.6399999,0.81999993,1.0,0.81999993,0.6399999,0.46000007,0.28000003,
0.1,0.27999982,0.46000007,0.6399999,0.8200002]
lrs = cyclic_learning_rate.TriangularCyclicalLearningRate(
initial_learning_rate=0.1,
maximal_learning_rate=1.0,
step_size=5,
)
scheduler = [lrs(i) for i in range(25)]
self.assertTrue(self.compare_lr_values(scheduler, gold))
def test_cyclic_tri_lr(self):
"""Test a triangular cyclic learning rate"""
gold = [0.1, 0.28000003, 0.45999995, 0.64000005, 0.81999993, 1.0, 0.81999993, 0.64000005, 0.45999995, 0.28000003,
0.1, 0.28000003, 0.46000007, 0.6399999, 0.81999993, 1.0, 0.81999993, 0.6399999, 0.46000007, 0.28000003,
0.1, 0.27999982, 0.46000007, 0.6399999, 0.8200002]
lrs = cyclic_learning_rate.TriangularCyclicalLearningRate(
initial_learning_rate=0.1,
maximal_learning_rate=1.0,
step_size=5,
)
for i in range(25):
assert round(float(lrs(i).numpy()), 5) == round(gold[i], 5)
def choose_scheduler(model_config):
"""
Define the optimizer used for training the RelevanceModel
Users have the option to define an ExponentialDecay learning rate schedule
Parameters
----------
model_config : dict
model configuration doctionary
Returns
-------
tensorflow learning rate scheduler
Notes
-----
References:
https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer
https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/ExponentialDecay
https://arxiv.org/pdf/1506.01186.pdf
"""
if 'lr_schedule' not in model_config:
# use constant lr schedule
learning_rate_schedule = ExponentialDecay(
initial_learning_rate=OptimizerDefaultValues.CONSTANT_LR,
decay_steps=10000000,
decay_rate=1.0,
)
else:
lr_schedule = model_config['lr_schedule']
lr_schedule_key = lr_schedule['key']
if lr_schedule_key == LearningRateScheduleKey.EXPONENTIAL:
learning_rate_schedule = ExponentialDecay(
initial_learning_rate=lr_schedule.get(
'learning_rate', OptimizerDefaultValues.CONSTANT_LR),
decay_steps=lr_schedule.get(
'learning_rate_decay_steps',
OptimizerDefaultValues.EXP_DECAY_STEPS),
decay_rate=lr_schedule.get(
'learning_rate_decay',
OptimizerDefaultValues.EXP_DECAY_RATE),
staircase=True,
)
elif lr_schedule_key == LearningRateScheduleKey.CONSTANT:
learning_rate_schedule = ExponentialDecay(
initial_learning_rate=lr_schedule.get(
'learning_rate', OptimizerDefaultValues.CONSTANT_LR),
decay_steps=10000000,
decay_rate=1.0,
)
elif lr_schedule_key == LearningRateScheduleKey.REDUCE_LR_ON_PLATEAU:
learning_rate_schedule = lr_schedule.get(
'learning_rate', OptimizerDefaultValues.CONSTANT_LR)
elif lr_schedule_key == LearningRateScheduleKey.CYCLIC:
lr_schedule_type = lr_schedule['type']
if lr_schedule_type == CyclicLearningRateType.TRIANGULAR:
learning_rate_schedule = cyclic_learning_rate.TriangularCyclicalLearningRate(
initial_learning_rate=lr_schedule.get(
'initial_learning_rate',
OptimizerDefaultValues.CYCLIC_INITIAL_LEARNING_RATE),
maximal_learning_rate=lr_schedule.get(
'maximal_learning_rate',
OptimizerDefaultValues.CYCLIC_MAXIMAL_LEARNING_RATE),
step_size=lr_schedule.get(
'step_size', OptimizerDefaultValues.CYCLIC_STEP_SIZE),
)
elif lr_schedule_type == CyclicLearningRateType.TRIANGULAR2:
learning_rate_schedule = cyclic_learning_rate.Triangular2CyclicalLearningRate(
initial_learning_rate=lr_schedule.get(
'initial_learning_rate',
OptimizerDefaultValues.CYCLIC_INITIAL_LEARNING_RATE),
maximal_learning_rate=lr_schedule.get(
'maximal_learning_rate',
OptimizerDefaultValues.CYCLIC_MAXIMAL_LEARNING_RATE),
step_size=lr_schedule.get(
'step_size', OptimizerDefaultValues.CYCLIC_STEP_SIZE),
)
elif lr_schedule_type == CyclicLearningRateType.EXPONENTIAL:
learning_rate_schedule = cyclic_learning_rate.ExponentialCyclicalLearningRate(
initial_learning_rate=lr_schedule.get(
'initial_learning_rate',
OptimizerDefaultValues.CYCLIC_INITIAL_LEARNING_RATE),
maximal_learning_rate=lr_schedule.get(
'maximal_learning_rate',
OptimizerDefaultValues.CYCLIC_MAXIMAL_LEARNING_RATE),
step_size=lr_schedule.get(
'step_size', OptimizerDefaultValues.CYCLIC_STEP_SIZE),
gamma=lr_schedule.get('gamma',
OptimizerDefaultValues.CYCLIC_GAMMA),
)
else:
raise ValueError(
"Unsupported cyclic learning rate schedule type key: " +
lr_schedule_type)
else:
raise ValueError("Unsupported learning rate schedule key: " +
lr_schedule_key)
return learning_rate_schedule
def choose_scheduler(model_config):
"""
Define the optimizer used for training the RelevanceModel
Users have the option to define an ExponentialDecay learning rate schedule
Parameters
----------
model_config : dict
model configuration doctionary
Returns
-------
tensorflow learning rate scheduler
Notes
-----
References:
https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer
https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/ExponentialDecay
https://arxiv.org/pdf/1506.01186.pdf
"""
if 'lr_schedule' not in model_config:
#use constant lr schedule
learning_rate_schedule = ExponentialDecay(
initial_learning_rate=0.01,
decay_steps=10000000,
decay_rate=1.0,
)
else:
lr_schedule = model_config['lr_schedule']
lr_schedule_key = lr_schedule['key']
if lr_schedule_key == LearningRateScheduleKey.EXPONENTIAL:
learning_rate_schedule = ExponentialDecay(
initial_learning_rate=lr_schedule['learning_rate']
if 'learning_rate' in lr_schedule else 0.01,
decay_steps=lr_schedule['learning_rate_decay_steps']
if 'learning_rate_decay_steps' in lr_schedule else 100000,
decay_rate=lr_schedule['learning_rate_decay']
if 'learning_rate_decay' in lr_schedule else 0.96,
staircase=True,
)
elif lr_schedule_key == LearningRateScheduleKey.CONSTANT:
learning_rate_schedule = ExponentialDecay(
initial_learning_rate=lr_schedule['learning_rate']
if 'learning_rate' in lr_schedule else 0.01,
decay_steps=10000000,
decay_rate=1.0,
)
elif lr_schedule_key == LearningRateScheduleKey.CYCLIC:
lr_schedule_type = lr_schedule['type']
if lr_schedule_type == CyclicLearningRateType.TRIANGULAR:
learning_rate_schedule = cyclic_learning_rate.TriangularCyclicalLearningRate(
initial_learning_rate=lr_schedule['initial_learning_rate']
if 'initial_learning_rate' in lr_schedule else 0.001,
maximal_learning_rate=lr_schedule['maximal_learning_rate']
if 'maximal_learning_rate' in lr_schedule else 0.01,
step_size=lr_schedule['step_size']
if 'step_size' in lr_schedule else 10,
)
elif lr_schedule_type == CyclicLearningRateType.TRIANGULAR2:
learning_rate_schedule = cyclic_learning_rate.Triangular2CyclicalLearningRate(
initial_learning_rate=lr_schedule['initial_learning_rate']
if 'initial_learning_rate' in lr_schedule else 0.001,
maximal_learning_rate=lr_schedule['maximal_learning_rate']
if 'maximal_learning_rate' in lr_schedule else 0.01,
step_size=lr_schedule['step_size']
if 'step_size' in lr_schedule else 10,
)
elif lr_schedule_type == CyclicLearningRateType.EXPONENTIAL:
learning_rate_schedule = cyclic_learning_rate.ExponentialCyclicalLearningRate(
initial_learning_rate=lr_schedule['initial_learning_rate']
if 'initial_learning_rate' in lr_schedule else 0.001,
maximal_learning_rate=lr_schedule['maximal_learning_rate']
if 'maximal_learning_rate' in lr_schedule else 0.01,
step_size=lr_schedule['step_size']
if 'step_size' in lr_schedule else 10,
gamma=lr_schedule['gamma']
if 'gamma' in lr_schedule else 1.0,
)
else:
raise ValueError(
"Unsupported cyclic learning rate schedule type key: " +
lr_schedule_type)
else:
raise ValueError("Unsupported learning rate schedule key: " +
lr_schedule_key)
return learning_rate_schedule