def test_cyclic_tri_lr(self): """Test a triangular cyclic learning rate""" gold = [0.1,0.28000003,0.45999995,0.64000005,0.81999993,1.0,0.81999993,0.64000005,0.45999995,0.28000003, 0.1,0.28000003,0.46000007,0.6399999,0.81999993,1.0,0.81999993,0.6399999,0.46000007,0.28000003, 0.1,0.27999982,0.46000007,0.6399999,0.8200002] lrs = cyclic_learning_rate.TriangularCyclicalLearningRate( initial_learning_rate=0.1, maximal_learning_rate=1.0, step_size=5, ) scheduler = [lrs(i) for i in range(25)] self.assertTrue(self.compare_lr_values(scheduler, gold))
def test_cyclic_tri_lr(self): """Test a triangular cyclic learning rate""" gold = [0.1, 0.28000003, 0.45999995, 0.64000005, 0.81999993, 1.0, 0.81999993, 0.64000005, 0.45999995, 0.28000003, 0.1, 0.28000003, 0.46000007, 0.6399999, 0.81999993, 1.0, 0.81999993, 0.6399999, 0.46000007, 0.28000003, 0.1, 0.27999982, 0.46000007, 0.6399999, 0.8200002] lrs = cyclic_learning_rate.TriangularCyclicalLearningRate( initial_learning_rate=0.1, maximal_learning_rate=1.0, step_size=5, ) for i in range(25): assert round(float(lrs(i).numpy()), 5) == round(gold[i], 5)
def choose_scheduler(model_config): """ Define the optimizer used for training the RelevanceModel Users have the option to define an ExponentialDecay learning rate schedule Parameters ---------- model_config : dict model configuration doctionary Returns ------- tensorflow learning rate scheduler Notes ----- References: https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/ExponentialDecay https://arxiv.org/pdf/1506.01186.pdf """ if 'lr_schedule' not in model_config: # use constant lr schedule learning_rate_schedule = ExponentialDecay( initial_learning_rate=OptimizerDefaultValues.CONSTANT_LR, decay_steps=10000000, decay_rate=1.0, ) else: lr_schedule = model_config['lr_schedule'] lr_schedule_key = lr_schedule['key'] if lr_schedule_key == LearningRateScheduleKey.EXPONENTIAL: learning_rate_schedule = ExponentialDecay( initial_learning_rate=lr_schedule.get( 'learning_rate', OptimizerDefaultValues.CONSTANT_LR), decay_steps=lr_schedule.get( 'learning_rate_decay_steps', OptimizerDefaultValues.EXP_DECAY_STEPS), decay_rate=lr_schedule.get( 'learning_rate_decay', OptimizerDefaultValues.EXP_DECAY_RATE), staircase=True, ) elif lr_schedule_key == LearningRateScheduleKey.CONSTANT: learning_rate_schedule = ExponentialDecay( initial_learning_rate=lr_schedule.get( 'learning_rate', OptimizerDefaultValues.CONSTANT_LR), decay_steps=10000000, decay_rate=1.0, ) elif lr_schedule_key == LearningRateScheduleKey.REDUCE_LR_ON_PLATEAU: learning_rate_schedule = lr_schedule.get( 'learning_rate', OptimizerDefaultValues.CONSTANT_LR) elif lr_schedule_key == LearningRateScheduleKey.CYCLIC: lr_schedule_type = lr_schedule['type'] if lr_schedule_type == CyclicLearningRateType.TRIANGULAR: learning_rate_schedule = cyclic_learning_rate.TriangularCyclicalLearningRate( initial_learning_rate=lr_schedule.get( 'initial_learning_rate', OptimizerDefaultValues.CYCLIC_INITIAL_LEARNING_RATE), maximal_learning_rate=lr_schedule.get( 'maximal_learning_rate', OptimizerDefaultValues.CYCLIC_MAXIMAL_LEARNING_RATE), step_size=lr_schedule.get( 'step_size', OptimizerDefaultValues.CYCLIC_STEP_SIZE), ) elif lr_schedule_type == CyclicLearningRateType.TRIANGULAR2: learning_rate_schedule = cyclic_learning_rate.Triangular2CyclicalLearningRate( initial_learning_rate=lr_schedule.get( 'initial_learning_rate', OptimizerDefaultValues.CYCLIC_INITIAL_LEARNING_RATE), maximal_learning_rate=lr_schedule.get( 'maximal_learning_rate', OptimizerDefaultValues.CYCLIC_MAXIMAL_LEARNING_RATE), step_size=lr_schedule.get( 'step_size', OptimizerDefaultValues.CYCLIC_STEP_SIZE), ) elif lr_schedule_type == CyclicLearningRateType.EXPONENTIAL: learning_rate_schedule = cyclic_learning_rate.ExponentialCyclicalLearningRate( initial_learning_rate=lr_schedule.get( 'initial_learning_rate', OptimizerDefaultValues.CYCLIC_INITIAL_LEARNING_RATE), maximal_learning_rate=lr_schedule.get( 'maximal_learning_rate', OptimizerDefaultValues.CYCLIC_MAXIMAL_LEARNING_RATE), step_size=lr_schedule.get( 'step_size', OptimizerDefaultValues.CYCLIC_STEP_SIZE), gamma=lr_schedule.get('gamma', OptimizerDefaultValues.CYCLIC_GAMMA), ) else: raise ValueError( "Unsupported cyclic learning rate schedule type key: " + lr_schedule_type) else: raise ValueError("Unsupported learning rate schedule key: " + lr_schedule_key) return learning_rate_schedule
def choose_scheduler(model_config): """ Define the optimizer used for training the RelevanceModel Users have the option to define an ExponentialDecay learning rate schedule Parameters ---------- model_config : dict model configuration doctionary Returns ------- tensorflow learning rate scheduler Notes ----- References: https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/ExponentialDecay https://arxiv.org/pdf/1506.01186.pdf """ if 'lr_schedule' not in model_config: #use constant lr schedule learning_rate_schedule = ExponentialDecay( initial_learning_rate=0.01, decay_steps=10000000, decay_rate=1.0, ) else: lr_schedule = model_config['lr_schedule'] lr_schedule_key = lr_schedule['key'] if lr_schedule_key == LearningRateScheduleKey.EXPONENTIAL: learning_rate_schedule = ExponentialDecay( initial_learning_rate=lr_schedule['learning_rate'] if 'learning_rate' in lr_schedule else 0.01, decay_steps=lr_schedule['learning_rate_decay_steps'] if 'learning_rate_decay_steps' in lr_schedule else 100000, decay_rate=lr_schedule['learning_rate_decay'] if 'learning_rate_decay' in lr_schedule else 0.96, staircase=True, ) elif lr_schedule_key == LearningRateScheduleKey.CONSTANT: learning_rate_schedule = ExponentialDecay( initial_learning_rate=lr_schedule['learning_rate'] if 'learning_rate' in lr_schedule else 0.01, decay_steps=10000000, decay_rate=1.0, ) elif lr_schedule_key == LearningRateScheduleKey.CYCLIC: lr_schedule_type = lr_schedule['type'] if lr_schedule_type == CyclicLearningRateType.TRIANGULAR: learning_rate_schedule = cyclic_learning_rate.TriangularCyclicalLearningRate( initial_learning_rate=lr_schedule['initial_learning_rate'] if 'initial_learning_rate' in lr_schedule else 0.001, maximal_learning_rate=lr_schedule['maximal_learning_rate'] if 'maximal_learning_rate' in lr_schedule else 0.01, step_size=lr_schedule['step_size'] if 'step_size' in lr_schedule else 10, ) elif lr_schedule_type == CyclicLearningRateType.TRIANGULAR2: learning_rate_schedule = cyclic_learning_rate.Triangular2CyclicalLearningRate( initial_learning_rate=lr_schedule['initial_learning_rate'] if 'initial_learning_rate' in lr_schedule else 0.001, maximal_learning_rate=lr_schedule['maximal_learning_rate'] if 'maximal_learning_rate' in lr_schedule else 0.01, step_size=lr_schedule['step_size'] if 'step_size' in lr_schedule else 10, ) elif lr_schedule_type == CyclicLearningRateType.EXPONENTIAL: learning_rate_schedule = cyclic_learning_rate.ExponentialCyclicalLearningRate( initial_learning_rate=lr_schedule['initial_learning_rate'] if 'initial_learning_rate' in lr_schedule else 0.001, maximal_learning_rate=lr_schedule['maximal_learning_rate'] if 'maximal_learning_rate' in lr_schedule else 0.01, step_size=lr_schedule['step_size'] if 'step_size' in lr_schedule else 10, gamma=lr_schedule['gamma'] if 'gamma' in lr_schedule else 1.0, ) else: raise ValueError( "Unsupported cyclic learning rate schedule type key: " + lr_schedule_type) else: raise ValueError("Unsupported learning rate schedule key: " + lr_schedule_key) return learning_rate_schedule