def __init__(self, decay_steps, warmup_steps=0, learning_rate=0.1, end_learning_rate=0.0001, power=1.0): super(BertLearningRate, self).__init__() self.warmup_lr = lr_schedules.WarmUpLR(learning_rate, warmup_steps) self.decay_lr = lr_schedules.PolynomialDecayLR(learning_rate, end_learning_rate, decay_steps, power) self.warmup_steps = Tensor(np.array([warmup_steps]).astype(np.float32)) self.greater = P.Greater() self.one = Tensor(np.array([1.0]).astype(np.float32)) self.cast = P.Cast()
def test_warmup(): lr_schedule = lr_schedules.WarmUpLR(learning_rate, warmup_steps) _executor.compile(lr_schedule, global_step)