def __init__(self,
decay_steps,
warmup_steps,
min_lr=0.0,
lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
kernel_weight_decay=0.,
bias_weight_decay=0.,
amsgrad=False,
**kwargs):
super(AdamWarmup, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.decay_steps = K.variable(decay_steps, name='decay_steps')
self.warmup_steps = K.variable(warmup_steps, name='warmup_steps')
self.min_lr = K.variable(min_lr, name='min_lr')
self.iterations = K.variable(0, dtype='int64', name='iterations')
self.lr = K.variable(lr, name='lr')
self.beta_1 = K.variable(beta_1, name='beta_1')
self.beta_2 = K.variable(beta_2, name='beta_2')
self.kernel_weight_decay = K.variable(kernel_weight_decay,
name='kernel_weight_decay')
self.bias_weight_decay = K.variable(bias_weight_decay,
name='bias_weight_decay')
if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_kernel_weight_decay = kernel_weight_decay
self.initial_bias_weight_decay = bias_weight_decay
self.amsgrad = amsgrad
def __init__(self,
decay_steps,
warmup_steps,
min_lr=0.0,
learning_rate=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
weight_decay=0.,
weight_decay_pattern=None,
amsgrad=False,
lr_mult=None,
**kwargs):
learning_rate = kwargs.pop('lr', learning_rate)
super(AdamWarmup, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.decay_steps = K.variable(decay_steps, name='decay_steps')
self.warmup_steps = K.variable(warmup_steps, name='warmup_steps')
self.min_lr = K.variable(min_lr, name='min_lr')
self.iterations = K.variable(0, dtype='int64', name='iterations')
self.learning_rate = K.variable(learning_rate,
name='learning_rate')
self.beta_1 = K.variable(beta_1, name='beta_1')
self.beta_2 = K.variable(beta_2, name='beta_2')
self.weight_decay = K.variable(weight_decay, name='weight_decay')
if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_weight_decay = weight_decay
self.weight_decay_pattern = weight_decay_pattern
self.amsgrad = amsgrad
self.lr_mult = lr_mult