def __init__(
self,
params,
lr: float = 1e-2,
momentum: float = 0.9,
weight_decay: float = 0,
eps: float = 1e-6,
):
if momentum < 0 or momentum >= 1:
raise ValueError(f"Momentum {momentum} must be in the range [0,1]")
if lr <= 0:
raise ValueError(f"Learning rate {lr} must be positive")
if weight_decay < 0:
raise ValueError(
f"Weight decay {weight_decay} must be non-negative")
if eps < 0:
raise ValueError("Eps must be non-negative")
defaults = {
"lr": lr,
"eps": eps,
"momentum": momentum,
"weight_decay": weight_decay,
"k": 0,
}
self.momentum = momentum
PT_Optimizer.__init__(self, params, defaults)
self.initialize_state()
def __init__(self,
model_params,
is_ml=True,
lr=0.1,
noise_r=0,
noise_T=-1,
noise_eps=0,
momentum=0,
NCE=False,
NCE_s=0,
NCE_gamma=0,
is_verbose=False):
self.is_ml = is_ml
if is_ml:
Optimizer.__init__(self, model_params, dict())
else:
self.state = defaultdict(dict)
self.lr = lr
self.noise_r = noise_r
self.noise_T = noise_T
self.noise_eps = noise_eps
self.momentum = momentum
self.NCE = NCE
self.NCE_s = NCE_s
self.NCE_gamma = pow((1 - momentum), 2) / lr
self.is_verbose = is_verbose
def __init__(
self,
params,
lr=1e-3,
betas=(0.9, 0.999),
eps=1e-8,
weight_decay=0,
amsgrad=False,
weight_decouple=False,
fixed_decay=False,
rectify=False,
):
if not 0.0 <= lr:
raise ValueError("Invalid learning rate: {}".format(lr))
if not 0.0 <= eps:
raise ValueError("Invalid epsilon value: {}".format(eps))
if not 0.0 <= betas[0] < 1.0:
raise ValueError("Invalid beta parameter at index 0: {}".format(
betas[0]))
if not 0.0 <= betas[1] < 1.0:
raise ValueError("Invalid beta parameter at index 1: {}".format(
betas[1]))
defaults = dict(lr=lr,
betas=betas,
eps=eps,
weight_decay=weight_decay,
amsgrad=amsgrad)
PT_Optimizer.__init__(self, params, defaults)
self.weight_decouple = weight_decouple
self.rectify = rectify
self.fixed_decay = fixed_decay
if self.weight_decouple:
print("Weight decoupling enabled in AdaBelief")
if self.fixed_decay:
print("Weight decay fixed")
if self.rectify:
print("Rectification enabled in AdaBelief")
if amsgrad:
print("AMS enabled in AdaBelief")
def __init__(self, params, lr=required):
defaults = dict(lr=lr)
Optimizer.__init__(self, params, defaults)
BaseComponent.__init__(self)
self.logger.info("Optimizer Constructed.")
