def __init__( self, params, lr: float = 1e-2, momentum: float = 0.9, weight_decay: float = 0, eps: float = 1e-6, ): if momentum < 0 or momentum >= 1: raise ValueError(f"Momentum {momentum} must be in the range [0,1]") if lr <= 0: raise ValueError(f"Learning rate {lr} must be positive") if weight_decay < 0: raise ValueError( f"Weight decay {weight_decay} must be non-negative") if eps < 0: raise ValueError("Eps must be non-negative") defaults = { "lr": lr, "eps": eps, "momentum": momentum, "weight_decay": weight_decay, "k": 0, } self.momentum = momentum PT_Optimizer.__init__(self, params, defaults) self.initialize_state()
def __init__(self, model_params, is_ml=True, lr=0.1, noise_r=0, noise_T=-1, noise_eps=0, momentum=0, NCE=False, NCE_s=0, NCE_gamma=0, is_verbose=False): self.is_ml = is_ml if is_ml: Optimizer.__init__(self, model_params, dict()) else: self.state = defaultdict(dict) self.lr = lr self.noise_r = noise_r self.noise_T = noise_T self.noise_eps = noise_eps self.momentum = momentum self.NCE = NCE self.NCE_s = NCE_s self.NCE_gamma = pow((1 - momentum), 2) / lr self.is_verbose = is_verbose
def __init__( self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, amsgrad=False, weight_decouple=False, fixed_decay=False, rectify=False, ): if not 0.0 <= lr: raise ValueError("Invalid learning rate: {}".format(lr)) if not 0.0 <= eps: raise ValueError("Invalid epsilon value: {}".format(eps)) if not 0.0 <= betas[0] < 1.0: raise ValueError("Invalid beta parameter at index 0: {}".format( betas[0])) if not 0.0 <= betas[1] < 1.0: raise ValueError("Invalid beta parameter at index 1: {}".format( betas[1])) defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, amsgrad=amsgrad) PT_Optimizer.__init__(self, params, defaults) self.weight_decouple = weight_decouple self.rectify = rectify self.fixed_decay = fixed_decay if self.weight_decouple: print("Weight decoupling enabled in AdaBelief") if self.fixed_decay: print("Weight decay fixed") if self.rectify: print("Rectification enabled in AdaBelief") if amsgrad: print("AMS enabled in AdaBelief")
def __init__(self, params, lr=required): defaults = dict(lr=lr) Optimizer.__init__(self, params, defaults) BaseComponent.__init__(self) self.logger.info("Optimizer Constructed.")