def __init__( self, parameters: Union[Iterator[Parameter], List[Dict]], lr: float = 1e-3, betas: Tuple[float, float] = (0.9, 0.999), eps: float = 1e-8, weight_decay: float = 0, amsgrad: bool = False, scale: float = 1.0, ): super().__init__() assert lr >= 0.0, f"Invalid learning rate: {lr}" assert eps >= 0.0, f"Invalid epsilon value: {eps}" assert ( betas[0] >= 0.0 and betas[0] < 1.0 ), f"Invalid beta parameter at index 0: {betas[0]}" assert ( betas[1] >= 0.0 and betas[1] < 1.0 ), f"Invalid beta parameter at index 1: {betas[1]}" assert weight_decay >= 0.0, f"Invalid weight_decay value: {weight_decay}" assert scale > 0.0, f"Invalid scale factor: {scale}" assert amsgrad is False, "Not support AMSGrad now!" self._default_options["lr"] = lr self._default_options["eps"] = eps self._default_options["betas"] = betas self._default_options["weight_decay"] = weight_decay self._default_options["amsgrad"] = amsgrad self._default_options["scale"] = scale # Add parameters if isinstance(parameters, GeneratorType): self._param_groups.append(ParamGroup(parameters, self._default_options)) else: # List[Dict] for param in parameters: self._param_groups.append(ParamGroup(param, self._default_options)) for param_group in self._param_groups: for param in param_group.parameters: assert param.is_leaf, "parameters must be leaf tensor" self._state[param] = dict() self._state[param]["exp_avg"] = flow.experimental.zeros_like(param) self._state[param]["exp_avg_sq"] = flow.experimental.zeros_like(param) self._op = ( flow.builtin_op("adam_update") .Input("model") .Input("model_diff") .Input("learning_rate") .Input("m") .Input("v") .Attr("l1", 0.0) .Attr("l2", 0.0) .Build() )
def __init__( self, parameters: Union[Iterator[Parameter], List[Dict]], lr: float = 1e-3, alpha: float = 0.99, eps: float = 1e-8, weight_decay: float = 0, momentum: float = 0.0, centered: bool = False, scale: float = 1.0, ): super().__init__() assert lr >= 0.0, f"Invalid learning rate: {lr}" assert alpha >= 0.0, f"Invalid alpha value: {alpha}" assert eps >= 0.0, f"Invalid epsilon value: {eps}" assert weight_decay >= 0.0, f"Invalid weight_decay value: {weight_decay}" assert scale > 0.0, f"Invalid scale factor: {scale}" assert momentum == 0.0, "Not support momentum greater than zeros now!" self._default_options["lr"] = lr self._default_options["alpha"] = alpha self._default_options["eps"] = eps self._default_options["weight_decay"] = weight_decay self._default_options["centered"] = centered self._default_options["scale"] = scale # Add parameters if isinstance(parameters, GeneratorType): self._param_groups.append( ParamGroup(parameters, self._default_options)) else: # List[Dict] for param in parameters: self._param_groups.append( ParamGroup(param, self._default_options)) for param_group in self._param_groups: for param in param_group.parameters: assert param.is_leaf, "parameters must be leaf tensor" self._state[param] = dict() self._state[param][ "square_avg"] = flow.experimental.zeros_like(param) if param_group.options["centered"]: self._state[param][ "grad_avg"] = flow.experimental.zeros_like(param) self._centered_rmsprop = (flow.builtin_op("rmsprop_update").Input( "model").Input("model_diff").Input("learning_rate").Input( "mean_square").Input("mean_gradient").Attr( "centered", True).Attr("l1", 0.0).Attr("l2", 0.0).Build()) self._rmsprop = ( flow.builtin_op("rmsprop_update").Input("model").Input( "model_diff").Input("learning_rate").Input("mean_square").Attr( "centered", False).Attr("l1", 0.0).Attr("l2", 0.0).Build())