def __init__(
self,
parameters: Union[Iterator[Parameter], List[Dict]],
lr: float = 1e-3,
betas: Tuple[float, float] = (0.9, 0.999),
eps: float = 1e-8,
weight_decay: float = 0,
amsgrad: bool = False,
scale: float = 1.0,
):
super().__init__()
assert lr >= 0.0, f"Invalid learning rate: {lr}"
assert eps >= 0.0, f"Invalid epsilon value: {eps}"
assert (
betas[0] >= 0.0 and betas[0] < 1.0
), f"Invalid beta parameter at index 0: {betas[0]}"
assert (
betas[1] >= 0.0 and betas[1] < 1.0
), f"Invalid beta parameter at index 1: {betas[1]}"
assert weight_decay >= 0.0, f"Invalid weight_decay value: {weight_decay}"
assert scale > 0.0, f"Invalid scale factor: {scale}"
assert amsgrad is False, "Not support AMSGrad now!"
self._default_options["lr"] = lr
self._default_options["eps"] = eps
self._default_options["betas"] = betas
self._default_options["weight_decay"] = weight_decay
self._default_options["amsgrad"] = amsgrad
self._default_options["scale"] = scale
# Add parameters
if isinstance(parameters, GeneratorType):
self._param_groups.append(ParamGroup(parameters, self._default_options))
else: # List[Dict]
for param in parameters:
self._param_groups.append(ParamGroup(param, self._default_options))
for param_group in self._param_groups:
for param in param_group.parameters:
assert param.is_leaf, "parameters must be leaf tensor"
self._state[param] = dict()
self._state[param]["exp_avg"] = flow.experimental.zeros_like(param)
self._state[param]["exp_avg_sq"] = flow.experimental.zeros_like(param)
self._op = (
flow.builtin_op("adam_update")
.Input("model")
.Input("model_diff")
.Input("learning_rate")
.Input("m")
.Input("v")
.Attr("l1", 0.0)
.Attr("l2", 0.0)
.Build()
)
def __init__(
self,
parameters: Union[Iterator[Parameter], List[Dict]],
lr: float = 1e-3,
alpha: float = 0.99,
eps: float = 1e-8,
weight_decay: float = 0,
momentum: float = 0.0,
centered: bool = False,
scale: float = 1.0,
):
super().__init__()
assert lr >= 0.0, f"Invalid learning rate: {lr}"
assert alpha >= 0.0, f"Invalid alpha value: {alpha}"
assert eps >= 0.0, f"Invalid epsilon value: {eps}"
assert weight_decay >= 0.0, f"Invalid weight_decay value: {weight_decay}"
assert scale > 0.0, f"Invalid scale factor: {scale}"
assert momentum == 0.0, "Not support momentum greater than zeros now!"
self._default_options["lr"] = lr
self._default_options["alpha"] = alpha
self._default_options["eps"] = eps
self._default_options["weight_decay"] = weight_decay
self._default_options["centered"] = centered
self._default_options["scale"] = scale
# Add parameters
if isinstance(parameters, GeneratorType):
self._param_groups.append(
ParamGroup(parameters, self._default_options))
else: # List[Dict]
for param in parameters:
self._param_groups.append(
ParamGroup(param, self._default_options))
for param_group in self._param_groups:
for param in param_group.parameters:
assert param.is_leaf, "parameters must be leaf tensor"
self._state[param] = dict()
self._state[param][
"square_avg"] = flow.experimental.zeros_like(param)
if param_group.options["centered"]:
self._state[param][
"grad_avg"] = flow.experimental.zeros_like(param)
self._centered_rmsprop = (flow.builtin_op("rmsprop_update").Input(
"model").Input("model_diff").Input("learning_rate").Input(
"mean_square").Input("mean_gradient").Attr(
"centered", True).Attr("l1", 0.0).Attr("l2", 0.0).Build())
self._rmsprop = (
flow.builtin_op("rmsprop_update").Input("model").Input(
"model_diff").Input("learning_rate").Input("mean_square").Attr(
"centered", False).Attr("l1", 0.0).Attr("l2", 0.0).Build())