def __init__(self, parameters, lr=0.01, beta1=0.9, beta2=0.999, epsilon=1e-8):
"""
Nadam optimizer.
"""
super().__init__(parameters, lr=lr)
self.beta1 = fn.to_tensor(beta1)
self.beta2 = fn.to_tensor(beta2)
self.epsilon = fn.to_tensor(epsilon)
self.ms = [fn.zeros_like(p) for p in self.parameters]
self.vs = [fn.zeros_like(p) for p in self.parameters]
def __init__(self, parameters, lr=0.01, momentum=0.9):
"""
Nesterov accelerated SGD optimizer.
"""
super().__init__(parameters, lr=lr)
self.momentum = fn.to_tensor(momentum)
self.vs = [fn.zeros_like(p) for p in self.parameters]
def __init__(self, parameters, lr=0.01, epsilon=1e-8):
"""
Adagrad optimizer.
"""
super().__init__(parameters, lr=lr)
self.epison = epsilon
self.G = [fn.zeros_like(p) for p in self.parameters]
def __init__(self, parameters, lr=0.01, momentum=0.9):
"""
SGD with Momentum optimizer.
"""
super().__init__(parameters, lr=lr)
self.momentum = fn.to_tensor(momentum)
self.vs = [fn.zeros_like(p) for p in self.parameters]
def __init__(self, parameters, lr=0.01, beta=0.99, epsilon=1e-8):
"""
RMSProp optimizer.
"""
super().__init__(parameters, lr=lr)
self.epsilon = epsilon
self.beta = fn.to_tensor(beta)
self.E = [fn.zeros_like(p) for p in self.parameters]
def __init__(self,
parameters,
lr=0.01,
beta1=0.9,
beta2=0.999,
final_lr=0.1,
gamma=1e-3,
epsilon=1e-8):
"""
Adabound optimizer.
"""
super().__init__(parameters, lr=lr)
self.beta1 = fn.to_tensor(beta1)
self.beta2 = fn.to_tensor(beta2)
self.epsilon = fn.to_tensor(epsilon)
self.final_lr = fn.to_tensor(final_lr)
self.gamma = fn.to_tensor(gamma)
self.ms = [fn.zeros_like(p) for p in self.parameters]
self.vs = [fn.zeros_like(p) for p in self.parameters]