def _step(self, epoch): t = fn.to_tensor(epoch) grads = [p.grad for p in self.parameters] for p, g, m, v in (zip(self.parameters, grads, self.ms, self.vs)): m = self.beta1*m + (1-self.beta1)*g v = self.beta2*v + (1-self.beta2)*fn.square(g) m_hat = m / (1 - fn.power(self.beta1, t)) v_hat = v / (1 - fn.power(self.beta2, t)) p -= (self.lr * (m_hat * self.beta1 + (1 - self.beta2) / (1 - fn.power(self.beta1, t))*g)) / (fn.sqrt(v_hat) + self.epsilon)
def _step(self, epoch): t = fn.to_tensor(epoch) step_size = self.lr * (fn.sqrt(1 - fn.power(self.beta2, t)) / (1 - fn.power(self.beta1, t))) lower_bound = self.final_lr * (1.0 - 1.0 / (self.gamma * t + 1)) upper_bound = self.final_lr * (1.0 + 1.0 / (self.gamma * t)) grads = [p.grad for p in self.parameters] for p, g, m, v in (zip(self.parameters, grads, self.ms, self.vs)): m = self.beta1 * m + (1 - self.beta1) * g v = self.beta2 * v + (1 - self.beta2) * fn.square(g) denom = fn.sqrt(v) + self.epsilon p -= m * fn.clip(step_size / denom, lower_bound.item(), upper_bound.item())
def _step(self, epoch): t = fn.to_tensor(epoch) grads = [p.grad for p in self.parameters] for p, g, m, v in (zip(self.parameters, grads, self.ms, self.vs)): m = self.beta1 * m + (1 - self.beta1) * g v = fn.maximum(self.beta2 * v, fn.abs(g)) lr_t = self.lr / (1 - fn.power(self.beta1, t)) p -= lr_t * m / (v + self.epsilon)
def __rpow__(self, t): from beacon.tensor.functions import power return power(self._to_tensor(t), self)