def get_config(self): return { "name": self.__class__.__name__, "lr": float(K.get_value(self.lr)), "rho": float(K.get_value(self.rho)), "epsilon": self.epsilon }
def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) accumulators = [ K.variable(np.zeros(K.get_value(p).shape)) for p in params ] delta_accumulators = [ K.variable(np.zeros(K.get_value(p).shape)) for p in params ] self.updates = [] for p, g, a, d_a, c in zip(params, grads, accumulators, delta_accumulators, constraints): # update accumulator new_a = self.rho * a + (1 - self.rho) * K.square(g) self.updates.append((a, new_a)) # use the new accumulator and the *old* delta_accumulator update = g * K.sqrt(d_a + self.epsilon) / K.sqrt(new_a + self.epsilon) new_p = p - self.lr * update self.updates.append((p, c(new_p))) # apply constraints # update delta_accumulator new_d_a = self.rho * d_a + (1 - self.rho) * K.square(update) self.updates.append((d_a, new_d_a)) return self.updates
def get_config(self): return { "name": self.__class__.__name__, "lr": float(K.get_value(self.lr)), "momentum": float(K.get_value(self.momentum)), "decay": float(K.get_value(self.decay)), "nesterov": self.nesterov }
def get_config(self): return { "name": self.__class__.__name__, "lr": float(K.get_value(self.lr)), "beta_1": float(K.get_value(self.beta_1)), "beta_2": float(K.get_value(self.beta_2)), "epsilon": self.epsilon }
def set_weights(self, weights): '''Set the weights of the unit. weights: a list of numpy arrays. The number of arrays and their shape must match number of the dimensions of the weights of the unit (i.e. it should match the output of `get_weights`). ''' assert len(self.params) == len(weights), ( 'Provided weight array does not match unit weights (' + str(len(self.params)) + ' unit params vs. ' + str(len(weights)) + ' provided weights)') for p, w in zip(self.params, weights): if K.get_value(p).shape != w.shape: raise Exception( 'Weight shape %s not compatible with weight shape %s.' % (K.get_value(p).shape, w.shape)) K.set_value(p, w)
def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) accumulators = [K.variable(np.zeros(K.get_value(p).shape)) for p in params] self.updates = [] for p, g, a, c in zip(params, grads, accumulators, constraints): new_a = a + K.square(g) # update accumulator self.updates.append((a, new_a)) new_p = p - self.lr * g / K.sqrt(new_a + self.epsilon) self.updates.append((p, c(new_p))) # apply constraints return self.updates
def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) self.updates = [(self.iterations, self.iterations + 1.)] t = self.iterations + 1 lr_t = self.lr / (1 - K.pow(self.beta_1, t)) for p, g, c in zip(params, grads, constraints): # zero init of 1st moment m = K.variable(np.zeros(K.get_value(p).shape)) # zero init of exponentially weighted infinity norm u = K.variable(np.zeros(K.get_value(p).shape)) m_t = (self.beta_1 * m) + (1 - self.beta_1) * g u_t = K.maximum(self.beta_2 * u, K.abs(g)) p_t = p - lr_t * m_t / (u_t + self.epsilon) self.updates.append((m, m_t)) self.updates.append((u, u_t)) self.updates.append((p, c(p_t))) # apply constraints return self.updates
def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) self.updates = [(self.iterations, self.iterations+1.)] t = self.iterations + 1 lr_t = self.lr / (1 - K.pow(self.beta_1, t)) for p, g, c in zip(params, grads, constraints): # zero init of 1st moment m = K.variable(np.zeros(K.get_value(p).shape)) # zero init of exponentially weighted infinity norm u = K.variable(np.zeros(K.get_value(p).shape)) m_t = (self.beta_1 * m) + (1 - self.beta_1) * g u_t = K.maximum(self.beta_2 * u, K.abs(g)) p_t = p - lr_t * m_t / (u_t + self.epsilon) self.updates.append((m, m_t)) self.updates.append((u, u_t)) self.updates.append((p, c(p_t))) # apply constraints return self.updates
def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) self.updates = [(self.iterations, self.iterations+1.)] t = self.iterations + 1 lr_t = self.lr * K.sqrt(1 - K.pow(self.beta_2, t)) / (1 - K.pow(self.beta_1, t)) for p, g, c in zip(params, grads, constraints): # zero init of moment m = K.variable(np.zeros(K.get_value(p).shape)) # zero init of velocity v = K.variable(np.zeros(K.get_value(p).shape)) m_t = (self.beta_1 * m) + (1 - self.beta_1) * g v_t = (self.beta_2 * v) + (1 - self.beta_2) * K.square(g) p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon) self.updates.append((m, m_t)) self.updates.append((v, v_t)) self.updates.append((p, c(p_t))) # apply constraints return self.updates
def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) accumulators = [K.variable(np.zeros(K.get_value(p).shape)) for p in params] delta_accumulators = [K.variable(np.zeros(K.get_value(p).shape)) for p in params] self.updates = [] for p, g, a, d_a, c in zip(params, grads, accumulators, delta_accumulators, constraints): # update accumulator new_a = self.rho * a + (1 - self.rho) * K.square(g) self.updates.append((a, new_a)) # use the new accumulator and the *old* delta_accumulator update = g * K.sqrt(d_a + self.epsilon) / K.sqrt(new_a + self.epsilon) new_p = p - self.lr * update self.updates.append((p, c(new_p))) # apply constraints # update delta_accumulator new_d_a = self.rho * d_a + (1 - self.rho) * K.square(update) self.updates.append((d_a, new_d_a)) return self.updates
def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) accumulators = [ K.variable(np.zeros(K.get_value(p).shape)) for p in params ] self.updates = [] for p, g, a, c in zip(params, grads, accumulators, constraints): new_a = a + K.square(g) # update accumulator self.updates.append((a, new_a)) new_p = p - self.lr * g / K.sqrt(new_a + self.epsilon) self.updates.append((p, c(new_p))) # apply constraints return self.updates
def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) self.updates = [(self.iterations, self.iterations + 1.)] t = self.iterations + 1 lr_t = self.lr * K.sqrt(1 - K.pow(self.beta_2, t)) / ( 1 - K.pow(self.beta_1, t)) for p, g, c in zip(params, grads, constraints): # zero init of moment m = K.variable(np.zeros(K.get_value(p).shape)) # zero init of velocity v = K.variable(np.zeros(K.get_value(p).shape)) m_t = (self.beta_1 * m) + (1 - self.beta_1) * g v_t = (self.beta_2 * v) + (1 - self.beta_2) * K.square(g) p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon) self.updates.append((m, m_t)) self.updates.append((v, v_t)) self.updates.append((p, c(p_t))) # apply constraints return self.updates
def set_weights(self, weights): '''Set the weights of the unit. weights: a list of numpy arrays. The number of arrays and their shape must match number of the dimensions of the weights of the unit (i.e. it should match the output of `get_weights`). ''' assert len(self.params) == len(weights), ('Provided weight array does not match unit weights (' + str(len(self.params)) + ' unit params vs. ' + str(len(weights)) + ' provided weights)') for p, w in zip(self.params, weights): if K.get_value(p).shape != w.shape: raise Exception('Weight shape %s not compatible with weight shape %s.' % (K.get_value(p).shape, w.shape)) K.set_value(p, w)
def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) lr = self.lr * (1.0 / (1.0 + self.decay * self.iterations)) self.updates = [(self.iterations, self.iterations + 1.)] for p, g, c in zip(params, grads, constraints): m = K.variable(np.zeros(K.get_value(p).shape)) # momentum v = self.momentum * m - lr * g # velocity self.updates.append((m, v)) if self.nesterov: new_p = p + self.momentum * v - lr * g else: new_p = p + v self.updates.append((p, c(new_p))) # apply constraints return self.updates
def get_config(self): return {"name": self.__class__.__name__, "lr": float(K.get_value(self.lr)), "rho": self.rho, "epsilon": self.epsilon}
def get_weights(self): '''Return the weights of the unit, as a list of numpy arrays.''' weights = [] for p in self.params: weights.append(K.get_value(p)) return weights
def get_state(self): return [K.get_value(u[0]) for u in self.updates]
def get_config(self): return {"name": self.__class__.__name__, "lr": float(K.get_value(self.lr)), "beta_1": float(K.get_value(self.beta_1)), "beta_2": float(K.get_value(self.beta_2)), "epsilon": self.epsilon}
def get_config(self): return {"name": self.__class__.__name__, "lr": float(K.get_value(self.lr)), "momentum": float(K.get_value(self.momentum)), "decay": float(K.get_value(self.decay)), "nesterov": self.nesterov}