def get_config(self):
return {
"name": self.__class__.__name__,
"lr": float(K.get_value(self.lr)),
"rho": float(K.get_value(self.rho)),
"epsilon": self.epsilon
}
def get_updates(self, params, constraints, loss):
grads = self.get_gradients(loss, params)
accumulators = [
K.variable(np.zeros(K.get_value(p).shape)) for p in params
]
delta_accumulators = [
K.variable(np.zeros(K.get_value(p).shape)) for p in params
]
self.updates = []
for p, g, a, d_a, c in zip(params, grads, accumulators,
delta_accumulators, constraints):
# update accumulator
new_a = self.rho * a + (1 - self.rho) * K.square(g)
self.updates.append((a, new_a))
# use the new accumulator and the *old* delta_accumulator
update = g * K.sqrt(d_a + self.epsilon) / K.sqrt(new_a +
self.epsilon)
new_p = p - self.lr * update
self.updates.append((p, c(new_p))) # apply constraints
# update delta_accumulator
new_d_a = self.rho * d_a + (1 - self.rho) * K.square(update)
self.updates.append((d_a, new_d_a))
return self.updates
def get_config(self):
return {
"name": self.__class__.__name__,
"lr": float(K.get_value(self.lr)),
"momentum": float(K.get_value(self.momentum)),
"decay": float(K.get_value(self.decay)),
"nesterov": self.nesterov
}
def get_config(self):
return {
"name": self.__class__.__name__,
"lr": float(K.get_value(self.lr)),
"beta_1": float(K.get_value(self.beta_1)),
"beta_2": float(K.get_value(self.beta_2)),
"epsilon": self.epsilon
}
def set_weights(self, weights):
'''Set the weights of the unit.
weights: a list of numpy arrays. The number of arrays and their shape must match
number of the dimensions of the weights of the unit (i.e. it should match the
output of `get_weights`).
'''
assert len(self.params) == len(weights), (
'Provided weight array does not match unit weights (' +
str(len(self.params)) + ' unit params vs. ' + str(len(weights)) +
' provided weights)')
for p, w in zip(self.params, weights):
if K.get_value(p).shape != w.shape:
raise Exception(
'Weight shape %s not compatible with weight shape %s.' %
(K.get_value(p).shape, w.shape))
K.set_value(p, w)
def get_updates(self, params, constraints, loss):
grads = self.get_gradients(loss, params)
accumulators = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
self.updates = []
for p, g, a, c in zip(params, grads, accumulators, constraints):
new_a = a + K.square(g) # update accumulator
self.updates.append((a, new_a))
new_p = p - self.lr * g / K.sqrt(new_a + self.epsilon)
self.updates.append((p, c(new_p))) # apply constraints
return self.updates
def get_updates(self, params, constraints, loss):
grads = self.get_gradients(loss, params)
self.updates = [(self.iterations, self.iterations + 1.)]
t = self.iterations + 1
lr_t = self.lr / (1 - K.pow(self.beta_1, t))
for p, g, c in zip(params, grads, constraints):
# zero init of 1st moment
m = K.variable(np.zeros(K.get_value(p).shape))
# zero init of exponentially weighted infinity norm
u = K.variable(np.zeros(K.get_value(p).shape))
m_t = (self.beta_1 * m) + (1 - self.beta_1) * g
u_t = K.maximum(self.beta_2 * u, K.abs(g))
p_t = p - lr_t * m_t / (u_t + self.epsilon)
self.updates.append((m, m_t))
self.updates.append((u, u_t))
self.updates.append((p, c(p_t))) # apply constraints
return self.updates
def get_updates(self, params, constraints, loss):
grads = self.get_gradients(loss, params)
self.updates = [(self.iterations, self.iterations+1.)]
t = self.iterations + 1
lr_t = self.lr / (1 - K.pow(self.beta_1, t))
for p, g, c in zip(params, grads, constraints):
# zero init of 1st moment
m = K.variable(np.zeros(K.get_value(p).shape))
# zero init of exponentially weighted infinity norm
u = K.variable(np.zeros(K.get_value(p).shape))
m_t = (self.beta_1 * m) + (1 - self.beta_1) * g
u_t = K.maximum(self.beta_2 * u, K.abs(g))
p_t = p - lr_t * m_t / (u_t + self.epsilon)
self.updates.append((m, m_t))
self.updates.append((u, u_t))
self.updates.append((p, c(p_t))) # apply constraints
return self.updates
def get_updates(self, params, constraints, loss):
grads = self.get_gradients(loss, params)
self.updates = [(self.iterations, self.iterations+1.)]
t = self.iterations + 1
lr_t = self.lr * K.sqrt(1 - K.pow(self.beta_2, t)) / (1 - K.pow(self.beta_1, t))
for p, g, c in zip(params, grads, constraints):
# zero init of moment
m = K.variable(np.zeros(K.get_value(p).shape))
# zero init of velocity
v = K.variable(np.zeros(K.get_value(p).shape))
m_t = (self.beta_1 * m) + (1 - self.beta_1) * g
v_t = (self.beta_2 * v) + (1 - self.beta_2) * K.square(g)
p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
self.updates.append((m, m_t))
self.updates.append((v, v_t))
self.updates.append((p, c(p_t))) # apply constraints
return self.updates
def get_updates(self, params, constraints, loss):
grads = self.get_gradients(loss, params)
accumulators = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
delta_accumulators = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
self.updates = []
for p, g, a, d_a, c in zip(params, grads, accumulators,
delta_accumulators, constraints):
# update accumulator
new_a = self.rho * a + (1 - self.rho) * K.square(g)
self.updates.append((a, new_a))
# use the new accumulator and the *old* delta_accumulator
update = g * K.sqrt(d_a + self.epsilon) / K.sqrt(new_a + self.epsilon)
new_p = p - self.lr * update
self.updates.append((p, c(new_p))) # apply constraints
# update delta_accumulator
new_d_a = self.rho * d_a + (1 - self.rho) * K.square(update)
self.updates.append((d_a, new_d_a))
return self.updates
def get_updates(self, params, constraints, loss):
grads = self.get_gradients(loss, params)
accumulators = [
K.variable(np.zeros(K.get_value(p).shape)) for p in params
]
self.updates = []
for p, g, a, c in zip(params, grads, accumulators, constraints):
new_a = a + K.square(g) # update accumulator
self.updates.append((a, new_a))
new_p = p - self.lr * g / K.sqrt(new_a + self.epsilon)
self.updates.append((p, c(new_p))) # apply constraints
return self.updates
def get_updates(self, params, constraints, loss):
grads = self.get_gradients(loss, params)
self.updates = [(self.iterations, self.iterations + 1.)]
t = self.iterations + 1
lr_t = self.lr * K.sqrt(1 - K.pow(self.beta_2, t)) / (
1 - K.pow(self.beta_1, t))
for p, g, c in zip(params, grads, constraints):
# zero init of moment
m = K.variable(np.zeros(K.get_value(p).shape))
# zero init of velocity
v = K.variable(np.zeros(K.get_value(p).shape))
m_t = (self.beta_1 * m) + (1 - self.beta_1) * g
v_t = (self.beta_2 * v) + (1 - self.beta_2) * K.square(g)
p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
self.updates.append((m, m_t))
self.updates.append((v, v_t))
self.updates.append((p, c(p_t))) # apply constraints
return self.updates
def set_weights(self, weights):
'''Set the weights of the unit.
weights: a list of numpy arrays. The number of arrays and their shape must match
number of the dimensions of the weights of the unit (i.e. it should match the
output of `get_weights`).
'''
assert len(self.params) == len(weights), ('Provided weight array does not match unit weights (' +
str(len(self.params)) + ' unit params vs. ' +
str(len(weights)) + ' provided weights)')
for p, w in zip(self.params, weights):
if K.get_value(p).shape != w.shape:
raise Exception('Weight shape %s not compatible with weight shape %s.' % (K.get_value(p).shape, w.shape))
K.set_value(p, w)
def get_updates(self, params, constraints, loss):
grads = self.get_gradients(loss, params)
lr = self.lr * (1.0 / (1.0 + self.decay * self.iterations))
self.updates = [(self.iterations, self.iterations + 1.)]
for p, g, c in zip(params, grads, constraints):
m = K.variable(np.zeros(K.get_value(p).shape)) # momentum
v = self.momentum * m - lr * g # velocity
self.updates.append((m, v))
if self.nesterov:
new_p = p + self.momentum * v - lr * g
else:
new_p = p + v
self.updates.append((p, c(new_p))) # apply constraints
return self.updates
def get_updates(self, params, constraints, loss):
grads = self.get_gradients(loss, params)
lr = self.lr * (1.0 / (1.0 + self.decay * self.iterations))
self.updates = [(self.iterations, self.iterations + 1.)]
for p, g, c in zip(params, grads, constraints):
m = K.variable(np.zeros(K.get_value(p).shape)) # momentum
v = self.momentum * m - lr * g # velocity
self.updates.append((m, v))
if self.nesterov:
new_p = p + self.momentum * v - lr * g
else:
new_p = p + v
self.updates.append((p, c(new_p))) # apply constraints
return self.updates
def get_config(self):
return {"name": self.__class__.__name__,
"lr": float(K.get_value(self.lr)),
"rho": self.rho,
"epsilon": self.epsilon}
def get_weights(self):
'''Return the weights of the unit, as a list of numpy arrays.'''
weights = []
for p in self.params:
weights.append(K.get_value(p))
return weights
def get_state(self):
return [K.get_value(u[0]) for u in self.updates]
def get_config(self):
return {"name": self.__class__.__name__,
"lr": float(K.get_value(self.lr)),
"beta_1": float(K.get_value(self.beta_1)),
"beta_2": float(K.get_value(self.beta_2)),
"epsilon": self.epsilon}
def get_state(self):
return [K.get_value(u[0]) for u in self.updates]
def get_config(self):
return {"name": self.__class__.__name__,
"lr": float(K.get_value(self.lr)),
"momentum": float(K.get_value(self.momentum)),
"decay": float(K.get_value(self.decay)),
"nesterov": self.nesterov}
def get_weights(self):
'''Return the weights of the unit, as a list of numpy arrays.'''
weights = []
for p in self.params:
weights.append(K.get_value(p))
return weights