def optimize(self):
for parameter in default_graph.compute_grads_of(self._fn):
dx = parameter._cumulative_consumers_grad
parameter.dx_cache = parameter.dx_cache * self._decay_rate + (
1 - self._decay_rate) * (dx**2)
parameter.value -= self._lr * dx / (np.sqrt(parameter.dx_cache) +
self._eps)
def optimize(self):
for parameter in default_graph.compute_grads_of(self._fn):
dx = parameter._cumulative_consumers_grad
v_prev = parameter.v
parameter.v = self._mu * parameter.v - self._lr * dx
parameter.value += -1.0 * self._mu * v_prev + (
1.0 + self._mu) * parameter.v
def optimize(self):
for parameter in default_graph.compute_grads_of(self._fn):
dx = parameter._cumulative_consumers_grad
m = self._beta1 * parameter.m + (1.0 - self._beta1) * dx
parameter.m = m
v = self._beta2 * parameter.v + (1.0 - self._beta2) * (dx**2)
parameter.v = v
parameter.value -= self._lr * m / (np.sqrt(v) + self._eps)
def fit(self, x_train, y_train, epochs=15):
total = 0
correct = 0
for _ in range(epochs):
for x, y in zip(x_train, y_train):
total += 1
self.cost_f.y = y
self.forward(x)
correct += (np.argmax([float(a) for a in self.layers[-1].a
]) == np.argmax(y))
default_graph.nullify_grads()
default_graph.compute_grads_of(self.cost_f)
self.optimizer.optimize()
print('Epoch {}/{}. Error: {:3f}. Accuracy: {:3f}'.format(
_ + 1, epochs, self.cost_f.value, correct / (total + 1)),
end='\r')
print()
from graph.graph import default_graph from graph.nodes import Parameter from graph.gates import AddGate from graph.gates import MulGate a = Parameter(3) b = Parameter(4) c = Parameter(5) product = MulGate([a, b, c]) y = AddGate([product, c]) default_graph.compute(y) default_graph.compute_grads_of(y) print(a, b, c, product, y)
def optimize(self):
for parameter in default_graph.compute_grads_of(self._fn):
dx = parameter._cumulative_consumers_grad
parameter.dx_cache += np.abs(dx)
parameter.value -= self._lr * dx / (parameter.dx_cache + self._eps)
def optimize(self):
for parameter in default_graph.compute_grads_of(self._fn):
dx = parameter._cumulative_consumers_grad
parameter.v = (parameter.v * self._mu) - (dx * self._lr)
parameter.value += parameter.v
def optimize(self):
for parameter in default_graph.compute_grads_of(self._fn):
dx = (parameter._cumulative_consumers_grad * self._lr)
parameter.value -= dx
