def backprop(self, x, y):
# initialize empty arrays to represent the change in the weights and biases
delta_b = [np.zeros(b.shape) for b in self.biases]
delta_w = [np.zeros(w.shape) for w in self.weights]
# Feedforward
a = x
activations = [x]
products = []
for i in range(self.num_layers - 1):
# z^l = w^l a^l-1 + b^l
z = np.dot(self.weights[i], a) + self.biases
products.append(z)
a = act.sigmoid(z)
activations.append(a)
# Output Error
delta = (activations[-1] - y) * act.sigmoid_derivative(products[-1])
delta_b[-1] = delta
delta_w[-1] = np.dot(delta, activations[-2].transpose())
# Backprop Error
errors = []
for l in range(self.num_layers - 2, 0):
delta = np.dot(self.weights[l + 1].transpose(), delta) * act.sigmoid_derivative(products[l])
delta_b[l] = delta
delta_w[l] = np.dot(delta, activations[l - 1])
return (delta_b, delta_w)
def partial_fit(self, x, y): # activation A = [x] # feedforward out = A[-1] for i in range(0, len(self.layers) - 1): out = sigmoid(out.dot(self.W[i]) + (self.B[i].T)) A.append(out) # backpropagation dA = [-(y/A[-1] - (1 - y)/(1 - A[-1]))] dW = [] dB = [] for i in reversed(range(0, len(self.layers) - 1)): dw = A[i].T.dot(dA[-1] * sigmoid_derivative(A[i+1])) db = (np.sum(dA[-1] * sigmoid_derivative(A[i+1]), 0)).reshape(-1, 1) da = dA[-1] * sigmoid_derivative(A[i+1]).dot(self.W[i].T) dW.append(dw) dB.append(db) dA.append(da) # reverse dW = dW[::-1] dB = dB[::-1] # gradient descent for i in range(0, len(self.layers) - 1): self.W[i] -= self.alpha * dW[i] self.B[i] -= self.alpha * dB[i]
def test_sigmoid_derivative(self):
self.assertEqual(sigmoid_derivative(0), 0.25)
self.assertLess(sigmoid_derivative(100), .001)
self.assertLess(sigmoid_derivative(-100), .001)
Z = np.array([1, 2, 3])
expected = np.array([0.1966119, 0.1049935, 0.04517666])
self.assertTrue(np.allclose(sigmoid_derivative(Z), expected))
def backpropagate(self, z, train_out):
z_list = []
activations_list = []
del_bias = [numpy.zeros(b.shape) for b in self.bias]
del_weights = [numpy.zeros(w.shape) for w in self.weights]
#print 'feed forwarding in back propagation.....'
activations_list.append(z)
for w, b in zip(self.weights, self.bias):
z = numpy.dot(w, z) + b
z_list.append(z)
activation = sigmoid(z)
activations_list.append(activation)
#print 'feed forwarding done.....'
#print 'calculating gradient and rho.....'
#print activations_list[-1] - train_out
gradient_c_wrt_a = activations_list[-1] - train_out
#'sd:' , sigmoid_derivative(z_list[-1])
rho = gradient_c_wrt_a * sigmoid_derivative(z_list[-1])
#print 'calculated gradient and rho.....'
#print activations_list
#print 'rho:', rho
del_bias[-1] = rho
del_weights[-1] = numpy.dot(rho, activations_list[-2].transpose())
# print 'back propagating the error.....'
for l in range(2, self.layers):
k = z_list[-l]
sd = sigmoid_derivative(k)
rho = numpy.dot(self.weights[-l + 1].transpose(), rho) * sd
del_bias[-l] = rho
del_weights[-l] = numpy.dot(rho,
activations_list[-l - 1].transpose())
#print 'back propagation completed.....'
#print ' debug:',del_bias
#print 'debug2', del_weights
#print 'done'
return del_bias, del_weights
def backpropagate(self,z, train_out):
z_list = []
activations_list = []
del_bias = [numpy.zeros(b.shape) for b in self.bias]
del_weights = [numpy.zeros(w.shape) for w in self.weights]
#print 'feed forwarding in back propagation.....'
activations_list.append(z)
for w, b in zip(self.weights, self.bias):
z = numpy.dot(w, z) + b
z_list.append(z)
activation = sigmoid(z)
activations_list.append(activation)
#print 'feed forwarding done.....'
#print 'calculating gradient and rho.....'
#print activations_list[-1] - train_out
gradient_c_wrt_a = activations_list[-1] - train_out
#'sd:' , sigmoid_derivative(z_list[-1])
rho = gradient_c_wrt_a * sigmoid_derivative(z_list[-1])
#print 'calculated gradient and rho.....'
#print activations_list
#print 'rho:', rho
del_bias[-1] = rho
del_weights[-1] = numpy.dot(rho, activations_list[-2].transpose())
# print 'back propagating the error.....'
for l in range(2, self.layers):
k = z_list[-l]
sd = sigmoid_derivative(k)
rho = numpy.dot(self.weights[-l+1].transpose(), rho) * sd
del_bias[-l] = rho
del_weights[-l] = numpy.dot(rho, activations_list[-l-1].transpose())
#print 'back propagation completed.....'
#print ' debug:',del_bias
#print 'debug2', del_weights
#print 'done'
return del_bias, del_weights
def backprop(self, x, y, n):
nabla_b = [np.zeros(b.shape) for b in self.model.bias]
nabla_w = [np.zeros(w.shape) for w in self.model.weights]
# activations, zs = self.model.forward(x)
activation = x
activations = [x] # list to store all the activations, layer by layer
zs = [] # list to store all the z vectors, layer by layer
layer = 0
for b, w in zip(self.model.bias, self.model.weights):
if layer != self.model.depth - 1 and self.model.dropout:
try:
for drop in self.model.dropout.dropout[layer]:
try:
w[drop] = np.zeros(w[drop].shape)
b[drop] = 0
except Exception as e:
print(e)
print(drop)
print(layer)
print(self.model.dropout.dropout)
except Exception as e:
print(e)
print(layer)
print(self.model.dropout.dropout)
z = np.dot(w, activation) + b
zs.append(z)
activation = sigmoid(z)
activations.append(activation)
layer += 1
cost = self.cost.eval(activations[-1], y, self.model.weights, n)
# delta = self.cost.derivative(activations[-1], y, zs[-1])
nabla_b[-1] = delta = self.cost.delta_b(activations[-1], y,
self.model.weights[-1], n)
nabla_w[-1] = self.cost.delta_w(activations[-1], activations[-2], y,
self.model.weights[-1], n)
for l in range(2, self.model.depth + 1):
z = zs[-l]
sp = sigmoid_derivative(z)
# delta = np.dot(self.model.weights[-l + 1].T, delta) * sp
delta = self.bpcost.delta_b(self.model.weights[-l + 1], delta, sp,
self.model.weights, n)
nabla_b[-l] = delta
# np.dot(delta, activations[-l - 1].T)
nabla_w[-l] = self.bpcost.delta_w(activations[-l - 1], delta,
self.model.weights[-l],
self.model.weights, n)
return nabla_b, nabla_w, cost
def _step_backward(dA, D, cache, regularization, lambd, keep_prob):
((A_prev, W, b), (Z, activation)) = cache
# dZ = dA * g[l]'(Z[l])
if activation == "relu":
activation_derivative = relu_derivative(Z)
elif activation == "sigmoid":
activation_derivative = sigmoid_derivative(Z)
dZ = dA * activation_derivative
m = A_prev.shape[1]
dA_prev = np.dot(np.transpose(W), dZ) * D / keep_prob
dW = np.divide(np.dot(dZ, np.transpose(A_prev)), m)
if regularization == "L2":
dW += (lambd/m) * W
db = np.divide(np.sum(dZ, axis=1, keepdims=True), m)
return dA_prev, dW, db
def sigmoid_backward(dA, activation_cache):
# function calculate dZ from dA, if last layer == sigmoid
dZ = np.multiply(dA, sigmoid_derivative(activation_cache))
return dZ