def back_prop(self,incoming_grad,lr = 0.01):
"""
:param incoming_grad: should be a 1d vector
:return:
"""
if self.activation !="softmax":
if self.activation == "sigmoid":
self.local_grad = A.grad_sigmoid(self.sum_of_incoming)
elif self.activation == "relu":
self.local_grad = A.grad_relu(self.sum_of_incoming)
self.local_grad *= incoming_grad #element wise multiplication
else:
self.local_grad = incoming_grad
temp_to_pass_back = [self.local_grad for _ in range(self.weights.shape[0])]
temp_to_pass_back = np.asarray(temp_to_pass_back)
bias_grad = self.local_grad
weight_grad = np.matmul(self.input.reshape((self.input.shape[0],1)),
self.local_grad.reshape((1,self.local_grad.shape[0])))
back_grad = self.weights * temp_to_pass_back
self.biases =self.biases - lr * bias_grad
self.weights = self.weights - lr * weight_grad
##propogate gradient to previous layer
return np.sum(back_grad,axis=1)
def store_grad(self,incoming_grad,lr = 0.01):
"""
:param incoming_grad: should be a 1d vector
:return:
"""
if self.activation !="softmax":
if self.activation == "sigmoid":
self.local_grad = A.grad_sigmoid(self.sum_of_incoming)
elif self.activation == "relu":
self.local_grad = A.grad_relu(self.sum_of_incoming)
elif self.activation == "swish":
self.local_grad = A.grad_swish(self.sum_of_incoming)
elif self.activation == "tanh":
self.local_grad = A.grad_tanh(self.sum_of_incoming)
# print("local grad is:",self.local_grad)
self.local_grad *= incoming_grad #element wise multiplication
# print("incoming grad is:",self.local_grad)
else:
self.local_grad = incoming_grad
# print("for softmax incoming grad is:",self.local_grad)
temp_to_pass_back = [self.local_grad for _ in range(self.weights.shape[0])]
temp_to_pass_back = np.asarray(temp_to_pass_back)
temp = np.matmul(self.input.reshape((self.input.shape[0],1)),
self.local_grad.reshape((1,self.local_grad.shape[0])))
self.bias_grad += self.local_grad
# print("temp is ",temp)
# print("weight grad is ",self.weight_grad)
self.weight_grad += temp
# print("weight grad is ",self.weight_grad)
##propogate gradient to previous layer
back_grad = self.weights * temp_to_pass_back
# if np.linalg.norm(back_grad)>1.0:
# print("grad explosion , inside store_grad function of layer")
return np.sum(back_grad,axis=1)
