def back_prop(self,incoming_grad,lr = 0.01): """ :param incoming_grad: should be a 1d vector :return: """ if self.activation !="softmax": if self.activation == "sigmoid": self.local_grad = A.grad_sigmoid(self.sum_of_incoming) elif self.activation == "relu": self.local_grad = A.grad_relu(self.sum_of_incoming) self.local_grad *= incoming_grad #element wise multiplication else: self.local_grad = incoming_grad temp_to_pass_back = [self.local_grad for _ in range(self.weights.shape[0])] temp_to_pass_back = np.asarray(temp_to_pass_back) bias_grad = self.local_grad weight_grad = np.matmul(self.input.reshape((self.input.shape[0],1)), self.local_grad.reshape((1,self.local_grad.shape[0]))) back_grad = self.weights * temp_to_pass_back self.biases =self.biases - lr * bias_grad self.weights = self.weights - lr * weight_grad ##propogate gradient to previous layer return np.sum(back_grad,axis=1)
def store_grad(self,incoming_grad,lr = 0.01): """ :param incoming_grad: should be a 1d vector :return: """ if self.activation !="softmax": if self.activation == "sigmoid": self.local_grad = A.grad_sigmoid(self.sum_of_incoming) elif self.activation == "relu": self.local_grad = A.grad_relu(self.sum_of_incoming) elif self.activation == "swish": self.local_grad = A.grad_swish(self.sum_of_incoming) elif self.activation == "tanh": self.local_grad = A.grad_tanh(self.sum_of_incoming) # print("local grad is:",self.local_grad) self.local_grad *= incoming_grad #element wise multiplication # print("incoming grad is:",self.local_grad) else: self.local_grad = incoming_grad # print("for softmax incoming grad is:",self.local_grad) temp_to_pass_back = [self.local_grad for _ in range(self.weights.shape[0])] temp_to_pass_back = np.asarray(temp_to_pass_back) temp = np.matmul(self.input.reshape((self.input.shape[0],1)), self.local_grad.reshape((1,self.local_grad.shape[0]))) self.bias_grad += self.local_grad # print("temp is ",temp) # print("weight grad is ",self.weight_grad) self.weight_grad += temp # print("weight grad is ",self.weight_grad) ##propogate gradient to previous layer back_grad = self.weights * temp_to_pass_back # if np.linalg.norm(back_grad)>1.0: # print("grad explosion , inside store_grad function of layer") return np.sum(back_grad,axis=1)