def _gradients(self, thetas, X, y): # Load vars from neural net nl = self.neuralnet.num_layers() # Init variables DELTA = [None] * (nl) GRADIENTS = [None] * nl delta = [None] * (nl + 1) m = X.shape[0] # Do forward propagation, but here we don't need the result but the built vars _, a, z = self._forwardpropagate(thetas, X) delta[nl] = npe.add_ones(a[nl] - y).T # Calculate deltas for i in reversed(range(1, nl)): delta[i] = np.dot(thetas[i], delta[i+1][1:]) \ * npe.add_ones(NeuralNetClassification.activate(z[i].T, True), 0) # Calculate DELTAS for i in range(0, nl): DELTA[i] = np.dot(delta[i + 1][1:], a[i]).T # Calculate GRADIENTS for i in range(0, nl): GRADIENTS[i] = (1 / m) * DELTA[i] return GRADIENTS
def train(self, X, y, learning_rate=0.1, cost_threshold=1e-6, diff_threshold=1e-16, max_iter=10000, min_iter=0, lambda_=3): nclass = NeuralNetClassification(self) #self.thetas = npe.reshape_for_neuralnet(fmin_cg( # f=nclass._flat_cost, # x0=npe.flatten(self.thetas), # fprime=nclass._flat_gradients, # args=(X, y), # epsilon=learning_rate, # # gtol=gtol, # disp=False, # maxiter=50), self) self.thetas = npe.reshape_for_neuralnet( gradient_descent(start_thetas=npe.flatten(self.thetas), cost_func=nclass._flat_cost, gradient_func=nclass._flat_gradients, args=(X, y, lambda_), learning_rate=learning_rate, min_iter=min_iter, max_iter=max_iter, cost_threshold=cost_threshold, diff_threshold=diff_threshold), self)
def _flat_forwardpropagate(self, flat_thetas, X, *args): thetas = npe.reshape_for_neuralnet(flat_thetas, self.neuralnet) r,_,_ = self._forwardpropagate(thetas, self.neuralnet.num_layers(), X) return npe.flatten(r)
def rand_init(self): self.thetas = [None] * self.num_layers() for i in range(0, self.num_layers()): input = self.input_variables if i == 0 else self.hidden_units output = self.output_variables if i == self.hidden_layers else self.hidden_units self.thetas[i] = npe.random_matrix(input, output, epsilon=self.epsilon)
def _regularized_gradients(self, thetas, X, y, l): # Calculate gradients as usual gradients = self._gradients(thetas, X, y) # Number of examples & layers m = X.shape[0] nl = self.neuralnet.num_layers() for i in range(0, nl): gradients[i] += npe.add_zeros((l / m) * thetas[i][:, 1:], 1) return gradients
def numerical_gradients(self, X, y, epsilon=1e-4, lambda_=3): flat_thetas = npe.flatten(self.thetas) numgrad = np.zeros(flat_thetas.shape) perturb = np.zeros(flat_thetas.shape) for i in range(0, flat_thetas.size): perturb[i] = epsilon loss1 = self.get_algorithms()._flat_cost(flat_thetas - perturb, X, y, lambda_) loss2 = self.get_algorithms()._flat_cost(flat_thetas + perturb, X, y, lambda_) numgrad[i] = (loss2 - loss1) / (2 * epsilon) perturb[i] = 0 return numgrad
def _forwardpropagate(self, thetas, X): # Load vars from neural net nl = self.neuralnet.num_layers() # Init variables a = [None] * ( nl + 1 ) # Activation values, equal to z with applied activation method z = [None] * (nl + 1) # z values a[0] = X # Starting with X as input # Iterate over values to calculate a,z for i in range(0, nl): a[i] = npe.add_ones(a[i]) z[i + 1] = np.dot(a[i], thetas[i]) a[i + 1] = NeuralNetClassification.activate( z[i + 1]) # z for next level, i+1 # Return last z values, a's, and z's return a[nl], a, z
def check_gradients(self, X, y, do_print=False, imprecision=1e-8, epsilon=1e-8, lambda_=3): flat_thetas = npe.flatten(self.thetas) algorithmic = self.get_algorithms()._flat_gradients( flat_thetas, X, y, lambda_) numerical = self.numerical_gradients(X, y, epsilon, lambda_) all_match = True for i in range(0, flat_thetas.size): is_match = abs(algorithmic[i] - numerical[i]) < imprecision all_match = all_match and is_match if (do_print): print(i, numerical[i], algorithmic[i], is_match) if (do_print and all_match): print("All gradients are matching") elif (do_print and not all_match): print("Not all gradients are matching") return all_match
def _flat_regularized_gradients(self, flat_thetas, X, y, l, *args): thetas = npe.reshape_for_neuralnet(flat_thetas, self.neuralnet) gradients = self._regularized_gradients(thetas, X, y, l) f = npe.flatten(gradients) return f
def _flat_regularized_cost(self, flat_thetas, X, y, l, *args): thetas = npe.reshape_for_neuralnet(flat_thetas, self.neuralnet) return self._regularized_cost(thetas, X, y, l)
def _flat_cost(self, flat_thetas, X, y, *args): thetas = npe.reshape_for_neuralnet(flat_thetas, self.neuralnet) return self._cost(thetas, X, y)