def update_weights(self, w, dEdW): """ Updates the parameters. Parameters ---------- w : numpy.ndarray The weight to be updated. dEdW : numpy.ndarray The derivative of error with respect to weight. """ self.accumulated_gradient = self.beta1 * self.accumulated_gradient + ( 1 - self.beta1) * dEdW self.accumulated_sq_gradient = self.beta2 * self.accumulated_sq_gradient + ( 1 - self.beta2) * np.square(dEdW) if self.bias_fix: self.t += 1 accumulated_gradient_t = self.accumulated_gradient / ( 1 - (self.beta1**self.t)) accumulated_sq_gradient_t = self.accumulated_sq_gradient / ( 1 - (self.beta2**self.t)) return w - self.learning_rate * accumulated_gradient_t / ( np.sqrt(accumulated_sq_gradient_t) + 1e-8) return w - self.learning_rate * self.accumulated_gradient / ( np.sqrt(self.accumulated_sq_gradient) + 1e-8)
def update_weights(self, w, dEdW): """ Updates the parameters. Parameters ---------- w : numpy.ndarray The weight to be updated. dEdW : numpy.ndarray The derivative of error with respect to weight. """ self.accumulated_sq_gradient += np.square(dEdW) return w - self.learning_rate * dEdW / ( np.sqrt(self.accumulated_sq_gradient) + 1e-8)
def mean_squared_error(y_predicted, y_target, return_deriv=False): """ Calculates the mean square error between the predicted and the target ouputs. Parameters ---------- y_predicted : numpy.ndarray The ouput predicted by the model. y_target : numpy.ndarray The expected output. return_deriv : bool, optional If set to true, the function returns derivative of the error along with the error. Returns ------- numpy.array : The error. numpy.array : The error's derivative, optional. """ if return_deriv: return np.sum(np.square(y_predicted - y_target)) / (2. * y_target.shape[0]), \ (y_predicted - y_target) / y_target.shape[0] return np.sum(np.square(y_predicted - y_target)) / (2. * y_target.shape[0])
def value(self, weights, batch_size): return self.l1 * np.absolute(weights) / (1.0 * batch_size) + (self.l2 / 2.0) *\ np.square(weights) / (1.0 * batch_size)
def value(self, weights, batch_size): return (self.lmda / 2.0) * np.square(weights) / (1.0 * batch_size)