def test_sigmoid_numpy_array(self): data = np.array([1, .85]) result = activations.sigmoid(data) np.testing.assert_array_almost_equal( result, np.array([0.7310585786300049, 0.7005671424739729]), decimal=14)
def predict(self, features): """ Predict the class labels for the provided data Parameters ---------- features : array-like, shape (n_samples, n_features) Returns ------- target : array of shape [n_samples] Class labels for each data sample. """ if self.fit_intercept: intercept = np.ones((features.shape[0], 1)) features = np.hstack((intercept, features)) final_scores = np.dot(features, self.weights) predictions = np.round(sigmoid(final_scores)) return predictions
def fit(self, features, target): """ Fit model to training data Parameters ---------- features : array-like, shape (n_samples, n_features) features matrix target : array-like, shape [n_samples] target array """ if not type(target) == np.dtype: try: # try to convert to ndarray target = target.values.ravel() except Exception: print("Error - Couldn't convert to ndarray") raise if self.fit_intercept: intercept = np.ones((features.shape[0], 1)) features = np.hstack((intercept, features)) # Initialize weights self.weights = np.zeros(features.shape[1]) if self.optimization == 'adam': # Constants for Adam Optimization (values are paper recommended values) beta1 = 0.9 beta2 = 0.999 eps = 1E-8 # first-moment vector Adam Optimization for W1 m = np.zeros_like(self.weights) # second-moment vector Adam Optimization for W1 v = np.zeros_like(self.weights) elif self.optimization == 'radam': # Constants for Rectified Adam Optimization (values are paper recommended values) beta1 = 0.9 beta2 = 0.999 eps = 1E-8 # first-moment vector Rectified Adam Optimization for W1 m = np.zeros_like(self.weights) # second-moment vector Recified Adam Optimization for W1 v = np.zeros_like(self.weights) # Maximum length of approximated SMA p_inf = 2 / ((1 - beta2) - 1) # Print settings if verbose if self.verbose > 0: print(f'Optimization: {self.optimization}') # Initialize Previous loss to a arbitrary high number to check for stopping previous_loss = 100000 stop = False iteration = 0 while not stop: iteration += 1 scores = np.dot(features, self.weights) predictions = sigmoid(scores) # Compute gradient # The gradient for Logistic Regression is the derivative of log loss # (Negative Log-Likelihood) # Note: Calculating gradient for Binary and Multi-Class is different gradient = -np.dot(features.T, target - predictions) # Update Weights if self.optimization == 'adam': # Update weights by using Adam Optimization # (as opposed to simply learning_rate * gradient) # https://arxiv.org/pdf/1412.6980.pdf # http://cs231n.github.io/neural-networks-3/ # (See Section: Per-parameter adaptive learning rate methods) m = beta1 * m + (1 - beta1) * gradient mt = m / (1 - beta1**iteration) v = beta2 * v + (1 - beta2) * (gradient**2) vt = v / (1 - beta2**iteration) self.weights += -self.learning_rate * mt / (np.sqrt(vt) + eps) elif self.optimization == 'radam': # Update exponetial 1st and 2nd moment m = beta1 * m + (1 - beta1) * gradient v = beta2 * v + (1 - beta2) * (gradient**2) # Compute bias-corrected moving average mt = m / (1 - beta1**iteration) # Compute length of the approximated SMA pt = p_inf - ((2 * iteration * (beta2**iteration)) / (1 - (beta2**iteration))) # if variance is tractable, updated with adapative momentum, else unadapated if pt > 4: # Compute bias-corrected moving 2nd moment vt = v / (1 - beta2**iteration) # Compute variance rectification term rt = np.sqrt(((pt - 4)(pt - 2)(p_inf)) / ((p_inf - 4)(p_inf - 2)(pt))) # Update weights with adaptive momentum self.weights += -self.learning_rate * rt * mt / vt else: # Update weights with un-adapted momentum self.weights += -self.learning_rate * mt else: self.weights -= self.learning_rate * gradient # Check to see if stopping criterion is reached loss = log_loss(features, target, self.weights) if loss > previous_loss - self.tol: stop = True else: previous_loss = loss # Verbose Output if self.verbose > 0 and iteration % self.output_freq == 0: print("Iteration: {0}, Log Loss: {1}".format(iteration, loss)) if stop: print("Stopping Criterion Reached.") return self
def test_sigmoid_int(self): data = 1 result = activations.sigmoid(data) np.testing.assert_almost_equal(result, 0.7310585786300049, decimal=14)
def test_sigmoid_python_array(self): data = [1, .85] result = activations.sigmoid(data) np.testing.assert_array_almost_equal( result, [0.7310585786300049, 0.7005671424739729], decimal=14)
def test_sigmoid_float(self): data = .85 result = activations.sigmoid(data) np.testing.assert_almost_equal(result, 0.7005671424739729, decimal=14)