def test_binaryclassifcation_regularized2(self): """ You have to be really careful with the reg parameter. If its to high (past 0.1), your network won't learn anything. With softmax regression and logistic regression, there existed leeway to make the reg parameter pretty high but that absolutely won"t work with neural networks multiple layers deep. """ # normal regularization multi_layer_perceptron3 = MultiLayerPerceptron( typeSupervised="binary", numberInputFeatures=self.x_train.shape[0], regularization="L1", reg_parameter=0.01) multi_layer_perceptron4 = MultiLayerPerceptron( typeSupervised="binary", numberInputFeatures=self.x_train.shape[0], regularization="L2", reg_parameter=0.01) # 10 features learnt in first hidden layer w/ ReLU multi_layer_perceptron3.add_layer(10, activation_function=ReLU()) # # 5 features learnt in second hidden layer w/ ReLU multi_layer_perceptron3.add_layer(5, activation_function=ReLU()) # # Output layer sigmoid activation multi_layer_perceptron3.add_layer(1, activation_function=Sigmoid()) # 10 features learnt in first hidden layer w/ ReLU multi_layer_perceptron4.add_layer(10, activation_function=ReLU()) # # 5 features learnt in second hidden layer w/ ReLU multi_layer_perceptron4.add_layer(5, activation_function=ReLU()) # # Output layer sigmoid activation multi_layer_perceptron4.add_layer(1, activation_function=Sigmoid()) multi_layer_perceptron3.fit(self.x_train, self.y_train, self.x_valid, self.y_valid, ret_train_loss=True, num_epochs=100, learn_rate=2.8) preds3 = multi_layer_perceptron3.predict_multi_layer_perceptron( self.x_test, 0.5) multi_layer_perceptron4.fit(self.x_train, self.y_train, self.x_valid, self.y_valid, ret_train_loss=True, num_epochs=100, learn_rate=2.6) preds4 = multi_layer_perceptron4.predict_multi_layer_perceptron( self.x_test, 0.5) acc3 = accuracy(self.y_test, preds3) acc4 = accuracy(self.y_test, preds4) self.assertGreaterEqual(acc3, 0.95) self.assertGreaterEqual(acc4, 0.95)
def test4(self): sklearn_mode = LR(C=1) sklearn_mode.fit(self.x_train.T, self.y_train.ravel()) preds_sk = sklearn_mode.predict(self.x_test.T) accuracy_sklearn = accuracy(self.y_test, preds_sk) obj4 = LogisticRegression(self.x_train.shape[0], classificationThreshold=0.5, regularization="L2", reg_parameter=1) obj4.fit(self.x_train, self.y_train, num_epochs=200, learn_rate=0.1) preds_model = obj4.predict(self.x_test) accuracy_model = accuracy(self.y_test, preds_model) self.assertLessEqual(abs(accuracy_model - accuracy_sklearn), 0.1)
def test_binaryclassification_regularized(self): # Sanity check - high regularization leads to very high losses. multi_layer_perceptron1 = MultiLayerPerceptron( typeSupervised="binary", numberInputFeatures=self.x_train.shape[0], regularization="L1", reg_parameter=500) multi_layer_perceptron2 = MultiLayerPerceptron( typeSupervised="binary", numberInputFeatures=self.x_train.shape[0], regularization="L2", reg_parameter=500) # 10 features learnt in first hidden layer w/ ReLU multi_layer_perceptron1.add_layer(10, activation_function=ReLU()) # # 5 features learnt in second hidden layer w/ ReLU multi_layer_perceptron1.add_layer(5, activation_function=ReLU()) # # Output layer sigmoid activation multi_layer_perceptron1.add_layer(1, activation_function=Sigmoid()) # 10 features learnt in first hidden layer w/ ReLU multi_layer_perceptron2.add_layer(10, activation_function=ReLU()) # # 5 features learnt in second hidden layer w/ ReLU multi_layer_perceptron2.add_layer(5, activation_function=ReLU()) # # Output layer sigmoid activation multi_layer_perceptron2.add_layer(1, activation_function=Sigmoid()) multi_layer_perceptron1.fit(self.x_train, self.y_train, self.x_valid, self.y_valid, ret_train_loss=True, num_epochs=10, learn_rate=2.6) preds1 = multi_layer_perceptron1.predict_multi_layer_perceptron( self.x_test, 0.5) multi_layer_perceptron2.fit(self.x_train, self.y_train, self.x_valid, self.y_valid, ret_train_loss=True, num_epochs=10, learn_rate=3) preds2 = multi_layer_perceptron2.predict_multi_layer_perceptron( self.x_test, 0.5) acc1 = accuracy(self.y_test, preds1) acc2 = accuracy(self.y_test, preds2) self.assertLessEqual(acc1, 0.69) self.assertLessEqual(acc2, 0.69)
def test_overfit_small_batch(self): multi_layer_perceptron = MultiLayerPerceptron( typeSupervised="multiclass", numberInputFeatures=784) multi_layer_perceptron.add_layer(num_neurons=100, activation_function=ReLU(), layer=DenseBatchNormLayer) multi_layer_perceptron.add_layer(num_neurons=10, activation_function=Softmax(), isSoftmax=True) train_loss1, train_acc1 = multi_layer_perceptron.fit( self.x_train[:, :100].reshape(784, -1), self.y_train[:, :100], num_epochs=150, ret_train_loss=True, optim=RMSProp(), learn_rate=0.001) predictions1 = multi_layer_perceptron.predict_multi_layer_perceptron( self.x_train[:, :100].reshape(784, -1)) acc = accuracy(self.saved_y[:, :100].reshape(1, -1), predictions1) print(train_loss1) print(acc) self.assertLessEqual(train_loss1[-1], 0.09) self.assertEqual(acc, 1) multi_layer_perceptron2 = MultiLayerPerceptron( typeSupervised="multiclass", numberInputFeatures=784) multi_layer_perceptron2.add_layer(num_neurons=100, activation_function=ReLU(), layer=DenseBatchNormLayer) multi_layer_perceptron2.add_layer(num_neurons=10, activation_function=Softmax(), isSoftmax=True) train_loss2, train_acc2 = multi_layer_perceptron2.fit( self.x_train[:, :100].reshape(784, -1), self.y_train[:, :100], num_epochs=150, ret_train_loss=True, optim=GradientDescentMomentum(), learn_rate=0.1) predictions2 = multi_layer_perceptron2.predict_multi_layer_perceptron( self.x_train[:, :100].reshape(784, -1)) acc2 = accuracy(self.saved_y[:, :100].reshape(1, -1), predictions2) print(train_loss2) print(acc2) self.assertLessEqual(train_loss2[-1], 0.09) self.assertEqual(acc2, 1)
def test_binaryclassification_unregularized(self): """ Learning rate KEY: If its set to high, you will diverge. If its set to low, your network won"t learn anything. The more layers you have, the higher the learning rate should be. Example seen here: 1 layer only - learn rate 0.1 suffices. With two layers ~ 0.7, with 3 layers ~2.5. """ MLP = MultiLayerPerceptron(typeSupervised="binary", numberInputFeatures=self.x_train.shape[0]) # Just add Dense Layers # 10 features learnt in first hidden layer w/ ReLU MLP.add_layer(10, activation_function=ReLU()) # # 5 features learnt in second hidden layer w/ ReLU MLP.add_layer(5, activation_function=ReLU()) # # Output layer sigmoid activation MLP.add_layer(1, activation_function=Sigmoid()) MLP.fit(self.x_train, self.y_train, self.x_valid, self.y_valid, ret_train_loss=True, num_epochs=100, learn_rate=2.6) preds = MLP.predict_multi_layer_perceptron(self.x_test, 0.5) acc = accuracy(self.y_test, preds) self.assertGreaterEqual(acc, 0.95)
def test_overall_model(self): #object should be able to get >= 20% percent accuracy on CIFAR-10 with k = 10 test_obj = KNearestNeighboursClassifier() test_obj.fit(self.x1_train, self.y_train) prediction_test = test_obj.predict(self.x1_test[:200]) acc = accuracy(self.y_test, prediction_test) self.assertEqual(prediction_test.shape, self.y_test[:200].shape) self.assertGreaterEqual(acc, 20)
def get_oob_score(self, xtrain: np.ndarray, ytrain: np.ndarray) -> Tuple[int, int]: """ This method gets the out of bag score for the model. The method loops over every single example in the training set, and for every example, if the current tree did NOT fit on it, then the tree will predict on it. The predictions for every tree will be amalgamated into one prediction. For classification, we will return the accuracy and the error on the out of bag sample. For regression, we will return the mean squared error(mse) and the root mean squared error (rmse) on the out of bag sample. N - number of features M - number of examples Args: xtrain: A (N,M) matrix containing the feature vectors to predict on ytrain: A (1,M) vector containing the labels for each feature vector Returns: If self.typeSupervised == 0, indicating this ensemble is performing classification, an integer representing accuracy and an integer representing error will be returned. Otherwise, if self.typeSupervised == 1, indicating the ensemble is performing regression, an integer represnting mean squared error and an integer representing root mean squared error will be returned. """ predictions = np.zeros((1, xtrain.shape[1])) for i in range(xtrain.shape[1]): feature_vector = xtrain[:, i].reshape(-1, 1) preds_curr_vector = [] for j in range(len(self.forest)): # if this tree trained on this example, then skip it if i in self.examplesUsedInBootstrap[j]: continue # otherwise predict on it prediction = self.forest[j].predict(feature_vector) preds_curr_vector.append(prediction) if self.typeSupervised == 0: overall_prediction = prediction_classification( preds_curr_vector) else: overall_prediction = prediction_regression(preds_curr_vector) predictions[:, i] = overall_prediction if self.typeSupervised == 0: acc = accuracy(ytrain, predictions) error = 1 - acc return acc, error else: mse = mean_squared_error(ytrain, predictions) rmse = root_mean_squared_error(ytrain, predictions) return mse, rmse
def test_multiclass_1(self): x, y, not_encoded_y = create_spiral_dataset() sm = SoftmaxRegression(inLayerNeuron=2, numClasses=3) train_loss_sm, _ = sm.fit(x, y, ret_train_loss=True, num_epochs=190, learn_rate=1) preds_sm = sm.predict(x) acc_sm = accuracy(not_encoded_y, preds_sm) self.assertLessEqual(train_loss_sm[-1], 0.786302) self.assertGreaterEqual(acc_sm, 0.40)
def test_binary(self): # The tree we are growing is unconstrained so it should be able to # fit the training set perfectly 100% (AKA overfitting :) ) classification_obj = ClassificationTree(entropy=False, minSamplesSplit=1) classification_obj.fit(self.x1, self.y1) predictions = classification_obj.predict(self.x1) acc = accuracy(self.y1, predictions) self.assertEqual(acc, 1) classification_obj2 = ClassificationTree(entropy=False, minSamplesSplit=1) kScore = self.k_cv.get_k_score(self.x1, self.y1, accuracy, classification_obj2) self.assertEqual(kScore, 1)
def test_sanity_checks(self): classification_obj2 = ClassificationTree(entropy=False, minSamplesSplit=5, maxDepth=3, min_impurity_decrease=0.09) classification_obj2.fit(self.x1, self.y1) predictions2 = classification_obj2.predict(self.x1) acc2 = accuracy(self.y1, predictions2) classification_obj3 = ClassificationTree(entropy=False, minSamplesSplit=10, maxDepth=0, min_impurity_decrease=0.15) classification_obj3.fit(self.x1, self.y1) predictions3 = classification_obj3.predict(self.x1) acc3 = accuracy(self.y1, predictions3) classification_obj4 = ClassificationTree(entropy=False, minSamplesSplit=1000, maxDepth=10, min_impurity_decrease=0.15) classification_obj4.fit(self.x1, self.y1) predictions4 = classification_obj4.predict(self.x1) acc4 = accuracy(self.y1, predictions4) classification_obj5 = ClassificationTree(entropy=False, minSamplesSplit=10, maxDepth=10, min_impurity_decrease=1) classification_obj5.fit(self.x1, self.y1) predictions5 = classification_obj4.predict(self.x1) acc5 = accuracy(self.y1, predictions5) self.assertLessEqual(acc2, 1) self.assertAlmostEqual(acc3, 0.6274165202108963) self.assertAlmostEqual(acc4, 0.6274165202108963) self.assertAlmostEqual(acc5, 0.6274165202108963)
def test_multi_class2(self): x, y, not_encoded_y = create_spiral_dataset() multi_layer_perceptron = MultiLayerPerceptron( typeSupervised="multiclass", numberInputFeatures=2) multi_layer_perceptron.add_layer(100, activation_function=ReLU()) multi_layer_perceptron.add_layer(3, activation_function=Softmax(), isSoftmax=1) train_loss6, _ = multi_layer_perceptron.fit(xtrain=x, ytrain=y, num_epochs=1000, learn_rate=1, ret_train_loss=True) preds = multi_layer_perceptron.predict_multi_layer_perceptron(x) acc_6 = accuracy(not_encoded_y, preds) # Performance without regularization should be above 90% self.assertLessEqual(train_loss6[-1], 0.245) self.assertGreaterEqual(acc_6, 0.90)
def test_multi_class(self): # Should be able to overfit multiiclasses easy # notice lack of preprocessing - no need to normalize features # no need to one hot encode labels # out of box model :D classification_obj = ClassificationTree(entropy=False, minSamplesSplit=1) classification_obj.fit(self.x2, self.y2) predictions = classification_obj.predict(self.x2) acc = accuracy(self.y2, predictions) self.assertEqual(acc, 1) classification_obj2 = ClassificationTree(entropy=False, minSamplesSplit=1) kScore = self.k_cv.get_k_score(self.x2, self.y2, accuracy, classification_obj2) self.assertEqual(kScore, 1)
def test_multi_class3(self): # Performance with L2 regularization should be much better x, y, not_encoded_y = create_spiral_dataset() multi_layer_perceptron = MultiLayerPerceptron( typeSupervised="multiclass", numberInputFeatures=2, regularization="L2", reg_parameter=1e-3) # Learn 100 features in first hidden layer multi_layer_perceptron.add_layer(100, activation_function=ReLU()) # Output layer learn 3 features for softmax multi_layer_perceptron.add_layer(3, activation_function=Softmax(), isSoftmax=1) train_loss7, _ = multi_layer_perceptron.fit(xtrain=x, ytrain=y, num_epochs=5000, learn_rate=1, ret_train_loss=True) preds = multi_layer_perceptron.predict_multi_layer_perceptron(x) acc_7 = accuracy(not_encoded_y, preds) self.assertLessEqual(train_loss7[-1], 0.40) self.assertGreaterEqual(acc_7, 0.98)
def fit( self, xtrain: np.ndarray, ytrain: np.ndarray, xvalid: Union[np.ndarray, None] = None, yvalid: Union[np.ndarray, None] = None, num_epochs: int = 10, batch_size: int = 32, ret_train_loss: bool = False, learn_rate: float = 0.1, optim: Optimizer = GradientDescent(), verbose: bool = False ) -> Union[Tuple[int, int, int, int], Tuple[int, int], None]: """ This method trains the neural network on the training set. M: number of training examples N: number of features in a single example Args: xtrain: Numpy matrix of shape (M, N) containing feature vectors ytrain: Numpy matrix of shape (M,1) containing the labels for the feature vectors xvalid: None or a numpy matrix of shape (M,N) containing the feature vectors used to validate the algorithm yvalid: None or a numpy vector of shape (M,1) containing the labels for xvalid num_epochs: Integer representing the number of epochs to train the model batch_size: Integer representing the number of examples to go through before performing a parameter update ret_train_loss: Boolean value indicating whether to return train loss and valid loss if validation set provided learn_rate: Floating point value indicating the learning rate to be used when optimizing the loss function optim: Object of type Optimizer. Used to optimize the loss function verbose: Boolean value indicating whether to provide updates when training Returns: If ret_train_loss is set to True and xvalid is not None, 4 integers will be returned in a tuple, indicating the training loss, validation loss, training accuracy, and validation accuracy respectively. If ret_train_loss is True and xvalid is None, then two integers will be returned in a tuple, indicating the training loss and training accuracy respectively. Otherwise, None will be returned. """ # Dealing with edge case where you have less than 32 examples, which # can happen maybe for k-fold cv. Just do batch gradient descent if # the number of examples is super small if xtrain.shape[1] <= 1000 and len(xtrain.shape) == 2: batch_size = xtrain.shape[1] num_batches = 1 # otherwise do mini batch gradient descent elif xtrain.shape[1] <= 1000 and len(xtrain.shape) == 2: num_batches = xtrain.shape[1] // batch_size else: num_batches = xtrain.shape[0] // batch_size train_loss = [] train_acc = [] validation_loss = [] val_acc = [] for epoch in range(num_epochs): curr_start = 0 curr_end = batch_size loss_epoch = [] for _ in range(num_batches): curr_y = ytrain[:, curr_start:curr_end] if len(xtrain.shape) == 2: curr_x = xtrain[:, curr_start:curr_end] else: # 3D pictures curr_x = xtrain[curr_start:curr_end, :, :, :] curr_start = curr_end curr_end += batch_size predictions_mini_batch = self._forward_propagate(curr_x) loss = self._calculateLoss(curr_y, predictions_mini_batch, self.layers) loss_epoch.append(loss) backprop_init = self.lossFunction.get_gradient_pred( curr_y, predictions_mini_batch) self._backward_propagate(backprop_init, learn_rate, optim, epoch, curr_x, curr_y) train_loss.append(np.mean(loss_epoch)) if ytrain.shape[0] > 1: accuracy_train_set = accuracy(convert_to_highest_pred(ytrain), self.predict(xtrain)) else: accuracy_train_set = accuracy(ytrain, self.predict(xtrain)) train_acc.append(accuracy_train_set) if xvalid is not None: if ytrain.shape[0] > 1: accuracy_validation_set = accuracy( convert_to_highest_pred(yvalid), self.predict(xvalid)) val_loss = self._calculateLoss( yvalid, self._forward_propagate(xvalid), self.layers) else: accuracy_validation_set = accuracy(yvalid, self.predict(xvalid)) val_loss = self._calculateLoss(yvalid, self.predict(xvalid), self.layers) val_acc.append(accuracy_validation_set) validation_loss.append(val_loss) # provide updates during training for sanitys sake if verbose: print("Finished epoch %s" % (epoch)) print("Train loss: %s, Train acc: %s" % (train_loss[-1], train_acc[-1])) if xvalid is not None: print("Valid loss: %s, Valid acc: %s" % (validation_loss[-1], val_acc[-1])) if ret_train_loss and xvalid is not None: return train_loss, validation_loss, train_acc, val_acc elif ret_train_loss: return train_loss, train_acc