def test_binaryclassifcation_regularized2(self):
""" You have to be really careful with the reg parameter. If its to high
(past 0.1), your network won't learn anything. With softmax regression
and logistic regression, there existed leeway to make the reg parameter
pretty high but that absolutely won"t work with neural networks multiple
layers deep.
"""
# normal regularization
multi_layer_perceptron3 = MultiLayerPerceptron(
typeSupervised="binary",
numberInputFeatures=self.x_train.shape[0],
regularization="L1",
reg_parameter=0.01)
multi_layer_perceptron4 = MultiLayerPerceptron(
typeSupervised="binary",
numberInputFeatures=self.x_train.shape[0],
regularization="L2",
reg_parameter=0.01)
# 10 features learnt in first hidden layer w/ ReLU
multi_layer_perceptron3.add_layer(10, activation_function=ReLU())
# # 5 features learnt in second hidden layer w/ ReLU
multi_layer_perceptron3.add_layer(5, activation_function=ReLU())
# # Output layer sigmoid activation
multi_layer_perceptron3.add_layer(1, activation_function=Sigmoid())
# 10 features learnt in first hidden layer w/ ReLU
multi_layer_perceptron4.add_layer(10, activation_function=ReLU())
# # 5 features learnt in second hidden layer w/ ReLU
multi_layer_perceptron4.add_layer(5, activation_function=ReLU())
# # Output layer sigmoid activation
multi_layer_perceptron4.add_layer(1, activation_function=Sigmoid())
multi_layer_perceptron3.fit(self.x_train,
self.y_train,
self.x_valid,
self.y_valid,
ret_train_loss=True,
num_epochs=100,
learn_rate=2.8)
preds3 = multi_layer_perceptron3.predict_multi_layer_perceptron(
self.x_test, 0.5)
multi_layer_perceptron4.fit(self.x_train,
self.y_train,
self.x_valid,
self.y_valid,
ret_train_loss=True,
num_epochs=100,
learn_rate=2.6)
preds4 = multi_layer_perceptron4.predict_multi_layer_perceptron(
self.x_test, 0.5)
acc3 = accuracy(self.y_test, preds3)
acc4 = accuracy(self.y_test, preds4)
self.assertGreaterEqual(acc3, 0.95)
self.assertGreaterEqual(acc4, 0.95)
def test4(self):
sklearn_mode = LR(C=1)
sklearn_mode.fit(self.x_train.T, self.y_train.ravel())
preds_sk = sklearn_mode.predict(self.x_test.T)
accuracy_sklearn = accuracy(self.y_test, preds_sk)
obj4 = LogisticRegression(self.x_train.shape[0],
classificationThreshold=0.5,
regularization="L2",
reg_parameter=1)
obj4.fit(self.x_train, self.y_train, num_epochs=200, learn_rate=0.1)
preds_model = obj4.predict(self.x_test)
accuracy_model = accuracy(self.y_test, preds_model)
self.assertLessEqual(abs(accuracy_model - accuracy_sklearn), 0.1)
def test_binaryclassification_regularized(self):
# Sanity check - high regularization leads to very high losses.
multi_layer_perceptron1 = MultiLayerPerceptron(
typeSupervised="binary",
numberInputFeatures=self.x_train.shape[0],
regularization="L1",
reg_parameter=500)
multi_layer_perceptron2 = MultiLayerPerceptron(
typeSupervised="binary",
numberInputFeatures=self.x_train.shape[0],
regularization="L2",
reg_parameter=500)
# 10 features learnt in first hidden layer w/ ReLU
multi_layer_perceptron1.add_layer(10, activation_function=ReLU())
# # 5 features learnt in second hidden layer w/ ReLU
multi_layer_perceptron1.add_layer(5, activation_function=ReLU())
# # Output layer sigmoid activation
multi_layer_perceptron1.add_layer(1, activation_function=Sigmoid())
# 10 features learnt in first hidden layer w/ ReLU
multi_layer_perceptron2.add_layer(10, activation_function=ReLU())
# # 5 features learnt in second hidden layer w/ ReLU
multi_layer_perceptron2.add_layer(5, activation_function=ReLU())
# # Output layer sigmoid activation
multi_layer_perceptron2.add_layer(1, activation_function=Sigmoid())
multi_layer_perceptron1.fit(self.x_train,
self.y_train,
self.x_valid,
self.y_valid,
ret_train_loss=True,
num_epochs=10,
learn_rate=2.6)
preds1 = multi_layer_perceptron1.predict_multi_layer_perceptron(
self.x_test, 0.5)
multi_layer_perceptron2.fit(self.x_train,
self.y_train,
self.x_valid,
self.y_valid,
ret_train_loss=True,
num_epochs=10,
learn_rate=3)
preds2 = multi_layer_perceptron2.predict_multi_layer_perceptron(
self.x_test, 0.5)
acc1 = accuracy(self.y_test, preds1)
acc2 = accuracy(self.y_test, preds2)
self.assertLessEqual(acc1, 0.69)
self.assertLessEqual(acc2, 0.69)
def test_overfit_small_batch(self):
multi_layer_perceptron = MultiLayerPerceptron(
typeSupervised="multiclass", numberInputFeatures=784)
multi_layer_perceptron.add_layer(num_neurons=100,
activation_function=ReLU(),
layer=DenseBatchNormLayer)
multi_layer_perceptron.add_layer(num_neurons=10,
activation_function=Softmax(),
isSoftmax=True)
train_loss1, train_acc1 = multi_layer_perceptron.fit(
self.x_train[:, :100].reshape(784, -1),
self.y_train[:, :100],
num_epochs=150,
ret_train_loss=True,
optim=RMSProp(),
learn_rate=0.001)
predictions1 = multi_layer_perceptron.predict_multi_layer_perceptron(
self.x_train[:, :100].reshape(784, -1))
acc = accuracy(self.saved_y[:, :100].reshape(1, -1), predictions1)
print(train_loss1)
print(acc)
self.assertLessEqual(train_loss1[-1], 0.09)
self.assertEqual(acc, 1)
multi_layer_perceptron2 = MultiLayerPerceptron(
typeSupervised="multiclass", numberInputFeatures=784)
multi_layer_perceptron2.add_layer(num_neurons=100,
activation_function=ReLU(),
layer=DenseBatchNormLayer)
multi_layer_perceptron2.add_layer(num_neurons=10,
activation_function=Softmax(),
isSoftmax=True)
train_loss2, train_acc2 = multi_layer_perceptron2.fit(
self.x_train[:, :100].reshape(784, -1),
self.y_train[:, :100],
num_epochs=150,
ret_train_loss=True,
optim=GradientDescentMomentum(),
learn_rate=0.1)
predictions2 = multi_layer_perceptron2.predict_multi_layer_perceptron(
self.x_train[:, :100].reshape(784, -1))
acc2 = accuracy(self.saved_y[:, :100].reshape(1, -1), predictions2)
print(train_loss2)
print(acc2)
self.assertLessEqual(train_loss2[-1], 0.09)
self.assertEqual(acc2, 1)
def test_binaryclassification_unregularized(self):
""" Learning rate KEY: If its set to high, you will diverge. If its set
to low, your network won"t learn anything. The more layers you have,
the higher the learning rate should be. Example seen here:
1 layer only - learn rate 0.1 suffices. With
two layers ~ 0.7, with 3 layers ~2.5. """
MLP = MultiLayerPerceptron(typeSupervised="binary",
numberInputFeatures=self.x_train.shape[0])
# Just add Dense Layers
# 10 features learnt in first hidden layer w/ ReLU
MLP.add_layer(10, activation_function=ReLU())
# # 5 features learnt in second hidden layer w/ ReLU
MLP.add_layer(5, activation_function=ReLU())
# # Output layer sigmoid activation
MLP.add_layer(1, activation_function=Sigmoid())
MLP.fit(self.x_train,
self.y_train,
self.x_valid,
self.y_valid,
ret_train_loss=True,
num_epochs=100,
learn_rate=2.6)
preds = MLP.predict_multi_layer_perceptron(self.x_test, 0.5)
acc = accuracy(self.y_test, preds)
self.assertGreaterEqual(acc, 0.95)
def test_overall_model(self):
#object should be able to get >= 20% percent accuracy on CIFAR-10 with k = 10
test_obj = KNearestNeighboursClassifier()
test_obj.fit(self.x1_train, self.y_train)
prediction_test = test_obj.predict(self.x1_test[:200])
acc = accuracy(self.y_test, prediction_test)
self.assertEqual(prediction_test.shape, self.y_test[:200].shape)
self.assertGreaterEqual(acc, 20)
def get_oob_score(self, xtrain: np.ndarray,
ytrain: np.ndarray) -> Tuple[int, int]:
""" This method gets the out of bag score for the model. The method
loops over every single example in the training set, and for every
example, if the current tree did NOT fit on it, then the tree will
predict on it. The predictions for every tree will be amalgamated
into one prediction.
For classification, we will return the accuracy and the error on the
out of bag sample. For regression, we will return the mean squared
error(mse) and the root mean squared error (rmse) on the out of bag
sample.
N - number of features
M - number of examples
Args:
xtrain:
A (N,M) matrix containing the feature vectors to predict on
ytrain:
A (1,M) vector containing the labels for each feature vector
Returns:
If self.typeSupervised == 0, indicating this ensemble is performing
classification, an integer representing accuracy and an integer
representing error will be returned.
Otherwise, if self.typeSupervised == 1, indicating the ensemble is
performing regression, an integer represnting mean squared error and
an integer representing root mean squared error will be returned.
"""
predictions = np.zeros((1, xtrain.shape[1]))
for i in range(xtrain.shape[1]):
feature_vector = xtrain[:, i].reshape(-1, 1)
preds_curr_vector = []
for j in range(len(self.forest)):
# if this tree trained on this example, then skip it
if i in self.examplesUsedInBootstrap[j]:
continue
# otherwise predict on it
prediction = self.forest[j].predict(feature_vector)
preds_curr_vector.append(prediction)
if self.typeSupervised == 0:
overall_prediction = prediction_classification(
preds_curr_vector)
else:
overall_prediction = prediction_regression(preds_curr_vector)
predictions[:, i] = overall_prediction
if self.typeSupervised == 0:
acc = accuracy(ytrain, predictions)
error = 1 - acc
return acc, error
else:
mse = mean_squared_error(ytrain, predictions)
rmse = root_mean_squared_error(ytrain, predictions)
return mse, rmse
def test_multiclass_1(self):
x, y, not_encoded_y = create_spiral_dataset()
sm = SoftmaxRegression(inLayerNeuron=2, numClasses=3)
train_loss_sm, _ = sm.fit(x,
y,
ret_train_loss=True,
num_epochs=190,
learn_rate=1)
preds_sm = sm.predict(x)
acc_sm = accuracy(not_encoded_y, preds_sm)
self.assertLessEqual(train_loss_sm[-1], 0.786302)
self.assertGreaterEqual(acc_sm, 0.40)
def test_binary(self):
# The tree we are growing is unconstrained so it should be able to
# fit the training set perfectly 100% (AKA overfitting :) )
classification_obj = ClassificationTree(entropy=False,
minSamplesSplit=1)
classification_obj.fit(self.x1, self.y1)
predictions = classification_obj.predict(self.x1)
acc = accuracy(self.y1, predictions)
self.assertEqual(acc, 1)
classification_obj2 = ClassificationTree(entropy=False,
minSamplesSplit=1)
kScore = self.k_cv.get_k_score(self.x1, self.y1, accuracy,
classification_obj2)
self.assertEqual(kScore, 1)
def test_sanity_checks(self):
classification_obj2 = ClassificationTree(entropy=False,
minSamplesSplit=5,
maxDepth=3,
min_impurity_decrease=0.09)
classification_obj2.fit(self.x1, self.y1)
predictions2 = classification_obj2.predict(self.x1)
acc2 = accuracy(self.y1, predictions2)
classification_obj3 = ClassificationTree(entropy=False,
minSamplesSplit=10,
maxDepth=0,
min_impurity_decrease=0.15)
classification_obj3.fit(self.x1, self.y1)
predictions3 = classification_obj3.predict(self.x1)
acc3 = accuracy(self.y1, predictions3)
classification_obj4 = ClassificationTree(entropy=False,
minSamplesSplit=1000,
maxDepth=10,
min_impurity_decrease=0.15)
classification_obj4.fit(self.x1, self.y1)
predictions4 = classification_obj4.predict(self.x1)
acc4 = accuracy(self.y1, predictions4)
classification_obj5 = ClassificationTree(entropy=False,
minSamplesSplit=10,
maxDepth=10,
min_impurity_decrease=1)
classification_obj5.fit(self.x1, self.y1)
predictions5 = classification_obj4.predict(self.x1)
acc5 = accuracy(self.y1, predictions5)
self.assertLessEqual(acc2, 1)
self.assertAlmostEqual(acc3, 0.6274165202108963)
self.assertAlmostEqual(acc4, 0.6274165202108963)
self.assertAlmostEqual(acc5, 0.6274165202108963)
def test_multi_class2(self):
x, y, not_encoded_y = create_spiral_dataset()
multi_layer_perceptron = MultiLayerPerceptron(
typeSupervised="multiclass", numberInputFeatures=2)
multi_layer_perceptron.add_layer(100, activation_function=ReLU())
multi_layer_perceptron.add_layer(3,
activation_function=Softmax(),
isSoftmax=1)
train_loss6, _ = multi_layer_perceptron.fit(xtrain=x,
ytrain=y,
num_epochs=1000,
learn_rate=1,
ret_train_loss=True)
preds = multi_layer_perceptron.predict_multi_layer_perceptron(x)
acc_6 = accuracy(not_encoded_y, preds)
# Performance without regularization should be above 90%
self.assertLessEqual(train_loss6[-1], 0.245)
self.assertGreaterEqual(acc_6, 0.90)
def test_multi_class(self):
# Should be able to overfit multiiclasses easy
# notice lack of preprocessing - no need to normalize features
# no need to one hot encode labels
# out of box model :D
classification_obj = ClassificationTree(entropy=False,
minSamplesSplit=1)
classification_obj.fit(self.x2, self.y2)
predictions = classification_obj.predict(self.x2)
acc = accuracy(self.y2, predictions)
self.assertEqual(acc, 1)
classification_obj2 = ClassificationTree(entropy=False,
minSamplesSplit=1)
kScore = self.k_cv.get_k_score(self.x2, self.y2, accuracy,
classification_obj2)
self.assertEqual(kScore, 1)
def test_multi_class3(self):
# Performance with L2 regularization should be much better
x, y, not_encoded_y = create_spiral_dataset()
multi_layer_perceptron = MultiLayerPerceptron(
typeSupervised="multiclass",
numberInputFeatures=2,
regularization="L2",
reg_parameter=1e-3)
# Learn 100 features in first hidden layer
multi_layer_perceptron.add_layer(100, activation_function=ReLU())
# Output layer learn 3 features for softmax
multi_layer_perceptron.add_layer(3,
activation_function=Softmax(),
isSoftmax=1)
train_loss7, _ = multi_layer_perceptron.fit(xtrain=x,
ytrain=y,
num_epochs=5000,
learn_rate=1,
ret_train_loss=True)
preds = multi_layer_perceptron.predict_multi_layer_perceptron(x)
acc_7 = accuracy(not_encoded_y, preds)
self.assertLessEqual(train_loss7[-1], 0.40)
self.assertGreaterEqual(acc_7, 0.98)
def fit(
self,
xtrain: np.ndarray,
ytrain: np.ndarray,
xvalid: Union[np.ndarray, None] = None,
yvalid: Union[np.ndarray, None] = None,
num_epochs: int = 10,
batch_size: int = 32,
ret_train_loss: bool = False,
learn_rate: float = 0.1,
optim: Optimizer = GradientDescent(),
verbose: bool = False
) -> Union[Tuple[int, int, int, int], Tuple[int, int], None]:
""" This method trains the neural network on the training set.
M: number of training examples
N: number of features in a single example
Args:
xtrain:
Numpy matrix of shape (M, N) containing feature vectors
ytrain:
Numpy matrix of shape (M,1) containing the labels for the
feature vectors
xvalid:
None or a numpy matrix of shape (M,N) containing the feature
vectors used to validate the algorithm
yvalid:
None or a numpy vector of shape (M,1) containing the labels for
xvalid
num_epochs:
Integer representing the number of epochs to train the model
batch_size:
Integer representing the number of examples to go through before
performing a parameter update
ret_train_loss:
Boolean value indicating whether to return train loss and valid
loss if validation set provided
learn_rate:
Floating point value indicating the learning rate to be used
when optimizing the loss function
optim:
Object of type Optimizer. Used to optimize the loss function
verbose:
Boolean value indicating whether to provide updates when
training
Returns:
If ret_train_loss is set to True and xvalid is not None,
4 integers will be returned in a tuple, indicating the training
loss, validation loss, training accuracy, and validation accuracy
respectively.
If ret_train_loss is True and xvalid is None, then two integers
will be returned in a tuple, indicating the training loss and
training accuracy respectively.
Otherwise, None will be returned.
"""
# Dealing with edge case where you have less than 32 examples, which
# can happen maybe for k-fold cv. Just do batch gradient descent if
# the number of examples is super small
if xtrain.shape[1] <= 1000 and len(xtrain.shape) == 2:
batch_size = xtrain.shape[1]
num_batches = 1
# otherwise do mini batch gradient descent
elif xtrain.shape[1] <= 1000 and len(xtrain.shape) == 2:
num_batches = xtrain.shape[1] // batch_size
else:
num_batches = xtrain.shape[0] // batch_size
train_loss = []
train_acc = []
validation_loss = []
val_acc = []
for epoch in range(num_epochs):
curr_start = 0
curr_end = batch_size
loss_epoch = []
for _ in range(num_batches):
curr_y = ytrain[:, curr_start:curr_end]
if len(xtrain.shape) == 2:
curr_x = xtrain[:, curr_start:curr_end]
else:
# 3D pictures
curr_x = xtrain[curr_start:curr_end, :, :, :]
curr_start = curr_end
curr_end += batch_size
predictions_mini_batch = self._forward_propagate(curr_x)
loss = self._calculateLoss(curr_y, predictions_mini_batch,
self.layers)
loss_epoch.append(loss)
backprop_init = self.lossFunction.get_gradient_pred(
curr_y, predictions_mini_batch)
self._backward_propagate(backprop_init, learn_rate, optim,
epoch, curr_x, curr_y)
train_loss.append(np.mean(loss_epoch))
if ytrain.shape[0] > 1:
accuracy_train_set = accuracy(convert_to_highest_pred(ytrain),
self.predict(xtrain))
else:
accuracy_train_set = accuracy(ytrain, self.predict(xtrain))
train_acc.append(accuracy_train_set)
if xvalid is not None:
if ytrain.shape[0] > 1:
accuracy_validation_set = accuracy(
convert_to_highest_pred(yvalid), self.predict(xvalid))
val_loss = self._calculateLoss(
yvalid, self._forward_propagate(xvalid), self.layers)
else:
accuracy_validation_set = accuracy(yvalid,
self.predict(xvalid))
val_loss = self._calculateLoss(yvalid,
self.predict(xvalid),
self.layers)
val_acc.append(accuracy_validation_set)
validation_loss.append(val_loss)
# provide updates during training for sanitys sake
if verbose:
print("Finished epoch %s" % (epoch))
print("Train loss: %s, Train acc: %s" %
(train_loss[-1], train_acc[-1]))
if xvalid is not None:
print("Valid loss: %s, Valid acc: %s" %
(validation_loss[-1], val_acc[-1]))
if ret_train_loss and xvalid is not None:
return train_loss, validation_loss, train_acc, val_acc
elif ret_train_loss:
return train_loss, train_acc