def __init__(self, optimizer:object, layers:list, loss_func:object=CrossEntropy()):
"""
A Convolutional Neural Network.
Parameters
----------
optimizer: (object) optimizer class to use (sgd, sgd_momentum, rms_prop, adam)
layers: (list) a list of sequential layers of cnn architecture.
loss_func: (object) the type of loss function we want to optimize.
"""
super().__init__(optimizer, layers, loss_func)
def __init__(self, optimizer:object, layers:list, loss_func:object=CrossEntropy()):
"""
Deep neural network architecture.
Parameters
----------
optimizer: (object) optimizer object uses to optimize the loss.
layers: (list) a list of sequential layers. For neural network, it should have [FCLayer, ActivationLayer, BatchnormLayer, DropoutLayer]
loss_func: (object) the type of loss function we want to optimize.
"""
self.optimizer = optimizer
self.loss_func = loss_func
self.layers = self._structure(layers)
def main():
load_dataset_mnist("../libs")
mndata = MNIST('../libs/data_mnist', gz=True)
weight_path = "nn_weights.pkl"
training_phase = weight_path not in os.listdir(".")
if training_phase:
images, labels = mndata.load_training()
images, labels = preprocess_data(images, labels)
epochs = 10
batch_size = 64
learning_rate = 0.01
optimizer = Adam(learning_rate)
loss_func = CrossEntropy()
archs = [
InputLayer(),
FCLayer(num_neurons=100, weight_init="he_normal"),
ActivationLayer(activation="relu"),
DropoutLayer(keep_prob=0.8),
FCLayer(num_neurons=125, weight_init="he_normal"),
ActivationLayer(activation="relu"),
DropoutLayer(keep_prob=0.8),
FCLayer(num_neurons=50, weight_init="he_normal"),
BatchNormLayer(),
ActivationLayer(activation="relu"),
FCLayer(num_neurons=labels.shape[1], weight_init="he_normal"),
ActivationLayer(activation="softmax"),
]
nn = NeuralNetwork(optimizer=optimizer,
layers=archs,
loss_func=loss_func)
trainer = Trainer(nn, batch_size, epochs)
trainer.train(images, labels)
trainer.save_model("nn_weights.pkl")
else:
import pickle
images_test, labels_test = mndata.load_testing()
images_test, labels_test = preprocess_data(images_test,
labels_test,
test=True)
with open(weight_path, "rb") as f:
nn = pickle.load(f)
pred = nn.predict(images_test)
print("Accuracy:", len(pred[labels_test == pred]) / len(pred))
from sklearn.metrics.classification import confusion_matrix
print("Confusion matrix: ")
print(confusion_matrix(labels_test, pred))
def __init__(self,
feature_dim: int,
num_classes: int,
optimizer: object,
loss_func: object = CrossEntropy()):
"""
Constructor for softmax regression. It can be seen as the simplest neural network with input layer and output layer.
Parameter
---------
feature_dim: (int) dimension of input feature.
num_classes: (int) number of output classes.
optimizer: (object) optimizer used to optimize loss w.r.t W.
loss_func: (object) the type of loss function to optimize.
"""
self.loss_func = loss_func
self.optimizer = optimizer
self.W = np.random.normal(size=(feature_dim, num_classes))
def main():
arch = [
ConvLayer(filter_size=(3, 3),
filters=6,
padding="SAME",
stride=1,
weight_init="he_normal"),
ActivationLayer(activation="relu"),
PoolingLayer(filter_size=(2, 2), stride=2, mode="max"),
ConvLayer(filter_size=(3, 3),
filters=16,
padding="SAME",
stride=1,
weight_init="he_normal"),
ActivationLayer(activation="relu"),
PoolingLayer(filter_size=(2, 2), stride=2, mode="max"),
ConvLayer(filter_size=(3, 3),
filters=32,
padding="SAME",
stride=1,
weight_init="he_normal"),
ActivationLayer(activation="relu"),
PoolingLayer(filter_size=(2, 2), stride=2, mode="max"),
FlattenLayer(),
FCLayer(num_neurons=128, weight_init="he_normal"),
ActivationLayer(activation="relu"),
FCLayer(num_neurons=64, weight_init="he_normal"),
ActivationLayer(activation="relu"),
FCLayer(num_neurons=10, weight_init="he_normal"),
ActivationLayer(activation="softmax")
]
print("Train MNIST dataset by CNN with pure Python: Numpy.")
weight_path = "cnn_weights.pkl"
training_phase = weight_path not in os.listdir(".")
load_dataset_mnist("../libs")
mndata = MNIST('../libs/data_mnist', gz=True)
if training_phase:
images_train, labels_train = mndata.load_training()
images_train, labels_train = preprocess_data(images_train,
labels_train,
nn=True)
epochs = 5
batch_size = 64
learning_rate = 0.006
optimizer = Adam(alpha=learning_rate)
loss_func = CrossEntropy()
cnn = CNN(optimizer=optimizer, layers=arch, loss_func=loss_func)
trainer = Trainer(cnn, batch_size, epochs)
trainer.train(images_train, labels_train)
trainer.save_model(weight_path)
else:
import pickle
images_test, labels_test = mndata.load_testing()
images_test, labels_test = preprocess_data(images_test,
labels_test,
nn=True,
test=True)
with open(weight_path, "rb") as f:
cnn = pickle.load(f)
pred = cnn.predict(images_test)
print("Accuracy:", len(pred[labels_test == pred]) / len(pred))
from sklearn.metrics.classification import confusion_matrix
print("Confusion matrix: ")
print(confusion_matrix(labels_test, pred))
from libs.utils import load_dataset_mnist, preprocess_data, Trainer, Evaluator
from libs.mnist_lib import MNIST
load_dataset_mnist("../libs")
mndata = MNIST('../libs/data_mnist', gz=True)
weight_path = "nn_weights.pkl"
training_phase = weight_path not in os.listdir(".")
if training_phase:
images, labels = mndata.load_training()
images, labels = preprocess_data(images, labels)
epochs = 10
batch_size = 64
learning_rate = 0.01
optimizer = Adam(learning_rate)
loss_func = CrossEntropy()
archs = [
FCLayer(num_neurons=100, weight_init="he_normal"),
ActivationLayer(activation="relu"),
DropoutLayer(keep_prob=0.8),
FCLayer(num_neurons=125, weight_init="he_normal"),
ActivationLayer(activation="relu"),
DropoutLayer(keep_prob=0.8),
FCLayer(num_neurons=50, weight_init="he_normal"),
ActivationLayer(activation="relu"),
BatchNormLayer(),
FCLayer(num_neurons=labels.shape[1], weight_init="he_normal"),
ActivationLayer(activation="softmax"),
]
nn = NeuralNetwork(optimizer=optimizer, layers=archs, loss_func=loss_func)