def _structure(self, nn_structure):
"""
Structure function that initializes neural network architecture.
Parameters
----------
nn_structure: (list) a list of dictionaries define Neural Network architecture.
- Each dict element in the list should have following key-pair value:
+ num_neurons: (int) define number of neurons in the dense layer.
+ weight_init: (str) choose which kind to initialize the weight, either `he` `xavier` or `std`.
+ activation (optional): (str) apply activation to the output of the layer. LINEAR -> ACTIVATION.
+ batch_norm (optional): (any) apply batch norm to the output of the layer. LINEAR -> BATCH NORM -> ACTIVATION.
+ drop_out (optional): (float) choose rate to drop out neurons.
"""
layers = []
for struct in nn_structure:
num_neurons = struct["num_neurons"]
weight_init = struct["weight_init"]
fc = FCLayer(num_neurons=num_neurons, weight_init=weight_init)
fc.initialize_optimizer(self.optimizer)
layers.append(fc)
if "batch_norm" in struct:
bn_layer = BatchNormLayer()
bn_layer.initialize_optimizer(self.optimizer)
layers.append(bn_layer)
if "activation" in struct:
activation = struct["activation"]
act_layer = ActivationLayer(activation=activation)
layers.append(act_layer)
if "drop_out" in struct:
drop_out = struct["drop_out"]
do_layer = DropoutLayer(drop_out)
layers.append(do_layer)
return layers
def main():
load_dataset_mnist("../libs")
mndata = MNIST('../libs/data_mnist', gz=True)
weight_path = "nn_weights.pkl"
training_phase = weight_path not in os.listdir(".")
if training_phase:
images, labels = mndata.load_training()
images, labels = preprocess_data(images, labels)
epochs = 10
batch_size = 64
learning_rate = 0.01
optimizer = Adam(learning_rate)
loss_func = CrossEntropy()
archs = [
InputLayer(),
FCLayer(num_neurons=100, weight_init="he_normal"),
ActivationLayer(activation="relu"),
DropoutLayer(keep_prob=0.8),
FCLayer(num_neurons=125, weight_init="he_normal"),
ActivationLayer(activation="relu"),
DropoutLayer(keep_prob=0.8),
FCLayer(num_neurons=50, weight_init="he_normal"),
BatchNormLayer(),
ActivationLayer(activation="relu"),
FCLayer(num_neurons=labels.shape[1], weight_init="he_normal"),
ActivationLayer(activation="softmax"),
]
nn = NeuralNetwork(optimizer=optimizer,
layers=archs,
loss_func=loss_func)
trainer = Trainer(nn, batch_size, epochs)
trainer.train(images, labels)
trainer.save_model("nn_weights.pkl")
else:
import pickle
images_test, labels_test = mndata.load_testing()
images_test, labels_test = preprocess_data(images_test,
labels_test,
test=True)
with open(weight_path, "rb") as f:
nn = pickle.load(f)
pred = nn.predict(images_test)
print("Accuracy:", len(pred[labels_test == pred]) / len(pred))
from sklearn.metrics.classification import confusion_matrix
print("Confusion matrix: ")
print(confusion_matrix(labels_test, pred))
def _structure(self, nn_structure):
"""
Structure function that initializes neural network architecture.
"""
layers = []
for struct in nn_structure:
num_neurons = struct["num_neurons"]
weight_init = struct["weight_init"]
fc = FCLayer(num_neurons=num_neurons, weight_init=weight_init)
layers.append(fc)
if "batch_norm" in struct:
bn_layer = BatchNormLayer()
layers.append(bn_layer)
if "activation" in struct:
activation = struct["activation"]
act_layer = ActivationLayer(activation=activation)
layers.append(act_layer)
return layers
def _structure(self, cnn_structure):
"""
Structure function that initializes cnn architecture.
"""
layers = []
for struct in cnn_structure:
if type(struct) is str and struct == "flatten":
flatten_layer = FlattenLayer()
layers.append(flatten_layer)
continue
if struct["type"] == "conv":
filter_size = struct["filter_size"]
filters = struct["filters"]
padding = struct["padding"]
stride = struct["stride"]
conv_layer = ConvLayer(filter_size, filters, padding, stride)
layers.append(conv_layer)
if "batch_norm" in struct:
bn_layer = BatchNormLayer()
layers.append(bn_layer)
if "activation" in struct:
activation = struct["activation"]
act_layer = ActivationLayer(activation=activation)
layers.append(act_layer)
elif struct["type"] == "pool":
filter_size = struct["filter_size"]
stride = struct["stride"]
mode = struct["mode"]
pool_layer = PoolingLayer(filter_size=filter_size,
stride=stride,
mode=mode)
layers.append(pool_layer)
else:
num_neurons = struct["num_neurons"]
weight_init = struct["weight_init"]
fc_layer = FCLayer(num_neurons=num_neurons,
weight_init=weight_init)
layers.append(fc_layer)
if "activation" in struct:
activation = struct["activation"]
act_layer = ActivationLayer(activation)
layers.append(act_layer)
return layers
epochs = 10
batch_size = 64
learning_rate = 0.01
optimizer = Adam(learning_rate)
loss_func = CrossEntropy()
archs = [
FCLayer(num_neurons=100, weight_init="he_normal"),
ActivationLayer(activation="relu"),
DropoutLayer(keep_prob=0.8),
FCLayer(num_neurons=125, weight_init="he_normal"),
ActivationLayer(activation="relu"),
DropoutLayer(keep_prob=0.8),
FCLayer(num_neurons=50, weight_init="he_normal"),
ActivationLayer(activation="relu"),
BatchNormLayer(),
FCLayer(num_neurons=labels.shape[1], weight_init="he_normal"),
ActivationLayer(activation="softmax"),
]
nn = NeuralNetwork(optimizer=optimizer, layers=archs, loss_func=loss_func)
trainer = Trainer(nn, batch_size, epochs)
trainer.train(images, labels)
trainer.save_model("nn_weights.pkl")
else:
import pickle
images_test, labels_test = mndata.load_testing()
images_test, labels_test = preprocess_data(images_test,
labels_test,
test=True)
with open(weight_path, "rb") as f:
def _structure(self, cnn_structure):
"""
Structure function that initializes CNN architecture.
Parameters
----------
cnn_structure: (list) a list of dictionaries define CNN architecture.
Each dictionary element is 1 kind of layer (ConvLayer, FCLayer, PoolingLayer, FlattenLayer, BatchNormLayer).
- Convolutional layer (`type: conv`) dict should have following key-value pair:
+ filter_size: (tuple) define conv filter size (fH, fW)
+ filters: (int) number of conv filters at the layer.
+ stride: (int) stride of conv filter.
+ weight_init: (str) choose which kind to initialize the filter, either `he` `xavier` or `std`.
+ padding: (str) padding type of input corresponding to the output, either `SAME` or `VALID`.
+ activation (optional): (str) apply activation to the output of the layer. LINEAR -> ACTIVATION.
+ batch_norm (optional): (any) apply batch norm to the output of the layer. LINEAR -> BATCH NORM -> ACTIVATION
- Pooling layer (`type: pool`) dict should have following key-value pair:
+ filter_size: (tuple) define pooling filter size (fH, fW).
+ mode: (str) choose the mode of pooling, either `max` or `avg`.
+ stride: (int) stride of pooling filter.
- Fully-connected layer (`type: fc`) dict should have following key-value pair:
+ num_neurons: (int) define number of neurons in the dense layer.
+ weight_init: (str) choose which kind to initialize the weight, either `he` `xavier` or `std`.
+ activation (optional): (str) apply activation to the output of the layer. LINEAR -> ACTIVATION.
+ batch_norm (optional): (any) apply batch norm to the output of the layer. LINEAR -> BATCH NORM -> ACTIVATION
"""
layers = []
for struct in cnn_structure:
if type(struct) is str and struct == "flatten":
flatten_layer = FlattenLayer()
layers.append(flatten_layer)
continue
if struct["type"] == "conv":
filter_size = struct["filter_size"]
filters = struct["filters"]
padding = struct["padding"]
stride = struct["stride"]
weight_init = struct["weight_init"]
conv_layer = ConvLayer(filter_size, filters, padding, stride,
weight_init)
conv_layer.initialize_optimizer(self.optimizer)
layers.append(conv_layer)
if "batch_norm" in struct:
bn_layer = BatchNormLayer()
bn_layer.initialize_optimizer(self.optimizer)
layers.append(bn_layer)
if "activation" in struct:
activation = struct["activation"]
act_layer = ActivationLayer(activation=activation)
layers.append(act_layer)
elif struct["type"] == "pool":
filter_size = struct["filter_size"]
stride = struct["stride"]
mode = struct["mode"]
pool_layer = PoolingLayer(filter_size=filter_size,
stride=stride,
mode=mode)
layers.append(pool_layer)
else:
num_neurons = struct["num_neurons"]
weight_init = struct["weight_init"]
fc_layer = FCLayer(num_neurons=num_neurons,
weight_init=weight_init)
fc_layer.initialize_optimizer(self.optimizer)
layers.append(fc_layer)
if "batch_norm" in struct:
bn_layer = BatchNormLayer()
bn_layer.initialize_optimizer(self.optimizer)
layers.append(bn_layer)
if "activation" in struct:
activation = struct["activation"]
act_layer = ActivationLayer(activation)
layers.append(act_layer)
return layers