def test():
import numpy as np
from nn_components.layers import ConvLayer, PoolingLayer
from optimizations_algorithms.optimizers import SGD
import tensorflow as tf
tf.enable_eager_execution()
filter_size = (3, 3)
filters = 16
padding = "SAME"
stride = 1
optimizer = SGD()
conv_layer = ConvLayer(filter_size=filter_size, filters=filters, padding=padding, stride=stride)
conv_layer.initialize_optimizer(optimizer)
conv_layer.debug = True
pool_filter_size = (2, 2)
pool_stride = 2
pool_mode = "max"
pool_layer = PoolingLayer(filter_size=pool_filter_size, stride=pool_stride, mode=pool_mode)
X = np.random.normal(size=(16, 12, 12, 3))
d_prev = np.random.normal(size=(16, 12, 12, 16))
my_conv_forward = conv_layer.forward(X)
my_dA, my_dW = conv_layer.backward(d_prev, X)
my_pool_forward = pool_layer.forward(X)
with tf.device("/cpu:0"):
tf_conv_forward = tf.nn.conv2d(X, conv_layer.W, strides=(stride, stride), padding=padding).numpy()
tf_dW = tf.nn.conv2d_backprop_filter(X, filter_sizes=filter_size + (X.shape[-1], filters), out_backprop=d_prev,
strides=(1, stride, stride, 1), padding=padding).numpy()
tf_dA = tf.nn.conv2d_backprop_input(input_sizes=X.shape, filter=conv_layer.W, out_backprop=d_prev,
strides=(1, stride, stride, 1), padding=padding).numpy()
tf_pool_forward = tf.nn.max_pool2d(X, ksize=pool_filter_size, strides=(pool_stride, pool_stride), padding="VALID")
blank = "----------------------"
print(blank + "TEST FORWARD CONVOLUTION" + blank)
forward_result = np.allclose(my_conv_forward, tf_conv_forward)
forward_out = "PASS" if forward_result else "FAIL"
print("====> " + forward_out)
print(blank + "TEST BACKWARD CONVOLUTION" + blank)
dW_result = np.allclose(my_dW, tf_dW)
dW_out = "PASS" if dW_result else "FAIL"
print("====> dW case: " + dW_out)
dA_result = np.allclose(my_dA, tf_dA)
dA_out = "PASS" if dA_result else "FAIL"
print("====> dA case: " + dA_out)
print(blank + "TEST FORWARD POOLING" + blank)
pool_result = np.allclose(my_pool_forward, tf_pool_forward)
pool_out = "PASS" if pool_result else "FAIL"
print("====> " + pool_out)
def _structure(self, cnn_structure):
"""
Structure function that initializes cnn architecture.
"""
layers = []
for struct in cnn_structure:
if type(struct) is str and struct == "flatten":
flatten_layer = FlattenLayer()
layers.append(flatten_layer)
continue
if struct["type"] == "conv":
filter_size = struct["filter_size"]
filters = struct["filters"]
padding = struct["padding"]
stride = struct["stride"]
conv_layer = ConvLayer(filter_size, filters, padding, stride)
layers.append(conv_layer)
if "batch_norm" in struct:
bn_layer = BatchNormLayer()
layers.append(bn_layer)
if "activation" in struct:
activation = struct["activation"]
act_layer = ActivationLayer(activation=activation)
layers.append(act_layer)
elif struct["type"] == "pool":
filter_size = struct["filter_size"]
stride = struct["stride"]
mode = struct["mode"]
pool_layer = PoolingLayer(filter_size=filter_size,
stride=stride,
mode=mode)
layers.append(pool_layer)
else:
num_neurons = struct["num_neurons"]
weight_init = struct["weight_init"]
fc_layer = FCLayer(num_neurons=num_neurons,
weight_init=weight_init)
layers.append(fc_layer)
if "activation" in struct:
activation = struct["activation"]
act_layer = ActivationLayer(activation)
layers.append(act_layer)
return layers
def main():
arch = [
ConvLayer(filter_size=(3, 3),
filters=6,
padding="SAME",
stride=1,
weight_init="he_normal"),
ActivationLayer(activation="relu"),
PoolingLayer(filter_size=(2, 2), stride=2, mode="max"),
ConvLayer(filter_size=(3, 3),
filters=16,
padding="SAME",
stride=1,
weight_init="he_normal"),
ActivationLayer(activation="relu"),
PoolingLayer(filter_size=(2, 2), stride=2, mode="max"),
ConvLayer(filter_size=(3, 3),
filters=32,
padding="SAME",
stride=1,
weight_init="he_normal"),
ActivationLayer(activation="relu"),
PoolingLayer(filter_size=(2, 2), stride=2, mode="max"),
FlattenLayer(),
FCLayer(num_neurons=128, weight_init="he_normal"),
ActivationLayer(activation="relu"),
FCLayer(num_neurons=64, weight_init="he_normal"),
ActivationLayer(activation="relu"),
FCLayer(num_neurons=10, weight_init="he_normal"),
ActivationLayer(activation="softmax")
]
print("Train MNIST dataset by CNN with pure Python: Numpy.")
weight_path = "cnn_weights.pkl"
training_phase = weight_path not in os.listdir(".")
load_dataset_mnist("../libs")
mndata = MNIST('../libs/data_mnist', gz=True)
if training_phase:
images_train, labels_train = mndata.load_training()
images_train, labels_train = preprocess_data(images_train,
labels_train,
nn=True)
epochs = 5
batch_size = 64
learning_rate = 0.006
optimizer = Adam(alpha=learning_rate)
loss_func = CrossEntropy()
cnn = CNN(optimizer=optimizer, layers=arch, loss_func=loss_func)
trainer = Trainer(cnn, batch_size, epochs)
trainer.train(images_train, labels_train)
trainer.save_model(weight_path)
else:
import pickle
images_test, labels_test = mndata.load_testing()
images_test, labels_test = preprocess_data(images_test,
labels_test,
nn=True,
test=True)
with open(weight_path, "rb") as f:
cnn = pickle.load(f)
pred = cnn.predict(images_test)
print("Accuracy:", len(pred[labels_test == pred]) / len(pred))
from sklearn.metrics.classification import confusion_matrix
print("Confusion matrix: ")
print(confusion_matrix(labels_test, pred))
def _structure(self, cnn_structure):
"""
Structure function that initializes CNN architecture.
Parameters
----------
cnn_structure: (list) a list of dictionaries define CNN architecture.
Each dictionary element is 1 kind of layer (ConvLayer, FCLayer, PoolingLayer, FlattenLayer, BatchNormLayer).
- Convolutional layer (`type: conv`) dict should have following key-value pair:
+ filter_size: (tuple) define conv filter size (fH, fW)
+ filters: (int) number of conv filters at the layer.
+ stride: (int) stride of conv filter.
+ weight_init: (str) choose which kind to initialize the filter, either `he` `xavier` or `std`.
+ padding: (str) padding type of input corresponding to the output, either `SAME` or `VALID`.
+ activation (optional): (str) apply activation to the output of the layer. LINEAR -> ACTIVATION.
+ batch_norm (optional): (any) apply batch norm to the output of the layer. LINEAR -> BATCH NORM -> ACTIVATION
- Pooling layer (`type: pool`) dict should have following key-value pair:
+ filter_size: (tuple) define pooling filter size (fH, fW).
+ mode: (str) choose the mode of pooling, either `max` or `avg`.
+ stride: (int) stride of pooling filter.
- Fully-connected layer (`type: fc`) dict should have following key-value pair:
+ num_neurons: (int) define number of neurons in the dense layer.
+ weight_init: (str) choose which kind to initialize the weight, either `he` `xavier` or `std`.
+ activation (optional): (str) apply activation to the output of the layer. LINEAR -> ACTIVATION.
+ batch_norm (optional): (any) apply batch norm to the output of the layer. LINEAR -> BATCH NORM -> ACTIVATION
"""
layers = []
for struct in cnn_structure:
if type(struct) is str and struct == "flatten":
flatten_layer = FlattenLayer()
layers.append(flatten_layer)
continue
if struct["type"] == "conv":
filter_size = struct["filter_size"]
filters = struct["filters"]
padding = struct["padding"]
stride = struct["stride"]
weight_init = struct["weight_init"]
conv_layer = ConvLayer(filter_size, filters, padding, stride,
weight_init)
conv_layer.initialize_optimizer(self.optimizer)
layers.append(conv_layer)
if "batch_norm" in struct:
bn_layer = BatchNormLayer()
bn_layer.initialize_optimizer(self.optimizer)
layers.append(bn_layer)
if "activation" in struct:
activation = struct["activation"]
act_layer = ActivationLayer(activation=activation)
layers.append(act_layer)
elif struct["type"] == "pool":
filter_size = struct["filter_size"]
stride = struct["stride"]
mode = struct["mode"]
pool_layer = PoolingLayer(filter_size=filter_size,
stride=stride,
mode=mode)
layers.append(pool_layer)
else:
num_neurons = struct["num_neurons"]
weight_init = struct["weight_init"]
fc_layer = FCLayer(num_neurons=num_neurons,
weight_init=weight_init)
fc_layer.initialize_optimizer(self.optimizer)
layers.append(fc_layer)
if "batch_norm" in struct:
bn_layer = BatchNormLayer()
bn_layer.initialize_optimizer(self.optimizer)
layers.append(bn_layer)
if "activation" in struct:
activation = struct["activation"]
act_layer = ActivationLayer(activation)
layers.append(act_layer)
return layers