def test(): import numpy as np from nn_components.layers import ConvLayer, PoolingLayer from optimizations_algorithms.optimizers import SGD import tensorflow as tf tf.enable_eager_execution() filter_size = (3, 3) filters = 16 padding = "SAME" stride = 1 optimizer = SGD() conv_layer = ConvLayer(filter_size=filter_size, filters=filters, padding=padding, stride=stride) pool_filter_size = (2, 2) pool_stride = 2 pool_mode = "max" pool_layer = PoolingLayer(filter_size=pool_filter_size, stride=pool_stride, mode=pool_mode) X = np.random.normal(size=(16, 12, 12, 3)) d_prev = np.random.normal(size=(16, 12, 12, 16)) my_conv_forward = conv_layer.forward(X) my_dA, my_dW = conv_layer.backward(d_prev, X) my_pool_forward = pool_layer.forward(X) with tf.device("/cpu:0"): tf_conv_forward = tf.nn.conv2d(X, conv_layer.W, strides=(stride, stride), padding=padding).numpy() tf_dW = tf.nn.conv2d_backprop_filter(X, filter_sizes=filter_size + (X.shape[-1], filters), out_backprop=d_prev, strides=(1, stride, stride, 1), padding=padding).numpy() tf_dA = tf.nn.conv2d_backprop_input(input_sizes=X.shape, filter=conv_layer.W, out_backprop=d_prev, strides=(1, stride, stride, 1), padding=padding).numpy() tf_pool_forward = tf.nn.max_pool2d(X, ksize=pool_filter_size, strides=(pool_stride, pool_stride), padding="VALID") blank = "----------------------" print(blank + "TEST FORWARD CONVOLUTION" + blank) forward_result = np.allclose(my_conv_forward, tf_conv_forward) forward_out = "PASS" if forward_result else "FAIL" print("====> " + forward_out) print(blank + "TEST BACKWARD CONVOLUTION" + blank) dW_result = np.allclose(my_dW, tf_dW) dW_out = "PASS" if dW_result else "FAIL" print("====> dW case: " + dW_out) dA_result = np.allclose(my_dA, tf_dA) dA_out = "PASS" if dA_result else "FAIL" print("====> dA case: " + dA_out) print(blank + "TEST FORWARD POOLING" + blank) pool_result = np.allclose(my_pool_forward, tf_pool_forward) pool_out = "PASS" if pool_result else "FAIL" print("====> " + pool_out)
def _structure(self, cnn_structure): """ Structure function that initializes cnn architecture. """ layers = [] for struct in cnn_structure: if type(struct) is str and struct == "flatten": flatten_layer = FlattenLayer() layers.append(flatten_layer) continue if struct["type"] == "conv": filter_size = struct["filter_size"] filters = struct["filters"] padding = struct["padding"] stride = struct["stride"] conv_layer = ConvLayer(filter_size, filters, padding, stride) layers.append(conv_layer) if "batch_norm" in struct: bn_layer = BatchNormLayer() layers.append(bn_layer) if "activation" in struct: activation = struct["activation"] act_layer = ActivationLayer(activation=activation) layers.append(act_layer) elif struct["type"] == "pool": filter_size = struct["filter_size"] stride = struct["stride"] mode = struct["mode"] pool_layer = PoolingLayer(filter_size=filter_size, stride=stride, mode=mode) layers.append(pool_layer) else: num_neurons = struct["num_neurons"] weight_init = struct["weight_init"] fc_layer = FCLayer(num_neurons=num_neurons, weight_init=weight_init) layers.append(fc_layer) if "activation" in struct: activation = struct["activation"] act_layer = ActivationLayer(activation) layers.append(act_layer) return layers
def main(): arch = [ ConvLayer(filter_size=(3, 3), filters=6, padding="SAME", stride=1, weight_init="he_normal"), ActivationLayer(activation="relu"), PoolingLayer(filter_size=(2, 2), stride=2, mode="max"), ConvLayer(filter_size=(3, 3), filters=16, padding="SAME", stride=1, weight_init="he_normal"), ActivationLayer(activation="relu"), PoolingLayer(filter_size=(2, 2), stride=2, mode="max"), ConvLayer(filter_size=(3, 3), filters=32, padding="SAME", stride=1, weight_init="he_normal"), ActivationLayer(activation="relu"), PoolingLayer(filter_size=(2, 2), stride=2, mode="max"), FlattenLayer(), FCLayer(num_neurons=128, weight_init="he_normal"), ActivationLayer(activation="relu"), FCLayer(num_neurons=64, weight_init="he_normal"), ActivationLayer(activation="relu"), FCLayer(num_neurons=10, weight_init="he_normal"), ActivationLayer(activation="softmax") ] print("Train MNIST dataset by CNN with pure Python: Numpy.") weight_path = "cnn_weights.pkl" training_phase = weight_path not in os.listdir(".") load_dataset_mnist("../libs") mndata = MNIST('../libs/data_mnist', gz=True) if training_phase: images_train, labels_train = mndata.load_training() images_train, labels_train = preprocess_data(images_train, labels_train, nn=True) epochs = 5 batch_size = 64 learning_rate = 0.006 optimizer = Adam(alpha=learning_rate) loss_func = CrossEntropy() cnn = CNN(optimizer=optimizer, layers=arch, loss_func=loss_func) trainer = Trainer(cnn, batch_size, epochs) trainer.train(images_train, labels_train) trainer.save_model(weight_path) else: import pickle images_test, labels_test = mndata.load_testing() images_test, labels_test = preprocess_data(images_test, labels_test, nn=True, test=True) with open(weight_path, "rb") as f: cnn = pickle.load(f) pred = cnn.predict(images_test) print("Accuracy:", len(pred[labels_test == pred]) / len(pred)) from sklearn.metrics.classification import confusion_matrix print("Confusion matrix: ") print(confusion_matrix(labels_test, pred))
def _structure(self, cnn_structure): """ Structure function that initializes CNN architecture. Parameters ---------- cnn_structure: (list) a list of dictionaries define CNN architecture. Each dictionary element is 1 kind of layer (ConvLayer, FCLayer, PoolingLayer, FlattenLayer, BatchNormLayer). - Convolutional layer (`type: conv`) dict should have following key-value pair: + filter_size: (tuple) define conv filter size (fH, fW) + filters: (int) number of conv filters at the layer. + stride: (int) stride of conv filter. + weight_init: (str) choose which kind to initialize the filter, either `he` `xavier` or `std`. + padding: (str) padding type of input corresponding to the output, either `SAME` or `VALID`. + activation (optional): (str) apply activation to the output of the layer. LINEAR -> ACTIVATION. + batch_norm (optional): (any) apply batch norm to the output of the layer. LINEAR -> BATCH NORM -> ACTIVATION - Pooling layer (`type: pool`) dict should have following key-value pair: + filter_size: (tuple) define pooling filter size (fH, fW). + mode: (str) choose the mode of pooling, either `max` or `avg`. + stride: (int) stride of pooling filter. - Fully-connected layer (`type: fc`) dict should have following key-value pair: + num_neurons: (int) define number of neurons in the dense layer. + weight_init: (str) choose which kind to initialize the weight, either `he` `xavier` or `std`. + activation (optional): (str) apply activation to the output of the layer. LINEAR -> ACTIVATION. + batch_norm (optional): (any) apply batch norm to the output of the layer. LINEAR -> BATCH NORM -> ACTIVATION """ layers = [] for struct in cnn_structure: if type(struct) is str and struct == "flatten": flatten_layer = FlattenLayer() layers.append(flatten_layer) continue if struct["type"] == "conv": filter_size = struct["filter_size"] filters = struct["filters"] padding = struct["padding"] stride = struct["stride"] weight_init = struct["weight_init"] conv_layer = ConvLayer(filter_size, filters, padding, stride, weight_init) conv_layer.initialize_optimizer(self.optimizer) layers.append(conv_layer) if "batch_norm" in struct: bn_layer = BatchNormLayer() bn_layer.initialize_optimizer(self.optimizer) layers.append(bn_layer) if "activation" in struct: activation = struct["activation"] act_layer = ActivationLayer(activation=activation) layers.append(act_layer) elif struct["type"] == "pool": filter_size = struct["filter_size"] stride = struct["stride"] mode = struct["mode"] pool_layer = PoolingLayer(filter_size=filter_size, stride=stride, mode=mode) layers.append(pool_layer) else: num_neurons = struct["num_neurons"] weight_init = struct["weight_init"] fc_layer = FCLayer(num_neurons=num_neurons, weight_init=weight_init) fc_layer.initialize_optimizer(self.optimizer) layers.append(fc_layer) if "batch_norm" in struct: bn_layer = BatchNormLayer() bn_layer.initialize_optimizer(self.optimizer) layers.append(bn_layer) if "activation" in struct: activation = struct["activation"] act_layer = ActivationLayer(activation) layers.append(act_layer) return layers