def _structure(self, nn_structure): """ Structure function that initializes neural network architecture. Parameters ---------- nn_structure: (list) a list of dictionaries define Neural Network architecture. - Each dict element in the list should have following key-pair value: + num_neurons: (int) define number of neurons in the dense layer. + weight_init: (str) choose which kind to initialize the weight, either `he` `xavier` or `std`. + activation (optional): (str) apply activation to the output of the layer. LINEAR -> ACTIVATION. + batch_norm (optional): (any) apply batch norm to the output of the layer. LINEAR -> BATCH NORM -> ACTIVATION. + drop_out (optional): (float) choose rate to drop out neurons. """ layers = [] for struct in nn_structure: num_neurons = struct["num_neurons"] weight_init = struct["weight_init"] fc = FCLayer(num_neurons=num_neurons, weight_init=weight_init) fc.initialize_optimizer(self.optimizer) layers.append(fc) if "batch_norm" in struct: bn_layer = BatchNormLayer() bn_layer.initialize_optimizer(self.optimizer) layers.append(bn_layer) if "activation" in struct: activation = struct["activation"] act_layer = ActivationLayer(activation=activation) layers.append(act_layer) if "drop_out" in struct: drop_out = struct["drop_out"] do_layer = DropoutLayer(drop_out) layers.append(do_layer) return layers
def _structure(self, cnn_structure): """ Structure function that initializes cnn architecture. """ layers = [] for struct in cnn_structure: if type(struct) is str and struct == "flatten": flatten_layer = FlattenLayer() layers.append(flatten_layer) continue if struct["type"] == "conv": filter_size = struct["filter_size"] filters = struct["filters"] padding = struct["padding"] stride = struct["stride"] conv_layer = ConvLayer(filter_size, filters, padding, stride) layers.append(conv_layer) if "batch_norm" in struct: bn_layer = BatchNormLayer() layers.append(bn_layer) if "activation" in struct: activation = struct["activation"] act_layer = ActivationLayer(activation=activation) layers.append(act_layer) elif struct["type"] == "pool": filter_size = struct["filter_size"] stride = struct["stride"] mode = struct["mode"] pool_layer = PoolingLayer(filter_size=filter_size, stride=stride, mode=mode) layers.append(pool_layer) else: num_neurons = struct["num_neurons"] weight_init = struct["weight_init"] fc_layer = FCLayer(num_neurons=num_neurons, weight_init=weight_init) layers.append(fc_layer) if "activation" in struct: activation = struct["activation"] act_layer = ActivationLayer(activation) layers.append(act_layer) return layers
def _structure(self, nn_structure): """ Structure function that initializes neural network architecture. """ layers = [] for struct in nn_structure: num_neurons = struct["num_neurons"] weight_init = struct["weight_init"] fc = FCLayer(num_neurons=num_neurons, weight_init=weight_init) layers.append(fc) if "batch_norm" in struct: bn_layer = BatchNormLayer() layers.append(bn_layer) if "activation" in struct: activation = struct["activation"] act_layer = ActivationLayer(activation=activation) layers.append(act_layer) return layers
def main(): arch = [ ConvLayer(filter_size=(3, 3), filters=6, padding="SAME", stride=1, weight_init="he_normal"), ActivationLayer(activation="relu"), PoolingLayer(filter_size=(2, 2), stride=2, mode="max"), ConvLayer(filter_size=(3, 3), filters=16, padding="SAME", stride=1, weight_init="he_normal"), ActivationLayer(activation="relu"), PoolingLayer(filter_size=(2, 2), stride=2, mode="max"), ConvLayer(filter_size=(3, 3), filters=32, padding="SAME", stride=1, weight_init="he_normal"), ActivationLayer(activation="relu"), PoolingLayer(filter_size=(2, 2), stride=2, mode="max"), FlattenLayer(), FCLayer(num_neurons=128, weight_init="he_normal"), ActivationLayer(activation="relu"), FCLayer(num_neurons=64, weight_init="he_normal"), ActivationLayer(activation="relu"), FCLayer(num_neurons=10, weight_init="he_normal"), ActivationLayer(activation="softmax") ] print("Train MNIST dataset by CNN with pure Python: Numpy.") weight_path = "cnn_weights.pkl" training_phase = weight_path not in os.listdir(".") load_dataset_mnist("../libs") mndata = MNIST('../libs/data_mnist', gz=True) if training_phase: images_train, labels_train = mndata.load_training() images_train, labels_train = preprocess_data(images_train, labels_train, nn=True) epochs = 5 batch_size = 64 learning_rate = 0.006 optimizer = Adam(alpha=learning_rate) loss_func = CrossEntropy() cnn = CNN(optimizer=optimizer, layers=arch, loss_func=loss_func) trainer = Trainer(cnn, batch_size, epochs) trainer.train(images_train, labels_train) trainer.save_model(weight_path) else: import pickle images_test, labels_test = mndata.load_testing() images_test, labels_test = preprocess_data(images_test, labels_test, nn=True, test=True) with open(weight_path, "rb") as f: cnn = pickle.load(f) pred = cnn.predict(images_test) print("Accuracy:", len(pred[labels_test == pred]) / len(pred)) from sklearn.metrics.classification import confusion_matrix print("Confusion matrix: ") print(confusion_matrix(labels_test, pred))
load_dataset_mnist("../libs") mndata = MNIST('../libs/data_mnist', gz=True) weight_path = "nn_weights.pkl" training_phase = weight_path not in os.listdir(".") if training_phase: images, labels = mndata.load_training() images, labels = preprocess_data(images, labels) epochs = 10 batch_size = 64 learning_rate = 0.01 optimizer = Adam(learning_rate) loss_func = CrossEntropy() archs = [ FCLayer(num_neurons=100, weight_init="he_normal"), ActivationLayer(activation="relu"), DropoutLayer(keep_prob=0.8), FCLayer(num_neurons=125, weight_init="he_normal"), ActivationLayer(activation="relu"), DropoutLayer(keep_prob=0.8), FCLayer(num_neurons=50, weight_init="he_normal"), ActivationLayer(activation="relu"), BatchNormLayer(), FCLayer(num_neurons=labels.shape[1], weight_init="he_normal"), ActivationLayer(activation="softmax"), ] nn = NeuralNetwork(optimizer=optimizer, layers=archs, loss_func=loss_func) trainer = Trainer(nn, batch_size, epochs) trainer.train(images, labels) trainer.save_model("nn_weights.pkl")
def _structure(self, cnn_structure): """ Structure function that initializes CNN architecture. Parameters ---------- cnn_structure: (list) a list of dictionaries define CNN architecture. Each dictionary element is 1 kind of layer (ConvLayer, FCLayer, PoolingLayer, FlattenLayer, BatchNormLayer). - Convolutional layer (`type: conv`) dict should have following key-value pair: + filter_size: (tuple) define conv filter size (fH, fW) + filters: (int) number of conv filters at the layer. + stride: (int) stride of conv filter. + weight_init: (str) choose which kind to initialize the filter, either `he` `xavier` or `std`. + padding: (str) padding type of input corresponding to the output, either `SAME` or `VALID`. + activation (optional): (str) apply activation to the output of the layer. LINEAR -> ACTIVATION. + batch_norm (optional): (any) apply batch norm to the output of the layer. LINEAR -> BATCH NORM -> ACTIVATION - Pooling layer (`type: pool`) dict should have following key-value pair: + filter_size: (tuple) define pooling filter size (fH, fW). + mode: (str) choose the mode of pooling, either `max` or `avg`. + stride: (int) stride of pooling filter. - Fully-connected layer (`type: fc`) dict should have following key-value pair: + num_neurons: (int) define number of neurons in the dense layer. + weight_init: (str) choose which kind to initialize the weight, either `he` `xavier` or `std`. + activation (optional): (str) apply activation to the output of the layer. LINEAR -> ACTIVATION. + batch_norm (optional): (any) apply batch norm to the output of the layer. LINEAR -> BATCH NORM -> ACTIVATION """ layers = [] for struct in cnn_structure: if type(struct) is str and struct == "flatten": flatten_layer = FlattenLayer() layers.append(flatten_layer) continue if struct["type"] == "conv": filter_size = struct["filter_size"] filters = struct["filters"] padding = struct["padding"] stride = struct["stride"] weight_init = struct["weight_init"] conv_layer = ConvLayer(filter_size, filters, padding, stride, weight_init) conv_layer.initialize_optimizer(self.optimizer) layers.append(conv_layer) if "batch_norm" in struct: bn_layer = BatchNormLayer() bn_layer.initialize_optimizer(self.optimizer) layers.append(bn_layer) if "activation" in struct: activation = struct["activation"] act_layer = ActivationLayer(activation=activation) layers.append(act_layer) elif struct["type"] == "pool": filter_size = struct["filter_size"] stride = struct["stride"] mode = struct["mode"] pool_layer = PoolingLayer(filter_size=filter_size, stride=stride, mode=mode) layers.append(pool_layer) else: num_neurons = struct["num_neurons"] weight_init = struct["weight_init"] fc_layer = FCLayer(num_neurons=num_neurons, weight_init=weight_init) fc_layer.initialize_optimizer(self.optimizer) layers.append(fc_layer) if "batch_norm" in struct: bn_layer = BatchNormLayer() bn_layer.initialize_optimizer(self.optimizer) layers.append(bn_layer) if "activation" in struct: activation = struct["activation"] act_layer = ActivationLayer(activation) layers.append(act_layer) return layers