def __build_model(self, n_features, n_classes): # Create layers of network layers = [] for l in range(self.n_layers): # First layer if(l == 0): layers.append(ReluLayer(n_features, self.neurons_per_layer, self.learning_rate, self.reg_lambda)) # Last layer elif(l == self.n_layers - 1): layers.append(SoftmaxLayer(self.neurons_per_layer, n_classes, self.learning_rate, self.reg_lambda)) # Middle layers else: layers.append(ReluLayer(self.neurons_per_layer, self.neurons_per_layer, self.learning_rate, self.reg_lambda)) return layers
def __init__(self, layers, l2_decay=0.001, debug=False, learning_rate=0.001): super(Network, self).__init__() mapping = { "input": lambda x: InputLayer(x), "fc": lambda x: FullyConnectedLayer(x), "convolution": lambda x: ConvLayer(x), "pool": lambda x: MaxPoolLayer(x), "squaredloss": lambda x: SquaredLossLayer(x), "softmax": lambda x: SoftmaxLayer(x), "relu": lambda x: ReLULayer(x), "dropout": lambda x: DropoutLayer(x) } self.layers = [] self.l2_decay = l2_decay self.debug = debug self.learning_rate = learning_rate
def __init__(self, layer_types, layer_shapes, conv_layer_types=None, layers=None): self.layer_types = layer_types self.layer_shapes = layer_shapes self.num_genes = 0 self.conv_layer_types = conv_layer_types if layers is not None: self.layers = layers for typ, shpe in zip(layer_types, layer_shapes): if typ == "conv": self.num_genes += shpe[1][0] elif typ == "dense": self.num_genes += shpe[0][0] elif typ == "soft": self.num_genes += shpe[0][0] else: self.layers = [] cntr = 0 n_conv_layer_types = -1 if conv_layer_types is not None: n_conv_layer_types = len(conv_layer_types) for typ, shpe in zip(layer_types, layer_shapes): if typ == "conv": if cntr <= n_conv_layer_types: self.layers.append(ConvLayer(image_shape=shpe[0], filter_shape=shpe[1], filter_method=conv_layer_types[cntr][0], zero_padding=conv_layer_types[cntr][1])) cntr += 1 else: self.layers.append(ConvLayer(image_shape=shpe[0], filter_shape=shpe[1])) self.num_genes += shpe[1][0] elif typ == "dense": self.layers.append(DenseLayer(layer_shape=shpe[0])) self.num_genes += shpe[0][0] elif typ == "soft": self.layers.append(SoftmaxLayer(layer_shape=shpe[0])) self.num_genes += shpe[0][0]
def crossover(father, mother, alpha=0.5): layers = [] for lt, ls, fl, ml in zip(father.get_layer_types(), father.get_layer_shapes(), father.get_layers(), mother.get_layers()): if lt == "conv": lyr = ConvLayer(image_shape=ls[0], filter_shape=ls[1]) # Loop for each filter for i in range(ls[1][0]): parent = ml # Randomly pick either mother or father gene based on alpha if random.random() < alpha: parent = fl lyr.set_filter(index=i, filtr=deepcopy(parent.get_filter(i))) layers.append(lyr) elif lt == "dense" or lt == "soft": weights = [] biases = [] lyr = DenseLayer(layer_shape=ls[0]) if lt == "soft": lyr = SoftmaxLayer(layer_shape=ls[0]) # Loop for each neuron for i in range(ls[0][0]): parent = ml # Randomly pick either mother or father gene based on alpha if random.random() < alpha: parent = fl weights.append(deepcopy(parent.get_weights(i))) biases.append(deepcopy(parent.get_biases(i))) lyr.set_weights_biases(weights, biases) layers.append(lyr) child = Individual(deepcopy(father.get_layer_types()), deepcopy(father.get_layer_shapes()), deepcopy(father.get_conv_layer_types()), layers) child = mutate_individual(child) return child
import mnist import numpy as np from conv_layer import ConvLayer from max_pool_layer import MaxPoolLayer from softmax_layer import SoftmaxLayer train_images = mnist.train_images()[:1000] train_labels = mnist.train_labels()[:1000] test_images = mnist.test_images()[:1000] test_labels = mnist.test_labels()[:1000] conv = ConvLayer(8) pool = MaxPoolLayer() softmax = SoftmaxLayer(13 * 13 * 8, 10) def check(image): out = conv.forward((image / 255) - 0.5) out = pool.forward(out) out = softmax.forward(out) return np.argmax(out) def forward(image, label): ''' Completes a forward pass of the CNN and calculates the accuracy and cross-entropy loss. - image is a 2d numpy array - label is a digit '''
def test_softmax_activation(): # Simple case lay = SoftmaxLayer(1, 1) input_arr = np.array([[7]]) expected_arr = np.array([[1]]) actual_arr = lay.activation(input_arr) np.testing.assert_array_equal(expected_arr, actual_arr) # Simple case with batches lay = SoftmaxLayer(1, 1) input_arr = np.array([[2], [7]]) expected_arr = np.array([[1], [1]]) actual_arr = lay.activation(input_arr) np.testing.assert_array_equal(expected_arr, actual_arr) # Multiple inputs lay = SoftmaxLayer(4, 4) input_arr = np.array([[1, 2, 3, 4]]) exp_sum = np.exp(1) + np.exp(2) + np.exp(3) + np.exp(4) expected_arr = np.exp(np.array([[1, 2, 3, 4]])) / exp_sum actual_arr = lay.activation(input_arr) # round down to nearest 10 decimal places # because floating point sucks expected_arr = np.round(expected_arr, 10) actual_arr = np.round(actual_arr, 10) np.testing.assert_array_equal(expected_arr, actual_arr) # Multiple inputs lay = SoftmaxLayer(4, 4) input_arr = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]) exp_sum = np.exp(1) + np.exp(2) + np.exp(3) + np.exp(4) expected_arr1 = np.exp(np.array([1, 2, 3, 4])) / exp_sum exp_sum = np.exp(5) + np.exp(6) + np.exp(7) + np.exp(8) expected_arr2 = np.exp(np.array([5, 6, 7, 8])) / exp_sum expected_arr = np.array([expected_arr1, expected_arr2]) actual_arr = lay.activation(input_arr) # round down to nearest 10 decimal places # because floating point sucks expected_arr = np.round(expected_arr, 10) actual_arr = np.round(actual_arr, 10) np.testing.assert_array_equal(expected_arr, actual_arr)
def load_population(file_name): counter = 0 file = open(file_name, 'r') #read line by line arr = file.read().splitlines() pop_size = int(arr[counter]) counter += 1 initial_pop = [] clt = [] for i in range(pop_size): layer_types = [] layer_shapes = [] layers = [] conv_layer_types = [] num_layers = int(arr[counter]) counter += 1 for j in range(num_layers): type = arr[counter] layer_types.append(type) counter += 1 if type == "conv": filter_method = arr[counter] counter += 1 zero_padding = int(arr[counter]) counter += 1 conv_layer_types.append((filter_method, zero_padding)) image_shape = (int(arr[counter]), int(arr[counter + 1]), int(arr[counter + 2])) counter += 3 filter_shape = (int(arr[counter]), int(arr[counter + 1]), int(arr[counter + 2]), int(arr[counter + 3])) counter += 4 layer_shapes.append([image_shape, filter_shape]) filters = [] for i in range(filter_shape[0]): weights = np.zeros(filter_shape[1:]) bias = 0 for i in range(filter_shape[1]): for j in range(filter_shape[2]): for k in range(filter_shape[3]): weights[i][j][k] = float(arr[counter]) counter += 1 bias = float(arr[counter]) counter += 1 filters.append(Filter(filter_shape[1:], weights, bias)) clt = conv_layer_types layers.append( ConvLayer(image_shape, filter_shape, filter_method, zero_padding, filters)) elif type == "dense" or type == "soft": shpe = (int(arr[counter]), int(arr[counter + 1])) layer_shapes.append([shpe]) counter += 2 weights = np.zeros(shpe) biases = np.zeros(shpe[0]) for cl in range(shpe[0]): for pl in range(shpe[1]): weights[cl][pl] = float(arr[counter]) counter += 1 for cl in range(shpe[0]): biases[cl] = float(arr[counter]) counter += 1 if type == "dense": layers.append(DenseLayer(shpe, weights, biases)) elif type == "soft": layers.append(SoftmaxLayer(shpe, weights, biases)) initial_pop.append(Individual(layer_types, layer_shapes, clt, layers)) population = Population(pop_size, initial_pop[0].get_layer_types(), initial_pop[0].get_layer_shapes(), clt, initial_pop) return population
class TestSoftmaxLayer(TestCase): def setUp(self): self.layer = SoftmaxLayer() self.layer.set_input_shape((4, )) self.predicted = np.array([0.2, 0.4, 0.4], dtype=np.float64) self.true = np.array([0, 1, 0], dtype=np.float64) def test_forward_prop_two_equal(self): input = np.array([43265, 0, 0, 43265], dtype=np.float64) output = self.layer.forward_prop(input) expected_output = np.array([0.5, 0, 0, 0.5], dtype=np.float64) numpy.testing.assert_array_equal(output, expected_output) def test_forward_prop_one_large(self): input = np.array([43265, 2, 54, 21], dtype=np.float64) output = self.layer.forward_prop(input) expected_output = np.array([1, 0, 0, 0], dtype=np.float64) numpy.testing.assert_array_equal(output, expected_output) def test_back_prop(self): out_grad = np.array([3, 3, 3, 3], dtype=np.float64) with self.assertRaises(NotImplementedError): self.layer.back_prop(out_grad) def test_initial_gradient(self): expected_gradient = np.array([0.2, -0.6, 0.4], dtype=np.float64) gradient = self.layer.initial_gradient(self.predicted, self.true) numpy.testing.assert_array_equal(expected_gradient, gradient) def test_loss(self): expected_loss = 0.9162907 loss = self.layer.loss(self.predicted, self.true) self.assertAlmostEqual(expected_loss, loss) def test_get_output_shape(self): self.layer.set_input_shape((5, )) self.assertEqual(self.layer.get_output_shape(), (5, ))
def setUp(self): self.layer = SoftmaxLayer() self.layer.set_input_shape((4, )) self.predicted = np.array([0.2, 0.4, 0.4], dtype=np.float64) self.true = np.array([0, 1, 0], dtype=np.float64)