def forward(self, input_volume): # check to make sure the input is what the layer expects self.check_input_dimensions(input_volume) # make instance input_volume an unmodified version of original input self.input_volume = volume(input_volume.volume_slices) # add padding to input volume if specified input_volume.volume_slices = self.add_padding(input_volume.volume_slices) # go through all the filters, storing the result of a convolution in an output array output_slice = [] stride = self.stride for filter_volume, bias in zip(self.filters, self.biases.volume_slices[0][0]): for i in range(self.out_height): # find the current 'y' position of the input image row = i * self.stride for j in range(self.out_width): # find the current 'x' position of the input image col = j * self.stride # element-wise multiply the depth column by the filters to yield a single value from the convolution # find the inputs for a single convolution step depth_column = input_volume.volume_slices[:,row:row + self.field_size, col:col + self.field_size] output_slice.append(np.sum(depth_column * filter_volume.volume_slices) + bias) # return the result of a convolution on the entire volume self.output_volume = volume(np.reshape(output_slice, (self.out_depth, self.out_height, self.out_width))) return self.output_volume
def load_params(self, net_name):
params = np.load("saved_networks/"+net_name+"/"+self.name + ".npz")
weights = []
for weight_values in params["weights"]:
weights.append(volume(weight_values))
self.weights = weights
self.biases = volume(params["biases"])
def load_params(self, net_name):
params = np.load("saved_networks/"+net_name+"/"+self.name + ".npz")
filters = []
for filter_values in params["filters"]:
filters.append(volume(filter_values))
self.filters = filters
self.biases = volume(params["biases"])
def initialize_biases(self): # make a matrix with the dimensions of the output volume biases = np.zeros((1,1,self.out_width)) # fill the matrix with 0.1 (initial bias value) and make it an instance variable biases.fill(0.1) self.biases = volume(biases)
def initialize_weights(self): weights = [] weight_count = self.in_height * self.in_width * self.in_depth # initialize weights for each neuron in the network for i in range(self.out_width): weight_gen = np.random.randn(weight_count) * np.sqrt(2.0 / weight_count) weight_gen = np.reshape(weight_gen, (self.in_depth, self.in_height, self.in_width)) weights.append(volume(weight_gen)) # set as an instance variable for later methods self.weights = weights
def backward(self): # create a padded matrix with zeros to hold input gradients input_padding = self.padding * 2 input_gradient = np.zeros((self.in_depth, self.in_height + input_padding, self.in_width + input_padding)) # do the same for the biases biases_gradient = np.zeros((1,1,self.out_depth)) # preserve the original input volume before making changes input_volume = volume(self.input_volume.volume_slices) input_volume.volume_slices = self.add_padding(input_volume.volume_slices) # iterate through the entire volume (height, width, depth); first along the depth dimension # this also functions to iterate through each filter individually for depth_index in range(self.out_depth): # iterate along the height dimension # create an empty volume of zeros for the gradient filter_gradient = np.zeros((self.in_depth, self.field_size, self.field_size)) for height_index in range(self.out_height): # iterate along the width dimension # make sure to find row position in input volume row = height_index * self.stride for width_index in range(self.out_width): # iterate along the height dimension # make sure to find column position in input volume col = width_index * self.stride # find the gradient from the previous layer chain_gradient = self.output_volume.gradient_slices[depth_index][height_index][width_index] # update filter gradient by multiplying previous layer's gradient # with the inputs within the receptive field depth_column = input_volume.volume_slices[:,row:row + self.field_size, col:col + self.field_size] filter_gradient += depth_column * chain_gradient # add the filter weights multiplied by the chain to the input input_gradient[:,row:row + self.field_size, col:col + self.field_size] += self.filters[depth_index].volume_slices * chain_gradient # update the biases biases_gradient[0][0][depth_index] += chain_gradient # set the new gradients to whatever filter we're currently iterating over # chnaged to += self.filters[depth_index].gradient_slices += filter_gradient # trim the padding off the gradient volume to match the input volume's original size input_gradient = self.trim_padding(input_gradient) # set the input and bias gradients to whatever gradients we just calculated self.input_volume.gradient_slices = input_gradient # changed to += self.biases.gradient_slices += biases_gradient return self.input_volume
def images_to_volumes(path): volumes = [] batch_data = unpickle_batch(file_path) # cifar-10 stores 32x32 (1024 total) pixel images as an array of length 3072 # first 1024 are red, next 1024 are green, final 1024 are blue. Pretty awesome! # each array of 3072 is put into an array of 10,000 (because 10,000 total images) for rgb_array in batch_data['data']: image_volume = volume(np.reshape(rgb_array, (3,32,32))) volumes.append(image_volume) # return the image volumed paired with their labels return (volumes, batch_data['labels'])
def forward(self, input_volume): self.input_volume = input_volume output = np.array([]) # each matrix of weights corresponds to a slice in the output volume for weight in self.weights: # print weight.volume_slices output = np.append(output, np.sum(weight.volume_slices * input_volume.volume_slices)) output = output + self.biases.volume_slices[0][0] # maybe redefine which axix the wights are on (height, width, or depth)? Something to consider self.output_volume = volume(np.reshape(output, (1,1, self.out_width))) return self.output_volume
def forward(self, input_volume): # subtract the max output from all output values to avoid exponential explosion max_input = np.max(input_volume.volume_slices) exp_matrix = input_volume.volume_slices - max_input # map e^x across list to find unnormalized probabilities. save them for backpropagation exp_matrix = np.exp(exp_matrix) # normalize the probabilities to be between 0 and 1 exp_matrix_sum = np.sum(exp_matrix) exp_matrix = exp_matrix / exp_matrix_sum # save volumes for backprop self.input_volume = input_volume self.output_volume = volume(exp_matrix) self.classify = np.argmax(self.output_volume.volume_slices) # print self.output_volume.volume_slices return self.output_volume
def initialize_filters(self): # a list to hold all the filter volumes - depth of volumes is the depth of the input volume # a filter will be paired with inputs from each layer of depth filters = [] # calculate the area of the receptive field weight_count = (self.field_size ** 2) * self.in_depth # create filter volumes for the number of sets the user specifies. If depth is 3, and the user specifies # 20 filter sets, then there will be 60 total filter. If the receptive field size is 3, then the # total number of weights will be 540. (3 * 20 * (3^2)) for i in range(self.filter_count): # initialize weights for every filter # recommended initalization function for ReLu: np.random.randn(n) * np.sqrt(2.0 / n) filter_volume = np.random.randn(weight_count) * np.sqrt(2.0 / weight_count) filter_volume = np.reshape(filter_volume, (self.in_depth, self.field_size, self.field_size)) # turn filters into a new volume filters.append(volume(filter_volume)) # assign as an instance variable self.filters = filters
def forward(self, input_volume): # save for backprop self.input_volume = input_volume # store the max's (x,y) position in the original input volume # this will later be used to transfer gradients in backprop max_row_positions = [] max_col_positions = [] # output volume of the pooling process output_volume = [] # iterate along the depth dimension of the volume for volume_slice in input_volume.volume_slices: # iterate along the rows (height dimension) for i in range(self.out_height): row = i * self.field_size # iterate along the columns (width dimension) for j in range(self.out_width): col = j * self.field_size # find the inputs within the current receptive field inputs = volume_slice[row:row+self.field_size,col:col+self.field_size] # find the max's (x,y) position local to the receptive field # add the column and row to find max's position within the input max_positions = np.unravel_index(inputs.argmax(), inputs.shape) max_row_positions.append(max_positions[0] + row) max_col_positions.append(max_positions[1] + col) # add the max value to the final output volume output_volume.append(np.max(inputs)) # reshape arrays into volumes (both saved as instance variables for backprop) self.output_volume = volume(np.reshape(output_volume, (self.out_depth, self.out_height, self.out_width))) self.max_row_positions = np.reshape(max_row_positions, (self.out_depth, self.out_height, self.out_width)) self.max_col_positions = np.reshape(max_col_positions, (self.out_depth, self.out_height, self.out_width)) return self.output_volume
def forward(self, input_volume): return volume(self.tanh_volume(input_volume.volume_slices))
def forward(self, input_volume): # save volumes for backprop self.input_volume = input_volume self.output_volume = volume(self.activate_volume(input_volume.volume_slices)) # print self.output_volume.volume_slices return self.output_volume
def forward(self, input_volume): # return input_volume return volume(self.normalize_volume(input_volume.volume_slices))
