def generate(self, images_type='both', batch_size=10): """ This is the main function of this class. :param batch_size: :param images_type: :return: """ images_type = images_type.lower() allowed_types = ['both', 'length', 'spacing'] if images_type not in allowed_types: raise Exception("Invalid Image Type") if images_type == 'both': n_c_len_images = batch_size >> 1 n_c_spacing_images = batch_size - n_c_len_images elif images_type == 'length': n_c_len_images = batch_size n_c_spacing_images = 0 else: n_c_len_images = 0 n_c_spacing_images = batch_size print( "Generated batches will have %d contour length and %d contour spacing images" % (n_c_len_images, n_c_spacing_images)) while True: image_arr = [] label_arr = [] if images_type is not 'spacing': for i in range(n_c_len_images): # select_idx = i select_idx = np.random.randint(0, len(self.c_len_arr)) test_image = np.zeros((227, 227, 3)) test_image = alex_net_utils.tile_image( test_image, self.frag, self.bg_tile_loc[0], rotate=True, gaussian_smoothing=True) test_image = alex_net_utils.tile_image( test_image, self.frag, self.c_len_cont_tile_loc[select_idx], rotate=False, gaussian_smoothing=True, ) # Image preprocessing test_image = test_image / 255.0 # Bring test_image back to the [0, 1] range. test_image = np.transpose( test_image, (2, 0, 1)) # Theano back-end expects channel first format image_arr.append(test_image) label_arr.append(self.c_len_expected_gains[select_idx]) if images_type is not 'length': for i in range(n_c_spacing_images): # select_idx = i select_idx = np.random.randint(0, len(self.c_spacing_arr)) test_image = np.zeros((227, 227, 3)) test_image = alex_net_utils.tile_image( test_image, self.frag, self.bg_tile_loc[select_idx], rotate=True, gaussian_smoothing=True) test_image = alex_net_utils.tile_image( test_image, self.frag, self.c_spacing_cont_tile_loc[select_idx], rotate=False, gaussian_smoothing=True, ) # Image preprocessing test_image = test_image / 255.0 # Bring test_image back to the [0, 1] range. test_image = np.transpose( test_image, (2, 0, 1)) # Theano back-end expects channel first format image_arr.append(test_image) label_arr.append(self.c_spacing_expected_gains[select_idx]) image_arr = np.stack(image_arr, axis=0) label_arr = np.reshape(np.array(label_arr), (len(label_arr), 1)) yield image_arr, label_arr
def optimize_contour_enhancement_layer_weights(model, tgt_filt_idx, frag, contour_generator_cb, n_runs, learning_rate=0.00025, offset=0, optimize_type='both', axis=None): """ Optimize the l2 kernel (contour integration) weights corresponding to the specified target L1 kernel. A loss function is defined that compares the model l2 gain (L2 activation /L1 activation) and compares with the expected contour integration gain from Li-2006. :param model: contour integration model :param tgt_filt_idx: :param frag: :param contour_generator_cb: :param n_runs: Number of loops to iterate over. If n_runs is < 5, input images for each run are shown. :param learning_rate: THe learning rate (the size of the step in the gradient direction) :param offset: the pixel offset by which each row should be shifted by as it moves away from the center row. Used by diagonal contour optimization. :param optimize_type: Optimize over [length, spacing, or both(Default)] :param axis: Figure axis on which the loss function over time should be plotted. If None, a new figure is created. Default = None. :return: None """ # Validate input parameters valid_optimize_type = ['length', 'spacing', 'both'] if optimize_type.lower() not in valid_optimize_type: raise Exception( "Invalid optimization type specified. Valid = [length, spacing, or both(Default)" ) optimize_type = optimize_type.lower() # Some Initialization tgt_n_loc = 27 # neuron looking @ center of RF tgt_n_visual_rf_start = tgt_n_loc * 4 # stride length of L1 conv Layer # 1. Extract the neural data to match # ----------------------------------- with open('.//neuro_data//Li2006.pickle', 'rb') as handle: data = pickle.load(handle) expected_gains = [] contour_len_arr = [] relative_colinear_dist_arr = [] if optimize_type == 'length' or optimize_type == 'both': expected_gains = np.append(expected_gains, data['contour_len_avg_gain']) contour_len_arr = data["contour_len_avg_len"] if optimize_type == 'spacing' or optimize_type == 'both': expected_gains = np.append(expected_gains, data['contour_separation_avg_gain']) relative_colinear_dist_arr = np.array( data["contour_separation_avg_rcd"]) # 2. Setup the optimization problem # ------------------------------------------------------ l1_output_cb = model.layers[1].output l2_output_cb = model.layers[2].output input_cb = model.input # Mean Square Error Loss current_gains = l2_output_cb[:, tgt_filt_idx, tgt_n_loc, tgt_n_loc] / \ (l1_output_cb[:, tgt_filt_idx, tgt_n_loc, tgt_n_loc] + 1e-8) loss = (expected_gains - current_gains)**2 / expected_gains.shape[0] # Callbacks for the weights (learnable parameters) w_cb = model.layers[2].raw_kernel b_cb = model.layers[2].bias # Gradients of weights and bias wrt to the loss function grads = K.gradients(loss, [w_cb, b_cb]) grads = [ gradient / (K.sqrt(K.mean(K.square(gradient))) + 1e-8) for gradient in grads ] iterate = K.function( [input_cb], [loss, grads[0], grads[1], l1_output_cb, l2_output_cb]) # 3. Initializations # ------------------- smooth_edges = True frag_len = frag.shape[0] contour_spacing_contour_len = 7 # Reference uses length 7 for relative spacing part # 4. Main Loop # --------------------------- old_loss = 10000000 losses = [] # ADAM Optimization starting parameters m_w = 0 v_w = 0 m_b = 0 v_b = 0 for run_idx in range(n_runs): # Create test set of images (new set for each run) # ------------------------------------------------ images = [] # Contour Lengths if optimize_type == 'length' or optimize_type == 'both': for c_len in contour_len_arr: test_image = np.zeros((227, 227, 3)) test_image_len = test_image.shape[0] # Background Tiles start_x, start_y = alex_net_utils.get_background_tiles_locations( frag_len, test_image_len, offset, 0, tgt_n_visual_rf_start) test_image = alex_net_utils.tile_image( test_image, frag, (start_x, start_y), rotate=True, gaussian_smoothing=smooth_edges) # Place contour in image start_x, start_y = contour_generator_cb( frag_len, bw_tile_spacing=0, cont_len=c_len, cont_start_loc=tgt_n_visual_rf_start, row_offset=offset) test_image = alex_net_utils.tile_image( test_image, frag, (start_x, start_y), rotate=False, gaussian_smoothing=smooth_edges) # Image preprocessing # ------------------- # # zero_mean and 1 standard deviation # test_image -= np.mean(test_image, axis=0) # test_image /= np.std(test_image, axis=0) # Normalize pixels to be in the range [0, 1] test_image = test_image / 255.0 # Theano back-end expects channel first format test_image = np.transpose(test_image, (2, 0, 1)) images.append(test_image) # Contour Spacing if optimize_type == 'spacing' or optimize_type == 'both': spacing_bw_tiles_arr = np.floor( relative_colinear_dist_arr * frag_len) - frag_len for spacing in spacing_bw_tiles_arr: spacing = int(spacing) test_image = np.zeros((227, 227, 3)) test_image_len = test_image.shape[0] # Background Tiles start_x, start_y = alex_net_utils.get_background_tiles_locations( frag_len, test_image_len, offset, spacing, tgt_n_visual_rf_start) test_image = alex_net_utils.tile_image( test_image, frag, (start_x, start_y), rotate=True, gaussian_smoothing=smooth_edges) # Place contour in image start_x, start_y = contour_generator_cb( frag_len, bw_tile_spacing=spacing, cont_len=contour_spacing_contour_len, cont_start_loc=tgt_n_visual_rf_start, row_offset=offset) test_image = alex_net_utils.tile_image( test_image, frag, (start_x, start_y), rotate=False, gaussian_smoothing=smooth_edges) # Image preprocessing test_image = test_image / 255.0 # Bring test_image back to the [0, 1] range. test_image = np.transpose( test_image, (2, 0, 1)) # Theano back-end expects channel first format images.append(test_image) images = np.stack(images, axis=0) # Plot the generated images # ------------------------- if n_runs <= 5: f = plt.figure() num_images_per_row = 5 if optimize_type == 'both': num_rows = 2 else: num_rows = 1 for img_idx, img in enumerate(images): display_img = np.transpose(img, (1, 2, 0)) f.add_subplot(num_rows, num_images_per_row, img_idx + 1) plt.imshow(display_img) # Now iterate loss_value, grad_w, grad_b, l1_out, l2_out = iterate([images]) print("%d: loss %s" % (run_idx, loss_value.mean())) w, b = model.layers[2].get_weights() if loss_value.mean() > old_loss: # step /= 2.0 # print("Lowering step value to %f" % step) pass else: m_w = 0.9 * m_w + (1 - 0.9) * grad_w v_w = 0.999 * v_w + (1 - 0.999) * grad_w**2 new_w = w - learning_rate * m_w / (np.sqrt(v_w) + 1e-8) m_b = 0.9 * m_b + (1 - 0.9) * grad_b v_b = 0.999 * v_b + (1 - 0.999) * grad_b**2 new_b = b - learning_rate * m_b / (np.sqrt(v_b) + 1e-8) # Print Contour Enhancement Gains print("Contour Enhancement Gain %s" % (l2_out[:, tgt_filt_idx, tgt_n_loc, tgt_n_loc] / l1_out[:, tgt_filt_idx, tgt_n_loc, tgt_n_loc])) model.layers[2].set_weights([new_w, new_b]) old_loss = loss_value.mean() losses.append(loss_value.mean()) # At the end of simulation plot loss vs iteration if axis is None: f, axis = plt.subplots() axis.plot(range(n_runs), losses, label='learning rate = %0.8f' % learning_rate) font_size = 20 axis.set_xlabel("Iteration", fontsize=font_size) axis.set_ylabel("Loss", fontsize=font_size) axis.tick_params(axis='x', labelsize=font_size) axis.tick_params(axis='y', labelsize=font_size)
# its unmasked neighbors # ------------------------------------------------------------------------- # The test image test_image = np.ones((227, 227, 3)) * 128 start_x_arr, start_y_arr = alex_net_utils.diagonal_contour_generator( tgt_filter_len, offset, fragment_spacing, 9, target_neuron_rf_start, ) test_image = alex_net_utils.tile_image(test_image, fragment, (start_x_arr, start_y_arr), rotate=False, gaussian_smoothing=False) # Plot the image test_image = test_image / 255.0 plt.figure() plt.imshow(test_image) plt.title("Input image") l2_masks = K.eval(contour_integration_model.layers[2].mask) tgt_l2_masks = l2_masks[tgt_feat_extract_kernel_idx, :, :] alex_net_utils.plot_l2_visual_field(tgt_neuron_loc, tgt_l2_masks, test_image)
def get_contour_responses(l1_act_cb, l2_act_cb, tgt_filt_idx, frag, contour_len, cont_gen_cb, space_bw_tiles=0, offset=0, smooth_tiles=True): """ Creates a test image of a sea of "contour fragments" by tiling random rotations of the contour fragment (frag), then inserts a vertical contour of the specified length into the center of the image Plot the l1 & l2 activations of the contour integration alexnet model. :param l1_act_cb: Callback function to get activations of L1 contour integration layer :param l2_act_cb: Callback function to get activations of L2 contour integration layer :param tgt_filt_idx: target neuron activation :param frag: Contour fragment (square) to use for creating the larger tiled image :param contour_len: length of contour to generate :param cont_gen_cb: Contour generating callback. Generates contours of the specified length and spacing between fragments. For format see vertical_contour_generator :param space_bw_tiles: Amount of spacing (in pixels) between inserted tiles :param offset: the offset by which a tile row should be shifted by as it moves away from center row :param smooth_tiles: Use gaussian smoothing on tiles to prevent tile edges from becoming part of the stimulus :return: l1 activation, l2 activation (of target filter) & generated image """ test_image = np.zeros((227, 227, 3)) test_image_len = test_image.shape[0] frag_len = frag.shape[0] # Output dimensions of the first convolutional layer of Alexnet [55x55x96] and a stride=4 was used center_neuron_loc = 108 # RF size of 11 at center of l1 activation # Sea of similar but randomly oriented fragments # ----------------------------------------------- # Rather than tiling starting from location (0, 0). Start at the center of the RF of a central neuron. # This ensures that this central neuron is looking directly at the fragment and has a maximal response. # This is similar to the Ref, where the center contour fragment was in the center of the RF of the neuron # being monitored and the origin of the visual field start_x, start_y = alex_net_utils.get_background_tiles_locations( frag_len, test_image_len, offset, space_bw_tiles, center_neuron_loc) test_image = alex_net_utils.tile_image(test_image, frag, (start_x, start_y), rotate=True, gaussian_smoothing=smooth_tiles) # Insert Contour # -------------- cont_coordinates = cont_gen_cb(frag_len, bw_tile_spacing=space_bw_tiles, cont_len=contour_len, cont_start_loc=center_neuron_loc, row_offset=offset) test_image = alex_net_utils.tile_image(test_image, frag, cont_coordinates, rotate=False, gaussian_smoothing=smooth_tiles) # Bring it back to the [0, 1] range (rotation fcn scales pixels to [0, 255]) test_image = test_image / 255.0 # Get the activations of the first convolutional and second contour integration layer # ----------------------------------------------------------------------------------- test_image = np.transpose( test_image, (2, 0, 1)) # Theano back-end expects channel first format test_image = np.reshape( test_image, [1, test_image.shape[0], test_image.shape[1], test_image.shape[2]]) # batch size is expected as the first dimension l1_act = l1_act_cb([test_image, 0]) l1_act = np.squeeze(np.array(l1_act), axis=0) l2_act = l2_act_cb([test_image, 0]) l2_act = np.squeeze(np.array(l2_act), axis=0) tgt_l1_act = l1_act[0, tgt_filt_idx, :, :] tgt_l2_act = l2_act[0, tgt_filt_idx, :, :] # return the test image back to its original format test_image = test_image[0, :, :, :] test_image = np.transpose(test_image, (1, 2, 0)) return tgt_l1_act, tgt_l2_act, test_image