def generate(self, images_type='both', batch_size=10):
"""
This is the main function of this class.
:param batch_size:
:param images_type:
:return:
"""
images_type = images_type.lower()
allowed_types = ['both', 'length', 'spacing']
if images_type not in allowed_types:
raise Exception("Invalid Image Type")
if images_type == 'both':
n_c_len_images = batch_size >> 1
n_c_spacing_images = batch_size - n_c_len_images
elif images_type == 'length':
n_c_len_images = batch_size
n_c_spacing_images = 0
else:
n_c_len_images = 0
n_c_spacing_images = batch_size
print(
"Generated batches will have %d contour length and %d contour spacing images"
% (n_c_len_images, n_c_spacing_images))
while True:
image_arr = []
label_arr = []
if images_type is not 'spacing':
for i in range(n_c_len_images):
# select_idx = i
select_idx = np.random.randint(0, len(self.c_len_arr))
test_image = np.zeros((227, 227, 3))
test_image = alex_net_utils.tile_image(
test_image,
self.frag,
self.bg_tile_loc[0],
rotate=True,
gaussian_smoothing=True)
test_image = alex_net_utils.tile_image(
test_image,
self.frag,
self.c_len_cont_tile_loc[select_idx],
rotate=False,
gaussian_smoothing=True,
)
# Image preprocessing
test_image = test_image / 255.0 # Bring test_image back to the [0, 1] range.
test_image = np.transpose(
test_image,
(2, 0,
1)) # Theano back-end expects channel first format
image_arr.append(test_image)
label_arr.append(self.c_len_expected_gains[select_idx])
if images_type is not 'length':
for i in range(n_c_spacing_images):
# select_idx = i
select_idx = np.random.randint(0, len(self.c_spacing_arr))
test_image = np.zeros((227, 227, 3))
test_image = alex_net_utils.tile_image(
test_image,
self.frag,
self.bg_tile_loc[select_idx],
rotate=True,
gaussian_smoothing=True)
test_image = alex_net_utils.tile_image(
test_image,
self.frag,
self.c_spacing_cont_tile_loc[select_idx],
rotate=False,
gaussian_smoothing=True,
)
# Image preprocessing
test_image = test_image / 255.0 # Bring test_image back to the [0, 1] range.
test_image = np.transpose(
test_image,
(2, 0,
1)) # Theano back-end expects channel first format
image_arr.append(test_image)
label_arr.append(self.c_spacing_expected_gains[select_idx])
image_arr = np.stack(image_arr, axis=0)
label_arr = np.reshape(np.array(label_arr), (len(label_arr), 1))
yield image_arr, label_arr
def optimize_contour_enhancement_layer_weights(model,
tgt_filt_idx,
frag,
contour_generator_cb,
n_runs,
learning_rate=0.00025,
offset=0,
optimize_type='both',
axis=None):
"""
Optimize the l2 kernel (contour integration) weights corresponding to the specified target L1 kernel.
A loss function is defined that compares the model l2 gain (L2 activation /L1 activation) and compares with the
expected contour integration gain from Li-2006.
:param model: contour integration model
:param tgt_filt_idx:
:param frag:
:param contour_generator_cb:
:param n_runs: Number of loops to iterate over. If n_runs is < 5, input images for each run are shown.
:param learning_rate: THe learning rate (the size of the step in the gradient direction)
:param offset: the pixel offset by which each row should be shifted by as it moves away from the center row.
Used by diagonal contour optimization.
:param optimize_type: Optimize over [length, spacing, or both(Default)]
:param axis: Figure axis on which the loss function over time should be plotted. If None, a new figure
is created. Default = None.
:return: None
"""
# Validate input parameters
valid_optimize_type = ['length', 'spacing', 'both']
if optimize_type.lower() not in valid_optimize_type:
raise Exception(
"Invalid optimization type specified. Valid = [length, spacing, or both(Default)"
)
optimize_type = optimize_type.lower()
# Some Initialization
tgt_n_loc = 27 # neuron looking @ center of RF
tgt_n_visual_rf_start = tgt_n_loc * 4 # stride length of L1 conv Layer
# 1. Extract the neural data to match
# -----------------------------------
with open('.//neuro_data//Li2006.pickle', 'rb') as handle:
data = pickle.load(handle)
expected_gains = []
contour_len_arr = []
relative_colinear_dist_arr = []
if optimize_type == 'length' or optimize_type == 'both':
expected_gains = np.append(expected_gains,
data['contour_len_avg_gain'])
contour_len_arr = data["contour_len_avg_len"]
if optimize_type == 'spacing' or optimize_type == 'both':
expected_gains = np.append(expected_gains,
data['contour_separation_avg_gain'])
relative_colinear_dist_arr = np.array(
data["contour_separation_avg_rcd"])
# 2. Setup the optimization problem
# ------------------------------------------------------
l1_output_cb = model.layers[1].output
l2_output_cb = model.layers[2].output
input_cb = model.input
# Mean Square Error Loss
current_gains = l2_output_cb[:, tgt_filt_idx, tgt_n_loc, tgt_n_loc] / \
(l1_output_cb[:, tgt_filt_idx, tgt_n_loc, tgt_n_loc] + 1e-8)
loss = (expected_gains - current_gains)**2 / expected_gains.shape[0]
# Callbacks for the weights (learnable parameters)
w_cb = model.layers[2].raw_kernel
b_cb = model.layers[2].bias
# Gradients of weights and bias wrt to the loss function
grads = K.gradients(loss, [w_cb, b_cb])
grads = [
gradient / (K.sqrt(K.mean(K.square(gradient))) + 1e-8)
for gradient in grads
]
iterate = K.function(
[input_cb], [loss, grads[0], grads[1], l1_output_cb, l2_output_cb])
# 3. Initializations
# -------------------
smooth_edges = True
frag_len = frag.shape[0]
contour_spacing_contour_len = 7 # Reference uses length 7 for relative spacing part
# 4. Main Loop
# ---------------------------
old_loss = 10000000
losses = []
# ADAM Optimization starting parameters
m_w = 0
v_w = 0
m_b = 0
v_b = 0
for run_idx in range(n_runs):
# Create test set of images (new set for each run)
# ------------------------------------------------
images = []
# Contour Lengths
if optimize_type == 'length' or optimize_type == 'both':
for c_len in contour_len_arr:
test_image = np.zeros((227, 227, 3))
test_image_len = test_image.shape[0]
# Background Tiles
start_x, start_y = alex_net_utils.get_background_tiles_locations(
frag_len, test_image_len, offset, 0, tgt_n_visual_rf_start)
test_image = alex_net_utils.tile_image(
test_image,
frag, (start_x, start_y),
rotate=True,
gaussian_smoothing=smooth_edges)
# Place contour in image
start_x, start_y = contour_generator_cb(
frag_len,
bw_tile_spacing=0,
cont_len=c_len,
cont_start_loc=tgt_n_visual_rf_start,
row_offset=offset)
test_image = alex_net_utils.tile_image(
test_image,
frag, (start_x, start_y),
rotate=False,
gaussian_smoothing=smooth_edges)
# Image preprocessing
# -------------------
# # zero_mean and 1 standard deviation
# test_image -= np.mean(test_image, axis=0)
# test_image /= np.std(test_image, axis=0)
# Normalize pixels to be in the range [0, 1]
test_image = test_image / 255.0
# Theano back-end expects channel first format
test_image = np.transpose(test_image, (2, 0, 1))
images.append(test_image)
# Contour Spacing
if optimize_type == 'spacing' or optimize_type == 'both':
spacing_bw_tiles_arr = np.floor(
relative_colinear_dist_arr * frag_len) - frag_len
for spacing in spacing_bw_tiles_arr:
spacing = int(spacing)
test_image = np.zeros((227, 227, 3))
test_image_len = test_image.shape[0]
# Background Tiles
start_x, start_y = alex_net_utils.get_background_tiles_locations(
frag_len, test_image_len, offset, spacing,
tgt_n_visual_rf_start)
test_image = alex_net_utils.tile_image(
test_image,
frag, (start_x, start_y),
rotate=True,
gaussian_smoothing=smooth_edges)
# Place contour in image
start_x, start_y = contour_generator_cb(
frag_len,
bw_tile_spacing=spacing,
cont_len=contour_spacing_contour_len,
cont_start_loc=tgt_n_visual_rf_start,
row_offset=offset)
test_image = alex_net_utils.tile_image(
test_image,
frag, (start_x, start_y),
rotate=False,
gaussian_smoothing=smooth_edges)
# Image preprocessing
test_image = test_image / 255.0 # Bring test_image back to the [0, 1] range.
test_image = np.transpose(
test_image,
(2, 0, 1)) # Theano back-end expects channel first format
images.append(test_image)
images = np.stack(images, axis=0)
# Plot the generated images
# -------------------------
if n_runs <= 5:
f = plt.figure()
num_images_per_row = 5
if optimize_type == 'both':
num_rows = 2
else:
num_rows = 1
for img_idx, img in enumerate(images):
display_img = np.transpose(img, (1, 2, 0))
f.add_subplot(num_rows, num_images_per_row, img_idx + 1)
plt.imshow(display_img)
# Now iterate
loss_value, grad_w, grad_b, l1_out, l2_out = iterate([images])
print("%d: loss %s" % (run_idx, loss_value.mean()))
w, b = model.layers[2].get_weights()
if loss_value.mean() > old_loss:
# step /= 2.0
# print("Lowering step value to %f" % step)
pass
else:
m_w = 0.9 * m_w + (1 - 0.9) * grad_w
v_w = 0.999 * v_w + (1 - 0.999) * grad_w**2
new_w = w - learning_rate * m_w / (np.sqrt(v_w) + 1e-8)
m_b = 0.9 * m_b + (1 - 0.9) * grad_b
v_b = 0.999 * v_b + (1 - 0.999) * grad_b**2
new_b = b - learning_rate * m_b / (np.sqrt(v_b) + 1e-8)
# Print Contour Enhancement Gains
print("Contour Enhancement Gain %s" %
(l2_out[:, tgt_filt_idx, tgt_n_loc, tgt_n_loc] /
l1_out[:, tgt_filt_idx, tgt_n_loc, tgt_n_loc]))
model.layers[2].set_weights([new_w, new_b])
old_loss = loss_value.mean()
losses.append(loss_value.mean())
# At the end of simulation plot loss vs iteration
if axis is None:
f, axis = plt.subplots()
axis.plot(range(n_runs),
losses,
label='learning rate = %0.8f' % learning_rate)
font_size = 20
axis.set_xlabel("Iteration", fontsize=font_size)
axis.set_ylabel("Loss", fontsize=font_size)
axis.tick_params(axis='x', labelsize=font_size)
axis.tick_params(axis='y', labelsize=font_size)
# its unmasked neighbors
# -------------------------------------------------------------------------
# The test image
test_image = np.ones((227, 227, 3)) * 128
start_x_arr, start_y_arr = alex_net_utils.diagonal_contour_generator(
tgt_filter_len,
offset,
fragment_spacing,
9,
target_neuron_rf_start,
)
test_image = alex_net_utils.tile_image(test_image,
fragment,
(start_x_arr, start_y_arr),
rotate=False,
gaussian_smoothing=False)
# Plot the image
test_image = test_image / 255.0
plt.figure()
plt.imshow(test_image)
plt.title("Input image")
l2_masks = K.eval(contour_integration_model.layers[2].mask)
tgt_l2_masks = l2_masks[tgt_feat_extract_kernel_idx, :, :]
alex_net_utils.plot_l2_visual_field(tgt_neuron_loc, tgt_l2_masks,
test_image)
def get_contour_responses(l1_act_cb,
l2_act_cb,
tgt_filt_idx,
frag,
contour_len,
cont_gen_cb,
space_bw_tiles=0,
offset=0,
smooth_tiles=True):
"""
Creates a test image of a sea of "contour fragments" by tiling random rotations of the contour
fragment (frag), then inserts a vertical contour of the specified length into the center of
the image Plot the l1 & l2 activations of the contour integration alexnet model.
:param l1_act_cb: Callback function to get activations of L1 contour integration layer
:param l2_act_cb: Callback function to get activations of L2 contour integration layer
:param tgt_filt_idx: target neuron activation
:param frag: Contour fragment (square) to use for creating the larger tiled image
:param contour_len: length of contour to generate
:param cont_gen_cb: Contour generating callback. Generates contours of the specified length
and spacing between fragments. For format see vertical_contour_generator
:param space_bw_tiles: Amount of spacing (in pixels) between inserted tiles
:param offset: the offset by which a tile row should be shifted by as it moves away from center row
:param smooth_tiles: Use gaussian smoothing on tiles to prevent tile edges from becoming part
of the stimulus
:return: l1 activation, l2 activation (of target filter) & generated image
"""
test_image = np.zeros((227, 227, 3))
test_image_len = test_image.shape[0]
frag_len = frag.shape[0]
# Output dimensions of the first convolutional layer of Alexnet [55x55x96] and a stride=4 was used
center_neuron_loc = 108 # RF size of 11 at center of l1 activation
# Sea of similar but randomly oriented fragments
# -----------------------------------------------
# Rather than tiling starting from location (0, 0). Start at the center of the RF of a central neuron.
# This ensures that this central neuron is looking directly at the fragment and has a maximal response.
# This is similar to the Ref, where the center contour fragment was in the center of the RF of the neuron
# being monitored and the origin of the visual field
start_x, start_y = alex_net_utils.get_background_tiles_locations(
frag_len, test_image_len, offset, space_bw_tiles, center_neuron_loc)
test_image = alex_net_utils.tile_image(test_image,
frag, (start_x, start_y),
rotate=True,
gaussian_smoothing=smooth_tiles)
# Insert Contour
# --------------
cont_coordinates = cont_gen_cb(frag_len,
bw_tile_spacing=space_bw_tiles,
cont_len=contour_len,
cont_start_loc=center_neuron_loc,
row_offset=offset)
test_image = alex_net_utils.tile_image(test_image,
frag,
cont_coordinates,
rotate=False,
gaussian_smoothing=smooth_tiles)
# Bring it back to the [0, 1] range (rotation fcn scales pixels to [0, 255])
test_image = test_image / 255.0
# Get the activations of the first convolutional and second contour integration layer
# -----------------------------------------------------------------------------------
test_image = np.transpose(
test_image, (2, 0, 1)) # Theano back-end expects channel first format
test_image = np.reshape(
test_image,
[1, test_image.shape[0], test_image.shape[1], test_image.shape[2]])
# batch size is expected as the first dimension
l1_act = l1_act_cb([test_image, 0])
l1_act = np.squeeze(np.array(l1_act), axis=0)
l2_act = l2_act_cb([test_image, 0])
l2_act = np.squeeze(np.array(l2_act), axis=0)
tgt_l1_act = l1_act[0, tgt_filt_idx, :, :]
tgt_l2_act = l2_act[0, tgt_filt_idx, :, :]
# return the test image back to its original format
test_image = test_image[0, :, :, :]
test_image = np.transpose(test_image, (1, 2, 0))
return tgt_l1_act, tgt_l2_act, test_image