def fit_dimensionality_reduction(model, data):
"""
Helper function to fit the model dimensionality reduction set in the config
:param data: input data
:return:
"""
# in the case of dimensionality reduction set up the pipeline
if model.dim_red is None:
print("[FIT] no dimensionality reduction")
preds = data
elif model.dim_red == 'PCA':
# perform PCA on this output
print("[FIT] Fitting PCA")
model.pca.fit(data)
print("[FIT] PCA: explained variance",
model.pca.explained_variance_ratio_)
preds = model.pca.transform(data)
elif model.dim_red == "semantic" or model.dim_red == "semantic-pattern" or model.dim_red == "pattern":
preds = model.feat_red.fit(data, activation='mean')
# apply filter post_processing
# todo modify this, I'm not really happy with this yet
preds = get_feat_map_filt_preds(
preds,
ref_type="self0",
norm=model.config['feat_map_filt_norm'],
activation=model.config['feat_map_filt_activation'],
filter=model.config['feat_map_filter_type'])
# allow to further reduce dimensionality by getting a 2 dim vector for each feature maps
if model.config['feat_map_position_mode'] != 'raw':
print("[FIT] Using position mode: {}".format(
model.config['feat_map_position_mode']))
preds = calculate_position(
preds,
mode=model.config['feat_map_position_mode'],
return_mode='xy float')
preds = np.reshape(preds, (len(preds), -1))
print("[FIT] Finished to find the semantic units")
else:
raise KeyError(f'model.dim_red={model.dim_red} is not a valid value')
return preds
def predict_dimensionality_reduction(model, data):
"""
Helper function to apply the dimensionality reduction as set in the configuation before the model
:param model: model instance
:param data:
:return: prediction
"""
if model.dim_red is None:
# get prediction after cnn, before dimensionality reduction
preds = data
elif model.dim_red == 'PCA':
# projection by PCA
preds = model.pca.transform(data)
elif model.dim_red == "semantic" or model.dim_red == "semantic-pattern" or model.dim_red == "pattern":
preds = model.feat_red.transform(data, activation='mean')
# apply filter post_processing
# todo modify this, I'm not really happy with this yet
preds = get_feat_map_filt_preds(
preds,
ref_type="self0",
norm=model.config['feat_map_filt_norm'],
activation=model.config['feat_map_filt_activation'],
filter=model.config['feat_map_filter_type'])
# allow to further reduce dimensionality by getting a 2 dim vector for each feature maps
if model.config['feat_map_position_mode'] != 'raw':
print("[PREDS] Using position mode: {}".format(
model.config['feat_map_position_mode']))
preds = calculate_position(
preds,
mode=model.config['feat_map_position_mode'],
return_mode='xy float')
preds = np.reshape(preds, (len(preds), -1))
else:
raise KeyError(f'invalid value self.dim_red={model.dim_red}')
return preds
config,
activation='relu')
# lips
dyn_lips_preds = ref_feature_map_neuron(lips_preds,
data[1],
config,
activation='relu')
# concatenate feature maps concepts
dyn_preds = np.concatenate([dyn_eyebrow_preds, dyn_lips_preds], axis=3)
print("[TRAIN] shape dyn preds", np.shape(dyn_preds))
print("[TRAIN] finished computing dynamic predictions")
# compute positions eyebrow
dyn_pos = calculate_position(dyn_preds,
mode="weighted average",
return_mode="xy float flat")
print("[TRAIN] shape dyn_pos", np.shape(dyn_pos))
# add weights
pos_weights = np.ones(np.shape(dyn_pos)[-1])
pos_weights[:8] = 1 / (2 * 2 * len(best_eyebrow_IoU_ft)
) # num_concept * x/y * num_ft/concept
pos_weights[8:] = 1 / (2 * 2 * len(best_lips_IoU_ft))
dyn_pos = np.multiply(dyn_pos, pos_weights)
nb_model.n_features = np.shape(dyn_pos)[-1] # todo add this to init
# train manually ref vector
nb_model.r = np.zeros(nb_model.n_features)
nb_model._fit_reference([dyn_pos, data[1]], config['batch_size'])
# train manually tuning vector
# for lips, create four mouth zones
left_lips = np.zeros(np.shape(max_lips_preds))
left_lips[:, 33:45, 19:26] = max_lips_preds[:, 33:45, 19:26]
middle_up_lips = np.zeros(np.shape(max_lips_preds))
middle_up_lips[:, 33:37, 24:34] = max_lips_preds[:, 33:37, 24:34]
middle_down_lips = np.zeros(np.shape(max_lips_preds))
middle_down_lips[:, 38:48, 24:33] = max_lips_preds[:, 38:48, 24:33]
right_lips = np.zeros(np.shape(max_lips_preds))
right_lips[:, 33:45, 32:37] = max_lips_preds[:, 33:45, 32:37]
preds = np.concatenate([left_ext_eyebrow, left_int_eyebrow, right_int_eyebrow, right_ext_eyebrow,
middle_up_lips, middle_down_lips, left_lips, right_lips], axis=3)
print("[TRAIN] preds", np.shape(preds))
# compute positions eyebrow
pos = calculate_position(preds, mode="weighted average", return_mode="xy float flat")
print("[TRAIN] shape pos", np.shape(pos))
nb_model.n_features = np.shape(pos)[-1] # todo add this to init
# train manually ref vector
nb_model.r = np.zeros(nb_model.n_features)
nb_model._fit_reference([pos, data[1]], config['batch_size'])
print("[TRAIN] model.r", np.shape(nb_model.r))
print(nb_model.r)
ref_train = np.copy(nb_model.r)
# train manually tuning vector
nb_model.t = np.zeros((nb_model.n_category, nb_model.n_features))
nb_model.t_mean = np.zeros((nb_model.n_category, nb_model.n_features))
nb_model._fit_tuning([pos, data[1]], config['batch_size'])
ref_tuning = np.copy(nb_model.t)
print("[TRAIN] ref_tuning[1]")
elif 'stride' in config["difference_option"]:
stride_length = int(config["difference_option"][6:])
reference = np.roll(response, shift=stride_length, axis=0)
reference[:stride_length, ...] = response[:stride_length,...]
elif config["difference_option"] == "reference":
# calculate and retrieve reference vector from norm_base
raise KeyError('reference not implemented yet')
else:
raise KeyError(f'config["difference_option"]={config["difference_option"]} is no valid option')
vector_plot = response - reference
>>>>>>> 11da6a3... restructured and started analysis of stimuli noise rendering
else:
try:
# config["resp_type"]: "maximum", "maximum{int}", "weighted average"
vector_plot = calculate_position(response=norm_base.evaluate_v4(data, flatten=False),
mode=resp_type,
return_mode="array")
except KeyError:
raise KeyError(f'config["plot_option"]={resp_type} is no valid key')
# calculate which feature maps to highlight
try:
if config["highlight_option"] == "maximum":
max_map = np.amax(np.abs(vector_plot), axis=(0,1,2)) # max over frames, and within feature map
highlight = np.flip(np.argsort(max_map)[-25:])
else:
raise KeyError(f'config["highlight_option"]={config["highlight_option"]} is no valid option')
except KeyError:
highlight = None
<<<<<<< HEAD
def predict_expression(expression, ft_idx, test_num=None):
# set config
config["train_expression"] = [expression]
# -----------------------------------------------------------------------------------------------------------------
# human avatar
config["train_avatar"] = "human_orig"
# load data
data = load_data(config)[0]
# predict and filter responses
print("[PE] Human {} loaded".format(expression))
print("[PE] shape data", np.shape(data))
print("[PE] Start predictions")
preds_human = model.predict(data)
preds_human = get_feat_map_filt_preds(preds_human,
ft_idx,
ref_type="self0",
norm=1000,
activation='ReLu',
filter='spatial_mean',
verbose=True)
print("[PE] shape preds Human", np.shape(preds_human))
print("[PE] min max preds Human", np.amin(preds_human),
np.amax(preds_human))
# get xy positions
preds_human_pos = calculate_position(preds_human,
mode="weighted average",
return_mode="xy float")
print("[PE] shape preds Human positions", np.shape(preds_human_pos))
# # filter feat maps
# preds_human_cleaned = filter_feature_maps(preds_human)
# preds_human_pos_cleaned = calculate_position(preds_human_cleaned, mode="weighted average", return_mode="xy float")
# -----------------------------------------------------------------------------------------------------------------
# Monkey Avatar
config["train_avatar"] = "monkey_orig"
# load data
data = load_data(config)[0]
print("[PE] Monkey {} loaded".format(expression))
print("[PE] shape data", np.shape(data))
print("[PE] Start predictions")
preds_monkey = model.predict(data)
preds_monkey = get_feat_map_filt_preds(preds_monkey,
ft_idx,
ref_type="self0",
norm=1000,
activation='ReLu',
filter='spatial_mean',
verbose=True)
print("[PE] shape preds Human", np.shape(preds_monkey))
print("[PE] min max preds Human", np.amin(preds_monkey),
np.amax(preds_monkey))
# get xy positions
preds_monkey_pos = calculate_position(preds_monkey,
mode="weighted average",
return_mode="xy float")
print("[PE] shape preds Monkey positions", np.shape(preds_monkey_pos))
# # filter feat maps
# preds_monkey_cleaned = filter_feature_maps(preds_monkey)
# preds_monkey_pos_cleaned = calculate_position(preds_monkey_cleaned, mode="weighted average", return_mode="xy float")
# -----------------------------------------------------------------------------------------------------------------
# plot raw feature maps
print("[PE] Create plot")
# plot human
cnn_output_name = 'test_human_cnn_output.gif'
if test_num is not None:
cnn_output_name = 'test{}_human_cnn_output.gif'.format(test_num)
plot_cnn_output(preds_human, save_path, cnn_output_name, video=True)
# # plot human filtered feature maps
# cnn_output_name = 'test_human_filtered_cnn_output.gif'
# if test_num is not None:
# cnn_output_name = 'test{}_human_filtered_cnn_output.gif'.format(test_num)
#
# plot_cnn_output(preds_human_cleaned, save_path, cnn_output_name, video=True)
# plot monkey
cnn_output_name = 'test_monkey_cnn_output.gif'
if test_num is not None:
cnn_output_name = 'test{}_monkey_cnn_output.gif'.format(test_num)
plot_cnn_output(preds_monkey, save_path, cnn_output_name, video=True)
# # plot monkey cleaned
# cnn_output_name = 'test_monkey_filtered_cnn_output.gif'
# if test_num is not None:
# cnn_output_name = 'test{}_monkey_filtered_cnn_output.gif'.format(test_num)
#
# plot_cnn_output(preds_monkey_cleaned, save_path, cnn_output_name, video=True)
# plot slice predictions
# plot raw responses of the dynamic
max_pred = np.amax([preds_human, preds_monkey])
plt.figure()
plt.subplot(2, 2, 1)
plt.plot(
preds_human[:, :, slice_pos_eyebrow,
0]) # slice over the 10 column to try to get the eyebrow
plt.ylim(0, max_pred)
plt.title("Human Avatar Eyebrow")
plt.subplot(2, 2, 2)
plt.plot(
preds_monkey[:, :, slice_pos_eyebrow,
0]) # slice over the 10 column to try to get the eyebrow
plt.ylim(0, max_pred)
plt.title("Monkey Avatar Eyebrow")
plt.subplot(2, 2, 3)
plt.plot(preds_human[:, :, slice_pos_lips,
1]) # slice over the 10 column to try to get the lips
plt.ylim(0, max_pred)
plt.title("Human Avatar Lips")
plt.subplot(2, 2, 4)
plt.plot(
preds_monkey[:, :, slice_pos_lips,
1]) # slice over the 10 column to try to get the lips
plt.ylim(0, max_pred)
plt.title("Monkey Avatar Lips")
plt.suptitle(expression)
if test_num is not None:
plt.savefig(
os.path.join(
save_path,
"test{}_Hum_vs_Monk_{}_expression_slice_eb{}_lips_eb{}".format(
test_num, expression, slice_pos_eyebrow, slice_pos_lips)))
else:
plt.savefig(
os.path.join(
save_path,
"test_Hum_vs_Monk_{}_expression_slice_eb{}_lips_eb{}".format(
expression, slice_pos_eyebrow, slice_pos_lips)))
# plot positions
# set color to represent time
color_seq = np.arange(len(preds_human_pos))
plt.figure()
plt.subplot(2, 2, 1)
plt.scatter(preds_human_pos[:, 1, 0],
preds_human_pos[:, 0, 0],
c=color_seq)
plt.xlim(12, 15)
plt.colorbar()
plt.title("Human Avatar Eyebrow")
plt.subplot(2, 2, 2)
plt.scatter(preds_monkey_pos[:, 1, 0],
preds_monkey_pos[:, 0, 0],
c=color_seq)
plt.xlim(12, 15)
plt.colorbar()
plt.title("Monkey Avatar Eyebrow")
plt.subplot(2, 2, 3)
plt.scatter(preds_human_pos[:, 1, 1],
preds_human_pos[:, 0, 1],
c=color_seq)
plt.xlim(12, 15)
plt.colorbar()
plt.title("Human Avatar Lips")
plt.subplot(2, 2, 4)
plt.scatter(preds_monkey_pos[:, 1, 1],
preds_monkey_pos[:, 0, 1],
c=color_seq)
plt.xlim(12, 15)
plt.colorbar()
plt.title("Monkey Avatar Lips")
plt.suptitle(expression + " xy-pos")
if test_num is not None:
plt.savefig(
os.path.join(
save_path,
"test{}_Hum_vs_Monk_{}_expression_pos_eb{}_lips_eb{}".format(
test_num, expression, slice_pos_eyebrow, slice_pos_lips)))
else:
plt.savefig(
os.path.join(
save_path,
"test_Hum_vs_Monk_{}_expression_pos_eb{}_lips_eb{}".format(
expression, slice_pos_eyebrow, slice_pos_lips)))
preds_lips = preds[:, :, :, feature_map_of_lips]
print("[TEST] num of feature map for lips", len(feature_map_of_lips))
# compute dynamic changes
preds_eyebrow_init = preds_eyebrow[0]
print("shape preds_eyebrow_init", np.shape(preds_eyebrow_init))
dyn_preds_eyebrow = preds_eyebrow - np.repeat(np.expand_dims(
preds_eyebrow_init, axis=0),
np.shape(preds_eyebrow)[0],
axis=0)
print("shape dyn_preds_eyebrow", np.shape(dyn_preds_eyebrow))
dyn_preds_eyebrow[dyn_preds_eyebrow < 0] = 0
# compute position vectors
eye_brow_pos = calculate_position(preds_eyebrow,
mode="weighted average",
return_mode="array")
print("shape eye_brow_pos", np.shape(eye_brow_pos))
dyn_eye_brow_pos = calculate_position(dyn_preds_eyebrow[1:],
mode="weighted average",
return_mode="array")
if do_plot:
# # plot feature maps
# plot_cnn_output(preds, os.path.join("models/saved", config["config_name"]),
# config['v4_layer'] + "_eye_brow.gif",
# image=raw_data,
# video=True,
# highlight=feature_map_of_eyebrow)
#
# plot_cnn_output(preds, os.path.join("models/saved", config["config_name"]),
plt.savefig(
os.path.join("models/saved", config["config_name"],
"test4_eyebrow_raw_slice_{}".format(slice_pos)))
# plot raw responses of the dynamic
plt.figure()
slice_pos = 9
plt.plot(dyn_eye_brow_mean_pred[:, :, slice_pos, 0]
) # slice over the 10 column to trz to get the eyebrow
plt.savefig(
os.path.join("models/saved", config["config_name"],
"test4_eyebrow_raw_dyn_slice_{}".format(slice_pos)))
# plot raw positions of first feature map
preds_pos = calculate_position(eye_brow_mean_pred,
mode="weighted average",
return_mode="xy float")
plt.figure()
plt.scatter(preds_pos[:, 1, 0], preds_pos[:, 0, 0],
c=color_seq) # plot first feature map
plt.xlim(13, 15)
plt.colorbar()
plt.savefig(
os.path.join("models/saved", config["config_name"],
"test4_eyebrow_mean_xy_float_pos"))
# plot mean positions over all eyebrow feature map
dyn_preds_pos = calculate_position(dyn_eye_brow_mean_pred,
mode="weighted average",
return_mode="xy float")
plt.figure()
# calculate mean freq
#mean_freq = np.sum(energy * freq) / np.sum(energy)
frequencies_mean = np.einsum("ij,i->j", power_spectrum, frequencies) / np.sum(power_spectrum, axis=0)
#--> produces some nan values but these are discarded by sorting later
print("len(frequencies_mean)", len(frequencies_mean))
# select lowest frequencies
n_filter_time = 256
features_sorted_freq = np.argsort(frequencies_mean)
selection_array = np.zeros(len(frequencies_mean), dtype=bool)
selection_array[features_sorted_freq[:n_filter_time]] = True
print("selection_array", selection_array)
# calculate selected positions
selection_indices = np.arange(len(selection_array))[selection_array]
print("selected positions", selection_indices)
# calculate selected feature map depending on whether x and y component from this feature map is selected
highlight_x = selection_indices[selection_indices<256]
highlight_y = selection_indices[selection_indices>256] % 256
highlight_union = list(set(highlight_x) | set(highlight_y))
highlight_intersect = list(set(highlight_x) & set(highlight_y))
print("len(highlight_union)", len(highlight_union))
print("len(highlight_intersect)", len(highlight_intersect))
# animate positions and highlight selected ones
positions_array = calculate_position( norm_base.evaluate_v4(images, flatten=False),
mode=config["position_method"], return_mode="array")
plot_cnn_output(positions_array, os.path.join("models/saved/", config["save_name"]), config["plot_name"]+"_video.mp4",
title=None, image=images, video=True, highlight=highlight_intersect)
images_human_fear[selection_human_expression],
images_monkey_threat[selection_monkey_expression]
]) # (8, 224, 224, 3)
# calculate response
norm_base = NormBase(config, (224, 224, 3))
response = norm_base.evaluate_v4(images, flatten=False) # (8, 28, 28, 256)
# reduce response to selected feature map
response = response[..., n_feature_map] # (8, 28, 28)
response = np.expand_dims(response, axis=-1) # (8,28,28,1)
# calculate arrays based on position for different modes
position_array_list = []
position_xy_list = []
for i, mode in enumerate(mode_list):
position_array_list.append(
calculate_position(response, mode=mode, return_mode="array"))
position_xy_list.append(
calculate_position(response, mode=mode, return_mode="xy float"))
# create plot
fig, axs = plt.subplots(8, 2 + len(position_array_list), figsize=[10, 10])
for _, ax in np.ndenumerate(axs):
#ax.axis('off')
ax.xaxis.set_ticks([])
ax.yaxis.set_ticks([])
for ax, col in zip(axs[0], ["picture", "response"] + mode_list):
ax.set_title(col)
# plot images
images = images / 255 # transform into correct range for imshow()
for i, image in enumerate(images):