def fit_dimensionality_reduction(model, data): """ Helper function to fit the model dimensionality reduction set in the config :param data: input data :return: """ # in the case of dimensionality reduction set up the pipeline if model.dim_red is None: print("[FIT] no dimensionality reduction") preds = data elif model.dim_red == 'PCA': # perform PCA on this output print("[FIT] Fitting PCA") model.pca.fit(data) print("[FIT] PCA: explained variance", model.pca.explained_variance_ratio_) preds = model.pca.transform(data) elif model.dim_red == "semantic" or model.dim_red == "semantic-pattern" or model.dim_red == "pattern": preds = model.feat_red.fit(data, activation='mean') # apply filter post_processing # todo modify this, I'm not really happy with this yet preds = get_feat_map_filt_preds( preds, ref_type="self0", norm=model.config['feat_map_filt_norm'], activation=model.config['feat_map_filt_activation'], filter=model.config['feat_map_filter_type']) # allow to further reduce dimensionality by getting a 2 dim vector for each feature maps if model.config['feat_map_position_mode'] != 'raw': print("[FIT] Using position mode: {}".format( model.config['feat_map_position_mode'])) preds = calculate_position( preds, mode=model.config['feat_map_position_mode'], return_mode='xy float') preds = np.reshape(preds, (len(preds), -1)) print("[FIT] Finished to find the semantic units") else: raise KeyError(f'model.dim_red={model.dim_red} is not a valid value') return preds
def predict_dimensionality_reduction(model, data): """ Helper function to apply the dimensionality reduction as set in the configuation before the model :param model: model instance :param data: :return: prediction """ if model.dim_red is None: # get prediction after cnn, before dimensionality reduction preds = data elif model.dim_red == 'PCA': # projection by PCA preds = model.pca.transform(data) elif model.dim_red == "semantic" or model.dim_red == "semantic-pattern" or model.dim_red == "pattern": preds = model.feat_red.transform(data, activation='mean') # apply filter post_processing # todo modify this, I'm not really happy with this yet preds = get_feat_map_filt_preds( preds, ref_type="self0", norm=model.config['feat_map_filt_norm'], activation=model.config['feat_map_filt_activation'], filter=model.config['feat_map_filter_type']) # allow to further reduce dimensionality by getting a 2 dim vector for each feature maps if model.config['feat_map_position_mode'] != 'raw': print("[PREDS] Using position mode: {}".format( model.config['feat_map_position_mode'])) preds = calculate_position( preds, mode=model.config['feat_map_position_mode'], return_mode='xy float') preds = np.reshape(preds, (len(preds), -1)) else: raise KeyError(f'invalid value self.dim_red={model.dim_red}') return preds
config, activation='relu') # lips dyn_lips_preds = ref_feature_map_neuron(lips_preds, data[1], config, activation='relu') # concatenate feature maps concepts dyn_preds = np.concatenate([dyn_eyebrow_preds, dyn_lips_preds], axis=3) print("[TRAIN] shape dyn preds", np.shape(dyn_preds)) print("[TRAIN] finished computing dynamic predictions") # compute positions eyebrow dyn_pos = calculate_position(dyn_preds, mode="weighted average", return_mode="xy float flat") print("[TRAIN] shape dyn_pos", np.shape(dyn_pos)) # add weights pos_weights = np.ones(np.shape(dyn_pos)[-1]) pos_weights[:8] = 1 / (2 * 2 * len(best_eyebrow_IoU_ft) ) # num_concept * x/y * num_ft/concept pos_weights[8:] = 1 / (2 * 2 * len(best_lips_IoU_ft)) dyn_pos = np.multiply(dyn_pos, pos_weights) nb_model.n_features = np.shape(dyn_pos)[-1] # todo add this to init # train manually ref vector nb_model.r = np.zeros(nb_model.n_features) nb_model._fit_reference([dyn_pos, data[1]], config['batch_size']) # train manually tuning vector
# for lips, create four mouth zones left_lips = np.zeros(np.shape(max_lips_preds)) left_lips[:, 33:45, 19:26] = max_lips_preds[:, 33:45, 19:26] middle_up_lips = np.zeros(np.shape(max_lips_preds)) middle_up_lips[:, 33:37, 24:34] = max_lips_preds[:, 33:37, 24:34] middle_down_lips = np.zeros(np.shape(max_lips_preds)) middle_down_lips[:, 38:48, 24:33] = max_lips_preds[:, 38:48, 24:33] right_lips = np.zeros(np.shape(max_lips_preds)) right_lips[:, 33:45, 32:37] = max_lips_preds[:, 33:45, 32:37] preds = np.concatenate([left_ext_eyebrow, left_int_eyebrow, right_int_eyebrow, right_ext_eyebrow, middle_up_lips, middle_down_lips, left_lips, right_lips], axis=3) print("[TRAIN] preds", np.shape(preds)) # compute positions eyebrow pos = calculate_position(preds, mode="weighted average", return_mode="xy float flat") print("[TRAIN] shape pos", np.shape(pos)) nb_model.n_features = np.shape(pos)[-1] # todo add this to init # train manually ref vector nb_model.r = np.zeros(nb_model.n_features) nb_model._fit_reference([pos, data[1]], config['batch_size']) print("[TRAIN] model.r", np.shape(nb_model.r)) print(nb_model.r) ref_train = np.copy(nb_model.r) # train manually tuning vector nb_model.t = np.zeros((nb_model.n_category, nb_model.n_features)) nb_model.t_mean = np.zeros((nb_model.n_category, nb_model.n_features)) nb_model._fit_tuning([pos, data[1]], config['batch_size']) ref_tuning = np.copy(nb_model.t) print("[TRAIN] ref_tuning[1]")
elif 'stride' in config["difference_option"]: stride_length = int(config["difference_option"][6:]) reference = np.roll(response, shift=stride_length, axis=0) reference[:stride_length, ...] = response[:stride_length,...] elif config["difference_option"] == "reference": # calculate and retrieve reference vector from norm_base raise KeyError('reference not implemented yet') else: raise KeyError(f'config["difference_option"]={config["difference_option"]} is no valid option') vector_plot = response - reference >>>>>>> 11da6a3... restructured and started analysis of stimuli noise rendering else: try: # config["resp_type"]: "maximum", "maximum{int}", "weighted average" vector_plot = calculate_position(response=norm_base.evaluate_v4(data, flatten=False), mode=resp_type, return_mode="array") except KeyError: raise KeyError(f'config["plot_option"]={resp_type} is no valid key') # calculate which feature maps to highlight try: if config["highlight_option"] == "maximum": max_map = np.amax(np.abs(vector_plot), axis=(0,1,2)) # max over frames, and within feature map highlight = np.flip(np.argsort(max_map)[-25:]) else: raise KeyError(f'config["highlight_option"]={config["highlight_option"]} is no valid option') except KeyError: highlight = None <<<<<<< HEAD
def predict_expression(expression, ft_idx, test_num=None): # set config config["train_expression"] = [expression] # ----------------------------------------------------------------------------------------------------------------- # human avatar config["train_avatar"] = "human_orig" # load data data = load_data(config)[0] # predict and filter responses print("[PE] Human {} loaded".format(expression)) print("[PE] shape data", np.shape(data)) print("[PE] Start predictions") preds_human = model.predict(data) preds_human = get_feat_map_filt_preds(preds_human, ft_idx, ref_type="self0", norm=1000, activation='ReLu', filter='spatial_mean', verbose=True) print("[PE] shape preds Human", np.shape(preds_human)) print("[PE] min max preds Human", np.amin(preds_human), np.amax(preds_human)) # get xy positions preds_human_pos = calculate_position(preds_human, mode="weighted average", return_mode="xy float") print("[PE] shape preds Human positions", np.shape(preds_human_pos)) # # filter feat maps # preds_human_cleaned = filter_feature_maps(preds_human) # preds_human_pos_cleaned = calculate_position(preds_human_cleaned, mode="weighted average", return_mode="xy float") # ----------------------------------------------------------------------------------------------------------------- # Monkey Avatar config["train_avatar"] = "monkey_orig" # load data data = load_data(config)[0] print("[PE] Monkey {} loaded".format(expression)) print("[PE] shape data", np.shape(data)) print("[PE] Start predictions") preds_monkey = model.predict(data) preds_monkey = get_feat_map_filt_preds(preds_monkey, ft_idx, ref_type="self0", norm=1000, activation='ReLu', filter='spatial_mean', verbose=True) print("[PE] shape preds Human", np.shape(preds_monkey)) print("[PE] min max preds Human", np.amin(preds_monkey), np.amax(preds_monkey)) # get xy positions preds_monkey_pos = calculate_position(preds_monkey, mode="weighted average", return_mode="xy float") print("[PE] shape preds Monkey positions", np.shape(preds_monkey_pos)) # # filter feat maps # preds_monkey_cleaned = filter_feature_maps(preds_monkey) # preds_monkey_pos_cleaned = calculate_position(preds_monkey_cleaned, mode="weighted average", return_mode="xy float") # ----------------------------------------------------------------------------------------------------------------- # plot raw feature maps print("[PE] Create plot") # plot human cnn_output_name = 'test_human_cnn_output.gif' if test_num is not None: cnn_output_name = 'test{}_human_cnn_output.gif'.format(test_num) plot_cnn_output(preds_human, save_path, cnn_output_name, video=True) # # plot human filtered feature maps # cnn_output_name = 'test_human_filtered_cnn_output.gif' # if test_num is not None: # cnn_output_name = 'test{}_human_filtered_cnn_output.gif'.format(test_num) # # plot_cnn_output(preds_human_cleaned, save_path, cnn_output_name, video=True) # plot monkey cnn_output_name = 'test_monkey_cnn_output.gif' if test_num is not None: cnn_output_name = 'test{}_monkey_cnn_output.gif'.format(test_num) plot_cnn_output(preds_monkey, save_path, cnn_output_name, video=True) # # plot monkey cleaned # cnn_output_name = 'test_monkey_filtered_cnn_output.gif' # if test_num is not None: # cnn_output_name = 'test{}_monkey_filtered_cnn_output.gif'.format(test_num) # # plot_cnn_output(preds_monkey_cleaned, save_path, cnn_output_name, video=True) # plot slice predictions # plot raw responses of the dynamic max_pred = np.amax([preds_human, preds_monkey]) plt.figure() plt.subplot(2, 2, 1) plt.plot( preds_human[:, :, slice_pos_eyebrow, 0]) # slice over the 10 column to try to get the eyebrow plt.ylim(0, max_pred) plt.title("Human Avatar Eyebrow") plt.subplot(2, 2, 2) plt.plot( preds_monkey[:, :, slice_pos_eyebrow, 0]) # slice over the 10 column to try to get the eyebrow plt.ylim(0, max_pred) plt.title("Monkey Avatar Eyebrow") plt.subplot(2, 2, 3) plt.plot(preds_human[:, :, slice_pos_lips, 1]) # slice over the 10 column to try to get the lips plt.ylim(0, max_pred) plt.title("Human Avatar Lips") plt.subplot(2, 2, 4) plt.plot( preds_monkey[:, :, slice_pos_lips, 1]) # slice over the 10 column to try to get the lips plt.ylim(0, max_pred) plt.title("Monkey Avatar Lips") plt.suptitle(expression) if test_num is not None: plt.savefig( os.path.join( save_path, "test{}_Hum_vs_Monk_{}_expression_slice_eb{}_lips_eb{}".format( test_num, expression, slice_pos_eyebrow, slice_pos_lips))) else: plt.savefig( os.path.join( save_path, "test_Hum_vs_Monk_{}_expression_slice_eb{}_lips_eb{}".format( expression, slice_pos_eyebrow, slice_pos_lips))) # plot positions # set color to represent time color_seq = np.arange(len(preds_human_pos)) plt.figure() plt.subplot(2, 2, 1) plt.scatter(preds_human_pos[:, 1, 0], preds_human_pos[:, 0, 0], c=color_seq) plt.xlim(12, 15) plt.colorbar() plt.title("Human Avatar Eyebrow") plt.subplot(2, 2, 2) plt.scatter(preds_monkey_pos[:, 1, 0], preds_monkey_pos[:, 0, 0], c=color_seq) plt.xlim(12, 15) plt.colorbar() plt.title("Monkey Avatar Eyebrow") plt.subplot(2, 2, 3) plt.scatter(preds_human_pos[:, 1, 1], preds_human_pos[:, 0, 1], c=color_seq) plt.xlim(12, 15) plt.colorbar() plt.title("Human Avatar Lips") plt.subplot(2, 2, 4) plt.scatter(preds_monkey_pos[:, 1, 1], preds_monkey_pos[:, 0, 1], c=color_seq) plt.xlim(12, 15) plt.colorbar() plt.title("Monkey Avatar Lips") plt.suptitle(expression + " xy-pos") if test_num is not None: plt.savefig( os.path.join( save_path, "test{}_Hum_vs_Monk_{}_expression_pos_eb{}_lips_eb{}".format( test_num, expression, slice_pos_eyebrow, slice_pos_lips))) else: plt.savefig( os.path.join( save_path, "test_Hum_vs_Monk_{}_expression_pos_eb{}_lips_eb{}".format( expression, slice_pos_eyebrow, slice_pos_lips)))
preds_lips = preds[:, :, :, feature_map_of_lips] print("[TEST] num of feature map for lips", len(feature_map_of_lips)) # compute dynamic changes preds_eyebrow_init = preds_eyebrow[0] print("shape preds_eyebrow_init", np.shape(preds_eyebrow_init)) dyn_preds_eyebrow = preds_eyebrow - np.repeat(np.expand_dims( preds_eyebrow_init, axis=0), np.shape(preds_eyebrow)[0], axis=0) print("shape dyn_preds_eyebrow", np.shape(dyn_preds_eyebrow)) dyn_preds_eyebrow[dyn_preds_eyebrow < 0] = 0 # compute position vectors eye_brow_pos = calculate_position(preds_eyebrow, mode="weighted average", return_mode="array") print("shape eye_brow_pos", np.shape(eye_brow_pos)) dyn_eye_brow_pos = calculate_position(dyn_preds_eyebrow[1:], mode="weighted average", return_mode="array") if do_plot: # # plot feature maps # plot_cnn_output(preds, os.path.join("models/saved", config["config_name"]), # config['v4_layer'] + "_eye_brow.gif", # image=raw_data, # video=True, # highlight=feature_map_of_eyebrow) # # plot_cnn_output(preds, os.path.join("models/saved", config["config_name"]),
plt.savefig( os.path.join("models/saved", config["config_name"], "test4_eyebrow_raw_slice_{}".format(slice_pos))) # plot raw responses of the dynamic plt.figure() slice_pos = 9 plt.plot(dyn_eye_brow_mean_pred[:, :, slice_pos, 0] ) # slice over the 10 column to trz to get the eyebrow plt.savefig( os.path.join("models/saved", config["config_name"], "test4_eyebrow_raw_dyn_slice_{}".format(slice_pos))) # plot raw positions of first feature map preds_pos = calculate_position(eye_brow_mean_pred, mode="weighted average", return_mode="xy float") plt.figure() plt.scatter(preds_pos[:, 1, 0], preds_pos[:, 0, 0], c=color_seq) # plot first feature map plt.xlim(13, 15) plt.colorbar() plt.savefig( os.path.join("models/saved", config["config_name"], "test4_eyebrow_mean_xy_float_pos")) # plot mean positions over all eyebrow feature map dyn_preds_pos = calculate_position(dyn_eye_brow_mean_pred, mode="weighted average", return_mode="xy float") plt.figure()
# calculate mean freq #mean_freq = np.sum(energy * freq) / np.sum(energy) frequencies_mean = np.einsum("ij,i->j", power_spectrum, frequencies) / np.sum(power_spectrum, axis=0) #--> produces some nan values but these are discarded by sorting later print("len(frequencies_mean)", len(frequencies_mean)) # select lowest frequencies n_filter_time = 256 features_sorted_freq = np.argsort(frequencies_mean) selection_array = np.zeros(len(frequencies_mean), dtype=bool) selection_array[features_sorted_freq[:n_filter_time]] = True print("selection_array", selection_array) # calculate selected positions selection_indices = np.arange(len(selection_array))[selection_array] print("selected positions", selection_indices) # calculate selected feature map depending on whether x and y component from this feature map is selected highlight_x = selection_indices[selection_indices<256] highlight_y = selection_indices[selection_indices>256] % 256 highlight_union = list(set(highlight_x) | set(highlight_y)) highlight_intersect = list(set(highlight_x) & set(highlight_y)) print("len(highlight_union)", len(highlight_union)) print("len(highlight_intersect)", len(highlight_intersect)) # animate positions and highlight selected ones positions_array = calculate_position( norm_base.evaluate_v4(images, flatten=False), mode=config["position_method"], return_mode="array") plot_cnn_output(positions_array, os.path.join("models/saved/", config["save_name"]), config["plot_name"]+"_video.mp4", title=None, image=images, video=True, highlight=highlight_intersect)
images_human_fear[selection_human_expression], images_monkey_threat[selection_monkey_expression] ]) # (8, 224, 224, 3) # calculate response norm_base = NormBase(config, (224, 224, 3)) response = norm_base.evaluate_v4(images, flatten=False) # (8, 28, 28, 256) # reduce response to selected feature map response = response[..., n_feature_map] # (8, 28, 28) response = np.expand_dims(response, axis=-1) # (8,28,28,1) # calculate arrays based on position for different modes position_array_list = [] position_xy_list = [] for i, mode in enumerate(mode_list): position_array_list.append( calculate_position(response, mode=mode, return_mode="array")) position_xy_list.append( calculate_position(response, mode=mode, return_mode="xy float")) # create plot fig, axs = plt.subplots(8, 2 + len(position_array_list), figsize=[10, 10]) for _, ax in np.ndenumerate(axs): #ax.axis('off') ax.xaxis.set_ticks([]) ax.yaxis.set_ticks([]) for ax, col in zip(axs[0], ["picture", "response"] + mode_list): ax.set_title(col) # plot images images = images / 255 # transform into correct range for imshow() for i, image in enumerate(images):