def calc_modeled_layers_mean_and_std(self, data):
"""For each conv layer calc the mean and std of the output data
mode - 1 - returns the mean of the data.
2 - returns K examples randomly picked as representatives. """
keras_model = self.keras_model
layer_gmm_params = {}
for layer_name, gmm_layer_name in self.gmm_dict.items():
layer_gmm_params[gmm_layer_name] = {}
K = keras_model.get_layer(gmm_layer_name).n_clusters
if layer_name == self.network_output_layer_name:
x = np.zeros((K, K))
np.fill_diagonal(x, 1)
layer_gmm_params[gmm_layer_name] = {}
layer_gmm_params[gmm_layer_name].update({'mu': x})
layer_gmm_params[gmm_layer_name].update({'std': 0.1})
else:
x = get_layer_output(keras_model, layer_name, data)
# K = keras_model.get_layer( gmm_layer_name ).K
if len(x.shape) == 4:
B_dim = x.shape[0]
H_dim = x.shape[1]
W_dim = x.shape[2]
D_dim = x.shape[3]
n_samples = B_dim * H_dim * W_dim
x = np.reshape(x, (n_samples, D_dim))
elif len(x.shape) == 2:
n_samples = x.shape[0]
else:
raise TypeError(
'Layer type must be either Dense or Conv2D')
chosen_reps = np.random.choice(n_samples, K)
mu = x[chosen_reps, :]
std = np.std(x, axis=0)
layer_gmm_params[gmm_layer_name].update({'mu': mu})
layer_gmm_params[gmm_layer_name].update({'std': std})
return layer_gmm_params
heat_map_dir = pjoin(results_dir, 'heat_maps')
# Create file for results
saving_dir = pjoin(results_dir, 'image_tree')
images_clusters_dir = pjoin(results_dir, 'images_clusters')
makedir(saving_dir)
RF = ReceptiveField(model.keras_model)
layers_clusters_dict = {}
for layer_name in layers_to_inference:
gmm_name = model.get_correspond_gmm_layer(layer_name)
n_clusters = model.keras_model.get_layer(gmm_name).n_clusters
layers_clusters_dict[layer_name] = n_clusters
test_pred = get_layer_output(keras_model=model.keras_model,
layer_name=layers_to_inference[-1],
input_data=x_val)
preds_ind = np.argmax(test_pred, axis=1)
preds_confident = np.max(test_pred, axis=1)
y_inds = np.argmax(y_val, axis=1)
if good_or_bad_image == 'good':
selected_inds = np.where(preds_ind == y_inds)[0]
elif good_or_bad_image == 'bad':
selected_inds = np.where(preds_ind != y_inds)[0]
else:
ValueError(f'{good_or_bad_image} should be either \'good\' or \'bad\'')
selected_conf = preds_confident[selected_inds]
sort_selected_args = np.argsort(-selected_conf)
def create_clusters_for_imageGraph(vis_image, image_array, image_false_label, true_label, gmm_model, rf,
connections_dict, clusters_dir, saving_dir, cluster_type, image_pos='r'):
makedir(saving_dir)
for layer_cluster_key in connections_dict:
# fig, ax = plt.subplots(nrows=1, ncols=2, gridspec_kw={'width_ratios': [4, 1], 'wspace': 0.001})
# fig, ax = plt.subplots(nrows=2, ncols=1, gridspec_kw={'height_ratios': [1, 3], 'hspace': 0.0})
# plt.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0)
# plt.margins(0, 0)
if layer_cluster_key == 'class':
layer_name = 'classification'
fig, ax = plt.subplots(nrows=1, ncols=2, gridspec_kw={'width_ratios': [3, 1], 'wspace': 0.0})
plt.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0)
k_name = 'cluster_' + str(image_false_label+1) + '.png'
k_dir = pjoin(*[clusters_dir, layer_name, k_name])
cluster_img = mpimg.imread(k_dir)
ax[0].imshow(cluster_img)
ax[1].imshow(vis_image)
ax[1].set_title(true_label, fontdict={'fontsize': 20.0})
else:
layer_type, layer_num, k_num = layer_cluster_key.split('_')
layer_name = layer_type + '_' + layer_num
k_name = 'cluster_' + k_num + '_' + cluster_type[layer_name] + '.png'
if image_pos == 'r':
fig, ax = plt.subplots(nrows=1, ncols=2, gridspec_kw={'width_ratios': [4, 1], 'wspace': 0.001})
elif image_pos == 't':
fig, ax = plt.subplots(nrows=2, ncols=1, gridspec_kw={'height_ratios': [1, 3], 'hspace': 0.0})
plt.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0)
k_dir = pjoin(*[clusters_dir, layer_name, k_name])
cluster_img = mpimg.imread(k_dir)
radius_size = 0.5
if image_pos == 't':
ax[1].imshow(cluster_img)
ax[0].imshow(vis_image)
elif image_pos == 'r':
ax[0].imshow(cluster_img)
ax[1].imshow(vis_image)
gmm_explaining_layer_name = gmm_model.get_correspond_gmm_layer(layer_name)
corr_activation_layer = gmm_model.gmm_activations_dict[gmm_explaining_layer_name]
img_pred = get_layer_output(keras_model=gmm_model.keras_model, layer_name=corr_activation_layer,
input_data=image_array)
H_dim = img_pred.shape[1]
W_dim = img_pred.shape[2]
K_dim = img_pred.shape[3]
img_pred = np.reshape(img_pred, (H_dim * W_dim, K_dim))
img_k = np.argmax(img_pred, axis=1)
img_values = np.max(img_pred, axis=1)
location_inds = np.where(img_k == (int(k_num)-1))[0]
rep_values = img_values[location_inds]
sor_location_inds = location_inds[np.argsort(-rep_values)]
n_samples = W_dim*H_dim
n_samples_in_k = len(sor_location_inds)
first_time_flag = False # for adding a rectangle around first point - this field should be true
for loc_i in sor_location_inds:
xPos = np.remainder(loc_i, W_dim)
yPos = np.floor_divide(loc_i, W_dim)
size, center, upper_left_pos, origin_center = rf.target_neuron_rf(layer_name, (yPos, xPos),
rf_layer_name='input_layer',
return_origin_center=True,
return_upper_left_pos=True)
# The upper and left rectangle coordinates
upper_left_row = upper_left_pos[0]
upper_left_col = upper_left_pos[1]
if first_time_flag:
rect = patches.Rectangle((upper_left_col, upper_left_row), size[1] - 1, size[0] - 1,
linewidth=2, edgecolor='r', facecolor='none')
# ax[0].add_patch(rect)
ax[1].add_patch(rect)
first_time_flag = False
dot = patches.Circle((origin_center[1], origin_center[0]), radius=radius_size,
linewidth=1, edgecolor='r', facecolor='r')
# ax[0].add_patch(dot)
ax[1].add_patch(dot)
# ax[0].set_title(f'{n_samples_in_k}/{n_samples}', size=12)
ax[1].set_title(f'{n_samples_in_k}/{n_samples}', size=12)
ax[0].axis('off')
ax[1].axis('off')
plt.savefig(pjoin(saving_dir, layer_name + '_' + k_name), bbox_inches='tight', pad_inches=0)
plt.close(fig)