def heatmap(self, path):
# Grad-CAM requires the category of the image as input. So we need predict image to get the label first.
self.img = utils.load_img(path, target_size=(224, 224))
self.x = image.img_to_array(self.img)
self.x = np.expand_dims(self.x, axis=0)
self.y = self.model.predict(self.x, verbose=1)
self.y = self.y.argmax(axis=-1)
self.label = [
'dyed-lifted-polyps', 'dyed-resection-margins', 'esophagitis',
'normal-cecum', 'normal-pylorus', 'normal-z-line', 'polyps',
'ulcerative-colitis'
]
self.label = np.array(self.label)
self.layer_idx = utils.find_layer_idx(self.model, 'dense_1')
# Swap softmax with linear
self.model.layers[self.layer_idx].activation = activations.linear
self.model = utils.apply_modifications(self.model)
for modifier in ['guided']:
f, ax = plt.subplots(1, 1)
plt.suptitle(self.label[self.y])
# Model, layer id, class ,image as input.
self.grads = visualize_cam(self.model,
self.layer_idx,
filter_indices=self.y,
seed_input=self.img,
backprop_modifier=modifier)
# Lets overlay the heatmap onto original image.
self.jet_heatmap = np.uint8(cm.jet(self.grads)[..., :3] * 255)
self.a = ax.imshow(overlay(self.jet_heatmap, self.img))
plt.savefig("image/attention.jpg")
return self.a
def vis_cam(self, x_test, scores, img_idx, fname, output_path):
json_fname = os.path.join(output_path, 'model.json')
layer_idx = utils.find_layer_idx(self.model, 'predictions')
self.model.layers[layer_idx].activation = activations.linear
self.model = utils.apply_modifications(self.model)
classlabel = ['PA', 'Genuine']
for class_idx in range(len(classlabel)):
seed_input = x_test[img_idx]
fname_original = fname.split('/')[:-6] + [
'3',
'origin_image',
] + fname.split('/')[-4:]
fname_original = '/'.join(fname_original)
fname_original = "{}.png".format(
os.path.splitext(fname_original)[0])
img = cv2.imread(fname_original, cv2.IMREAD_COLOR)[:, :, ::-1]
output_fname = os.path.join(
output_path,
'vis_cam',
classlabel[class_idx],
os.path.relpath(fname, self.output_path).replace('../', ''),
)
output_fname = "{}.png".format(os.path.splitext(output_fname)[0])
safe_create_dir(os.path.dirname(output_fname))
grad_top = visualize_cam(self.model,
layer_idx,
class_idx,
seed_input,
penultimate_layer_idx=None,
backprop_modifier=None,
grad_modifier=None)
print('-- saving visualizations in', output_fname)
self.plot_map(grad_top, img, classlabel[class_idx],
scores[img_idx, class_idx], output_fname)
image_path = visualization_dir / "vis_epoch_{0:04d}.png".format(
epoch)
img.save(str(image_path))
elif args.visualize == "layers":
visualization_path /= "layers/ACGAN"
# visualize each class
for t in range(len(tags)):
save_path = visualization_path / "out_{}.png".format(tags[t])
if save_path.exists():
# print("{} already visualized".format(tags[t]))
continue
output_layer_name = acgan.generator.outputs[0].name
layer_idx = utils.find_layer_idx(acgan.generator,
output_layer_name)
# Swap softmax with linear
acgan.generator.layers[layer_idx].activation = activations.linear
acgan.generator = utils.apply_modifications(acgan.generator)
img = visualize_activation(acgan.generator,
layer_idx,
filter_indices=t,
verbose=True)
array_to_img(img).save(save_path)
print("saved to {}".format(str(save_path)))
# visualize each layer
for layer_name in [layer.name for layer in acgan.generator.layers]:
save_path = visualization_path / "{}.png".format(layer_name)
# Generate generalization metrics
score = model.evaluate(input_test, target_test, verbose=0)
print(f'Test loss: {score[0]} / Test accuracy: {score[1]}')
# =============================================
# Grad-CAM code
# =============================================
from vis.visualization import visualize_cam, overlay
from vis.utils import utils
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.cm as cm
# Find the index of the to be visualized layer above
layer_index = utils.find_layer_idx(model, 'visualized_layer')
# Swap softmax with linear
model.layers[layer_index].activation = activations.linear
model = utils.apply_modifications(model)
# Numbers to visualize
indices_to_visualize = [0, 12, 38, 83, 112, 74, 190]
# Visualize
for index_to_visualize in indices_to_visualize:
input_image = input_test[index_to_visualize]
input_class = np.argmax(target_test[index_to_visualize])
# Matplotlib preparations
fig, axes = plt.subplots(1, 3)
# Generate visualization
#
# We load the original model, with the top FC layers included.
# In[4]:
model_origin = InceptionV3(weights='imagenet', include_top=True)
# We replace the activation of the last layer by a linear activation, as softmax introduces dependencies between output nodes
#
# This operation may take several minutes...
# In[7]:
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
layer_idx = utils.find_layer_idx(model_origin, 'predictions')
print("Remove Activation from Last Layer")
# Swap softmax with linear
model_origin.layers[layer_idx].activation = activations.linear
print("Done. Now Applying changes to the model ...")
model_origin = utils.apply_modifications(model_origin)
# We import an image and preprocess it according to the pre-proc function of the InceptionV3 model. Then we compute the activation map for a specific class. The image is an husky, we then compute the activation map for the husky/eskimo-dog class which corresponds to label 248 for ImageNet db.
# In[8]:
#CAM on images for InceptionV3 network.
im_file = "husky.jpg"
img1 = image.load_img(im_file, target_size=(299, 299))
img1 = image.img_to_array(img1)
img1 = np.expand_dims(img1, axis=0)
sio.savemat('y_pred_cifar10_new_way_nov12.mat', {'y_pred': y_pred})
y_pred_arg = np.argmax(y_pred, axis=1)
sio.savemat('y_pred_arg_cifar10_new_way_nov12.mat', {'y_pred_arg': y_pred_arg})
from vis.visualization import visualize_activation
from vis.utils import utils
from keras import activations
import numpy as np
from matplotlib import pyplot as plt
#%matplotlib inline
plt.rcParams['figure.figsize'] = (18, 6)
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
layer_idx = utils.find_layer_idx(model, 'dense_3')
# Swap softmax with linear
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
# This is the output node we want to maximize.
filter_idx = 0
img = visualize_activation(model, layer_idx, filter_indices=filter_idx)
plt.imshow(img[..., 0])
for tv_weight in [1e-3, 1e-2, 1e-1, 1, 10]:
# Lets turn off verbose output this time to avoid clutter and just see the output.
img = visualize_activation(model,
layer_idx,
filter_indices=filter_idx,
def main():
parser = argparse.ArgumentParser(description='test AD recognition')
parser.add_argument('--input',
type=str,
required=True,
help="path to test data")
parser.add_argument('--model',
type=str,
required=True,
help="path to pre-trained model")
parser.add_argument('--id', type=int, required=True, help="data id")
args = parser.parse_args()
model_dir = os.path.join(os.path.dirname(os.getcwd()), args.model)
data = loaddata(args.input, 'testa.h5')
print('data_shape:{}'.format(data.shape))
print("[INFO] loading pre-trained network...")
json_file = open(model_dir + 'AD_3dcnnmodel.json', 'r')
model_json = json_file.read()
json_file.close()
model = model_from_json(model_json)
# load weights into new model
model.load_weights(model_dir + "AD_3dcnnmodel.hd5")
model.summary()
layer_idx = utils.find_layer_idx(model, 'activation_10')
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
grads = visualize_saliency(model,
layer_idx,
filter_indices=0,
seed_input=data[args.id],
backprop_modifier='guided',
keepdims=True)
grads = np.squeeze(grads, axis=0)
fig, ax = plt.subplots(nrows=1, ncols=3)
ax[0].set_axis_off()
ax[1].set_axis_off()
ax[2].set_axis_off()
extent1 = 0, grads.shape[1], 0, grads.shape[0]
extent2 = 0, grads.shape[2], 0, grads.shape[1]
extent3 = 0, grads.shape[2], 0, grads.shape[0]
ax[0].imshow(data[args.id][0, :, :, 45],
cmap='gray',
interpolation='nearest',
extent=extent1)
ax[0].imshow(grads[:, :, 45],
cmap='hot',
alpha=.5,
interpolation='bilinear',
extent=extent1)
ax[0].set_title('A')
ax[1].imshow(data[args.id][0, 39, :, :],
cmap='gray',
interpolation='nearest',
extent=extent2)
ax[1].imshow(grads[39, :, :],
cmap='hot',
alpha=.5,
interpolation='bilinear',
extent=extent2)
ax[1].set_title('C')
ax[2].imshow(data[args.id][0, :, 60, :],
cmap='gray',
interpolation='nearest',
extent=extent3)
ax[2].imshow(grads[:, 60, :],
cmap='hot',
alpha=.5,
interpolation='bilinear',
extent=extent3)
ax[2].set_title('S')
plt.show()
import pdb
pdb.set_trace()
volume = np.squeeze(grads, axis=0)
volume = np.swapaxes(volume, 0, 2)
r, c = volume.shape[1], volume.shape[2]
# Define frames
nb_frames = volume.shape[0]
fig = go.Figure(frames=[
go.Frame(
data=go.Surface(z=((nb_frames - 1) * 0.1 - k * 0.1) *
np.ones((r, c)),
surfacecolor=np.flipud(volume[nb_frames - 1 - k]),
cmin=0,
cmax=1),
name=str(
k
) # you need to name the frame for the animation to behave properly
) for k in range(nb_frames)
])
# Add data to be displayed before animation starts
fig.add_trace(
go.Surface(z=(nb_frames - 1) * 0.1 * np.ones((r, c)),
surfacecolor=np.flipud(volume[nb_frames - 1]),
colorscale='hot',
cmin=0,
cmax=1,
colorbar=dict(thickness=20, ticklen=4)))
def frame_args(duration):
return {
"frame": {
"duration": duration
},
"mode": "immediate",
"fromcurrent": True,
"transition": {
"duration": duration,
"easing": "linear"
},
}
sliders = [{
"pad": {
"b": 10,
"t": 60
},
"len":
0.9,
"x":
0.1,
"y":
0,
"steps": [{
"args": [[f.name], frame_args(0)],
"label": str(k),
"method": "animate",
} for k, f in enumerate(fig.frames)],
}]
# Layout
fig.update_layout(
title='Transverse View Saliency',
width=600,
height=600,
scene=dict(
zaxis=dict(range=[-0.1, nb_frames * 0.1], autorange=False),
aspectratio=dict(x=1, y=1, z=1),
),
updatemenus=[{
"buttons": [
{
"args": [None, frame_args(50)],
"label": "▶", # play symbol
"method": "animate",
},
{
"args": [[None], frame_args(0)],
"label": "◼", # pause symbol
"method": "animate",
},
],
"direction":
"left",
"pad": {
"r": 10,
"t": 70
},
"type":
"buttons",
"x":
0.1,
"y":
0,
}],
sliders=sliders)
fig.show()
def main(model_path, image_file_path):
image_process = ImageProcess()
image = cv2.imread(image_file_path)
# if collected data is not cropped then crop here
# otherwise do not crop.
if Config.data_collection['crop'] is not True:
image = image[Config.data_collection['image_crop_y1']:Config.
data_collection['image_crop_y2'],
Config.data_collection['image_crop_x1']:Config.
data_collection['image_crop_x2']]
image = cv2.resize(image, (Config.neural_net['input_image_width'],
Config.neural_net['input_image_height']))
image = image_process.process(image)
drive_run = DriveRun(model_path)
measurement = drive_run.run((image, ))
""" grad modifier doesn't work somehow
fig, axs = plt.subplots(1, 3)
fig.suptitle('Saliency Visualization' + str(measurement))
titles = ['left steering', 'right steering', 'maintain steering']
modifiers = [None, 'negate', 'small_values']
for i, modifier in enumerate(modifiers):
layer_idx = utils.find_layer_idx(drive_run.net_model.model, 'conv2d_last')
heatmap = visualize_cam(drive_run.net_model.model, layer_idx,
filter_indices=None, seed_input=image, backprop_modifier='guided',
grad_modifier=modifier)
axs[i].set(title = titles[i])
axs[i].imshow(image)
axs[i].imshow(heatmap, cmap='jet', alpha=0.3)
"""
plt.figure()
#plt.title('Saliency Visualization' + str(measurement))
plt.title('Steering Angle Prediction: ' + str(measurement[0][0]))
layer_idx = utils.find_layer_idx(drive_run.net_model.model, 'conv2d_last')
heatmap = visualize_cam(drive_run.net_model.model,
layer_idx,
filter_indices=None,
seed_input=image,
backprop_modifier='guided')
plt.imshow(image)
plt.imshow(heatmap, cmap='jet', alpha=0.5)
# file name
loc_slash = image_file_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
image_file_name = image_file_path[loc_slash + 1:]
saliency_file_path = model_path + '_' + image_file_name + '_saliency.png'
saliency_file_path_pdf = model_path + '_' + image_file_name + '_saliency.pdf'
plt.tight_layout()
# save fig
plt.savefig(saliency_file_path, dpi=150)
plt.savefig(saliency_file_path_pdf, dpi=150)
print('Saved ' + saliency_file_path + ' & .pdf')
def vis(model, args):
from vis.utils import utils
from keras import activations
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
# Anyway, we are interested in the last layer, where the prediction happens
layer_idx = utils.find_layer_idx(model, 'predictions')
#To visualize activation over final dense layer outputs, we need to switch the softmax activation out for linear
#since gradient of output node will depend on all the other node activations.
# Swap softmax with linear
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
layer_idx = args.layer_idx
#We define the softmax function to translate the output of the CNN into a probability for each class.
def softmax(x):
"""
Compute softmax values for each sets of scores in x.
Rows are scores for each class.
Columns are predictions (samples).
"""
scoreMatExp = np.exp(np.asarray(x))
return scoreMatExp / scoreMatExp.sum(0)
def predictImage(args):
from os.path import basename
load_count = args.cpc
(x_test, y_test, imgfiles) = load_xray_test(args, load_count)
images = []
outs = []
if not args.noshowpredict:
#plt.figure()
f, ax = plt.subplots(nb_classes, load_count, figsize=(15, 15))
ax = ax.reshape((nb_classes * load_count))
plt.suptitle('predicted classes')
i = 0
for im, real_y, fn in zip(x_test, y_test, imgfiles):
images.append(im)
out = softmax(
model.predict(
im.reshape(-1, args.input_size, args.input_size,
3).astype('float32') / 255.)[0])
print(out)
print(fn)
outs.append(out)
classKey = np.argmax(out)
# Look in the dictionary for the specific term for the image identification.
certainty = out[classKey]
# green to gray
#from skimage.color import rgb2gray
#im=rgb2gray(im)
if not args.noshowpredict:
if len(y_test) > 1:
ax[i].imshow(im / 255.)
ax[i].set_title(
basename(fn) + " pred: " + str(classKey) + '(' +
str(round(certainty * 100, 3)) + '%)')
i += 1
else:
ax.imshow(im / 255.)
ax.set_title(
basename(fn) + " pred: " + str(classKey) + '(' +
str(round(certainty * 100, 3)) + '%)')
return images, outs
images, outs = predictImage(args)
if not args.noshowpredict:
plt.show()
if args.vis == "act" or args.vis == "all":
show_activation(model, layer_idx)
elif args.vis == "sal" or args.vis == "all":
show_saliency(model, layer_idx, images, outs)
elif args.vis == "cam" or args.vis == "all":
show_cam(model, layer_idx, images, outs)
elif args.vis == "salcam" or args.vis == "all":
sal = show_saliency(model, layer_idx, images, outs)
cam = show_cam(model, layer_idx, images, outs)
show_salcam(sal, cam, images, outs)
# Utility to overlay text on image.
img = utils.draw_text(img, 'Filter {}'.format(idx))
new_vis_images.append(img)
# Generate stitched image palette with 5 cols so we get 2 rows.
stitched = utils.stitch_images(new_vis_images, cols=5)
plt.figure()
plt.axis('off')
plt.imshow(stitched)
plt.show()
from vis.visualization import get_num_filters
selected_indices = []
for layer_name in ['block_6_conv_3']:
layer_idx = utils.find_layer_idx(model, layer_name)
# Visualize all filters in this layer.
filters = np.random.permutation(get_num_filters(
model.layers[layer_idx]))[:10]
selected_indices.append(filters)
# Generate input image for each filter.
vis_images = []
for idx in filters:
img = visualize_activation(model, layer_idx, filter_indices=idx)
# Utility to overlay text on image.
img = utils.draw_text(img, 'Filter {}'.format(idx))
vis_images.append(img)
import math
from scipy.ndimage import convolve
import warnings
warnings.filterwarnings('ignore')
# In[2]:
model = load_model('1/weights-improvement-47-0.78.h5')
model.summary()
# In[3]:
layer_idx = utils.find_layer_idx(model, 'dense_1')
model.layers[layer_idx].activation = activations.linear
modelnew = utils.apply_modifications(model)
penultimate_layer = utils.find_layer_idx(modelnew, 'add_12')
# In[90]:
img1 = utils.load_img('1344_1_D_CC_R.png', target_size=(450, 450))
mean = img1.mean()
std = img1.std()
img1 = (img1 - mean) / std
print(img1.shape)
scipy.misc.imsave('1344_1_D_CC_R_450.png', img1)
img1_resized = img1[np.newaxis, ...]
img1_resized = img1_resized[..., np.newaxis]
def get_importance(tab_dir,
fig_dir,
model_info,
label_node,
label,
x_sample,
softmax=False,
save=True,
fn=None,
in_min=0.,
in_max=1.,
act_max_weight=1,
tv_weight=10.,
lp_norm_weight=10.,
max_iter=200,
backprop_modifier=None,
grad_modifier='absolute'):
"""
Args:
tab_dir: dir to save tabular data
fig_dir: dir to save fig data
model_info: output of get_model_data()
label_node:
label: name of label for output files
softmax: was softmax used?
save: save info?
in_min:
in_max:
act_max_weight:
tv_weight:
lp_norm_weight:
max_iter:
x_sample:
backprop_modifier:
grad_modifier:
for others see visualize_activation() from kera-vis
Returns: dataframe, rows are genes, col is importance
"""
# model
# classes = model_info['le'].classes_
#
# print('original class names: ', classes) # check classes
# print('transformed class names: ', model_info['le'].transform(classes))
model = model_info.get('nn')
layer_idx = utils.find_layer_idx(model, 'preds') # layer of interest
if softmax:
model.layers[
layer_idx].activation = activations.linear # swap softmax with linear
model = utils.apply_modifications(model)
filter_idx = label_node # moe of interest, output node we want to maximize.
print('getting saliency')
# print(x_sample.shape)
sals = visualize_saliency(model=model,
layer_idx=layer_idx,
filter_indices=filter_idx,
seed_input=x_sample,
backprop_modifier=backprop_modifier,
grad_modifier=grad_modifier)
sals = sals.reshape(len(model_info.get('gene_ids')))
sals = pd.DataFrame(data=sals,
index=model_info.get('gene_ids'),
columns=['activations'])
if save:
sals.to_csv(os.path.join(tab_dir, fn + '.csv')) # save activations
a = sals.values
plt.figure()
plt.hist(a)
plt.savefig(os.path.join(fig_dir, fn + '.tiff'))
return sals
batch_x = H5_file['data'][:]
H5_file.close()
batch_x = np.transpose(batch_x, (1, 2, 0))
data_input_c[0, :, :, :, 0] = batch_x[:, :, :]
# 保存或加載權重
# pre = model.predict_on_batch(data_input_c)
# model.save('G:\qweqweqweqwe\model.h5')
model.load_weights('G:\qweqweqweqwe\model.h5')
# 查看計算圖
print(model.summary())
# 查询自己要做saliency的評分层的index：即求梯度時候的因變量層，一般選最後一個全連接層
layer_idx = utils.find_layer_idx(model, 'predictions')
# 查詢要可視化的層的index：即加權的特征圖所在的層，也是求梯度時候的自變量，一般選最後一個卷積層或池化層
layer_idx_conv = utils.find_layer_idx(model, 'block2_pool')
# 按照套路，首先將評分層激活函數轉換為線性激活
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
# 開始CAM
# cam方法的激活 ============================================================================================================
# visualize_cam()用法：
# model：即你想可视化的keras模型
# layer_index：The layer index within `model.layers` whose filters needs to be visualized.
# 官方注释如上，按我的理解，一般这个层应该指定为最后一个全连接层，即最终softmax激活层的前一层，
# 这层的神经元数量等于总类别数，并且各个神经元的输出值相当于对应的每个类别的评分（评分经过softmax激活之后
]
xloc = np.arange(25)
yloc = np.arange(23)
yloc = yloc[::-1]
for i in range(len(dtints)):
dtints[i] = round(dtints[i], 3)
yloc = yloc - 0.5
xloc = xloc - 0.5
thres = np.where(prob >= 0.8)
ind_thres = thres[0]
class_thres = thres[1]
#plt.rcParams['figure.figsize'] = (18, 6)
layer_idx = utils.find_layer_idx(model, 'preds')
penultimate_layer_idx = utils.find_layer_idx(model, 'conv2d2')
# os.mkdir("/gdrive/My Drive/workflow/periodic/code/experiments/cnn/keras-vis/grad_CAM/dmdt/")
# os.mkdir("/gdrive/My Drive/workflow/periodic/code/experiments/cnn/keras-vis/grad_CAM/gradcam/")
###os.mkdir("keras-vis/")
###os.mkdir("keras-vis/grad_CAM/")
###os.mkdir("keras-vis/grad_CAM/dmdt/")
###os.mkdir("keras-vis/grad_CAM/gradcam/")
###os.mkdir("keras-vis/saliency/")
for i in range(len(classes)):
# This is the output node we want to maximize.
filter_idx = filterid[classes[i]]
###os.mkdir("keras-vis/grad_CAM/dmdt/"+pd[classes[i]]+"/")
from keras.applications import VGG16
from vis.utils import utils
from keras import activations
from vis.visualization import visualize_activation
from matplotlib import pyplot as plt
from vis.input_modifiers import Jitter
import numpy as np
import os
import cv2
categorias = np.random.permutation(1000)[:15]
model = VGG16(weights='imagenet', include_top=True)
layer_idx = utils.find_layer_idx(
model, 'predictions') # -1 tambien vale (es la ultima)
#Softmax a linear
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
if not os.path.isdir("fc"):
os.mkdir("fc")
for categoria in categorias:
img = visualize_activation(model,
layer_idx,
filter_indices=categoria,
max_iter=500,
input_modifiers=[Jitter(16)])
img = utils.draw_text(img, utils.get_imagenet_label(categoria))
# score = model.evaluate(x_test, y_test, verbose=0)
# print('Test loss:', score)
# print('Test accuracy:', score)
class_idx = 0
indices = np.where(y_test[:, class_idx] == 1.)[0]
# pick some random input from here.
idx = indices[0]
# Lets sanity check the picked image.
cv2.imwrite("test/raw.jpg", x_test[idx][0] * 255.)
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
layer_idx = utils.find_layer_idx(model, 'predictions')
# Swap softmax with linear
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
penultimate_layer = utils.find_layer_idx(model, 'block5_conv3')
for modifier in [None, 'guided', 'relu']:
plt.suptitle("vanilla" if modifier is None else modifier)
for i, img in enumerate([x_test[indices[0]], x_test[indices[1]]]):
grads = visualize_cam(model,
layer_idx,
filter_indices=0,
seed_input=img,
penultimate_layer_idx=penultimate_layer,
IMG_DIR = r'/content/result_images' # img-directory for saving results
'''lower this in case you run into problems (L.75-78)'''
analyzer_batchsize = 32
'''dont change the following unless you have a custom cnn-structure'''
name_softmax_layer = 'predictions' # used for vis-functions
name_last_conv_layer = 'block2_conv2' # used for gradcam-function
layerlist_for_layered_actmax = [
'block1_conv1', 'block1_conv2', 'block1_pool', 'block2_conv1',
'block2_conv2', 'block2_pool', 'fc1', 'fc2'
] # used for layered actmax-function
'''settable parameters end'''
'''loading and reshaping data'''
print('loading data...')
model = keras.models.load_model(model_PATH)
linear_model = keras.models.load_model(model_PATH)
linear_model.layers[utils.find_layer_idx(
linear_model, 'predictions')].activation = keras.activations.linear
linear_model = utils.apply_modifications(linear_model)
X_train = np.load(X_train_PATH)
X_train = X_train / 255
y_train = np.load(y_train_PATH)
X_test = np.load(X_test_PATH)
X_test = X_test / 255
y_test = np.load(y_test_PATH)
print('loaded data...')
y_test = keras.utils.to_categorical(
y_test) # reshapes labels to categorical format
IMG_SIZE = X_test.shape[
1] # sets to image dimensions for plotting equal to those of used data
def saliency_adv(model, data, labels, y_attack, eps_num, adv_all_eps):
from vis.utils import utils
from vis.visualization import visualize_saliency
import scipy.ndimage as ndimage
import os
adv_all_eps = np.asarray(adv_all_eps, dtype='float32')
cwd = os.getcwd()
adversarial_data_predictions = model.predict(adv_all_eps)
#print('predictions on adversarial data')
#print(adversarial_data_predictions)
clean_data_predictions = model.predict(data)
#print('predictions on clean data')
#print(clean_data_predictions)
#most_likely_label = outputs.argmax()
most_likely_label_adv = np.argmax(adversarial_data_predictions, axis=1)
most_likely_label_clean = np.argmax(clean_data_predictions, axis=1)
# get index of classification layer
#model.layers[-1].activation = keras.activations.linear
#layer_index = utils.find_layer_idx(model, model.layers[-1])
layer_index = utils.find_layer_idx(model, 'dense_1')
# Swap softmax with linear
model.layers[layer_index].activation = keras.activations.linear
model = utils.apply_modifications(model)
for i in range(len(eps_num)):
print('epsilon values in saliency function: {}'.format(eps_num))
adv_single_eps = adv_all_eps[i * test_size:test_size * (i + 1)]
attack_labels_single_eps = most_likely_label_adv[i *
test_size:test_size *
(i + 1)]
adv_pred_proba = adversarial_data_predictions[i * test_size:test_size *
(i + 1)]
test_idx = [0, 150]
for idx in test_idx:
#for idx in range(300):
attn_map_clean = visualize_saliency(
model,
layer_index,
filter_indices=most_likely_label_clean[idx],
#filter_indices = None,
seed_input=data[idx])
gaussian_attn_map_clean = ndimage.gaussian_filter(attn_map_clean,
sigma=5)
attn_map_adv = visualize_saliency(
model,
layer_index,
filter_indices=attack_labels_single_eps[idx],
#filter_indices = None,
seed_input=adv_single_eps[idx])
gaussian_attn_map_adv = ndimage.gaussian_filter(attn_map_adv,
sigma=5)
if (attack_labels_single_eps[idx] == 0):
predicted_class_name_adv = 'Normal'
elif (attack_labels_single_eps[idx] == 1):
predicted_class_name_adv = 'Pneumonia'
if (most_likely_label_clean[idx] == 0):
predicted_class_name_clean = 'Normal'
elif (most_likely_label_clean[idx] == 1):
predicted_class_name_clean = 'Pneumonia'
if (labels[idx] == 0):
true_class_name = 'Normal'
elif (labels[idx] == 1):
true_class_name = 'Pneumonia'
if (y_attack[idx] == 0):
target_attack_class = 'Normal'
elif (y_attack[idx] == 1):
target_attack_class = 'Pneumonia'
adversarial_highest_score = adv_pred_proba[idx].max()
print(adv_pred_proba[idx])
adversarial_highest_score = adversarial_highest_score * 100
print('highest accuracy score on adversarial data for index {}'.
format(idx))
adversarial_highest_score = round(adversarial_highest_score, 1)
print(adversarial_highest_score)
clean_highest_score = clean_data_predictions[idx].max()
print(clean_data_predictions[idx])
clean_highest_score = clean_highest_score * 100
print(
'highest predicted accuracy score on clean data for index {}'.
format(idx))
clean_highest_score = round(clean_highest_score, 1)
print(clean_highest_score)
if (idx == 0) | (idx == 150):
fig = plt.figure(figsize=(12, 12))
fig.suptitle('True Class: {}\nEpsilon: {}'.format(
true_class_name, eps_num[i]),
size=25)
gs1 = gridspec.GridSpec(2, 2)
gs1.update(wspace=0.1, hspace=0.1)
ax1 = plt.subplot(gs1[0, 0])
ax1.set_title('Clean: {}'.format(predicted_class_name_clean),
size=25)
ax1.imshow(data[idx])
ax1.text(0.04,
0.05,
'{}%'.format(clean_highest_score),
color="white",
size=25,
bbox=dict(facecolor='green', alpha=0.8),
transform=ax1.transAxes,
horizontalalignment='left')
ax2 = plt.subplot(gs1[0, 1])
ax2.set_title('Clean Saliency', size=25)
ax2.imshow(data[idx])
ax2.imshow(gaussian_attn_map_clean, cmap="jet", alpha=.7)
ax3 = plt.subplot(gs1[1, 0])
ax3.set_title(
'Adversarial: {}'.format(predicted_class_name_adv),
size=25)
ax3.imshow(adv_single_eps[idx])
ax3.text(0.04,
0.05,
'{}%'.format(adversarial_highest_score),
color="white",
size=25,
bbox=dict(facecolor='red', alpha=0.8),
transform=ax3.transAxes,
horizontalalignment='left')
#axes[1,0].text(0,0, '{}%'.format(predicted_accuracy), bbox=dict(facecolor='red', alpha=0.7), fontsize=10, horizontalalignment='center', verticalalignment='center', transform = axes[1,0].transAxes)
ax4 = plt.subplot(gs1[1, 1])
ax4.set_title('Adversarial Saliency', size=25)
ax4.imshow(adv_single_eps[idx])
ax4.imshow(gaussian_attn_map_adv, cmap="jet", alpha=.7)
ax1.axis('off')
ax2.axis('off')
ax3.axis('off')
ax4.axis('off')
plt.show()
fig.savefig(cwd +
"/pgd_saliency/pdf/epsilon_{}_index_{}.pdf".format(
eps_num[i], idx))
plt.clf()
return
# # 另外Jitter的參數如果大於1則代表像素個數（或體素個數），如果小於1大於0則代表圖像size的比例（乘以長和寬)
# max_activition_better = visualize_activation(model, layer_idx_fc, filter_indices=1, max_iter=5000, input_modifiers=[Jitter(8)], verbose=True)
# max_activition_better = np.squeeze(max_activition_better, axis=-1)
# for ii in range(16):
# plt.figure()
# max_activition_piece = max_activition_better[:, :, ii]
# plt.imshow(max_activition_piece, cmap=plt.cm.jet)
# plt.show()
#
#
# # 我們還可以指定多各類，這樣最大激活input的觀感就有點像“四不像”，即多各類的物體組成的某種東西，很有趣
# max_activition_better = visualize_activation(model, layer_idx_fc, filter_indices=[0,1], max_iter=500, input_modifiers=[Jitter(16)], verbose=True)
#
#
layer_idx_conv = utils.find_layer_idx(model, 'predictions')
model.layers[layer_idx_conv].activation = activations.linear
model = utils.apply_modifications(model)
filter_indices = 392
# 卷積層的卷積核的最大激活input===========================================================================
# 觀察第一個卷積層的第2個卷積核(filter_indices=1)的最大激活input，這個最大激活input即可表示這個卷積核所關注並提取的特征（人為可理解的）
# data_input_c[:,:,:,0]=max_activition_norm
# max_activition_norm = visualize_activation(model, layer_idx_conv, filter_indices=filter_indices, max_iter=1000, input_modifiers=[Jitter(9)], verbose=True,seed_input=data_input_c)
max_activition_norm = visualize_activation(model,
layer_idx_conv,
filter_indices=filter_indices,
max_iter=5000,
input_modifiers=[Jitter(16)],
verbose=True)
# max_activition_norm = np.squeeze(max_activition_norm, axis=-1)
for ii in range(1):
def _main():
# Load settings or use args or defaults
s = load_settings_or_use_args(FLAGS)
img_mode = s['img_mode']
all_imgs = []
# load image
img = cv2.imread(img_name, cv2.IMREAD_COLOR)
img = cv2.resize(img, out_size, interpolation=cv2.INTER_AREA)
img_s = cv2.resize(img.copy(), (s['img_width'], s['img_height']), interpolation=cv2.INTER_AREA)
#img_s = util.central_image_crop(img_s, s['crop_img_height'], s['crop_img_width'])
# prep array for input
img_out = np.asarray(img_s, dtype=np.float32) * np.float32(1.0 / 255.0)
if img_mode == "rgb":
carry = np.array(img_out)[np.newaxis, ...].astype(np.float32)
else:
img_out = cv2.cvtColor(img_out, cv2.COLOR_BGR2GRAY)
carry = np.array(img_out)[np.newaxis, :, :, np.newaxis]
print(carry.shape)
# Load json and create model
json_model_path = os.path.join(FLAGS.experiment_rootdir, s['model_dir'], s['json_model_fname'])
# Get weights paths
weights_2_load = sorted(glob.glob(os.path.join(FLAGS.experiment_rootdir, s['model_dir'], 'model_weights*')), key=os.path.getmtime)
# prep output directory
outfolder = os.path.join(FLAGS.experiment_rootdir, 'vis_epochs')
del_and_recreate_folder(outfolder)
# iterate through trained models
for i, weights_load_path in enumerate(weights_2_load):
k.clear_session()
model = get_model(json_model_path, weights_load_path)
penultimate_layer = vutils.find_layer_idx(model, 'conv2d_9')
img_out, dur = get_grad_cam(img, carry, model, penultimate_layer)
# make and store predictions
theta, p_t, img_out = make_prediction(img_out, carry, model)
# ouput of results
stats = ["Epoch: {:d}".format(i), "Predictions:", "[C: {:4.3f}] [SA: {:4.3f}]".format(float(p_t), float(theta))]
# place output on image
for idx, stat in enumerate(stats):
text = stat.lstrip()
cv2.putText(img_out, text, (0, 30 + (idx * 30)),
cv2.FONT_HERSHEY_SIMPLEX,
1.0, (255, 255, 255), 2, lineType=30)
# store img
all_imgs.append(img_out.astype(np.uint8))
# show output
cv2.imshow('frame', img_out)
if cv2.waitKey(10) & 0xFF == ord('q'):
break
# prep output video
out_name = os.path.splitext(os.path.split(img_name)[1])[0] + ".mp4"
output_name = os.path.join(outfolder, out_name)
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
video = cv2.VideoWriter(output_name, fourcc, 1, (out_size[0] + 20, out_size[1]))
# write video
for img in all_imgs:
video.write(img)
video.release()
val_loss_values = history_dict['val_loss']
acc_values = history_dict['acc']
val_acc_values = history_dict['val_acc']
f = open(train_info_record + 'AMP_acc_result.txt', 'a')
epochs = range(1, len(history_dict['acc']) + 1)
for i in range(0, len(history_dict['acc'])):
f.write(
'epoch:{}, train_loss:{:.4f}, train_acc:{:.2f}, val_loss:{:.4f}, val_acc:{:.2f}'
.format(i + 1, loss_values[i], acc_values[i], val_loss_values[i],
val_acc_values[i]) + '\n')
_, acc = model.evaluate(X_val, Y_val)
print(acc)
# saliency map
trained_model = load_model(train_info_record + "save_best_model")
layer_idx = utils.find_layer_idx(trained_model, 'prediction')
trained_model.layers[layer_idx].activation = activations.linear
new_model = utils.apply_modifications(trained_model)
indices = np.where(Y_test[:, 0] == 1.)[0]
idx = indices[0]
grads = visualize_saliency(
new_model,
layer_idx,
filter_indices=0,
seed_input=X_test[idx],
backprop_modifier="guided",
)
print("1")
penultimate_layer = utils.find_layer_idx(new_model, 'conv3')
batch_x = np.transpose(batch_x, (1, 2, 0))
data_input_c[0, :, :, :, 0] = batch_x[:, :, :]
# 保存或加載權重
# pre = model.predict_on_batch(data_input_c)
# model.save('G:\qweqweqweqwe\model.h5')
model.load_weights(
filepath=
'/data/XS_Aug_model_result/model_templete/qianyi/model_qianyi_new/2LEVEL/m_80000_model.h5',
by_name=True)
# 查看計算圖
print(model.summary())
# 查询自己要做saliency的評分层的index：即求梯度時候的因變量層，一般選最後一個全連接層
layer_idx = utils.find_layer_idx(model, 'fc1')
# 查詢要可視化的層的index：即加權的特征圖所在的層，也是求梯度時候的自變量，一般選最後一個卷積層或池化層
layer_idx_conv = utils.find_layer_idx(model, 'L4_block3conv3')
# 按照套路，首先將評分層激活函數轉換為線性激活
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
# 開始CAM
# cam方法的激活 ============================================================================================================
# visualize_cam()用法：
# model：即你想可视化的keras模型
# layer_index：The layer index within `model.layers` whose filters needs to be visualized.
# 官方注释如上，按我的理解，一般这个层应该指定为最后一个全连接层，即最终softmax激活层的前一层，
# 这层的神经元数量等于总类别数，并且各个神经元的输出值相当于对应的每个类别的评分（评分经过softmax激活之后
# 就变成概率值啦）。但是实际可以指定卷积层，这个需要在研究一下代码看看。
plt.figure(figsize=(20, 20)) cnf_matrix = confusion_matrix(validation_generator.classes, y_pred) plt.imshow(cnf_matrix, interpolation='nearest') plt.colorbar() tick_marks = np.arange(len(classes)) _ = plt.xticks(tick_marks, classes, rotation=90) _ = plt.yticks(tick_marks, classes) # In[] from vis.visualization import visualize_activation from vis.utils import utils from keras import activations from matplotlib import pyplot as plt plt.rcParams['figure.figsize'] = (18, 6) # Utility to search for layer index by name. # Alternatively we can specify this as -1 since it corresponds to the last layer. layer_idx = utils.find_layer_idx(model, 'preds') # Swap softmax with linear model.layers[layer_idx].activation = activations.linear model = utils.apply_modifications(model) # This is the output node we want to maximize. filter_idx = 0 img = visualize_activation(model, layer_idx, filter_indices=filter_idx) plt.imshow(img[..., 0])
def get_conv_layer(self,conv_layer_name): conv_layer=utils.find_layer_idx(self.model, conv_layer_name) return conv_layer
import keras
import numpy as np
import cv2
import sys
import os
from vis.utils import utils
from vis.visualization import get_num_filters
if not os.path.exists("single_neuron/block1_conv1"):
os.makedirs("single_neuron/block1_conv1")
model = keras.applications.VGG16()
layer_idx = utils.find_layer_idx(model, 'block1_conv1')
num_filters = get_num_filters(model.layers[layer_idx])
print(model.layers[layer_idx].get_weights()[0].shape)
max_v = np.amax(model.layers[layer_idx].get_weights()[0])
min_v = np.amin(model.layers[layer_idx].get_weights()[0])
print(max_v)
print(min_v)
pesos = model.layers[layer_idx].get_weights()[0].copy()
if min_v < 0:
pesos = pesos + abs(min_v)
max_v = max_v + abs(min_v)
pesos = pesos * (255.0 / max_v)
from keras.applications import ResNet50
from vis.utils import utils
from keras import activations
# Build the ResNet50 network with ImageNet weights
model = ResNet50(weights='imagenet', include_top=True)
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
layer_idx = utils.find_layer_idx(model, 'fc1000')
# Swap softmax with linear
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
from vis.utils import utils
from matplotlib import pyplot as plt
plt.rcParams['figure.figsize'] = (18, 6)
img1 = utils.load_img('../vggnet/images/ouzel1.jpg', target_size=(224, 224))
img2 = utils.load_img('../vggnet/images/ouzel2.jpg', target_size=(224, 224))
f, ax = plt.subplots(1, 2)
ax[0].imshow(img1)
ax[1].imshow(img2)
from vis.visualization import visualize_saliency, overlay
from vis.utils import utils
from keras import activations
topK_ndx = []
imagenet_ndx = [] # indexes into the softmax entries of final layer
for i, name in enumerate(class_names):
ndx = topK_names.index(name)
topK_ndx.append(ndx)
imagenet_ndx.append(np.argwhere(preds[0] == topK_scores[ndx])[0][0])
# 282 = Tiger cat, 242 = Boxer (0 indexed)
img = utils.load_img(img_path, target_size=(224, 224))
N = len(class_names)
fig, ax = plt.subplots(1, N + 1)
ax[0].imshow(img)
ax[0].axis('off')
# Lets overlay the heatmap for each desired class onto original image.
for i in range(N):
ndx = imagenet_ndx[i]
layer_idx = utils.find_layer_idx(model, 'predictions') # final layer
grads = visualize_cam(model,
layer_idx,
filter_indices=ndx,
seed_input=img,
backprop_modifier='guided')
jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255)
ax[i + 1].imshow(overlay(jet_heatmap, img))
ax[i + 1].axis('off')
ax[i + 1].set_title(class_names[i])
plt.show()
plt.savefig(os.path.join('figures', 'grad-cam-keras.pdf'))
import numpy as np
rom matplotlib import pyplot as plt
from vis.utils import utils
from vis.visualization import visualize_cam, overlay
def plot_heatmap(arr, figsize=(15, 15)):
n = arr.shape[0]
fig, ax = plt.subplots(1, n, figsize=figsize)
ax = ax.ravel()
for i in range(n):
ax[i].imshow(arr[i])
ax[i].set_xticks([])
ax[i].set_yticks([])
def attention_dia(idx):
pre_img = x_train[idx]
filter_idx = np.argmax(my_model.predict(pre_img[np.newaxis]))
heatmap = visualize_cam(my_model, layer_idx, filter_indices=filter_idx, seed_input=pre_img, backprop_modifier='guided')
return overlay(churches[idx], heatmap)
num_idx = 5
layer_idx = utils.find_layer_idx(my_model, 'predictions')
dia_idx = random.sample(range(100), 10)
dia_attention_maps = np.array(map(attention_dia, dia_idx))
plot_heatmap(dia_attention_maps)
from matplotlib import pyplot as plt
from vis.input_modifiers import Jitter
model = load_model('my_cifar10_ep10.h5')
model.summary()
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
#layer_idx = utils.find_layer_idx(model, 'preds')
# Swap softmax with linear
plt.rcParams['figure.figsize'] = (50, 50)
# The name of the layer we want to visualize
# You can see this in the model definition.
layer_name = 'preds'
layer_idx = utils.find_layer_idx(model, layer_name)
# Swap softmax with linear
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
# Visualize all filters in this layer.
filters = np.arange(get_num_filters(model.layers[layer_idx]))
#model.layers[layer_idx].activation = activations.linear
#model = utils.apply_modifications(model)
#This is the output node we want to maximize.
#Generate input image for each filter.
#for output_idx in filters[0:10]:
# # Lets turn off verbose output this time to avoid clutter and just see the output.
# img = visualize_activation(model, layer_idx, filter_indices=output_idx, input_range=(0., 1.),input_modifiers=[Jitter(16)])
# plt.figure()
# 1. read the model & pretrained weights
model = read_model("saved/final_model.txt", "saved/final_weights.txt")
# 2. print summary and plot model architecture
print(model.summary())
plot_model(model, to_file='model.png')
# 3. get some data to visualize through activations & normalize
(X_train, y_train), (X_test, y_test_orig) = cifar10.load_data()
X_test = X_test.astype('float32')
X_test = X_test / 255.0
y_test = np_utils.to_categorical(y_test_orig)
# 3. visualize
conv_layer_idx = utils.find_layer_idx(model, "conv2d_1")
conv_layer_idx2 = utils.find_layer_idx(model, "conv2d_2")
final_layer_idx = utils.find_layer_idx(model, "dense_2")
airplanes = get_class_data(X_test, y_test_orig, 0)
ships = get_class_data(X_test, y_test_orig, 8)
inputs = X_test[0:50]
input = ships[0:1]
# first conv layer
v2 = visualize_activations(inputs, model, conv_layer_idx, 32, 32)
v1 = visualize_activations(input, model, conv_layer_idx, 32, 32)
# second conv layer
v4 = visualize_activations(inputs, model, conv_layer_idx2, 10, 16)
v3 = visualize_activations(input, model, conv_layer_idx2, 10, 16)
