def showSaliencyMap(showModel, layerName, showImage, imageSize):
layer_idx = utils.find_layer_idx(showModel, layerName)
showModel.layers[layer_idx].activation = activations.linear
showModel = utils.apply_modifications(showModel)
plt.rcParams['figure.figsize'] = (18, 6)
for modifier in [None, 'guided', 'relu']:
plt.figure()
showImage = np.array(showImage)
f, ax = plt.subplots(1, len(showImage))
plt.suptitle("vanilla" if modifier is None else modifier)
for i, img in enumerate(showImage):
img = np.array(img)
grads = visualize_cam(showModel,
layer_idx,
filter_indices=3,
seed_input=img,
backprop_modifier=modifier)
# overlay the heatmap onto original image.
jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255)
jet_heatmap = np.array(np.reshape(jet_heatmap,
(imageSize, imageSize, 3)),
dtype=np.uint8)
if len(showImage) == 1:
# for single image
ax.imshow(overlay(jet_heatmap, img))
else:
# for multi images
ax[i].imshow(overlay(jet_heatmap, img))
plt.show()
def visualize(img_dir):
#TODO: test this method
print('Loading Model...')
model = keras.models.load_model(MODEL_PATH)
model.layers.pop()
model.add(Activation('linear'))
utils.apply_modifications(model)
for img_path in glob.glob(img_dir + '*.png'):
print(f'Working on {img_path}')
plt.figure()
f, ax = plt.subplots(1, 4)
img = load_img(img_path,
color_mode='grayscale',
target_size=(IMAGE_DIM, IMAGE_DIM),
interpolation='lanczos')
background = img.convert('RGB')
img = img_to_array(img)
saliency_grads = visualize_saliency(model,
-1,
filter_indices=0,
seed_input=img,
backprop_modifier='guided')
ax[0].imshow(background)
ax[1].imshow(saliency_grads, cmap='jet')
cam_grads = visualize_cam(model,
-1,
filter_indices=0,
seed_input=img,
backprop_modifier='guided')
cam_heatmap = np.uint8(cm.jet(cam_grads)[..., :3] * 255)
saliency_heatmap = np.uint8(cm.jet(saliency_grads)[..., :3] * 255)
ax[2].imshow(overlay(saliency_heatmap, img_to_array(background)))
ax[3].imshow(overlay(cam_heatmap, img_to_array(background)))
plt.show()
def visualize_attention(model, img_path, target):
"""Visualize attention for self driving `model`.
Args:
model: The keras model.
img_path: The image path to use as input.
target: One of 'right', 'left', 'same' to indicate whether regression target should 'increase', 'decrease'
or remain 'same'.
"""
img = utils.load_img(img_path, target_size=(FRAME_H, FRAME_W))
# Convert to BGR, create input with batch_size: 1.
bgr_img = utils.bgr2rgb(img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred = model.predict(img_input)[0][0]
print('Predicted {}'.format(pred))
if target == 'right':
t = pred + 1
elif target == 'left':
t = pred - 1
else:
t = pred
# Generate saliency visualization.
saliency = visualize_regression_saliency(model,
layer_idx=-1,
output_indices=0,
targets=t,
seed_input=bgr_img)
saliency = overlay(img, saliency, alpha=0.7)
plt.figure()
plt.axis('off')
plt.title('Saliency to steer: {}'.format(target))
plt.imshow(saliency)
# Generate grad-CAM visualization.
cam = visualize_regression_cam(model,
layer_idx=-1,
output_indices=0,
targets=t,
seed_input=bgr_img,
penultimate_layer_idx=5)
cam = overlay(img, cam, alpha=0.7)
plt.figure()
plt.axis('off')
plt.title('grad-CAM to steer: {}'.format(target))
plt.imshow(cam)
plt.show()
def generate_cam(show=True):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in [
'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg'
]:
seed_img = utils.load_img(path, target_size=(224, 224))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(seed_img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_class_cam(model, layer_idx, [pred_class], bgr_img)
heatmap = overlay(seed_img, heatmap)
if show:
plt.axis('off')
plt.imshow(heatmap)
plt.title('Attention - {}'.format(
utils.get_imagenet_label(pred_class)))
plt.show()
def plot(predictions,
targets,
images,
original_images,
path_info,
prefix=''):
for i in range(len(targets)):
# if i < 140: continue
file_path = '../results/CAM/{}/{}{}'.format(
args['sherpa_trial'], prefix, i)
cam = visualize_cam(model,
len(model.layers) - 1, predictions[i],
images[i][np.newaxis])
plt.subplot(1, 2, 1)
plt.imshow(original_images[i])
plt.title('Original Image')
plt.tight_layout()
plt.axis('off')
plt.subplot(1, 2, 2)
plt.imshow(overlay(cam, (original_images[i]) * 255.))
plt.title('Heat Map')
plt.tight_layout()
plt.axis('off')
plt.suptitle(path_info.iloc[i]['path'].replace(
'/baldig/physicstest/ValleyFeverDogs/', ''))
plt.savefig(file_path)
def model_vis(model):
images = np.asarray(load_anomaly_example(anomaly=False))
input_img = images[0]
plt_img = cv2.cvtColor(input_img, cv2.COLOR_YUV2BGR)
titles = ["Input", "Attention"]
subplot_args = {
'nrows': 1,
'ncols': 2,
'figsize': (9, 3),
'subplot_kw': {
'xticks': [],
'yticks': []
}
}
f, ax = plt.subplots(**subplot_args)
heatmap = visualize_cam(model,
layer_idx=-1,
filter_indices=0,
seed_input=input_img,
grad_modifier=None)
jet_heatmap = np.uint8(cm.jet(heatmap)[..., :3] * 255)
for i, modifier in enumerate(titles):
ax[i].set_title(titles[i], fontsize=14)
ax[0].imshow(plt_img)
ax[1].imshow(overlay(plt_img, jet_heatmap, alpha=0.75))
plt.tight_layout()
plt.show()
def plt_cam(model, img, ax, idx, layer_idx=None):
"""
Plot Class Activation Map(CAM), which represents the activation
at the end of all convolutional layer;
Reference: https://arxiv.org/pdf/1610.02391v1.pdf
Args:
model: Model to plot.
img: Seed image.
ax: Matplotlib axis.
idx: Index of the plot to be shown on the axis.
layer_idx: Index of the layer to plot Grad-CAM. Optional.
Returns: None
"""
pred_layer_idx = vutils.find_layer_idx(model, "predictions")
hmap = vvis.visualize_cam(model,
pred_layer_idx,
filter_indices=None,
seed_input=img)
if img.shape[2] == 1:
input_img = np.stack((img.reshape(img.shape[0:2]), ) * 3, -1)
else:
input_img = img
ax[idx].imshow(vvis.overlay(hmap, input_img))
ax[idx].set_title("Heatmap")
def apply(self, model, image, layer, filter):
# Changer l'activation softmax par linear
model.layers[layer].activation = activations.linear
model = utils.apply_modifications(model)
penultimate_layer_idx = None
if layer == 2 and isinstance(
model.layers[1], (_Conv, _Pooling1D, _Pooling2D, _Pooling3D)):
penultimate_layer_idx = 1
elif layer <= 2 and isinstance(
model.layers[0], (_Conv, _Pooling1D, _Pooling2D, _Pooling3D)):
penultimate_layer_idx = 0
grads = visualize_cam(model,
layer,
filter_indices=filter,
backprop_modifier="guided",
grad_modifier=None,
seed_input=image,
penultimate_layer_idx=penultimate_layer_idx)
shape = model.layers[0].input_shape
if shape[len(shape) - 1] == 1:
img = None
if K.image_data_format() == 'channels_first':
img = np.zeros(shape=(int(model.input.shape[2]),
int(model.input.shape[3]), 3))
else:
img = np.zeros(shape=(int(model.input.shape[1]),
int(model.input.shape[2]), 3))
for i in range(len(image)):
for j in range(len(image[i])):
img[i][j][0] = image[i][j][0]
img[i][j][1] = image[i][j][0]
img[i][j][2] = image[i][j][0]
image = img
grads = overlay(grads, image)
return grads
def plot_multiple_saliency(images,
model,
layer,
filter_idx=None,
backprop_modifier=None,
grad_modifier=None):
fig, ax = plt.subplots(2, len(images), figsize=(4 * len(images), 4))
ax = ax.flatten()
for i, filename in enumerate(images):
image = load_img(filename,
target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
ax[i].imshow(image)
ax[i].axis('off')
for i, filename in enumerate(images):
grad = visualize_saliency(model,
find_layer_idx(model, layer),
filter_idx,
normalize(image),
backprop_modifier=backprop_modifier,
grad_modifier=grad_modifier)
image = load_img(filename,
target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
ax[i + len(images)].imshow(overlay(grad, image))
ax[i + len(images)].axis('off')
return fig
def plotAttention(model, layer_idx, im, im_idx, label, fold,
output_folder_path):
grads = visualize_saliency(
model,
layer_idx,
filter_indices=None,
seed_input=im,
backprop_modifier="guided",
)
jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255)
jet_heatmap = cv2.cvtColor(jet_heatmap, cv2.COLOR_BGR2RGB)
cv2.imwrite(
os.path.join(output_folder_path,
label + "-" + fold + "-sm-" + str(im_idx) + ".png"),
jet_heatmap,
)
grads = visualize_cam(
model,
layer_idx,
filter_indices=None,
seed_input=im,
backprop_modifier="guided",
penultimate_layer_idx=utils.find_layer_idx(model, "block5_pool"),
)
jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255)
jet_heatmap = cv2.cvtColor(jet_heatmap, cv2.COLOR_BGR2RGB)
cv2.imwrite(
os.path.join(output_folder_path,
label + "-" + fold + "-cam-" + str(im_idx) + ".png"),
overlay(jet_heatmap, cv2.cvtColor(im, cv2.COLOR_GRAY2RGB), 0.2),
)
def generate_saliceny_map(show=True):
"""Generates a heatmap indicating the pixels that contributed the most towards
maximizing the filter output. First, the class prediction is determined, then we generate heatmap
to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in ['../resources/ouzel.jpg', '../resources/ouzel_1.jpg']:
seed_img = utils.load_img(path, target_size=(224, 224))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(seed_img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_class_saliency(model, layer_idx, [pred_class],
bgr_img)
heatmap = overlay(seed_img, heatmap)
if show:
plt.axis('off')
plt.imshow(heatmap)
plt.title('Saliency - {}'.format(
utils.get_imagenet_label(pred_class)))
plt.show()
def main():
if (len(sys.argv) != 2):
print('Give the model path.')
return
drive = DriveRun(sys.argv[1])
config = Config()
csv_fname = '/home/mir-alb/Ninad_Thesis/Test/Test.csv'
csv_header = ['image_fname', 'steering_angle']
df = pd.read_csv(csv_fname, names=csv_header, index_col=False)
num_data = len(df)
text = open('/home/mir-lab/Ninad_Thesis/Test/Salient/Salient.txt', 'w+')
bar = ProgressBar()
image_process = ImageProcess()
for i in bar(range(num_data)):
image_name = df.loc[i]['image_fname']
steering = df.loc[i]['steering_angle']
image_path = '/home/mir-lab/Ninad_Thesis/Test/' + image_name + '.jpg'
image = utils.load_img(image_path, target_size=(config.image_size[1],
config.image_size[0]))
image = image_process.process(image)
prediction = drive.run(image)
text.write(str(image_name) + '\t' + str(steering) + '\t' + str(prediction))
modifiers = [None, 'negate', 'small_values']
for i, modifier in enumerate(modifiers):
heatmap = visualize_saliency(drive.net_model.model, layer_idx=-1,
filter_indices=0, seed_input=image,
grad_modifier=modifier, keepdims=True)
final = overlay(image, heatmap, alpha=0.5)
cv2.imwrite('/home/mir-lab/Ninad_Thesis/Test/Salient/' + image_name + '_' + str(i) + '.jpg', final)
def test():
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
layer_idx = utils.find_layer_idx(model, 'predictions')
# Swap softmax with linear
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
plt.rcParams['figure.figsize'] = (18, 6)
img1 = utils.load_img('images/ouzel1.jpg', target_size=(224, 224))
img2 = utils.load_img('images/ouzel2.jpg', target_size=(224, 224))
# f, ax = plt.subplots(1, 2)
# ax[0].imshow(img1)
# ax[1].imshow(img2)
f, ax = plt.subplots(1, 2)
for i, img in enumerate([img1, img2]):
# 20 is the imagenet index corresponding to `ouzel`
# heatmap = saliency.visualize_cam(model, layer_idx, filter_indices=20, seed_input=img,backprop_modifier='guided')
heatmap = saliency.visualize_saliency(model, layer_idx, filter_indices=20, seed_input=img,backprop_modifier=None)
print (np.shape(heatmap))
# Lets overlay the heatmap onto original image.
ax[i].imshow(overlay(heatmap,img))
plt.show()
def visualize_saliency_with_losses(input_tensor, losses, seed_input,original_img, grad_modifier='absolute',save_path=''):
if not os.path.exists(save_path):
os.mkdir(save_path)
opt = Optimizer(input_tensor, losses, norm_grads=False)
grads = opt.minimize(seed_input=seed_input, max_iter=1, grad_modifier=grad_modifier, verbose=False)[1]
# print (np.shape(grads))
channel_idx = 1 if K.image_data_format() == 'channels_first' else -1
grads = np.max(grads, axis=channel_idx)
# if not os.path.exists('./image'):
# os.mkdir('./images')
print (np.shape(grads))
for i in range(np.shape(grads)[1]):
temp_grads = utils.normalize(grads[:,i,...])
# print ('temp_grads',np.shape(temp_grads))
heatmap = np.uint8(cm.jet(temp_grads)[..., :3] * 255)[0]
img = original_img[i,...]
temp = image.array_to_img(overlay(img, heatmap,alpha=0.5))
pil.Image.save(temp,save_path+'overlay{}.jpg'.format(i))
temp = image.array_to_img(heatmap)
pil.Image.save(temp,save_path+'heatmap{}.jpg'.format(i))
temp = image.array_to_img(img)
pil.Image.save(temp,save_path+'original{}.jpg'.format(i))
def run(input_img, model):
# preprocess
img = Image.fromarray(input_img)
img_resized = resize(img)
img_scaled = transform_image(img_resized)
# labels
labels = [
'Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration', 'Mass',
'Nodule', 'Pneumonia', 'Pneumothorax', 'Consolidation', 'Edema',
'Emphysema', 'Fibrosis', 'Pleural_Thickening', 'Hernia'
]
# predict for image
prediction = model.predict(np.array([img_scaled]))[0]
pred_idx = prediction.argmax()
prob = prediction.max()
label = labels[pred_idx]
# create region of interest map
layer_idx = len(model.layers) - 1
cam = visualization.visualize_cam(model, layer_idx, pred_idx,
np.array([img_scaled]))
img_overlay = visualization.overlay(cam, img_resized, .3)
return [label, prob, img_overlay]
def visualize_attention_maps():
model = load_regression_model()
print(model.summary())
plt.figure(figsize=(20, 4))
for ind, img_name in enumerate(SAMPLES):
img, bgr_img = load_image(img_name)
img_input = np.expand_dims(img_to_array(img), axis=0)
pred = model.predict(img_input)[0][0]
print('Predicted {}'.format(pred))
heatmap = visualize_cam(model,
layer_idx=-1,
filter_indices=0,
seed_input=bgr_img,
grad_modifier='small_values')
jet_heatmap = np.uint8(cm.jet(heatmap)[..., :3] * 255)
ax = plt.subplot(2, len(SAMPLES), ind + 1)
plt.imshow(img)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
ax = plt.subplot(2, len(SAMPLES), ind + len(SAMPLES) + 1)
plt.imshow(overlay(img, jet_heatmap, alpha=0.3))
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
# plt.show()
plt.savefig('heatmaps_c3.png', bbox_inches='tight')
def get_guided_cam(model,
img1,
layer_idx=None,
predict_class=None,
penultimate_layer_idx=None,
backprop_modifier='guided'):
if predict_class is None:
prob = model.predict(np.expand_dims(img1, axis=0))
predict_class = np.argmax(prob)
if layer_idx is None:
layer_idx = len(model.layers) - 1
if penultimate_layer_idx is None:
for i in range(len(model.layers) - 1, -1, -1):
if isinstance(model.layers[i], Conv2D) or \
isinstance(model.layers[i], MaxPool2D) or\
isinstance(model.layers[i], SeparableConv2D):
penultimate_layer_idx = i
break
# penultimate_layer_idx = LIBS.find_layer_idx(model, 'mixed10')
grads = visualize_cam(model,
layer_idx,
filter_indices=[predict_class],
seed_input=img1,
penultimate_layer_idx=penultimate_layer_idx,
backprop_modifier=backprop_modifier)
str_uuid = str(uuid.uuid1())
filename = '/tmp/' + str_uuid + '.png'
jet_heatmap = cv2.applyColorMap(np.uint8(255 * grads), cv2.COLORMAP_JET)
cv2.imwrite('1111.png', jet_heatmap)
img1 /= 2.0
img1 += 0.5
img1 *= 255.
img1 = img1.astype(np.uint8)
# x_train /= 255.
# x_train -= 0.5
# x_train *= 2.
img_cam = overlay(jet_heatmap, img1)
cv2.imwrite(filename, img_cam)
# plt.title('Guided-CAM')
# plt.imshow(overlay(jet_heatmap, img1))
# fig = plt.gcf()
# fig.set_size_inches(3, 3)
# fig.savefig(filename, dpi=100)
# plt.savefig('Guided_CAM1.png')
# plt.close()
return filename
def plot_convs_heatmap(self):
layer_idx = -1
self.model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(self.model)
names = []
for layer in self.model.layers:
if isinstance(layer, _Conv):
names.append(layer.name)
pred_class = np.argmax(self.model.predict(self.x))
fig = plt.figure()
for i in range(len(names)):
name = names[i]
print('Calculating heatmap for ', name)
penult_layer_idx = utils.find_layer_idx(model, name)
heatmap = visualize_cam(model, layer_idx, filter_indices=[pred_class], seed_input=self.seed_img,penultimate_layer_idx=penult_layer_idx, backprop_modifier=None)
sp = fig.add_subplot(6, 9, i + 1)
sp.set_title(name,fontsize=7)
sp.imshow(overlay(self.seed_img, heatmap))
sp.get_xaxis().set_visible(False)
sp.get_yaxis().set_visible(False)
plt.show()
def main():
parser = argparse.ArgumentParser(
description='test AD recognition')
parser.add_argument('--input', type=str, required=True, help="path to test data")
parser.add_argument('--model', type=str, required=True, help="path to pre-trained model")
parser.add_argument('--id', type=int, required=True, help="data id")
args = parser.parse_args()
model_dir = os.path.join(os.path.dirname(os.getcwd()), args.model)
data = loaddata(args.input, 'testa.h5')
print('data_shape:{}'.format(data.shape))
print("[INFO] loading pre-trained network...")
json_file = open(model_dir+'AD_3dcnnmodel.json', 'r')
model_json = json_file.read()
json_file.close()
model = model_from_json(model_json)
# load weights into new model
model.load_weights(model_dir+"AD_3dcnnmodel.hd5")
model.summary()
layer_idx = utils.find_layer_idx(model, 'activation_10')
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
grads = visualize_saliency(model, layer_idx, filter_indices=0, seed_input=data[args.id], backprop_modifier='guided', keepdims=True)
volume1 = np.squeeze(data[args.id], axis=0)
volume1 = (volume1 - np.min(volume1)) / (np.max(volume1) - np.min(volume1))
fig, axs = plt.subplots(1, 2, figsize=(16, 10), constrained_layout=True)
axs[1].imshow(volume1[39,:,:], cmap='gray')
plt.show()
import pdb
pdb.set_trace()
volume2 = np.squeeze(grads, axis=0)
vol= overlay(volume1, volume2)
axial_planes = []
for i in range(vol.shape[0]):
axial_planes.append(vol[i,:,:])
# Matplotlib animate heart
Hz = np.zeros([vol.shape[1], vol.shape[2]])
im = ax.imshow(Hz)
def init():
im.set_data(np.zeros(Hz.shape))
return [im]
def animate(i):
im.set_data(axial_planes[i])
im.autoscale()
return [im]
anim = animation.FuncAnimation(fig,
animate,
init_func=init,
frames=len(axial_planes),
interval=100,
blit=True)
plt.show()
def predict_and_visualize(model, indices, results_dir):
f = h5py.File(workdir + 'ibis.hdf5', 'r')
images = f['ibis_t1']
labels = f['qc_label']
filenames = f['filename']
predictions = []
with open(results_dir + 'test_images.csv', 'w') as output_file:
output_writer = csv.writer(output_file)
output_writer.writerow(['Filename', 'Probability'])
for index in indices:
img = images[index, target_size[0]//2, ...][np.newaxis, ..., np.newaxis]
label = labels[index, ...]
prediction = model.predict(img, batch_size=1)
print('probs:', prediction[0])
output_writer.writerow([filenames[index, ...], prediction[0][0], np.argmax(label)])
predictions.append(np.argmax(prediction[0]))
for i, (index, prediction) in enumerate(zip(indices, predictions)):
layer_idx = utils.find_layer_idx(model, 'predictions')
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
grads = visualize_cam(model, layer_idx, filter_indices=prediction, seed_input=img[0, ...], backprop_modifier='guided')
heatmap = np.uint8(cm.jet(grads)[:,:,0,:3]*255)
gray = np.uint8(img[0, :, :, :]*255)
gray3 = np.dstack((gray,)*3)
print('image shape, heatmap shape', gray3.shape, heatmap.shape)
plt.imshow(overlay(heatmap, gray3, alpha=0.25))
actual = np.argmax(labels[index, ...])
if prediction == actual:
decision = '_right_'
else:
decision = '_wrong_'
if actual == 1:
qc_status = 'PASS'
else:
qc_status = 'FAIL'
# filename = qc_status + decision + filenames[index, ...][:-4] + '.png'
filename = str(i) + decision + qc_status + '.png'
plt.axis('off')
plt.savefig(results_dir + filename, bbox_inches='tight')
plt.clf()
f.close()
def plot_multiple_grad_cam(
images,
model,
layer,
penultimate_layer=None,
filter_idx=None,
backprop_modifier=None,
grad_modifier=None,
experts=None,
expert_spacing=0.1,
):
rows = 2
if experts is not None:
rows = 3
fig, ax = plt.subplots(
rows, len(images), figsize=(4 * len(images), 4 * rows))
ax = ax.flatten()
penultimate_layer_idx = None
if penultimate_layer:
penultimate_layer_idx = find_layer_idx(model, penultimate_layer)
for i, filename in enumerate(images):
image = load_img(
filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
ax[i].imshow(image)
ax[i].axis('off')
for i, filename in enumerate(images):
image = load_img(
filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
grad = visualize_cam(
model,
find_layer_idx(model, layer),
filter_idx,
normalize(image),
penultimate_layer_idx=penultimate_layer_idx,
backprop_modifier=backprop_modifier,
grad_modifier=grad_modifier)
ax[i + len(images)].imshow(overlay(grad, image))
ax[i + len(images)].axis('off')
if experts:
for i, filename in enumerate(images):
for j, expert in enumerate(experts):
if i == 0:
message = "expert {}: {}".format(j + 1, expert[i])
ax[i + 2 * len(images)].text(
0.3,
1 - (expert_spacing * j),
message,
horizontalalignment='left',
verticalalignment='center')
else:
message = "{}".format(expert[i])
ax[i + 2 * len(images)].text(
0.5,
1 - (expert_spacing * j),
message,
horizontalalignment='center',
verticalalignment='center')
ax[i + 2 * len(images)].axis('off')
return fig, ax
def reason(model, image, layer, filters):
# Changer l'activation softmax par linear
model.layers[layer].activation = activations.linear
model = apply_modifications(model)
grads = visualize_cam(model, layer, filter_indices=filters, backprop_modifier="guided", grad_modifier=None, seed_input=image)
grads = overlay(grads, image)
return grads
def plot_cam(model, image, class_index, labels):
# TODO: this is a right old mess
layer_indices = [
i for i, l in enumerate(model.layers) if 'conv' in l.name
][1:]
from_index = max(0, class_index - 2)
to_index = min(class_index + 3, len(labels))
num_classes = to_index - from_index
fig, axes = plt.subplots(figsize=(15, 4 * num_classes),
ncols=len(layer_indices),
nrows=num_classes)
class_indices = list(range(from_index, to_index))
class_names = [labels[i] for i in class_indices]
entries = zip(
axes.reshape((1, -1)).squeeze().tolist(),
list(product(class_indices, layer_indices)))
for axis, (_class_index, layer_index) in tqdm_notebook(
list(entries), 'Generating class activation maps'):
grads = visualize_cam(model,
layer_idx=layer_index,
filter_indices=_class_index,
seed_input=image,
backprop_modifier='guided')
jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255)
axis.imshow(overlay(jet_heatmap, image * 255, alpha=.4))
axis.get_xaxis().set_ticks([])
axis.get_yaxis().set_ticks([])
for ax, col in zip(axes[0], [model.layers[i].name for i in layer_indices]):
ax.set_title(col, fontsize=20, y=1.02)
for ax, row in zip(axes[:, 0], class_names):
if row == labels[class_index]:
weight = 'bold'
else:
weight = 'normal'
ax.set_ylabel(row,
rotation=90,
fontsize=20,
labelpad=10,
weight=weight)
fig.suptitle('Label: {}'.format(labels[class_index]), fontsize=22, y=1.02)
plt.tight_layout()
return fig, axes
def on_epoch_end(self, epoch, logs={}):
clear_output(wait = True)
index = 1
print('------------------------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------------------------')
if self.verbose is True:
for image in self.test_img_list:
if self.color_mode == "rgb":
input_tensor = image_to_tensor(image, self.img_height, self.img_width)
elif self.color_mode == "grayscale":
input_tensor = image_to_tensor(image, self.img_height, self.img_width, color_mode="grayscale")
detection = self.model.predict(input_tensor)[0]
layer_idx = utils.find_layer_idx(self.model, 'predictions')
test_label = image.split("/")[-2]
filter_index = self.labels.index(test_label)
print(filter_index)
img1 = utils.load_img(image, target_size=(self.img_height, self.img_width))
grads = visualize_cam(
self.model,
layer_idx,
filter_indices=None,
seed_input=img1,
backprop_modifier='guided'
)
print('\nIndex' + str(index))
print(detection)
a = np.array(detection)
print('Estimation：' + self.labels[a.argmax(0)])
fig = plt.figure(figsize=(10, 5))
ax1 = fig.add_subplot(1, 2, 1)
ax1.tick_params(labelbottom="off",bottom="off")
ax1.tick_params(labelleft="off",left="off")
ax1.set_xticklabels([])
ax1.imshow(overlay(grads, img1))
ax2 = fig.add_subplot(1, 2, 2)
ax2.tick_params(labelbottom="off",bottom="off")
ax2.tick_params(labelleft="off",left="off")
ax2.set_xticklabels([])
ax2.imshow(Image.open(image))
plt.show()
sns.set(style="white", context="talk")
f, ax1 = plt.subplots(1, 1, figsize=(8, 6), sharex=True)
sns.barplot(self.labels, detection, palette="PiYG", ax=ax1)
ax1.set_ylabel("Value")
plt.show()
index = index + 1
if self.now_epoch % 5 == 0 or self.now_epoch == 1:
_index = str(self.now_epoch)
if self.now_epoch < 10:
_index = "0" + _index
self.model.save("./models/"+ self.task_name+"/"+"epoch_"+ _index +".hdf5")
self.now_epoch = self.now_epoch + 1
def overlay_gradcam(fm: FineModel,
image: np.ndarray,
penultimate_layer_idx: int = None,
class_index: int = None):
"""
Penultimate layer does not need to be manually specified for our current models.
"""
gradcam = generate_gradcam(fm,
image,
penultimate_layer_idx,
color=True,
class_index=class_index)
return overlay(gradcam, image)
def _plot(label_id):
print(pd.DataFrame(predictions, columns=label_list))
label_id = int(label_id)
text_label = label_list[label_id]
label_proba = np.round(predictions[0, label_id], 4)
heatmap = vis_func(img, label_id)
plt.figure(figsize=figsize)
plt.subplot(1, 2, 1)
plt.title('label:%s\nscore:%s' % (text_label, label_proba))
plt.imshow(overlay(heatmap, img))
plt.subplot(1, 2, 2)
plt.imshow(img)
plt.show()
def _plot(label_id):
label_id = int(label_id)
text_label = get_pred_text_label(label_id)
label_proba = np.round(predictions[0, label_id], 4)
heatmap = vis_func(img, label_id)
for p in prediction_text:
print(p[1:])
plt.figure(figsize=figsize)
plt.subplot(1, 2, 1)
plt.title('label:%s\nscore:%s' % (text_label, label_proba))
plt.imshow(overlay(heatmap, img))
plt.subplot(1, 2, 2)
plt.imshow(img)
plt.show()
def visualize_heat_map(self):
self.logger.info('Visualizing heat map')
# create folder for saving visualization
save_path = os.path.join(constants.MODEL_DIR, 'Visualization',
self.model_name)
if not os.path.exists(save_path):
os.makedirs(save_path)
# search the last dense layer with the name 'preds'
layer_idx = utils.find_layer_idx(self.model, 'preds')
# Swap softmax with linear
self.model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(self.model)
for class_idx in np.arange(len(constants.CATEGORIES)):
# choose a random image from test data
indices = np.where(self.testY[:, class_idx] == 1.)[0]
idx = random.choice(indices)
f, ax = plt.subplots(1, 4)
ax[0].imshow(self.testX[idx][..., 0])
for i, modifier in enumerate([None, 'guided', 'relu']):
grads = visualize_cam(model,
layer_idx,
filter_indices=None,
seed_input=self.testX[idx],
backprop_modifier=modifier)
# create heat map to overlay on image
jet_heat_map = np.uint8(cm.jet(grads)[..., :3] * 255)
image = np.asarray(self.testX[idx] * 255, np.uint8)
if constants.USE_GRAY_SCALE:
image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
if modifier is None:
modifier = 'vanilla'
ax[i + 1].set_title(modifier)
ax[i + 1].imshow(overlay(jet_heat_map, image))
# save the plot
plot_name = 'heat-map-{}.png'.format(
constants.CATEGORIES[class_idx])
plt.savefig(os.path.join(save_path, plot_name))
plt.show()
def model_vis(model):
images = np.asarray(load_anomaly_example(anomaly=True))
input_img = images[0]
titles = ['right steering', 'left steering', 'maintain steering']
modifiers = [None, 'negate', 'small_values']
for i, modifier in enumerate(modifiers):
heatmap = visualize_cam(model,
layer_idx=-1,
filter_indices=0,
seed_input=input_img,
grad_modifier=modifier)
plt.figure()
plt.title(titles[i])
# Overlay is used to alpha blend heatmap onto img.
jet_heatmap = np.uint8(cm.jet(heatmap)[..., :3] * 255)
plt.imshow(overlay(input_img, jet_heatmap, alpha=0.7))
def plot_last_heatmap(self):
layer_idx = -1
pred_class = np.argmax(self.model.predict(self.x))
print(pred_class)
print(imagenet_utils.decode_predictions(self.model.predict(self.x)))
self.model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(self.model)
names =[]
for layer in self.model.layers:
if isinstance(layer, _Conv):
names.append(layer.name)
penult_layer_idx = utils.find_layer_idx(model, names[-1])
print(names[-1])
heatmap = visualize_cam(model, layer_idx, filter_indices=[pred_class], seed_input=self.seed_img,penultimate_layer_idx=penult_layer_idx, backprop_modifier=None)
plt.imshow(overlay(self.seed_img, heatmap))
plt.show()
