def showSaliencyMap(showModel, layerName, showImage, imageSize): layer_idx = utils.find_layer_idx(showModel, layerName) showModel.layers[layer_idx].activation = activations.linear showModel = utils.apply_modifications(showModel) plt.rcParams['figure.figsize'] = (18, 6) for modifier in [None, 'guided', 'relu']: plt.figure() showImage = np.array(showImage) f, ax = plt.subplots(1, len(showImage)) plt.suptitle("vanilla" if modifier is None else modifier) for i, img in enumerate(showImage): img = np.array(img) grads = visualize_cam(showModel, layer_idx, filter_indices=3, seed_input=img, backprop_modifier=modifier) # overlay the heatmap onto original image. jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255) jet_heatmap = np.array(np.reshape(jet_heatmap, (imageSize, imageSize, 3)), dtype=np.uint8) if len(showImage) == 1: # for single image ax.imshow(overlay(jet_heatmap, img)) else: # for multi images ax[i].imshow(overlay(jet_heatmap, img)) plt.show()
def visualize(img_dir): #TODO: test this method print('Loading Model...') model = keras.models.load_model(MODEL_PATH) model.layers.pop() model.add(Activation('linear')) utils.apply_modifications(model) for img_path in glob.glob(img_dir + '*.png'): print(f'Working on {img_path}') plt.figure() f, ax = plt.subplots(1, 4) img = load_img(img_path, color_mode='grayscale', target_size=(IMAGE_DIM, IMAGE_DIM), interpolation='lanczos') background = img.convert('RGB') img = img_to_array(img) saliency_grads = visualize_saliency(model, -1, filter_indices=0, seed_input=img, backprop_modifier='guided') ax[0].imshow(background) ax[1].imshow(saliency_grads, cmap='jet') cam_grads = visualize_cam(model, -1, filter_indices=0, seed_input=img, backprop_modifier='guided') cam_heatmap = np.uint8(cm.jet(cam_grads)[..., :3] * 255) saliency_heatmap = np.uint8(cm.jet(saliency_grads)[..., :3] * 255) ax[2].imshow(overlay(saliency_heatmap, img_to_array(background))) ax[3].imshow(overlay(cam_heatmap, img_to_array(background))) plt.show()
def visualize_attention(model, img_path, target): """Visualize attention for self driving `model`. Args: model: The keras model. img_path: The image path to use as input. target: One of 'right', 'left', 'same' to indicate whether regression target should 'increase', 'decrease' or remain 'same'. """ img = utils.load_img(img_path, target_size=(FRAME_H, FRAME_W)) # Convert to BGR, create input with batch_size: 1. bgr_img = utils.bgr2rgb(img) img_input = np.expand_dims(img_to_array(bgr_img), axis=0) pred = model.predict(img_input)[0][0] print('Predicted {}'.format(pred)) if target == 'right': t = pred + 1 elif target == 'left': t = pred - 1 else: t = pred # Generate saliency visualization. saliency = visualize_regression_saliency(model, layer_idx=-1, output_indices=0, targets=t, seed_input=bgr_img) saliency = overlay(img, saliency, alpha=0.7) plt.figure() plt.axis('off') plt.title('Saliency to steer: {}'.format(target)) plt.imshow(saliency) # Generate grad-CAM visualization. cam = visualize_regression_cam(model, layer_idx=-1, output_indices=0, targets=t, seed_input=bgr_img, penultimate_layer_idx=5) cam = overlay(img, cam, alpha=0.7) plt.figure() plt.axis('off') plt.title('grad-CAM to steer: {}'.format(target)) plt.imshow(cam) plt.show()
def generate_cam(show=True): """Generates a heatmap via grad-CAM method. First, the class prediction is determined, then we generate heatmap to visualize that class. """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] for path in [ 'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg' ]: seed_img = utils.load_img(path, target_size=(224, 224)) # Convert to BGR, create input with batch_size: 1, and predict. bgr_img = utils.bgr2rgb(seed_img) img_input = np.expand_dims(img_to_array(bgr_img), axis=0) pred_class = np.argmax(model.predict(img_input)) heatmap = visualize_class_cam(model, layer_idx, [pred_class], bgr_img) heatmap = overlay(seed_img, heatmap) if show: plt.axis('off') plt.imshow(heatmap) plt.title('Attention - {}'.format( utils.get_imagenet_label(pred_class))) plt.show()
def plot(predictions, targets, images, original_images, path_info, prefix=''): for i in range(len(targets)): # if i < 140: continue file_path = '../results/CAM/{}/{}{}'.format( args['sherpa_trial'], prefix, i) cam = visualize_cam(model, len(model.layers) - 1, predictions[i], images[i][np.newaxis]) plt.subplot(1, 2, 1) plt.imshow(original_images[i]) plt.title('Original Image') plt.tight_layout() plt.axis('off') plt.subplot(1, 2, 2) plt.imshow(overlay(cam, (original_images[i]) * 255.)) plt.title('Heat Map') plt.tight_layout() plt.axis('off') plt.suptitle(path_info.iloc[i]['path'].replace( '/baldig/physicstest/ValleyFeverDogs/', '')) plt.savefig(file_path)
def model_vis(model): images = np.asarray(load_anomaly_example(anomaly=False)) input_img = images[0] plt_img = cv2.cvtColor(input_img, cv2.COLOR_YUV2BGR) titles = ["Input", "Attention"] subplot_args = { 'nrows': 1, 'ncols': 2, 'figsize': (9, 3), 'subplot_kw': { 'xticks': [], 'yticks': [] } } f, ax = plt.subplots(**subplot_args) heatmap = visualize_cam(model, layer_idx=-1, filter_indices=0, seed_input=input_img, grad_modifier=None) jet_heatmap = np.uint8(cm.jet(heatmap)[..., :3] * 255) for i, modifier in enumerate(titles): ax[i].set_title(titles[i], fontsize=14) ax[0].imshow(plt_img) ax[1].imshow(overlay(plt_img, jet_heatmap, alpha=0.75)) plt.tight_layout() plt.show()
def plt_cam(model, img, ax, idx, layer_idx=None): """ Plot Class Activation Map(CAM), which represents the activation at the end of all convolutional layer; Reference: https://arxiv.org/pdf/1610.02391v1.pdf Args: model: Model to plot. img: Seed image. ax: Matplotlib axis. idx: Index of the plot to be shown on the axis. layer_idx: Index of the layer to plot Grad-CAM. Optional. Returns: None """ pred_layer_idx = vutils.find_layer_idx(model, "predictions") hmap = vvis.visualize_cam(model, pred_layer_idx, filter_indices=None, seed_input=img) if img.shape[2] == 1: input_img = np.stack((img.reshape(img.shape[0:2]), ) * 3, -1) else: input_img = img ax[idx].imshow(vvis.overlay(hmap, input_img)) ax[idx].set_title("Heatmap")
def apply(self, model, image, layer, filter): # Changer l'activation softmax par linear model.layers[layer].activation = activations.linear model = utils.apply_modifications(model) penultimate_layer_idx = None if layer == 2 and isinstance( model.layers[1], (_Conv, _Pooling1D, _Pooling2D, _Pooling3D)): penultimate_layer_idx = 1 elif layer <= 2 and isinstance( model.layers[0], (_Conv, _Pooling1D, _Pooling2D, _Pooling3D)): penultimate_layer_idx = 0 grads = visualize_cam(model, layer, filter_indices=filter, backprop_modifier="guided", grad_modifier=None, seed_input=image, penultimate_layer_idx=penultimate_layer_idx) shape = model.layers[0].input_shape if shape[len(shape) - 1] == 1: img = None if K.image_data_format() == 'channels_first': img = np.zeros(shape=(int(model.input.shape[2]), int(model.input.shape[3]), 3)) else: img = np.zeros(shape=(int(model.input.shape[1]), int(model.input.shape[2]), 3)) for i in range(len(image)): for j in range(len(image[i])): img[i][j][0] = image[i][j][0] img[i][j][1] = image[i][j][0] img[i][j][2] = image[i][j][0] image = img grads = overlay(grads, image) return grads
def plot_multiple_saliency(images, model, layer, filter_idx=None, backprop_modifier=None, grad_modifier=None): fig, ax = plt.subplots(2, len(images), figsize=(4 * len(images), 4)) ax = ax.flatten() for i, filename in enumerate(images): image = load_img(filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE)) ax[i].imshow(image) ax[i].axis('off') for i, filename in enumerate(images): grad = visualize_saliency(model, find_layer_idx(model, layer), filter_idx, normalize(image), backprop_modifier=backprop_modifier, grad_modifier=grad_modifier) image = load_img(filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE)) ax[i + len(images)].imshow(overlay(grad, image)) ax[i + len(images)].axis('off') return fig
def plotAttention(model, layer_idx, im, im_idx, label, fold, output_folder_path): grads = visualize_saliency( model, layer_idx, filter_indices=None, seed_input=im, backprop_modifier="guided", ) jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255) jet_heatmap = cv2.cvtColor(jet_heatmap, cv2.COLOR_BGR2RGB) cv2.imwrite( os.path.join(output_folder_path, label + "-" + fold + "-sm-" + str(im_idx) + ".png"), jet_heatmap, ) grads = visualize_cam( model, layer_idx, filter_indices=None, seed_input=im, backprop_modifier="guided", penultimate_layer_idx=utils.find_layer_idx(model, "block5_pool"), ) jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255) jet_heatmap = cv2.cvtColor(jet_heatmap, cv2.COLOR_BGR2RGB) cv2.imwrite( os.path.join(output_folder_path, label + "-" + fold + "-cam-" + str(im_idx) + ".png"), overlay(jet_heatmap, cv2.cvtColor(im, cv2.COLOR_GRAY2RGB), 0.2), )
def generate_saliceny_map(show=True): """Generates a heatmap indicating the pixels that contributed the most towards maximizing the filter output. First, the class prediction is determined, then we generate heatmap to visualize that class. """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] for path in ['../resources/ouzel.jpg', '../resources/ouzel_1.jpg']: seed_img = utils.load_img(path, target_size=(224, 224)) # Convert to BGR, create input with batch_size: 1, and predict. bgr_img = utils.bgr2rgb(seed_img) img_input = np.expand_dims(img_to_array(bgr_img), axis=0) pred_class = np.argmax(model.predict(img_input)) heatmap = visualize_class_saliency(model, layer_idx, [pred_class], bgr_img) heatmap = overlay(seed_img, heatmap) if show: plt.axis('off') plt.imshow(heatmap) plt.title('Saliency - {}'.format( utils.get_imagenet_label(pred_class))) plt.show()
def main(): if (len(sys.argv) != 2): print('Give the model path.') return drive = DriveRun(sys.argv[1]) config = Config() csv_fname = '/home/mir-alb/Ninad_Thesis/Test/Test.csv' csv_header = ['image_fname', 'steering_angle'] df = pd.read_csv(csv_fname, names=csv_header, index_col=False) num_data = len(df) text = open('/home/mir-lab/Ninad_Thesis/Test/Salient/Salient.txt', 'w+') bar = ProgressBar() image_process = ImageProcess() for i in bar(range(num_data)): image_name = df.loc[i]['image_fname'] steering = df.loc[i]['steering_angle'] image_path = '/home/mir-lab/Ninad_Thesis/Test/' + image_name + '.jpg' image = utils.load_img(image_path, target_size=(config.image_size[1], config.image_size[0])) image = image_process.process(image) prediction = drive.run(image) text.write(str(image_name) + '\t' + str(steering) + '\t' + str(prediction)) modifiers = [None, 'negate', 'small_values'] for i, modifier in enumerate(modifiers): heatmap = visualize_saliency(drive.net_model.model, layer_idx=-1, filter_indices=0, seed_input=image, grad_modifier=modifier, keepdims=True) final = overlay(image, heatmap, alpha=0.5) cv2.imwrite('/home/mir-lab/Ninad_Thesis/Test/Salient/' + image_name + '_' + str(i) + '.jpg', final)
def test(): # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) # Utility to search for layer index by name. # Alternatively we can specify this as -1 since it corresponds to the last layer. layer_idx = utils.find_layer_idx(model, 'predictions') # Swap softmax with linear model.layers[layer_idx].activation = activations.linear model = utils.apply_modifications(model) plt.rcParams['figure.figsize'] = (18, 6) img1 = utils.load_img('images/ouzel1.jpg', target_size=(224, 224)) img2 = utils.load_img('images/ouzel2.jpg', target_size=(224, 224)) # f, ax = plt.subplots(1, 2) # ax[0].imshow(img1) # ax[1].imshow(img2) f, ax = plt.subplots(1, 2) for i, img in enumerate([img1, img2]): # 20 is the imagenet index corresponding to `ouzel` # heatmap = saliency.visualize_cam(model, layer_idx, filter_indices=20, seed_input=img,backprop_modifier='guided') heatmap = saliency.visualize_saliency(model, layer_idx, filter_indices=20, seed_input=img,backprop_modifier=None) print (np.shape(heatmap)) # Lets overlay the heatmap onto original image. ax[i].imshow(overlay(heatmap,img)) plt.show()
def visualize_saliency_with_losses(input_tensor, losses, seed_input,original_img, grad_modifier='absolute',save_path=''): if not os.path.exists(save_path): os.mkdir(save_path) opt = Optimizer(input_tensor, losses, norm_grads=False) grads = opt.minimize(seed_input=seed_input, max_iter=1, grad_modifier=grad_modifier, verbose=False)[1] # print (np.shape(grads)) channel_idx = 1 if K.image_data_format() == 'channels_first' else -1 grads = np.max(grads, axis=channel_idx) # if not os.path.exists('./image'): # os.mkdir('./images') print (np.shape(grads)) for i in range(np.shape(grads)[1]): temp_grads = utils.normalize(grads[:,i,...]) # print ('temp_grads',np.shape(temp_grads)) heatmap = np.uint8(cm.jet(temp_grads)[..., :3] * 255)[0] img = original_img[i,...] temp = image.array_to_img(overlay(img, heatmap,alpha=0.5)) pil.Image.save(temp,save_path+'overlay{}.jpg'.format(i)) temp = image.array_to_img(heatmap) pil.Image.save(temp,save_path+'heatmap{}.jpg'.format(i)) temp = image.array_to_img(img) pil.Image.save(temp,save_path+'original{}.jpg'.format(i))
def run(input_img, model): # preprocess img = Image.fromarray(input_img) img_resized = resize(img) img_scaled = transform_image(img_resized) # labels labels = [ 'Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration', 'Mass', 'Nodule', 'Pneumonia', 'Pneumothorax', 'Consolidation', 'Edema', 'Emphysema', 'Fibrosis', 'Pleural_Thickening', 'Hernia' ] # predict for image prediction = model.predict(np.array([img_scaled]))[0] pred_idx = prediction.argmax() prob = prediction.max() label = labels[pred_idx] # create region of interest map layer_idx = len(model.layers) - 1 cam = visualization.visualize_cam(model, layer_idx, pred_idx, np.array([img_scaled])) img_overlay = visualization.overlay(cam, img_resized, .3) return [label, prob, img_overlay]
def visualize_attention_maps(): model = load_regression_model() print(model.summary()) plt.figure(figsize=(20, 4)) for ind, img_name in enumerate(SAMPLES): img, bgr_img = load_image(img_name) img_input = np.expand_dims(img_to_array(img), axis=0) pred = model.predict(img_input)[0][0] print('Predicted {}'.format(pred)) heatmap = visualize_cam(model, layer_idx=-1, filter_indices=0, seed_input=bgr_img, grad_modifier='small_values') jet_heatmap = np.uint8(cm.jet(heatmap)[..., :3] * 255) ax = plt.subplot(2, len(SAMPLES), ind + 1) plt.imshow(img) ax.get_xaxis().set_visible(False) ax.get_yaxis().set_visible(False) ax = plt.subplot(2, len(SAMPLES), ind + len(SAMPLES) + 1) plt.imshow(overlay(img, jet_heatmap, alpha=0.3)) ax.get_xaxis().set_visible(False) ax.get_yaxis().set_visible(False) # plt.show() plt.savefig('heatmaps_c3.png', bbox_inches='tight')
def get_guided_cam(model, img1, layer_idx=None, predict_class=None, penultimate_layer_idx=None, backprop_modifier='guided'): if predict_class is None: prob = model.predict(np.expand_dims(img1, axis=0)) predict_class = np.argmax(prob) if layer_idx is None: layer_idx = len(model.layers) - 1 if penultimate_layer_idx is None: for i in range(len(model.layers) - 1, -1, -1): if isinstance(model.layers[i], Conv2D) or \ isinstance(model.layers[i], MaxPool2D) or\ isinstance(model.layers[i], SeparableConv2D): penultimate_layer_idx = i break # penultimate_layer_idx = LIBS.find_layer_idx(model, 'mixed10') grads = visualize_cam(model, layer_idx, filter_indices=[predict_class], seed_input=img1, penultimate_layer_idx=penultimate_layer_idx, backprop_modifier=backprop_modifier) str_uuid = str(uuid.uuid1()) filename = '/tmp/' + str_uuid + '.png' jet_heatmap = cv2.applyColorMap(np.uint8(255 * grads), cv2.COLORMAP_JET) cv2.imwrite('1111.png', jet_heatmap) img1 /= 2.0 img1 += 0.5 img1 *= 255. img1 = img1.astype(np.uint8) # x_train /= 255. # x_train -= 0.5 # x_train *= 2. img_cam = overlay(jet_heatmap, img1) cv2.imwrite(filename, img_cam) # plt.title('Guided-CAM') # plt.imshow(overlay(jet_heatmap, img1)) # fig = plt.gcf() # fig.set_size_inches(3, 3) # fig.savefig(filename, dpi=100) # plt.savefig('Guided_CAM1.png') # plt.close() return filename
def plot_convs_heatmap(self): layer_idx = -1 self.model.layers[layer_idx].activation = activations.linear model = utils.apply_modifications(self.model) names = [] for layer in self.model.layers: if isinstance(layer, _Conv): names.append(layer.name) pred_class = np.argmax(self.model.predict(self.x)) fig = plt.figure() for i in range(len(names)): name = names[i] print('Calculating heatmap for ', name) penult_layer_idx = utils.find_layer_idx(model, name) heatmap = visualize_cam(model, layer_idx, filter_indices=[pred_class], seed_input=self.seed_img,penultimate_layer_idx=penult_layer_idx, backprop_modifier=None) sp = fig.add_subplot(6, 9, i + 1) sp.set_title(name,fontsize=7) sp.imshow(overlay(self.seed_img, heatmap)) sp.get_xaxis().set_visible(False) sp.get_yaxis().set_visible(False) plt.show()
def main(): parser = argparse.ArgumentParser( description='test AD recognition') parser.add_argument('--input', type=str, required=True, help="path to test data") parser.add_argument('--model', type=str, required=True, help="path to pre-trained model") parser.add_argument('--id', type=int, required=True, help="data id") args = parser.parse_args() model_dir = os.path.join(os.path.dirname(os.getcwd()), args.model) data = loaddata(args.input, 'testa.h5') print('data_shape:{}'.format(data.shape)) print("[INFO] loading pre-trained network...") json_file = open(model_dir+'AD_3dcnnmodel.json', 'r') model_json = json_file.read() json_file.close() model = model_from_json(model_json) # load weights into new model model.load_weights(model_dir+"AD_3dcnnmodel.hd5") model.summary() layer_idx = utils.find_layer_idx(model, 'activation_10') model.layers[layer_idx].activation = activations.linear model = utils.apply_modifications(model) grads = visualize_saliency(model, layer_idx, filter_indices=0, seed_input=data[args.id], backprop_modifier='guided', keepdims=True) volume1 = np.squeeze(data[args.id], axis=0) volume1 = (volume1 - np.min(volume1)) / (np.max(volume1) - np.min(volume1)) fig, axs = plt.subplots(1, 2, figsize=(16, 10), constrained_layout=True) axs[1].imshow(volume1[39,:,:], cmap='gray') plt.show() import pdb pdb.set_trace() volume2 = np.squeeze(grads, axis=0) vol= overlay(volume1, volume2) axial_planes = [] for i in range(vol.shape[0]): axial_planes.append(vol[i,:,:]) # Matplotlib animate heart Hz = np.zeros([vol.shape[1], vol.shape[2]]) im = ax.imshow(Hz) def init(): im.set_data(np.zeros(Hz.shape)) return [im] def animate(i): im.set_data(axial_planes[i]) im.autoscale() return [im] anim = animation.FuncAnimation(fig, animate, init_func=init, frames=len(axial_planes), interval=100, blit=True) plt.show()
def predict_and_visualize(model, indices, results_dir): f = h5py.File(workdir + 'ibis.hdf5', 'r') images = f['ibis_t1'] labels = f['qc_label'] filenames = f['filename'] predictions = [] with open(results_dir + 'test_images.csv', 'w') as output_file: output_writer = csv.writer(output_file) output_writer.writerow(['Filename', 'Probability']) for index in indices: img = images[index, target_size[0]//2, ...][np.newaxis, ..., np.newaxis] label = labels[index, ...] prediction = model.predict(img, batch_size=1) print('probs:', prediction[0]) output_writer.writerow([filenames[index, ...], prediction[0][0], np.argmax(label)]) predictions.append(np.argmax(prediction[0])) for i, (index, prediction) in enumerate(zip(indices, predictions)): layer_idx = utils.find_layer_idx(model, 'predictions') model.layers[layer_idx].activation = activations.linear model = utils.apply_modifications(model) grads = visualize_cam(model, layer_idx, filter_indices=prediction, seed_input=img[0, ...], backprop_modifier='guided') heatmap = np.uint8(cm.jet(grads)[:,:,0,:3]*255) gray = np.uint8(img[0, :, :, :]*255) gray3 = np.dstack((gray,)*3) print('image shape, heatmap shape', gray3.shape, heatmap.shape) plt.imshow(overlay(heatmap, gray3, alpha=0.25)) actual = np.argmax(labels[index, ...]) if prediction == actual: decision = '_right_' else: decision = '_wrong_' if actual == 1: qc_status = 'PASS' else: qc_status = 'FAIL' # filename = qc_status + decision + filenames[index, ...][:-4] + '.png' filename = str(i) + decision + qc_status + '.png' plt.axis('off') plt.savefig(results_dir + filename, bbox_inches='tight') plt.clf() f.close()
def plot_multiple_grad_cam( images, model, layer, penultimate_layer=None, filter_idx=None, backprop_modifier=None, grad_modifier=None, experts=None, expert_spacing=0.1, ): rows = 2 if experts is not None: rows = 3 fig, ax = plt.subplots( rows, len(images), figsize=(4 * len(images), 4 * rows)) ax = ax.flatten() penultimate_layer_idx = None if penultimate_layer: penultimate_layer_idx = find_layer_idx(model, penultimate_layer) for i, filename in enumerate(images): image = load_img( filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE)) ax[i].imshow(image) ax[i].axis('off') for i, filename in enumerate(images): image = load_img( filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE)) grad = visualize_cam( model, find_layer_idx(model, layer), filter_idx, normalize(image), penultimate_layer_idx=penultimate_layer_idx, backprop_modifier=backprop_modifier, grad_modifier=grad_modifier) ax[i + len(images)].imshow(overlay(grad, image)) ax[i + len(images)].axis('off') if experts: for i, filename in enumerate(images): for j, expert in enumerate(experts): if i == 0: message = "expert {}: {}".format(j + 1, expert[i]) ax[i + 2 * len(images)].text( 0.3, 1 - (expert_spacing * j), message, horizontalalignment='left', verticalalignment='center') else: message = "{}".format(expert[i]) ax[i + 2 * len(images)].text( 0.5, 1 - (expert_spacing * j), message, horizontalalignment='center', verticalalignment='center') ax[i + 2 * len(images)].axis('off') return fig, ax
def reason(model, image, layer, filters): # Changer l'activation softmax par linear model.layers[layer].activation = activations.linear model = apply_modifications(model) grads = visualize_cam(model, layer, filter_indices=filters, backprop_modifier="guided", grad_modifier=None, seed_input=image) grads = overlay(grads, image) return grads
def plot_cam(model, image, class_index, labels): # TODO: this is a right old mess layer_indices = [ i for i, l in enumerate(model.layers) if 'conv' in l.name ][1:] from_index = max(0, class_index - 2) to_index = min(class_index + 3, len(labels)) num_classes = to_index - from_index fig, axes = plt.subplots(figsize=(15, 4 * num_classes), ncols=len(layer_indices), nrows=num_classes) class_indices = list(range(from_index, to_index)) class_names = [labels[i] for i in class_indices] entries = zip( axes.reshape((1, -1)).squeeze().tolist(), list(product(class_indices, layer_indices))) for axis, (_class_index, layer_index) in tqdm_notebook( list(entries), 'Generating class activation maps'): grads = visualize_cam(model, layer_idx=layer_index, filter_indices=_class_index, seed_input=image, backprop_modifier='guided') jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255) axis.imshow(overlay(jet_heatmap, image * 255, alpha=.4)) axis.get_xaxis().set_ticks([]) axis.get_yaxis().set_ticks([]) for ax, col in zip(axes[0], [model.layers[i].name for i in layer_indices]): ax.set_title(col, fontsize=20, y=1.02) for ax, row in zip(axes[:, 0], class_names): if row == labels[class_index]: weight = 'bold' else: weight = 'normal' ax.set_ylabel(row, rotation=90, fontsize=20, labelpad=10, weight=weight) fig.suptitle('Label: {}'.format(labels[class_index]), fontsize=22, y=1.02) plt.tight_layout() return fig, axes
def on_epoch_end(self, epoch, logs={}): clear_output(wait = True) index = 1 print('------------------------------------------------------------------------------------------------') print('------------------------------------------------------------------------------------------------') if self.verbose is True: for image in self.test_img_list: if self.color_mode == "rgb": input_tensor = image_to_tensor(image, self.img_height, self.img_width) elif self.color_mode == "grayscale": input_tensor = image_to_tensor(image, self.img_height, self.img_width, color_mode="grayscale") detection = self.model.predict(input_tensor)[0] layer_idx = utils.find_layer_idx(self.model, 'predictions') test_label = image.split("/")[-2] filter_index = self.labels.index(test_label) print(filter_index) img1 = utils.load_img(image, target_size=(self.img_height, self.img_width)) grads = visualize_cam( self.model, layer_idx, filter_indices=None, seed_input=img1, backprop_modifier='guided' ) print('\nIndex' + str(index)) print(detection) a = np.array(detection) print('Estimation:' + self.labels[a.argmax(0)]) fig = plt.figure(figsize=(10, 5)) ax1 = fig.add_subplot(1, 2, 1) ax1.tick_params(labelbottom="off",bottom="off") ax1.tick_params(labelleft="off",left="off") ax1.set_xticklabels([]) ax1.imshow(overlay(grads, img1)) ax2 = fig.add_subplot(1, 2, 2) ax2.tick_params(labelbottom="off",bottom="off") ax2.tick_params(labelleft="off",left="off") ax2.set_xticklabels([]) ax2.imshow(Image.open(image)) plt.show() sns.set(style="white", context="talk") f, ax1 = plt.subplots(1, 1, figsize=(8, 6), sharex=True) sns.barplot(self.labels, detection, palette="PiYG", ax=ax1) ax1.set_ylabel("Value") plt.show() index = index + 1 if self.now_epoch % 5 == 0 or self.now_epoch == 1: _index = str(self.now_epoch) if self.now_epoch < 10: _index = "0" + _index self.model.save("./models/"+ self.task_name+"/"+"epoch_"+ _index +".hdf5") self.now_epoch = self.now_epoch + 1
def overlay_gradcam(fm: FineModel, image: np.ndarray, penultimate_layer_idx: int = None, class_index: int = None): """ Penultimate layer does not need to be manually specified for our current models. """ gradcam = generate_gradcam(fm, image, penultimate_layer_idx, color=True, class_index=class_index) return overlay(gradcam, image)
def _plot(label_id): print(pd.DataFrame(predictions, columns=label_list)) label_id = int(label_id) text_label = label_list[label_id] label_proba = np.round(predictions[0, label_id], 4) heatmap = vis_func(img, label_id) plt.figure(figsize=figsize) plt.subplot(1, 2, 1) plt.title('label:%s\nscore:%s' % (text_label, label_proba)) plt.imshow(overlay(heatmap, img)) plt.subplot(1, 2, 2) plt.imshow(img) plt.show()
def _plot(label_id): label_id = int(label_id) text_label = get_pred_text_label(label_id) label_proba = np.round(predictions[0, label_id], 4) heatmap = vis_func(img, label_id) for p in prediction_text: print(p[1:]) plt.figure(figsize=figsize) plt.subplot(1, 2, 1) plt.title('label:%s\nscore:%s' % (text_label, label_proba)) plt.imshow(overlay(heatmap, img)) plt.subplot(1, 2, 2) plt.imshow(img) plt.show()
def visualize_heat_map(self): self.logger.info('Visualizing heat map') # create folder for saving visualization save_path = os.path.join(constants.MODEL_DIR, 'Visualization', self.model_name) if not os.path.exists(save_path): os.makedirs(save_path) # search the last dense layer with the name 'preds' layer_idx = utils.find_layer_idx(self.model, 'preds') # Swap softmax with linear self.model.layers[layer_idx].activation = activations.linear model = utils.apply_modifications(self.model) for class_idx in np.arange(len(constants.CATEGORIES)): # choose a random image from test data indices = np.where(self.testY[:, class_idx] == 1.)[0] idx = random.choice(indices) f, ax = plt.subplots(1, 4) ax[0].imshow(self.testX[idx][..., 0]) for i, modifier in enumerate([None, 'guided', 'relu']): grads = visualize_cam(model, layer_idx, filter_indices=None, seed_input=self.testX[idx], backprop_modifier=modifier) # create heat map to overlay on image jet_heat_map = np.uint8(cm.jet(grads)[..., :3] * 255) image = np.asarray(self.testX[idx] * 255, np.uint8) if constants.USE_GRAY_SCALE: image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB) if modifier is None: modifier = 'vanilla' ax[i + 1].set_title(modifier) ax[i + 1].imshow(overlay(jet_heat_map, image)) # save the plot plot_name = 'heat-map-{}.png'.format( constants.CATEGORIES[class_idx]) plt.savefig(os.path.join(save_path, plot_name)) plt.show()
def model_vis(model): images = np.asarray(load_anomaly_example(anomaly=True)) input_img = images[0] titles = ['right steering', 'left steering', 'maintain steering'] modifiers = [None, 'negate', 'small_values'] for i, modifier in enumerate(modifiers): heatmap = visualize_cam(model, layer_idx=-1, filter_indices=0, seed_input=input_img, grad_modifier=modifier) plt.figure() plt.title(titles[i]) # Overlay is used to alpha blend heatmap onto img. jet_heatmap = np.uint8(cm.jet(heatmap)[..., :3] * 255) plt.imshow(overlay(input_img, jet_heatmap, alpha=0.7))
def plot_last_heatmap(self): layer_idx = -1 pred_class = np.argmax(self.model.predict(self.x)) print(pred_class) print(imagenet_utils.decode_predictions(self.model.predict(self.x))) self.model.layers[layer_idx].activation = activations.linear model = utils.apply_modifications(self.model) names =[] for layer in self.model.layers: if isinstance(layer, _Conv): names.append(layer.name) penult_layer_idx = utils.find_layer_idx(model, names[-1]) print(names[-1]) heatmap = visualize_cam(model, layer_idx, filter_indices=[pred_class], seed_input=self.seed_img,penultimate_layer_idx=penult_layer_idx, backprop_modifier=None) plt.imshow(overlay(self.seed_img, heatmap)) plt.show()