def make_gradcam(image, prediction, model): prediction = prediction.squeeze() if prediction >= 0.5: # highlight the regions contributing towards '1-damaged' classification grad_modifier = None else: # highlight the regions contributing towards '0-non-damaged' classification grad_modifier = 'negate' # squeeze out the 'batch' dimension for keras-vis image = image.squeeze() gradient = visualize_cam(model, layer_idx=-1, filter_indices=0, seed_input=image, backprop_modifier=None, grad_modifier=grad_modifier) image = normalize_tf_image(image) gradient_tmp = np.expand_dims(gradient, axis=2) # print(image.shape, gradient_tmp.shape) image_gradient = np.concatenate((image, gradient_tmp), axis=2) # print(image_gradient.shape) figure, ax = plt.subplots(1, 1) # figure.suptitle('Prediction: {:.2f}'.format(prediction), y=0.8, horizontalalignment='center') ax.imshow(image_gradient) ax.contour(gradient, 3, colors='k') plt.axis('off') ax.get_xaxis().set_visible(False) ax.get_yaxis().set_visible(False) return figure
def visualize_activations(self, stacked_shreds): # TODO: hard-coded left and right indexes. change if you like. don't want to spend time. if self._comparator.t > 2: left, right = self._comparator.t + 1, self._comparator.t + 2 else: left, right = 0, 1 tensor = ComparatorCNN._prepare_left_right_check( stacked_shreds[0][left], stacked_shreds[0][right]) root = os.path.join(os.path.dirname(__file__), 'class_activation_maps') os.makedirs('class_activation_maps', exist_ok=True) for i, layer in enumerate(self._comparator._model.layers[1:]): try: layer_name = layer.name if 'conv' not in layer_name: continue cam = visualize_cam(self._comparator._model, i, None, tensor) plt.imshow(stacked_shreds[0][left], cmap='gray') plt.imshow(cam, cmap='hot', interpolation='nearest', alpha=0.15) plt.title('#{}:{}'.format(i, layer_name)) current_milli_time = lambda: int(round(time.time() * 1000)) plt.savefig( os.path.join( root, '{}_{}_{}.png'.format(i, layer_name, current_milli_time()))) plt.clf() print(cam) except: print("Exception!")
def run(input_img, model): # preprocess img = Image.fromarray(input_img) img_resized = resize(img) img_scaled = transform_image(img_resized) # labels labels = [ 'Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration', 'Mass', 'Nodule', 'Pneumonia', 'Pneumothorax', 'Consolidation', 'Edema', 'Emphysema', 'Fibrosis', 'Pleural_Thickening', 'Hernia' ] # predict for image prediction = model.predict(np.array([img_scaled]))[0] pred_idx = prediction.argmax() prob = prediction.max() label = labels[pred_idx] # create region of interest map layer_idx = len(model.layers) - 1 cam = visualization.visualize_cam(model, layer_idx, pred_idx, np.array([img_scaled])) img_overlay = visualization.overlay(cam, img_resized, .3) return [label, prob, img_overlay]
def save_img(path, savepath, origimg, typeimg, layeridx): img = load_img(path, target_size=(224,224)) x = img_to_array(img) #numpy array x = x.reshape(x.shape) #adds on dimension for keras model.layers[layeridx].activation = activations.linear if typeimg == 'activation': img = visualize_activation(model, layeridx, 20, x) if typeimg == 'saliency': img = visualize_saliency(model, layeridx, 1, x) if typeimg == 'cam': img = visualize_cam(model, layeridx, 1, x) if not os.path.exists('layer-' + savepath): os.makedirs('layer-' + savepath) if not os.path.exists('image-' + savepath): os.makedirs('image-' + savepath) combined = str(savepath) + '/' + str(origimg) plt.imshow(img) plt.savefig('layer-' + combined, dpi=600)
def get_guided_cam(model, img1, layer_idx=None, predict_class=None, penultimate_layer_idx=None, backprop_modifier='guided'): if predict_class is None: prob = model.predict(np.expand_dims(img1, axis=0)) predict_class = np.argmax(prob) if layer_idx is None: layer_idx = len(model.layers) - 1 if penultimate_layer_idx is None: for i in range(len(model.layers) - 1, -1, -1): if isinstance(model.layers[i], Conv2D) or \ isinstance(model.layers[i], MaxPool2D) or\ isinstance(model.layers[i], SeparableConv2D): penultimate_layer_idx = i break # penultimate_layer_idx = LIBS.find_layer_idx(model, 'mixed10') grads = visualize_cam(model, layer_idx, filter_indices=[predict_class], seed_input=img1, penultimate_layer_idx=penultimate_layer_idx, backprop_modifier=backprop_modifier) str_uuid = str(uuid.uuid1()) filename = '/tmp/' + str_uuid + '.png' jet_heatmap = cv2.applyColorMap(np.uint8(255 * grads), cv2.COLORMAP_JET) cv2.imwrite('1111.png', jet_heatmap) img1 /= 2.0 img1 += 0.5 img1 *= 255. img1 = img1.astype(np.uint8) # x_train /= 255. # x_train -= 0.5 # x_train *= 2. img_cam = overlay(jet_heatmap, img1) cv2.imwrite(filename, img_cam) # plt.title('Guided-CAM') # plt.imshow(overlay(jet_heatmap, img1)) # fig = plt.gcf() # fig.set_size_inches(3, 3) # fig.savefig(filename, dpi=100) # plt.savefig('Guided_CAM1.png') # plt.close() return filename
def generate_cam_from_image(image, model=None, returnAsImage=True, layer=None): """Generates a heatmap via grad-CAM method. First, the class prediction is determined, then we generate heatmap to visualize that class. """ # Build the VGG16 network with ImageNet weights if model is None: model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) if layer is None: layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] else: layer_idx = layer #seed_img = Image.open(image) seed_img = Image.fromarray(image) seed_img = seed_img.resize((configs.img_width, configs.img_height)) # Convert to BGR, create input with batch_size: 1, and predict. bgr_img = utils.bgr2rgb(np.asarray(seed_img)) img_input = np.expand_dims(bgr_img, axis=0) pred_class = np.argmax(model.predict(img_input)) heatmap = visualize_cam(model, layer_idx, [pred_class], np.asarray(seed_img)) if returnAsImage is True: return Image.fromarray(heatmap) else: return heatmap
def generate_gradcam(fm: FineModel, image: np.ndarray, penultimate_layer_idx: int = None, color=True, class_index: int = None): """ Generate gradcam image for given image, classification and class. If `class_index` is unspecified, we run the image through the classifier and use the predicted class. Penultimate layer does not need to be manually specified for our current models. """ if not class_index: class_index = fm.predict(image) preprocess = fm._get_preprocess_func() processed = preprocess(image) gradcam = visualize_cam(fm.get_model(), fm.get_depths()[0], filter_indices=class_index, seed_input=processed, penultimate_layer_idx=penultimate_layer_idx, backprop_modifier=None) if color: gradcam = np.uint8(cm.jet(gradcam)[..., :3] * 255) return gradcam
def visualize(img_dir): #TODO: test this method print('Loading Model...') model = keras.models.load_model(MODEL_PATH) model.layers.pop() model.add(Activation('linear')) utils.apply_modifications(model) for img_path in glob.glob(img_dir + '*.png'): print(f'Working on {img_path}') plt.figure() f, ax = plt.subplots(1, 4) img = load_img(img_path, color_mode='grayscale', target_size=(IMAGE_DIM, IMAGE_DIM), interpolation='lanczos') background = img.convert('RGB') img = img_to_array(img) saliency_grads = visualize_saliency(model, -1, filter_indices=0, seed_input=img, backprop_modifier='guided') ax[0].imshow(background) ax[1].imshow(saliency_grads, cmap='jet') cam_grads = visualize_cam(model, -1, filter_indices=0, seed_input=img, backprop_modifier='guided') cam_heatmap = np.uint8(cm.jet(cam_grads)[..., :3] * 255) saliency_heatmap = np.uint8(cm.jet(saliency_grads)[..., :3] * 255) ax[2].imshow(overlay(saliency_heatmap, img_to_array(background))) ax[3].imshow(overlay(cam_heatmap, img_to_array(background))) plt.show()
def generate_cam(show=True): """Generates a heatmap via grad-CAM method. First, the class prediction is determined, then we generate heatmap to visualize that class. """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] for path in [ 'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg' ]: seed_img = utils.load_img(path, target_size=(224, 224)) pred_class = np.argmax( model.predict(np.array([img_to_array(seed_img)]))) heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img) if show: cv2.imshow( 'Attention - {}'.format(utils.get_imagenet_label(pred_class)), heatmap) cv2.waitKey(0)
def plot_convs_heatmap(self): layer_idx = -1 self.model.layers[layer_idx].activation = activations.linear model = utils.apply_modifications(self.model) names = [] for layer in self.model.layers: if isinstance(layer, _Conv): names.append(layer.name) pred_class = np.argmax(self.model.predict(self.x)) fig = plt.figure() for i in range(len(names)): name = names[i] print('Calculating heatmap for ', name) penult_layer_idx = utils.find_layer_idx(model, name) heatmap = visualize_cam(model, layer_idx, filter_indices=[pred_class], seed_input=self.seed_img,penultimate_layer_idx=penult_layer_idx, backprop_modifier=None) sp = fig.add_subplot(6, 9, i + 1) sp.set_title(name,fontsize=7) sp.imshow(overlay(self.seed_img, heatmap)) sp.get_xaxis().set_visible(False) sp.get_yaxis().set_visible(False) plt.show()
def explain(self, image, target_class): # generate images with removed parts try: grad_cam_res = visualize_cam( model=self.model, layer_idx=self.layer, filter_indices=target_class, seed_input=image) except TypeError: print("GradCam need to have input dimension of the model defined." "Please define it or use other approach.") raise guided_bprop = GuidedBackprop(self.model, output_index=target_class) guided_backprop_res = guided_bprop.get_mask(image) # ignoring negative activations since they come only from the # last layer, since guided backprop does not propagate negative values # through the ReLU guided_backprop_res[guided_backprop_res < 0] = 0 guided_backprop_res = np.sum(guided_backprop_res, axis=2) # mask = np.abs(mask) # values should be between 0 and 1 guided_backprop_res /= guided_backprop_res.max() # pintvise multiplication res = guided_backprop_res * grad_cam_res # values should be between 0 and 1 res /= res.max() return res, None
def showSaliencyMap(showModel, layerName, showImage, imageSize): layer_idx = utils.find_layer_idx(showModel, layerName) showModel.layers[layer_idx].activation = activations.linear showModel = utils.apply_modifications(showModel) plt.rcParams['figure.figsize'] = (18, 6) for modifier in [None, 'guided', 'relu']: plt.figure() showImage = np.array(showImage) f, ax = plt.subplots(1, len(showImage)) plt.suptitle("vanilla" if modifier is None else modifier) for i, img in enumerate(showImage): img = np.array(img) grads = visualize_cam(showModel, layer_idx, filter_indices=3, seed_input=img, backprop_modifier=modifier) # overlay the heatmap onto original image. jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255) jet_heatmap = np.array(np.reshape(jet_heatmap, (imageSize, imageSize, 3)), dtype=np.uint8) if len(showImage) == 1: # for single image ax.imshow(overlay(jet_heatmap, img)) else: # for multi images ax[i].imshow(overlay(jet_heatmap, img)) plt.show()
def visualize_attention_maps(): model = load_regression_model() print(model.summary()) plt.figure(figsize=(20, 4)) for ind, img_name in enumerate(SAMPLES): img, bgr_img = load_image(img_name) img_input = np.expand_dims(img_to_array(img), axis=0) pred = model.predict(img_input)[0][0] print('Predicted {}'.format(pred)) heatmap = visualize_cam(model, layer_idx=-1, filter_indices=0, seed_input=bgr_img, grad_modifier='small_values') jet_heatmap = np.uint8(cm.jet(heatmap)[..., :3] * 255) ax = plt.subplot(2, len(SAMPLES), ind + 1) plt.imshow(img) ax.get_xaxis().set_visible(False) ax.get_yaxis().set_visible(False) ax = plt.subplot(2, len(SAMPLES), ind + len(SAMPLES) + 1) plt.imshow(overlay(img, jet_heatmap, alpha=0.3)) ax.get_xaxis().set_visible(False) ax.get_yaxis().set_visible(False) # plt.show() plt.savefig('heatmaps_c3.png', bbox_inches='tight')
def plot_map(self): # Utility to search for layer index by name. # Alternatively we can specify this as -1 since it corresponds to the last layer. if self.arch == 'vgg16': layer_idx = vis_utils.find_layer_idx(self.model, layer_name='conv5_3') elif self.arch == 'resnet50': layer_idx = vis_utils.find_layer_idx(self.model, layer_name='conv5_3_3x3') elif self.arch == 'senet50': layer_idx = vis_utils.find_layer_idx(self.model, layer_name='conv5_3_3x3') # o también conv5_3_1x1_up ó conv5_3_1x1_down class_idxs_sorted = np.argsort(self.scores.flatten())[::-1] class_idx = class_idxs_sorted[0] seed_input = self.img grad_top = visualize_cam(self.model, layer_idx, class_idx, seed_input, penultimate_layer_idx = layer_idx, backprop_modifier = None, grad_modifier = None) fig, axes = plt.subplots(1, 2, figsize=(14,5)) axes[0].imshow(self._img) axes[1].imshow(self._img) i = axes[1].imshow(grad_top/255., cmap="jet", alpha=0.8) fig.colorbar(i) plt.suptitle("Pr[img|label={}] = {:5.2f}".format( self.inv_classes[class_idx], self.scores[0,class_idx])) plt.show()
def plt_cam(model, img, ax, idx, layer_idx=None): """ Plot Class Activation Map(CAM), which represents the activation at the end of all convolutional layer; Reference: https://arxiv.org/pdf/1610.02391v1.pdf Args: model: Model to plot. img: Seed image. ax: Matplotlib axis. idx: Index of the plot to be shown on the axis. layer_idx: Index of the layer to plot Grad-CAM. Optional. Returns: None """ pred_layer_idx = vutils.find_layer_idx(model, "predictions") hmap = vvis.visualize_cam(model, pred_layer_idx, filter_indices=None, seed_input=img) if img.shape[2] == 1: input_img = np.stack((img.reshape(img.shape[0:2]), ) * 3, -1) else: input_img = img ax[idx].imshow(vvis.overlay(hmap, input_img)) ax[idx].set_title("Heatmap")
def model_vis(model): images = np.asarray(load_anomaly_example(anomaly=False)) input_img = images[0] plt_img = cv2.cvtColor(input_img, cv2.COLOR_YUV2BGR) titles = ["Input", "Attention"] subplot_args = { 'nrows': 1, 'ncols': 2, 'figsize': (9, 3), 'subplot_kw': { 'xticks': [], 'yticks': [] } } f, ax = plt.subplots(**subplot_args) heatmap = visualize_cam(model, layer_idx=-1, filter_indices=0, seed_input=input_img, grad_modifier=None) jet_heatmap = np.uint8(cm.jet(heatmap)[..., :3] * 255) for i, modifier in enumerate(titles): ax[i].set_title(titles[i], fontsize=14) ax[0].imshow(plt_img) ax[1].imshow(overlay(plt_img, jet_heatmap, alpha=0.75)) plt.tight_layout() plt.show()
def generate_cam(show=True): """Generates a heatmap via grad-CAM method. First, the class prediction is determined, then we generate heatmap to visualize that class. """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] for path in [ 'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg' ]: seed_img = utils.load_img(path, target_size=(224, 224)) # Convert to BGR, create input with batch_size: 1, and predict. bgr_img = utils.bgr2rgb(seed_img) img_input = np.expand_dims(img_to_array(bgr_img), axis=0) pred_class = np.argmax(model.predict(img_input)) heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img) if show: plt.axis('off') plt.imshow(heatmap) plt.title('Attention - {}'.format( utils.get_imagenet_label(pred_class))) plt.show()
def anomaly_detection_attention(model, logdir="../tests/ducks.log"): model = cbcNetv2.get_anomaly_inference(model) observation, prediction, anomaly = read_dataset(logdir) writer = imageio.get_writer("../TrainingDataVisualization.mp4", format='mp4', mode='I', fps=60.0) for a_observation, a_gt in zip(observation, anomaly): observation = np.expand_dims(a_observation, axis=0) output = model.predict(observation) heatmap = visualize_cam(model, layer_idx=-1, filter_indices=0, seed_input=a_observation, grad_modifier=None) jet_heatmap = cv2.cvtColor(np.uint8(cm.jet(heatmap)[..., :3] * 255), cv2.COLOR_BGR2RGB) plt_img = cv2.cvtColor(a_observation, cv2.COLOR_YUV2RGB) fin = cv2.addWeighted(plt_img, 0.6, jet_heatmap, 0.4, 0) string_to_show = "Guess: {}, GT: {}".format( round(float(output[0][0]), 2), a_gt) x, y, w, h = 0, 0, 30, 30 # Add text cv2.putText(fin, string_to_show, (x + int(w / 10), y + int(h / 2)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2) cv2.imshow("Output", fin) if cv2.waitKey(1) & 0xFF == ord('q'): break writer.append_data(cv2.cvtColor(fin, cv2.COLOR_RGB2BGR)) writer.close()
def apply(self, model, image, layer, filter): # Changer l'activation softmax par linear model.layers[layer].activation = activations.linear model = utils.apply_modifications(model) penultimate_layer_idx = None if layer == 2 and isinstance( model.layers[1], (_Conv, _Pooling1D, _Pooling2D, _Pooling3D)): penultimate_layer_idx = 1 elif layer <= 2 and isinstance( model.layers[0], (_Conv, _Pooling1D, _Pooling2D, _Pooling3D)): penultimate_layer_idx = 0 grads = visualize_cam(model, layer, filter_indices=filter, backprop_modifier="guided", grad_modifier=None, seed_input=image, penultimate_layer_idx=penultimate_layer_idx) shape = model.layers[0].input_shape if shape[len(shape) - 1] == 1: img = None if K.image_data_format() == 'channels_first': img = np.zeros(shape=(int(model.input.shape[2]), int(model.input.shape[3]), 3)) else: img = np.zeros(shape=(int(model.input.shape[1]), int(model.input.shape[2]), 3)) for i in range(len(image)): for j in range(len(image[i])): img[i][j][0] = image[i][j][0] img[i][j][1] = image[i][j][0] img[i][j][2] = image[i][j][0] image = img grads = overlay(grads, image) return grads
def show_attention_evolution(self, num_epoch, class_name, image_path): if class_name not in self.labels: print("ラベル が間違っているよ" + str(class_name)) class_index = self.labels.index(class_name) two_ep = int(num_epoch / 4) thr_ep = int(num_epoch / 2) four_ep = int(num_epoch * 3 / 4) model_epochs = [1, two_ep, thr_ep, four_ep, num_epoch] for idx, epoch in enumerate(model_epochs): print("epoch" + str(epoch)) self.load_model(epoch, silent=True) layer_idx = vis_utils.find_layer_idx(self.model, 'predictions') # Swap softmax with linear self.model.layers[layer_idx].activation = activations.linear vis_model = vis_utils.apply_modifications(self.model) x = vis_utils.load_img(image_path, target_size=(self.img_width, self.img_height)) grads = visualize_cam(vis_model, layer_idx, filter_indices=class_index, seed_input=x, backprop_modifier="guided") show_img_array(grads) show_img_array(np.squeeze(x))
def plotAttention(model, layer_idx, im, im_idx, label, fold, output_folder_path): grads = visualize_saliency( model, layer_idx, filter_indices=None, seed_input=im, backprop_modifier="guided", ) jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255) jet_heatmap = cv2.cvtColor(jet_heatmap, cv2.COLOR_BGR2RGB) cv2.imwrite( os.path.join(output_folder_path, label + "-" + fold + "-sm-" + str(im_idx) + ".png"), jet_heatmap, ) grads = visualize_cam( model, layer_idx, filter_indices=None, seed_input=im, backprop_modifier="guided", penultimate_layer_idx=utils.find_layer_idx(model, "block5_pool"), ) jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255) jet_heatmap = cv2.cvtColor(jet_heatmap, cv2.COLOR_BGR2RGB) cv2.imwrite( os.path.join(output_folder_path, label + "-" + fold + "-cam-" + str(im_idx) + ".png"), overlay(jet_heatmap, cv2.cvtColor(im, cv2.COLOR_GRAY2RGB), 0.2), )
def view_grad_cam(model, X, prediction_layer_name, cnn_layer_name, filters=None): pred_layer_idx = utils.find_layer_idx(cnn_model, prediction_layer_name) last_cnn_layer_idx = utils.find_layer_idx(cnn_model, cnn_layer_name) grads = visualize_cam( cnn_model, layer_idx=pred_layer_idx, filter_indices= filters, # filter/class indices within the layer to be maximized (None = all filters/classes are visualized). penultimate_layer_idx=last_cnn_layer_idx, seed_input=X, backprop_modifier=None) # the channel-wise mean of the resulting feature map is the heatmap of the class activation heatmap = np.mean(grads, axis=-1) # heatmap post processing for visualisation purposes heatmap = np.maximum(heatmap, 0) heatmap /= np.max(heatmap) plt.matshow(heatmap) plt.show() img2 = cv2.imread(args.test_image) heatmap = cv2.resize(heatmap, (img2.shape[1], img2.shape[0])) heatmap = np.uint8(255 * heatmap) heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET) superimposed_img = np.uint8(heatmap * 0.6 + img2 * 0.4) cv2.imshow('heatmap superimposed on image', superimposed_img) cv2.waitKey(100)
def predict_and_visualize(model, indices, results_dir): f = h5py.File(workdir + 'ibis.hdf5', 'r') images = f['ibis_t1'] labels = f['qc_label'] filenames = f['filename'] predictions = [] with open(results_dir + 'test_images.csv', 'w') as output_file: output_writer = csv.writer(output_file) output_writer.writerow(['Filename', 'Probability']) for index in indices: img = images[index, target_size[0]//2, ...][np.newaxis, ..., np.newaxis] label = labels[index, ...] prediction = model.predict(img, batch_size=1) print('probs:', prediction[0]) output_writer.writerow([filenames[index, ...], prediction[0][0], np.argmax(label)]) predictions.append(np.argmax(prediction[0])) for i, (index, prediction) in enumerate(zip(indices, predictions)): layer_idx = utils.find_layer_idx(model, 'predictions') model.layers[layer_idx].activation = activations.linear model = utils.apply_modifications(model) grads = visualize_cam(model, layer_idx, filter_indices=prediction, seed_input=img[0, ...], backprop_modifier='guided') heatmap = np.uint8(cm.jet(grads)[:,:,0,:3]*255) gray = np.uint8(img[0, :, :, :]*255) gray3 = np.dstack((gray,)*3) print('image shape, heatmap shape', gray3.shape, heatmap.shape) plt.imshow(overlay(heatmap, gray3, alpha=0.25)) actual = np.argmax(labels[index, ...]) if prediction == actual: decision = '_right_' else: decision = '_wrong_' if actual == 1: qc_status = 'PASS' else: qc_status = 'FAIL' # filename = qc_status + decision + filenames[index, ...][:-4] + '.png' filename = str(i) + decision + qc_status + '.png' plt.axis('off') plt.savefig(results_dir + filename, bbox_inches='tight') plt.clf() f.close()
def plot_multiple_grad_cam( images, model, layer, penultimate_layer=None, filter_idx=None, backprop_modifier=None, grad_modifier=None, experts=None, expert_spacing=0.1, ): rows = 2 if experts is not None: rows = 3 fig, ax = plt.subplots( rows, len(images), figsize=(4 * len(images), 4 * rows)) ax = ax.flatten() penultimate_layer_idx = None if penultimate_layer: penultimate_layer_idx = find_layer_idx(model, penultimate_layer) for i, filename in enumerate(images): image = load_img( filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE)) ax[i].imshow(image) ax[i].axis('off') for i, filename in enumerate(images): image = load_img( filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE)) grad = visualize_cam( model, find_layer_idx(model, layer), filter_idx, normalize(image), penultimate_layer_idx=penultimate_layer_idx, backprop_modifier=backprop_modifier, grad_modifier=grad_modifier) ax[i + len(images)].imshow(overlay(grad, image)) ax[i + len(images)].axis('off') if experts: for i, filename in enumerate(images): for j, expert in enumerate(experts): if i == 0: message = "expert {}: {}".format(j + 1, expert[i]) ax[i + 2 * len(images)].text( 0.3, 1 - (expert_spacing * j), message, horizontalalignment='left', verticalalignment='center') else: message = "{}".format(expert[i]) ax[i + 2 * len(images)].text( 0.5, 1 - (expert_spacing * j), message, horizontalalignment='center', verticalalignment='center') ax[i + 2 * len(images)].axis('off') return fig, ax
def cam(model, img, gt=None, nclasses = 2, save_path = None, layer_idx = -1, threshold = 0.5, dice = True, modifier = 'guided'): """ """ model.layers[layer_idx].activation = activations.linear # model = utils.apply_modifications(model) # print(model.summary()) num_layers = sum([model.layers[layer].name.__contains__('conv') for layer in range(1, len(model.layers))]) layer_dice = np.zeros((num_layers, nclasses)) layer_cams = [] counter = 0 for layer in range(1, len(model.layers)): if 'conv' not in model.layers[layer].name: continue nclass_cam = [] if save_path: plt.figure(figsize=(30, 10)) gs = gridspec.GridSpec(1, nclasses) gs.update(wspace=0.025, hspace=0.05) for class_ in range(nclasses): grads_ = visualize_cam(model, layer_idx, filter_indices=class_, penultimate_layer_idx = layer, seed_input = img[None, ...], backprop_modifier = modifier) nclass_cam.append(grads_) if save_path: ax = plt.subplot(gs[class_]) im = ax.imshow(grads_, cmap='jet') ax.set_xticklabels([]) ax.set_yticklabels([]) ax.set_aspect('equal') ax.tick_params(bottom='off', top='off', labelbottom='off' ) if class_ == nclasses: divider = make_axes_locatable(ax) cax = divider.append_axes("right", size="5%", pad=0.2) cb = plt.colorbar(im, ax=ax, cax=cax ) if dice: thresh_image = grads_ > threshold gt_mask = gt == class_ score = (np.sum(thresh_image*gt_mask))*2.0/(np.sum(gt_mask*1. + thresh_image*1.) + 1.e-3) layer_dice[counter][class_ -1] += score counter += 1 if save_path: os.makedirs(save_path, exist_ok = True) plt.savefig(os.path.join(save_path, model.layers[layer].name.replace('/', '_')+'.png'), bbox_inches='tight') layer_cams.append(nclass_cam) if dice: return np.array(layer_cams), layer_dice return np.array(layer_cams)
def visualize_class_activation_map(model, image): image = cv2.resize(image, (320, 160)) heatmap = visualize_cam(model, layer_idx=-1, filter_indices=0, seed_input=image, grad_modifier=None) heatmap = cv2.cvtColor(heatmap, cv2.COLOR_RGB2BGR) heatmap = cv2.addWeighted(image, 1.0, heatmap, 0.5, 0) heatmap = cv2.resize(heatmap, (640, 480)) return heatmap
def reason(model, image, layer, filters): # Changer l'activation softmax par linear model.layers[layer].activation = activations.linear model = apply_modifications(model) grads = visualize_cam(model, layer, filter_indices=filters, backprop_modifier="guided", grad_modifier=None, seed_input=image) grads = overlay(grads, image) return grads
def server_grad_cam(model_no, img_source, pred, preprocess=True, blend_original_image=True): model = dicts_models[model_no]['model_original'] image_size = dicts_models[model_no]['image_size'] if preprocess: img_preprocess = my_preprocess.do_preprocess(img_source, crop_size=384) img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor( img_preprocess, image_shape=(image_size, image_size, 3)) else: img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor( img_source, image_shape=(image_size, image_size, 3)) penultimate_layer = get_last_conv_layer_number(model) layer_idx = len(model.layers) - 1 modifier = 'guided' # [None, 'guided', 'relu'] # too slow grads = visualize_cam(model, layer_idx, filter_indices=[pred], seed_input=img_input, penultimate_layer_idx=penultimate_layer, backprop_modifier=modifier) cam = cv2.applyColorMap(np.uint8(255 * grads), cv2.COLORMAP_JET) if blend_original_image: # Return to BGR [0..255] from the preprocessed image image_original = img_input[0, :] from LIBS.ImgPreprocess.my_image_norm import input_norm_reverse image_original = input_norm_reverse(image_original) image_original = image_original.astype(np.uint8) image_original -= np.min(image_original) image_original = np.minimum(image_original, 255) cam = np.float32(cam) + np.float32(image_original) cam = 255 * cam / np.max(cam) # 传过来的是web目录 str_uuid = str(uuid.uuid1()) filename_CAM = os.path.join(BASE_DIR_SAVE, str_uuid, 'Grad_CAM{}.jpg'.format(pred)) if not os.path.exists(os.path.dirname(filename_CAM)): os.makedirs(os.path.dirname(filename_CAM)) cv2.imwrite(filename_CAM, cam) return filename_CAM
def detect(self, filename, answer_text, verbose=False, layer_index=-1, mode="local"): if mode == "local": input_tensor = image_to_tensor(filename, self.img_height, self.img_width) else: # if True: try: input_tensor = image_to_tensor(filename, self.img_height, self.img_width) except: raise ValueError("This URL is not supported!! Use other one.") detection = self.model.predict(input_tensor)[0] a = np.array(detection) detect_label = self.labels[a.argmax(0)] if verbose is True: print("結果 .... " + str(answer_text[detect_label])) img1 = vis_utils.load_img(filename, target_size=(self.img_height, self.img_width)) # Swap softmax with linear layer_idx = vis_utils.find_layer_idx(self.model, 'predictions') self.model.layers[layer_idx].activation = activations.linear vis_model = vis_utils.apply_modifications(self.model) filter_index = a.argmax(0) grads = visualize_cam( vis_model, layer_idx, filter_index, #クラス番号 img1[:, :, :], backprop_modifier='guided') a = np.array(detection) fig = plt.figure(figsize=(10, 5)) ax1 = fig.add_subplot(1, 2, 1) ax1.tick_params(labelbottom="off", bottom="off") ax1.grid(False) ax1.tick_params(labelleft="off", left=False) plt.yticks(color="None") ax1.set_xticklabels([]) ax1.imshow(img1) ax1.imshow(grads, cmap='jet', alpha=0.6) ax1.set_title("Heat Map") sns.set(style="white", context="talk") f, ax1 = plt.subplots(1, 1, figsize=(8, 6), sharex=True) sns.barplot(self.labels, detection, palette="PiYG", ax=ax1) ax1.set_ylabel("Value") plt.tick_params(length=0) plt.grid(False) plt.show() else: print(detect_label) print(detection) print("detectメソッドが完了しました.")
def grad_cam(self,model_input,kcc_id): final_layer=-1 grad_top = visualize_cam(self.model, final_layer,kcc_id, model_input, penultimate_layer_idx = self.conv_layer,#None, backprop_modifier = None, grad_modifier = None) return grad_top