def test(): # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) # Utility to search for layer index by name. # Alternatively we can specify this as -1 since it corresponds to the last layer. layer_idx = utils.find_layer_idx(model, 'predictions') # Swap softmax with linear model.layers[layer_idx].activation = activations.linear model = utils.apply_modifications(model) plt.rcParams['figure.figsize'] = (18, 6) img1 = utils.load_img('images/ouzel1.jpg', target_size=(224, 224)) img2 = utils.load_img('images/ouzel2.jpg', target_size=(224, 224)) # f, ax = plt.subplots(1, 2) # ax[0].imshow(img1) # ax[1].imshow(img2) f, ax = plt.subplots(1, 2) for i, img in enumerate([img1, img2]): # 20 is the imagenet index corresponding to `ouzel` # heatmap = saliency.visualize_cam(model, layer_idx, filter_indices=20, seed_input=img,backprop_modifier='guided') heatmap = saliency.visualize_saliency(model, layer_idx, filter_indices=20, seed_input=img,backprop_modifier=None) print (np.shape(heatmap)) # Lets overlay the heatmap onto original image. ax[i].imshow(overlay(heatmap,img)) plt.show()
def plot_multiple_saliency(images, model, layer, filter_idx=None, backprop_modifier=None, grad_modifier=None): fig, ax = plt.subplots(2, len(images), figsize=(4 * len(images), 4)) ax = ax.flatten() for i, filename in enumerate(images): image = load_img(filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE)) ax[i].imshow(image) ax[i].axis('off') for i, filename in enumerate(images): grad = visualize_saliency(model, find_layer_idx(model, layer), filter_idx, normalize(image), backprop_modifier=backprop_modifier, grad_modifier=grad_modifier) image = load_img(filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE)) ax[i + len(images)].imshow(overlay(grad, image)) ax[i + len(images)].axis('off') return fig
def plot_multiple_grad_cam( images, model, layer, penultimate_layer=None, filter_idx=None, backprop_modifier=None, grad_modifier=None, experts=None, expert_spacing=0.1, ): rows = 2 if experts is not None: rows = 3 fig, ax = plt.subplots( rows, len(images), figsize=(4 * len(images), 4 * rows)) ax = ax.flatten() penultimate_layer_idx = None if penultimate_layer: penultimate_layer_idx = find_layer_idx(model, penultimate_layer) for i, filename in enumerate(images): image = load_img( filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE)) ax[i].imshow(image) ax[i].axis('off') for i, filename in enumerate(images): image = load_img( filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE)) grad = visualize_cam( model, find_layer_idx(model, layer), filter_idx, normalize(image), penultimate_layer_idx=penultimate_layer_idx, backprop_modifier=backprop_modifier, grad_modifier=grad_modifier) ax[i + len(images)].imshow(overlay(grad, image)) ax[i + len(images)].axis('off') if experts: for i, filename in enumerate(images): for j, expert in enumerate(experts): if i == 0: message = "expert {}: {}".format(j + 1, expert[i]) ax[i + 2 * len(images)].text( 0.3, 1 - (expert_spacing * j), message, horizontalalignment='left', verticalalignment='center') else: message = "{}".format(expert[i]) ax[i + 2 * len(images)].text( 0.5, 1 - (expert_spacing * j), message, horizontalalignment='center', verticalalignment='center') ax[i + 2 * len(images)].axis('off') return fig, ax
def generate_cam(show=True): """Generates a heatmap via grad-CAM method. First, the class prediction is determined, then we generate heatmap to visualize that class. """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] for path in [ 'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg' ]: seed_img = utils.load_img(path, target_size=(224, 224)) pred_class = np.argmax( model.predict(np.array([img_to_array(seed_img)]))) heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img) if show: cv2.imshow( 'Attention - {}'.format(utils.get_imagenet_label(pred_class)), heatmap) cv2.waitKey(0)
def generate_saliceny_map(show=True): """Generates a heatmap indicating the pixels that contributed the most towards maximizing the filter output. First, the class prediction is determined, then we generate heatmap to visualize that class. """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] for path in ['../resources/ouzel.jpg', '../resources/ouzel_1.jpg']: seed_img = utils.load_img(path, target_size=(224, 224)) pred_class = np.argmax( model.predict(np.array([img_to_array(seed_img)]))) heatmap = visualize_saliency(model, layer_idx, [pred_class], seed_img) if show: cv2.imshow( 'Saliency - {}'.format(utils.get_imagenet_label(pred_class)), heatmap) cv2.waitKey(0)
def generate_saliceny_map(show=True): """Generates a heatmap indicating the pixels that contributed the most towards maximizing the filter output. First, the class prediction is determined, then we generate heatmap to visualize that class. """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] for path in ['../resources/ouzel.jpg', '../resources/ouzel_1.jpg']: seed_img = utils.load_img(path, target_size=(224, 224)) # Convert to BGR, create input with batch_size: 1, and predict. bgr_img = utils.bgr2rgb(seed_img) img_input = np.expand_dims(img_to_array(bgr_img), axis=0) pred_class = np.argmax(model.predict(img_input)) heatmap = visualize_saliency(model, layer_idx, [pred_class], seed_img) if show: plt.axis('off') plt.imshow(heatmap) plt.title('Saliency - {}'.format( utils.get_imagenet_label(pred_class))) plt.show()
def show_attention_evolution(self, num_epoch, class_name, image_path): if class_name not in self.labels: print("ラベル が間違っているよ" + str(class_name)) class_index = self.labels.index(class_name) two_ep = int(num_epoch / 4) thr_ep = int(num_epoch / 2) four_ep = int(num_epoch * 3 / 4) model_epochs = [1, two_ep, thr_ep, four_ep, num_epoch] for idx, epoch in enumerate(model_epochs): print("epoch" + str(epoch)) self.load_model(epoch, silent=True) layer_idx = vis_utils.find_layer_idx(self.model, 'predictions') # Swap softmax with linear self.model.layers[layer_idx].activation = activations.linear vis_model = vis_utils.apply_modifications(self.model) x = vis_utils.load_img(image_path, target_size=(self.img_width, self.img_height)) grads = visualize_cam(vis_model, layer_idx, filter_indices=class_index, seed_input=x, backprop_modifier="guided") show_img_array(grads) show_img_array(np.squeeze(x))
def recognition(img, conn, x): Stime = time.time() img = 'upload/' + x + ".jpg" print(img) img1 = utils.load_img(img, target_size=(299, 299)) im_normal2 = img1.reshape(1, img1.shape[0], img1.shape[1], img1.shape[2]).astype('float32') / 255 probabilities = model.predict(im_normal2) predict = np.argmax(probabilities, axis=1) i = predict '''print("class: %s, acc: %.2f" % (list(load_dict.keys())[list(load_dict.values()).index(i)], (probabilities[0][i])))''' data_to_client = { 'class': list(load_dict.keys())[list(load_dict.values()).index(i)], 'acc': (probabilities[0][i]) } conn.send(bytes(data_to_client['class'], encoding="utf8")) print(data_to_client['class'], data_to_client['acc']) #寫入資料庫 inserst(i, img) Etime = time.time() print("spend: %f" % (Etime - Stime) + 's') return
def main(): if (len(sys.argv) != 2): print('Give the model path.') return drive = DriveRun(sys.argv[1]) config = Config() csv_fname = '/home/mir-alb/Ninad_Thesis/Test/Test.csv' csv_header = ['image_fname', 'steering_angle'] df = pd.read_csv(csv_fname, names=csv_header, index_col=False) num_data = len(df) text = open('/home/mir-lab/Ninad_Thesis/Test/Salient/Salient.txt', 'w+') bar = ProgressBar() image_process = ImageProcess() for i in bar(range(num_data)): image_name = df.loc[i]['image_fname'] steering = df.loc[i]['steering_angle'] image_path = '/home/mir-lab/Ninad_Thesis/Test/' + image_name + '.jpg' image = utils.load_img(image_path, target_size=(config.image_size[1], config.image_size[0])) image = image_process.process(image) prediction = drive.run(image) text.write(str(image_name) + '\t' + str(steering) + '\t' + str(prediction)) modifiers = [None, 'negate', 'small_values'] for i, modifier in enumerate(modifiers): heatmap = visualize_saliency(drive.net_model.model, layer_idx=-1, filter_indices=0, seed_input=image, grad_modifier=modifier, keepdims=True) final = overlay(image, heatmap, alpha=0.5) cv2.imwrite('/home/mir-lab/Ninad_Thesis/Test/Salient/' + image_name + '_' + str(i) + '.jpg', final)
def activation_vis(layer_name, overlay_image): # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] # Images corresponding to tiger, penguin, dumbbell, speedboat, spider # image_paths = [ # "http://www.tigerfdn.com/wp-content/uploads/2016/05/How-Much-Does-A-Tiger-Weigh.jpg", # "http://www.slate.com/content/dam/slate/articles/health_and_science/wild_things/2013/10/131025_WILD_AdeliePenguin.jpg.CROP.promo-mediumlarge.jpg", # "https://www.kshs.org/cool2/graphics/dumbbell1lg.jpg", # "http://tampaspeedboatadventures.com/wp-content/uploads/2010/10/DSC07011.jpg", # "http://ichef-1.bbci.co.uk/news/660/cpsprodpb/1C24/production/_85540270_85540265.jpg" # ] # Predict the corresponding class for use in `visualize_saliency`. seed_img = utils.load_img(overlay_image, target_size=target_size) pred_class = np.argmax( model.predict(np.array([img_to_array(seed_img)]))) # Here we are asking it to show attention such that prob of `pred_class` is maximized. heatmap = visualize_saliency(model, layer_idx, [pred_class], seed_img) filename = general_dict["model_definition_id"] + "_" + general_dict["model_training_id"] + \ "_" + layer_name + "_" + overlay_image print "Saving activation map for layer %s overlaid onto image %s" % ( layer_name, overlay_image) imsave(os.path.join(activation_maps_dir, filename), heatmap, 'png')
def generate_cam(show=True): """Generates a heatmap via grad-CAM method. First, the class prediction is determined, then we generate heatmap to visualize that class. """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] for path in [ 'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg' ]: seed_img = utils.load_img(path, target_size=(224, 224)) # Convert to BGR, create input with batch_size: 1, and predict. bgr_img = utils.bgr2rgb(seed_img) img_input = np.expand_dims(img_to_array(bgr_img), axis=0) pred_class = np.argmax(model.predict(img_input)) heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img) if show: plt.axis('off') plt.imshow(heatmap) plt.title('Attention - {}'.format( utils.get_imagenet_label(pred_class))) plt.show()
def load_image(img_name): img = utils.load_img(f'../../scraped_database_tags_new/{img_name}', target_size=IMAGE_SIZE) # plt.imshow(img) bgr_img = utils.bgr2rgb(img) return img, bgr_img
def detect(self, filename, answer_text, verbose=False, layer_index=-1, mode="local"): if mode == "local": input_tensor = image_to_tensor(filename, self.img_height, self.img_width) else: # if True: try: input_tensor = image_to_tensor(filename, self.img_height, self.img_width) except: raise ValueError("This URL is not supported!! Use other one.") detection = self.model.predict(input_tensor)[0] a = np.array(detection) detect_label = self.labels[a.argmax(0)] if verbose is True: print("結果 .... " + str(answer_text[detect_label])) img1 = vis_utils.load_img(filename, target_size=(self.img_height, self.img_width)) # Swap softmax with linear layer_idx = vis_utils.find_layer_idx(self.model, 'predictions') self.model.layers[layer_idx].activation = activations.linear vis_model = vis_utils.apply_modifications(self.model) filter_index = a.argmax(0) grads = visualize_cam( vis_model, layer_idx, filter_index, #クラス番号 img1[:, :, :], backprop_modifier='guided') a = np.array(detection) fig = plt.figure(figsize=(10, 5)) ax1 = fig.add_subplot(1, 2, 1) ax1.tick_params(labelbottom="off", bottom="off") ax1.grid(False) ax1.tick_params(labelleft="off", left=False) plt.yticks(color="None") ax1.set_xticklabels([]) ax1.imshow(img1) ax1.imshow(grads, cmap='jet', alpha=0.6) ax1.set_title("Heat Map") sns.set(style="white", context="talk") f, ax1 = plt.subplots(1, 1, figsize=(8, 6), sharex=True) sns.barplot(self.labels, detection, palette="PiYG", ax=ax1) ax1.set_ylabel("Value") plt.tick_params(length=0) plt.grid(False) plt.show() else: print(detect_label) print(detection) print("detectメソッドが完了しました.")
def check_load_image(model, path): if os.path.exists(path): if K.image_data_format() == 'channels_first': image = utils.load_img(path, target_size=(int(model.input.shape[2]), int(model.input.shape[3]))) image = image.reshape(model.input.shape[1], model.input.shape[2], model.input.shape[3]) elif K.image_data_format() == 'channels_last': image = utils.load_img(path, target_size=(int(model.input.shape[1]), int(model.input.shape[2]))) image = image.reshape(model.input.shape[1], model.input.shape[2], model.input.shape[3]) return image else: print("The path for the image doesn't exist.") sys.exit(1)
def __init__(self,model_path,img_path): K.set_image_data_format('channels_last') self.model_path = model_path self.img_path = img_path self.model = load_model(model_path) self.inv_dico = {v: k for k, v in get_dico().items()} self.seed_img = utils.load_img(img_path, target_size=(224, 224)) x = image.img_to_array(self.seed_img) x = np.expand_dims(x, axis=0) x = preprocess_input(x) self.x = x
def on_epoch_end(self, epoch, logs={}): clear_output(wait = True) index = 1 print('------------------------------------------------------------------------------------------------') print('------------------------------------------------------------------------------------------------') if self.verbose is True: for image in self.test_img_list: if self.color_mode == "rgb": input_tensor = image_to_tensor(image, self.img_height, self.img_width) elif self.color_mode == "grayscale": input_tensor = image_to_tensor(image, self.img_height, self.img_width, color_mode="grayscale") detection = self.model.predict(input_tensor)[0] layer_idx = utils.find_layer_idx(self.model, 'predictions') test_label = image.split("/")[-2] filter_index = self.labels.index(test_label) print(filter_index) img1 = utils.load_img(image, target_size=(self.img_height, self.img_width)) grads = visualize_cam( self.model, layer_idx, filter_indices=None, seed_input=img1, backprop_modifier='guided' ) print('\nIndex' + str(index)) print(detection) a = np.array(detection) print('Estimation:' + self.labels[a.argmax(0)]) fig = plt.figure(figsize=(10, 5)) ax1 = fig.add_subplot(1, 2, 1) ax1.tick_params(labelbottom="off",bottom="off") ax1.tick_params(labelleft="off",left="off") ax1.set_xticklabels([]) ax1.imshow(overlay(grads, img1)) ax2 = fig.add_subplot(1, 2, 2) ax2.tick_params(labelbottom="off",bottom="off") ax2.tick_params(labelleft="off",left="off") ax2.set_xticklabels([]) ax2.imshow(Image.open(image)) plt.show() sns.set(style="white", context="talk") f, ax1 = plt.subplots(1, 1, figsize=(8, 6), sharex=True) sns.barplot(self.labels, detection, palette="PiYG", ax=ax1) ax1.set_ylabel("Value") plt.show() index = index + 1 if self.now_epoch % 5 == 0 or self.now_epoch == 1: _index = str(self.now_epoch) if self.now_epoch < 10: _index = "0" + _index self.model.save("./models/"+ self.task_name+"/"+"epoch_"+ _index +".hdf5") self.now_epoch = self.now_epoch + 1
def visualize_attention(model, img_path, target): """Visualize attention for self driving `model`. Args: model: The keras model. img_path: The image path to use as input. target: One of 'right', 'left', 'same' to indicate whether regression target should 'increase', 'decrease' or remain 'same'. """ img = utils.load_img(img_path, target_size=(FRAME_H, FRAME_W)) # Convert to BGR, create input with batch_size: 1. bgr_img = utils.bgr2rgb(img) img_input = np.expand_dims(img_to_array(bgr_img), axis=0) pred = model.predict(img_input)[0][0] print('Predicted {}'.format(pred)) if target == 'right': t = pred + 1 elif target == 'left': t = pred - 1 else: t = pred # Generate saliency visualization. saliency = visualize_regression_saliency(model, layer_idx=-1, output_indices=0, targets=t, seed_input=bgr_img) saliency = overlay(img, saliency, alpha=0.7) plt.figure() plt.axis('off') plt.title('Saliency to steer: {}'.format(target)) plt.imshow(saliency) # Generate grad-CAM visualization. cam = visualize_regression_cam(model, layer_idx=-1, output_indices=0, targets=t, seed_input=bgr_img, penultimate_layer_idx=5) cam = overlay(img, cam, alpha=0.7) plt.figure() plt.axis('off') plt.title('grad-CAM to steer: {}'.format(target)) plt.imshow(cam) plt.show()
def generate_cam(show=True): """Generates a heatmap via grad-CAM method. First, the class prediction is determined, then we generate heatmap to visualize that class. """ # # Build the VGG16 network with ImageNet weights # model = VGG16(weights='imagenet', include_top=True) # print('Model loaded.') # Build the InceptionV3 network with ImageNet weights # model = InceptionV3(weights='imagenet', include_top=True) # print('Model loaded.') json_file = open('inception_v3_tf_cat_dog_top_layer.json', 'r') loaded_model_json = json_file.read() json_file.close() loaded_model = model_from_json(loaded_model_json) # load weights into new model # loaded_model.load_weights("vgg16_tf_cat_dog_final_dense2.h5") loaded_model.load_weights("inception_v3_tf_cat_dog_top_layer.h5") print("Loaded model from disk") model = loaded_model # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] for path in [ '../resources/tiger.jpg', '../resources/ouzel.jpg', '../resources/ouzel_1.jpg' ]: # seed_img = utils.load_img(path, target_size=(224, 224)) seed_img = utils.load_img(path, target_size=(299, 299)) # Convert to BGR, create input with batch_size: 1, and predict. bgr_img = utils.bgr2rgb(seed_img) img_input = np.expand_dims(img_to_array(bgr_img), axis=0) pred_class = np.argmax(model.predict(img_input)) heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img) if show: plt.axis('off') plt.imshow(heatmap) plt.title('Attention - {}'.format( utils.get_imagenet_label(pred_class))) plt.show()
def plot_grad_cam(image_file, model, layer, filter_idx=None, backprop_modifier="relu"): image = load_img(image_file, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE)) grad = visualize_cam(model, find_layer_idx(model, layer), filter_idx, normalize(image), backprop_modifier=backprop_modifier) fig, ax = plt.subplots(1, 2) ax[0].imshow(overlay(grad, image)) ax[0].axis('off') ax[1].imshow(image) ax[1].axis('off') return fig
def plot(self, vis_func, img_path, label_list, figsize): img = utils.load_img(img_path, target_size=self.img_shape_) img = img[:, :, :3] predictions = self.model_.predict(img2tensor(img, self.img_shape_)) predictions = softmax(predictions) if not label_list: prediction_text = decode_predictions(predictions)[0] def _plot(label_id): label_id = int(label_id) text_label = get_pred_text_label(label_id) label_proba = np.round(predictions[0, label_id], 4) heatmap = vis_func(img, label_id) for p in prediction_text: print(p[1:]) plt.figure(figsize=figsize) plt.subplot(1, 2, 1) plt.title('label:%s\nscore:%s' % (text_label, label_proba)) plt.imshow(overlay(heatmap, img)) plt.subplot(1, 2, 2) plt.imshow(img) plt.show() else: def _plot(label_id): print(pd.DataFrame(predictions, columns=label_list)) label_id = int(label_id) text_label = label_list[label_id] label_proba = np.round(predictions[0, label_id], 4) heatmap = vis_func(img, label_id) plt.figure(figsize=figsize) plt.subplot(1, 2, 1) plt.title('label:%s\nscore:%s' % (text_label, label_proba)) plt.imshow(overlay(heatmap, img)) plt.subplot(1, 2, 2) plt.imshow(img) plt.show() return interact(_plot, label_id='352')
def main(): """Generates a heatmap indicating the pixels that contributed the most towards maximizing the filter output. """ # Build the VGG16 network with ImageNet weights model = VGG16(weights='imagenet', include_top=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'predictions' layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]]) for path in ['../resources/ouzel.jpg', '../resources/ouzel_1.jpg']: seed_img = utils.load_img(path, target_size=(224, 224)) # 20 is the imagenet category for 'ouzel' heatmap = visualize_saliency(model.input, layer_dict[layer_name], [20], seed_img) cv2.imshow('Importance map', heatmap) cv2.waitKey(0)
def segmentation(self, path): self.path = 'model/unet.h5' self.model1 = load_model(self.path, custom_objects={ 'generalized_dice_loss': generalized_dice_loss, 'mean_iou': mean_iou, 'dice_coef': dice_coef }) self.img = utils.load_img(path, target_size=(224, 224)) self.img = image.img_to_array(self.img) self.img = np.expand_dims(self.img, axis=0) self.img = self.img.astype('float32') self.img /= 255. self.img_mask = self.model1.predict(self.img, verbose=1) self.imgs = self.img_mask for i in range(self.imgs.shape[0]): self.img = self.imgs[i] self.img = array_to_img(self.img) self.img.save("image/mask.jpg")
def visualize(): model = load_model('fisheries.h5') path = './input/train/ALB/img_00003.jpg' img = utils.load_img(path, target_size=(dataset.SIZE, dataset.SIZE)) X = dataset.load_image(path) print(X.shape) # print(X[0]) preds = model.predict(X) pred_class = preds.argmax() print('Predicted:' + str(preds)) print('Predicted:' + dataset.TYPES[pred_class]) plt.imshow(img) plt.show() idx = [2, 6, 8, 13] for i in idx: print(model.layers[i]) heatmap = visualize_cam(model, i, [pred_class], img) plt.imshow(heatmap) plt.show()
def heatmap(self, path): # Grad-CAM requires the category of the image as input. So we need predict image to get the label first. self.img = utils.load_img(path, target_size=(224, 224)) self.x = image.img_to_array(self.img) self.x = np.expand_dims(self.x, axis=0) self.y = self.model.predict(self.x, verbose=1) self.y = self.y.argmax(axis=-1) self.label = [ 'dyed-lifted-polyps', 'dyed-resection-margins', 'esophagitis', 'normal-cecum', 'normal-pylorus', 'normal-z-line', 'polyps', 'ulcerative-colitis' ] self.label = np.array(self.label) self.layer_idx = utils.find_layer_idx(self.model, 'dense_1') # Swap softmax with linear self.model.layers[self.layer_idx].activation = activations.linear self.model = utils.apply_modifications(self.model) for modifier in ['guided']: f, ax = plt.subplots(1, 1) plt.suptitle(self.label[self.y]) # Model, layer id, class ,image as input. self.grads = visualize_cam(self.model, self.layer_idx, filter_indices=self.y, seed_input=self.img, backprop_modifier=modifier) # Lets overlay the heatmap onto original image. self.jet_heatmap = np.uint8(cm.jet(self.grads)[..., :3] * 255) self.a = ax.imshow(overlay(self.jet_heatmap, self.img)) plt.savefig("image/attention.jpg") return self.a
print('Predicted:', decoded) topK_synsets = [triple[0] for triple in decoded] topK_names = [triple[1] for triple in decoded] topK_scores = [triple[2] for triple in decoded] class_names = ['boxer', 'tiger_cat'] topK_ndx = [] imagenet_ndx = [] # indexes into the softmax entries of final layer for i, name in enumerate(class_names): ndx = topK_names.index(name) topK_ndx.append(ndx) imagenet_ndx.append(np.argwhere(preds[0] == topK_scores[ndx])[0][0]) # 282 = Tiger cat, 242 = Boxer (0 indexed) img = utils.load_img(img_path, target_size=(224, 224)) N = len(class_names) fig, ax = plt.subplots(1, N + 1) ax[0].imshow(img) ax[0].axis('off') # Lets overlay the heatmap for each desired class onto original image. for i in range(N): ndx = imagenet_ndx[i] layer_idx = utils.find_layer_idx(model, 'predictions') # final layer grads = visualize_cam(model, layer_idx, filter_indices=ndx, seed_input=img, backprop_modifier='guided') jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255)
#CAM on images for InceptionV3 network. im_file = "husky.jpg" img1 = image.load_img(im_file, target_size=(299, 299)) img1 = image.img_to_array(img1) img1 = np.expand_dims(img1, axis=0) img1 = preprocess_input(img1) layer_idx = utils.find_layer_idx(model_origin, 'predictions') heatmap = visualize_cam(model_origin, layer_idx, filter_indices=248, seed_input=img1[0, :, :, :]) # In[11]: img_init = utils.load_img(im_file, target_size=(299, 299)) plt.imshow(overlay(img_init, heatmap)) plt.show() # The red region represents the area of the image on which the network focuses. This result is very nice as main attention is given at the head of the dog. # # ### Custom Model : # # We run the same process over the custom model trained for car crash detection. # # # The activation of the last FC layer is removed : # In[5]: #With Custom Model
from vis.visualization import visualize_activation from matplotlib import pyplot as plt #%matplotlib inline plt.rcParams['figure.figsize'] = (18, 6) # 20 is the imagenet category for 'ouzel' img = visualize_activation(model, layer_idx, filter_indices=20, verbose=True) plt.imshow(img) """ https://github.com/raghakot/keras-vis/blob/master/examples/vggnet/attention.ipynb """ img1 = utils.load_img( '/home/innereye/keras-vis/examples/vggnet/images/ouzel1.jpg', target_size=(224, 224)) img2 = utils.load_img( '/home/innereye/keras-vis/examples/vggnet/images/ouzel2.jpg', target_size=(224, 224)) f, ax = plt.subplots(1, 2) ax[0].imshow(img1) ax[1].imshow(img2) from vis.visualization import visualize_saliency # Utility to search for layer index by name. # Alternatively we can specify this as -1 since it corresponds to the last layer. layer_idx = utils.find_layer_idx(model, 'predictions')
model = ResNet50(weights='imagenet', include_top=True) # Utility to search for layer index by name. # Alternatively we can specify this as -1 since it corresponds to the last layer. layer_idx = utils.find_layer_idx(model, 'fc1000') # Swap softmax with linear model.layers[layer_idx].activation = activations.linear model = utils.apply_modifications(model) from vis.utils import utils from matplotlib import pyplot as plt plt.rcParams['figure.figsize'] = (18, 6) img1 = utils.load_img('../vggnet/images/ouzel1.jpg', target_size=(224, 224)) img2 = utils.load_img('../vggnet/images/ouzel2.jpg', target_size=(224, 224)) f, ax = plt.subplots(1, 2) ax[0].imshow(img1) ax[1].imshow(img2) from vis.visualization import visualize_saliency, overlay from vis.utils import utils from keras import activations # Utility to search for layer index by name. # Alternatively we can specify this as -1 since it corresponds to the last layer. layer_idx = utils.find_layer_idx(model, 'fc1000') f, ax = plt.subplots(1, 2)
# Images corresponding to tiger, penguin, dumbbell, speedboat, spider image_paths = [ "http://www.tigerfdn.com/wp-content/uploads/2016/05/How-Much-Does-A-Tiger-Weigh.jpg", "http://www.slate.com/content/dam/slate/articles/health_and_science/wild_things/2013/10/131025_WILD_AdeliePenguin.jpg.CROP.promo-mediumlarge.jpg", "http://www.kshs.org/cool2/graphics/dumbbell1lg.jpg", "http://media1.britannica.com/eb-media/80/150980-004-EE46999B.jpg", ] heatmaps = [] # vis_images = [] for path in image_paths: # For InceptionV3 seed_img = utils.load_img(path, target_size=(299, 299)) pred = model.predict(preprocess_input(np.expand_dims(img_to_array(seed_img), axis=0))) # For VGG16 # seed_img = utils.load_img(path, target_size=(224, 224)) # pred_class = np.argmax(model.predict(np.array([img_to_array(seed_img)]))) # print(utils.get_imagenet_label(pred_class)) # seed_img = utils.load_img(path, target_size=(224, 224)) # pred_class = np.argmax(model.predict(np.array([img_to_array(seed_img)]))) print('Predicted:', decode_predictions(pred)) print('Predicted:', decode_predictions(pred)[0][0][1]) # pred_class = np.argmax(model.predict(preprocess_input(np.array([img_to_array(seed_img)]))))
model = cnn.define_CNN(x_train, num_classes, dropouts, 1) print(model.summary()) model.load_weights(filepath, by_name=True) print('Model loaded.') # The name of the layer we want to visualize # (see model definition in vggnet.py) layer_name = 'fc2_out' layer_idx = [ idx for idx, layer in enumerate(model.layers) if layer.name == layer_name ][0] heatmaps = [] #for path in image_paths: for i in range(5): seed_img = utils.load_img(path, target_size=(32, 32)) print(seed_img.shape) seed_img = x_train[np.random.randint(i * 13, 50000)] print(seed_img.shape) x = np.expand_dims(img_to_array(seed_img), axis=0) x = preprocess_input(x) pred_class = np.argmax(model.predict(x)) # Here we are asking it to show attention such that prob of `pred_class` is maximized. heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img) heatmaps.append(heatmap) plt.axis('off') plt.imshow(utils.stitch_images(heatmaps)) plt.title('Activation map') plt.savefig('activation_map_cnn')