def display_map_and_predictions(self, heatmap, image_non_normalized, fileName_postfix, fileName, img, patches_mask, layerName, plot=True, use_gpu=False): title = fileName.replace('_', '\_') if plot: fig = plt.figure(figsize=(8, 2.5), dpi=300) plt.imshow(self.getGrayScale( format_for_plotting( image_non_normalized).cpu().detach().numpy()), cmap='gray', alpha=0.6) # [:, :, 2] to show a channel plt.imshow(self.getGrayScale( format_for_plotting(heatmap).cpu().detach().numpy()), cmap='seismic', alpha=0.5) plt.imshow( format_for_plotting(patches_mask).cpu().detach().numpy(), cmap='seismic', alpha=0.3) plt.xticks([]) plt.yticks([]) fig.tight_layout() fig.show() path = os.path.join(self.experimentName, "models", self.trial_hash, 'saliency_map', fileName) if not os.path.exists(path): os.makedirs(path) fig.savefig(os.path.join(path, fileName + fileName_postfix + ".pdf"), bbox_inches='tight', pad_inches=0) # fig.suptitle("Saliency Map - " + title) if torch.cuda.is_available() and use_gpu: img = img.cuda() if plot: activatins_rows = 1 A = PlotNetwork.plot_activations(self.model.model, layerName, img, path, self.experiment_params, self.dataset, fileName + fileName_postfix, activatins_rows) else: activation = PlotNetwork.model_activations(self.model.model, layerName, self.dataset) A = activation(img) return A
def test_detach_tensor_from_computational_graph(): input_ = torch.zeros((1, 224, 224)) input_.requires_grad = True formatted = format_for_plotting(input_) assert not formatted.requires_grad
def visualizeAllClasses(self, image_normalized, input_non_normalized, listOfClasses, guided=False, use_gpu=False, cmap='viridis', alpha=.5): w = input_non_normalized.shape[3] h = input_non_normalized.shape[2] result = torch.zeros(1, 3, h, w * len(listOfClasses)) idx = 0 for i in listOfClasses: result[:, :, :, idx * w:(idx + 1) * w] = self.calculate_gradients( image_normalized, i, guided=guided, take_max=True, #True use_gpu=use_gpu) idx = idx + 1 stdized = standardize_and_clip(result) stdized = torch.cat((stdized, input_non_normalized), 3) output = (format_for_plotting(stdized), cmap, alpha) return output
def _visualize_filters(self, layer, filter_idxs, num_iter, num_subplots, title): # Prepare the main plot num_cols = 4 num_rows = int(np.ceil(num_subplots / num_cols)) fig = plt.figure(figsize=(16, num_rows * 5)) plt.title(title) plt.axis('off') self.output = [] # Plot subplots for i, filter_idx in enumerate(filter_idxs): output = self.optimize(layer, filter_idx, num_iter=num_iter) self.output.append(output) ax = fig.add_subplot(num_rows, num_cols, i+1) ax.set_xticks([]) ax.set_yticks([]) ax.set_title(f'filter {filter_idx}') ax.imshow(format_for_plotting( standardize_and_clip(output[-1], saturation=0.15, brightness=0.7))) plt.subplots_adjust(wspace=0, hspace=0);
def get_input_gradient(model, fname,guided=True, take_max=False, use_gpu=True): '''Warper for getting image gradient + cool visualize for it Input: model: (__main__.Model) Model fname: (str) Image file path Output: gradient for fname ''' ## Load image file img = load_img(fname) backprop = Backprop(model) output, target_class = backprop.calculate_gradients(img, None, take_max, guided, use_gpu) # Reshaping img gradient - Remove batch + move channel to last dim x = output output = format_for_plotting(output) clip_grad = standardize_and_clip(output) #Clip gradient of image from 0 -> 1 print(f'Model predicted {target_class}') fig = plt.figure() ax = fig.add_subplot(1, 3, 1) ax.set_axis_off() ax.imshow(clip_grad) ax.set_title('Gradient') ax = fig.add_subplot(1, 3, 2) ax.set_axis_off() ax.imshow(format_for_plotting(img)) ax.set_title('Original image') ax = fig.add_subplot(1, 3, 3) ax.set_axis_off() ax.imshow(format_for_plotting(img)) ax.imshow(clip_grad, alpha=0.3) ax.set_title('Blend grad and\n original image') fig.show() return output, x
def deepdream(self, img_path, layer, filter_idx, lr=.1, num_iter=20, figsize=(4, 4), title='DeepDream', return_output=False): """Creates DeepDream. It applies the optimization on the image provided. The image is loaded and made into a torch.Tensor that is compatible as the input to the network. Read the original blog post by Google for more information on `DeepDream <https://ai.googleblog.com/2015/06/inceptionism-going-deeper-into-neural.html>`_. Args: img_path (str): A path to the image you want to apply DeepDream on layer (torch.nn.modules.conv.Conv2d): The target Conv2d layer from which the filter to be chosen, based on `filter_idx`. filter_idx (int): The index of the target filter. lr (float, optional, default=.1): The step size of optimization. num_iter (int, optional, default=30): The number of iteration for the gradient ascent operation. figsize (tuple, optional, default=(4, 4)): The size of the plot. Relevant in case 1 above. title (str, optional default='Conv2d'): The title of the plot. return_output (bool, optional, default=False): Returns the output(s) of optimization if set to True. Returns: output (list of torch.Tensor): With dimentions :math:`(num_iter, C, H, W)`. The size of the image is determined by `img_size` attribute which defaults to 224. """ # noqa input_ = apply_transforms(load_image(img_path), self.img_size) self._lr = lr output = self.optimize(layer, filter_idx, input_, num_iter=num_iter) plt.figure(figsize=figsize) plt.axis('off') plt.title(title) plt.imshow( format_for_plotting( standardize_and_clip(output[-1], saturation=0.15, brightness=0.7))) # noqa if return_output: return output
def _visualize_filter(self, layer, filter_idx, num_iter, figsize, title): self.output = self.optimize(layer, filter_idx, num_iter=num_iter) plt.figure(figsize=figsize) plt.axis('off') plt.title(title) plt.imshow(format_for_plotting( standardize_and_clip(self.output[-1], saturation=0.15, brightness=0.7)));
def visualizeOverlay(self, input_, denormalizedInput_, target_class, guided=False, use_gpu=False, cmap='viridis', alpha=.5): # Calculate gradients max_gradients = self.calculate_gradients(input_, target_class, guided=guided, take_max=False, use_gpu=use_gpu) clipped_gradients = format_for_plotting( standardize_and_clip(max_gradients, saturation=1)) #return two entries of type (image, cmap, alpha) return [(format_for_plotting(denormalizedInput_), None, None), (clipped_gradients, cmap, alpha)]
def _visualize_filter(self, layer, filter_idx, num_iter, figsize, title): self.output = self.optimize(layer, filter_idx, num_iter=num_iter) if self.plot: plt.figure(figsize=figsize) plt.axis('off') plt.title(title) temp = format_for_plotting(standardize_and_clip(self.output[-1], saturation=0.15, brightness=0.7)) if temp.shape[0] == 3: plt.imshow(temp) else: fig, axes = plt.subplots(1, temp.shape[0]) for i in range(temp.shape[0]): axes[i].imshow(temp[i])
def visualizeHeatmap(self, input_, target_class, guided=False, use_gpu=False, cmap='viridis', alpha=.5): # Calculate gradients max_gradients = self.calculate_gradients( input_, target_class, guided=guided, take_max= True, # We are not taking max because the interplay between channels is important! use_gpu=use_gpu) #return image, cmap, alpha output = (format_for_plotting( standardize_and_clip(max_gradients, saturation=1)), cmap, alpha) return [output, output]
def test_format_multi_channel_tensor_without_batch_dimension(): input_ = torch.zeros((3, 224, 224)) formatted = format_for_plotting(input_) assert formatted.shape == (224, 224, 3)
def visualize(self, input_, target_class, guided=False, use_gpu=False, figsize=(16, 4), cmap='viridis', alpha=.5, return_output=False): """Calculates gradients and visualizes the output. A method that combines the backprop operation and visualization. It also returns the gradients, if specified with `return_output=True`. Args: input_ (torch.Tensor): With shape :math:`(N, C, H, W)`. target_class (int, optional, default=None) take_max (bool, optional, default=False): If True, take the maximum gradients across colour channels for each pixel. guided (bool, optional, default=Fakse): If True, perform guided backpropagation. See `Striving for Simplicity: The All Convolutional Net <https://arxiv.org/pdf/1412.6806.pdf>`_. use_gpu (bool, optional, default=False): Use GPU if set to True and `torch.cuda.is_available()`. figsize (tuple, optional, default=(16, 4)): The size of the plot. cmap (str, optional, default='viridis): The color map of the gradients plots. See avaialable color maps `here <https://matplotlib.org/3.1.0/tutorials/colors/colormaps.html>`_. alpha (float, optional, default=.5): The alpha value of the max gradients to be jaxaposed on top of the input image. return_output (bool, optional, default=False): Returns the output(s) of optimization if set to True. Returns: gradients (torch.Tensor): With shape :math:`(C, H, W)`. """ # Calculate gradients gradients = self.calculate_gradients(input_, target_class, guided=guided) max_gradients = self.calculate_gradients(input_, target_class, guided=guided, take_max=True) # Setup subplots subplots = [ # (title, [(image1, cmap, alpha), (image2, cmap, alpha)]) ('Input image', [(format_for_plotting(denormalize(input_)), None, None)]), ('Gradients across RGB channels', [(format_for_plotting(standardize_and_clip(gradients)), None, None)]), ('Max gradients', [(format_for_plotting(standardize_and_clip(max_gradients)), cmap, None)]), ('Overlay', [(format_for_plotting(denormalize(input_)), None, None), (format_for_plotting(standardize_and_clip(max_gradients)), cmap, alpha)]) ] fig = plt.figure(figsize=figsize) for i, (title, images) in enumerate(subplots): ax = fig.add_subplot(1, len(subplots), i + 1) ax.set_axis_off() ax.set_title(title) for image, cmap, alpha in images: ax.imshow(image, cmap=cmap, alpha=alpha) if return_output: return gradients, max_gradients
def process_single_image_1(fname, model, target_class=None, lr=9e-4, output_path=None, ilen=5, jlen=201): img = load_img(fname) img_mean = img.mean() img_std = img.std() tensor = img.cuda() model.cuda() prev_tensor = tensor model.eval() optimizer = None flag = False ## Load image one time # print(torch.argmax(model(tensor))) if target_class is None: target_class = int(torch.argmax(model(tensor))) for i in range(ilen): with torch.no_grad(): if not optimizer is None: tensor = optimizer.param_groups[0]['params'][0] tensor = tensor.sub(tensor.mean()).div( tensor.std()).mul(img_std).add(img_mean) # tensor = tensor.clamp(0, 1) tensor.requires_grad_(True) optimizer = torch.optim.Adam([tensor], lr) for j in range(jlen): model_output = model(tensor) target = torch.zeros_like(model_output) target[0, target_class] = 1 optimizer.zero_grad() model_output.backward(target) optimizer.step() if j == 0: print(F.softmax(model_output[0])[target_class]) if torch.argmax(model_output[0]) != target_class: print(j) print(F.softmax(model_output[0])[target_class]) flag = True break if flag: break with torch.no_grad(): if not optimizer is None: tensor = optimizer.param_groups[0]['params'][0] tensor = tensor.sub(tensor.mean()).div( tensor.std()).mul(img_std).add(img_mean) # tensor = tensor.clamp(0, 1) print(tensor.max()) tensor = do_inverse_IMNET(tensor) plt.imshow(format_for_plotting(tensor.cpu()[0]))
model.load_state_dict(torch.load(r'C:\Users\johan\iCloudDrive\DTU\KID\4. semester\Fagprojekt\Resultater\CNN_b\model1_l5_b.pt')) model.eval() def get_image(image,num): img1 = image[num, :, :, :].unsqueeze(0).requires_grad_(requires_grad=True) return img1 img1 = get_image(images,0) img2 = get_image(images,1) img3 = get_image(images,2) img4 = get_image(images,3) img5 = get_image(images,4) img6 = get_image(images,5) img7 = get_image(images,6) img8 = get_image(images,7) plt.imshow(format_for_plotting(denormalize(img2))) plt.show() def visualize_saliency(model, tensor, k=1, guide=True): backprop = Backprop(model) backprop.visualize(tensor, k, guided=guide) plt.show() def get_image(image,num): img1 = image[num, :, :, :].unsqueeze(0).requires_grad_(requires_grad=True) return img1 start = 0 def multiple_saliency_maps(model, tensors, start, num, k=0, guide = True): backprop = Backprop(model) for i in range(start,start+num): cur_image = tensors[i,:,:,:].unsqueeze(0).requires_grad_(requires_grad=True) backprop.visualize(cur_image, k, guided=guide)