def display_map_and_predictions(self,
heatmap,
image_non_normalized,
fileName_postfix,
fileName,
img,
patches_mask,
layerName,
plot=True,
use_gpu=False):
title = fileName.replace('_', '\_')
if plot:
fig = plt.figure(figsize=(8, 2.5), dpi=300)
plt.imshow(self.getGrayScale(
format_for_plotting(
image_non_normalized).cpu().detach().numpy()),
cmap='gray',
alpha=0.6) # [:, :, 2] to show a channel
plt.imshow(self.getGrayScale(
format_for_plotting(heatmap).cpu().detach().numpy()),
cmap='seismic',
alpha=0.5)
plt.imshow(
format_for_plotting(patches_mask).cpu().detach().numpy(),
cmap='seismic',
alpha=0.3)
plt.xticks([])
plt.yticks([])
fig.tight_layout()
fig.show()
path = os.path.join(self.experimentName, "models", self.trial_hash,
'saliency_map', fileName)
if not os.path.exists(path):
os.makedirs(path)
fig.savefig(os.path.join(path,
fileName + fileName_postfix + ".pdf"),
bbox_inches='tight',
pad_inches=0)
# fig.suptitle("Saliency Map - " + title)
if torch.cuda.is_available() and use_gpu:
img = img.cuda()
if plot:
activatins_rows = 1
A = PlotNetwork.plot_activations(self.model.model, layerName, img,
path, self.experiment_params,
self.dataset,
fileName + fileName_postfix,
activatins_rows)
else:
activation = PlotNetwork.model_activations(self.model.model,
layerName, self.dataset)
A = activation(img)
return A
def test_detach_tensor_from_computational_graph():
input_ = torch.zeros((1, 224, 224))
input_.requires_grad = True
formatted = format_for_plotting(input_)
assert not formatted.requires_grad
def visualizeAllClasses(self,
image_normalized,
input_non_normalized,
listOfClasses,
guided=False,
use_gpu=False,
cmap='viridis',
alpha=.5):
w = input_non_normalized.shape[3]
h = input_non_normalized.shape[2]
result = torch.zeros(1, 3, h, w * len(listOfClasses))
idx = 0
for i in listOfClasses:
result[:, :, :, idx * w:(idx + 1) * w] = self.calculate_gradients(
image_normalized,
i,
guided=guided,
take_max=True, #True
use_gpu=use_gpu)
idx = idx + 1
stdized = standardize_and_clip(result)
stdized = torch.cat((stdized, input_non_normalized), 3)
output = (format_for_plotting(stdized), cmap, alpha)
return output
def _visualize_filters(self, layer, filter_idxs, num_iter, num_subplots,
title):
# Prepare the main plot
num_cols = 4
num_rows = int(np.ceil(num_subplots / num_cols))
fig = plt.figure(figsize=(16, num_rows * 5))
plt.title(title)
plt.axis('off')
self.output = []
# Plot subplots
for i, filter_idx in enumerate(filter_idxs):
output = self.optimize(layer, filter_idx, num_iter=num_iter)
self.output.append(output)
ax = fig.add_subplot(num_rows, num_cols, i+1)
ax.set_xticks([])
ax.set_yticks([])
ax.set_title(f'filter {filter_idx}')
ax.imshow(format_for_plotting(
standardize_and_clip(output[-1],
saturation=0.15,
brightness=0.7)))
plt.subplots_adjust(wspace=0, hspace=0);
def get_input_gradient(model, fname,guided=True, take_max=False, use_gpu=True):
'''Warper for getting image gradient + cool visualize for it
Input:
model: (__main__.Model) Model
fname: (str) Image file path
Output:
gradient for fname
'''
## Load image file
img = load_img(fname)
backprop = Backprop(model)
output, target_class = backprop.calculate_gradients(img, None, take_max, guided, use_gpu)
# Reshaping img gradient - Remove batch + move channel to last dim
x = output
output = format_for_plotting(output)
clip_grad = standardize_and_clip(output) #Clip gradient of image from 0 -> 1
print(f'Model predicted {target_class}')
fig = plt.figure()
ax = fig.add_subplot(1, 3, 1)
ax.set_axis_off()
ax.imshow(clip_grad)
ax.set_title('Gradient')
ax = fig.add_subplot(1, 3, 2)
ax.set_axis_off()
ax.imshow(format_for_plotting(img))
ax.set_title('Original image')
ax = fig.add_subplot(1, 3, 3)
ax.set_axis_off()
ax.imshow(format_for_plotting(img))
ax.imshow(clip_grad, alpha=0.3)
ax.set_title('Blend grad and\n original image')
fig.show()
return output, x
def deepdream(self,
img_path,
layer,
filter_idx,
lr=.1,
num_iter=20,
figsize=(4, 4),
title='DeepDream',
return_output=False):
"""Creates DeepDream.
It applies the optimization on the image provided. The image is loaded
and made into a torch.Tensor that is compatible as the input to the
network.
Read the original blog post by Google for more information on
`DeepDream <https://ai.googleblog.com/2015/06/inceptionism-going-deeper-into-neural.html>`_.
Args:
img_path (str): A path to the image you want to apply DeepDream on
layer (torch.nn.modules.conv.Conv2d): The target Conv2d layer from
which the filter to be chosen, based on `filter_idx`.
filter_idx (int): The index of the target filter.
lr (float, optional, default=.1): The step size of optimization.
num_iter (int, optional, default=30): The number of iteration for
the gradient ascent operation.
figsize (tuple, optional, default=(4, 4)): The size of the plot.
Relevant in case 1 above.
title (str, optional default='Conv2d'): The title of the plot.
return_output (bool, optional, default=False): Returns the
output(s) of optimization if set to True.
Returns:
output (list of torch.Tensor): With dimentions
:math:`(num_iter, C, H, W)`. The size of the image is
determined by `img_size` attribute which defaults to 224.
""" # noqa
input_ = apply_transforms(load_image(img_path), self.img_size)
self._lr = lr
output = self.optimize(layer, filter_idx, input_, num_iter=num_iter)
plt.figure(figsize=figsize)
plt.axis('off')
plt.title(title)
plt.imshow(
format_for_plotting(
standardize_and_clip(output[-1],
saturation=0.15,
brightness=0.7)))
# noqa
if return_output:
return output
def _visualize_filter(self, layer, filter_idx, num_iter, figsize, title):
self.output = self.optimize(layer, filter_idx, num_iter=num_iter)
plt.figure(figsize=figsize)
plt.axis('off')
plt.title(title)
plt.imshow(format_for_plotting(
standardize_and_clip(self.output[-1],
saturation=0.15,
brightness=0.7)));
def visualizeOverlay(self,
input_,
denormalizedInput_,
target_class,
guided=False,
use_gpu=False,
cmap='viridis',
alpha=.5):
# Calculate gradients
max_gradients = self.calculate_gradients(input_,
target_class,
guided=guided,
take_max=False,
use_gpu=use_gpu)
clipped_gradients = format_for_plotting(
standardize_and_clip(max_gradients, saturation=1))
#return two entries of type (image, cmap, alpha)
return [(format_for_plotting(denormalizedInput_), None, None),
(clipped_gradients, cmap, alpha)]
def _visualize_filter(self, layer, filter_idx, num_iter, figsize, title):
self.output = self.optimize(layer, filter_idx, num_iter=num_iter)
if self.plot:
plt.figure(figsize=figsize)
plt.axis('off')
plt.title(title)
temp = format_for_plotting(standardize_and_clip(self.output[-1],
saturation=0.15,
brightness=0.7))
if temp.shape[0] == 3:
plt.imshow(temp)
else:
fig, axes = plt.subplots(1, temp.shape[0])
for i in range(temp.shape[0]):
axes[i].imshow(temp[i])
def visualizeHeatmap(self,
input_,
target_class,
guided=False,
use_gpu=False,
cmap='viridis',
alpha=.5):
# Calculate gradients
max_gradients = self.calculate_gradients(
input_,
target_class,
guided=guided,
take_max=
True, # We are not taking max because the interplay between channels is important!
use_gpu=use_gpu)
#return image, cmap, alpha
output = (format_for_plotting(
standardize_and_clip(max_gradients, saturation=1)), cmap, alpha)
return [output, output]
def test_format_multi_channel_tensor_without_batch_dimension():
input_ = torch.zeros((3, 224, 224))
formatted = format_for_plotting(input_)
assert formatted.shape == (224, 224, 3)
def visualize(self,
input_,
target_class,
guided=False,
use_gpu=False,
figsize=(16, 4),
cmap='viridis',
alpha=.5,
return_output=False):
"""Calculates gradients and visualizes the output.
A method that combines the backprop operation and visualization.
It also returns the gradients, if specified with `return_output=True`.
Args:
input_ (torch.Tensor): With shape :math:`(N, C, H, W)`.
target_class (int, optional, default=None)
take_max (bool, optional, default=False): If True, take the maximum
gradients across colour channels for each pixel.
guided (bool, optional, default=Fakse): If True, perform guided
backpropagation. See `Striving for Simplicity: The All
Convolutional Net <https://arxiv.org/pdf/1412.6806.pdf>`_.
use_gpu (bool, optional, default=False): Use GPU if set to True and
`torch.cuda.is_available()`.
figsize (tuple, optional, default=(16, 4)): The size of the plot.
cmap (str, optional, default='viridis): The color map of the
gradients plots. See avaialable color maps `here <https://matplotlib.org/3.1.0/tutorials/colors/colormaps.html>`_.
alpha (float, optional, default=.5): The alpha value of the max
gradients to be jaxaposed on top of the input image.
return_output (bool, optional, default=False): Returns the
output(s) of optimization if set to True.
Returns:
gradients (torch.Tensor): With shape :math:`(C, H, W)`.
"""
# Calculate gradients
gradients = self.calculate_gradients(input_,
target_class,
guided=guided)
max_gradients = self.calculate_gradients(input_,
target_class,
guided=guided,
take_max=True)
# Setup subplots
subplots = [
# (title, [(image1, cmap, alpha), (image2, cmap, alpha)])
('Input image', [(format_for_plotting(denormalize(input_)), None,
None)]),
('Gradients across RGB channels',
[(format_for_plotting(standardize_and_clip(gradients)), None,
None)]),
('Max gradients',
[(format_for_plotting(standardize_and_clip(max_gradients)), cmap,
None)]),
('Overlay',
[(format_for_plotting(denormalize(input_)), None, None),
(format_for_plotting(standardize_and_clip(max_gradients)), cmap,
alpha)])
]
fig = plt.figure(figsize=figsize)
for i, (title, images) in enumerate(subplots):
ax = fig.add_subplot(1, len(subplots), i + 1)
ax.set_axis_off()
ax.set_title(title)
for image, cmap, alpha in images:
ax.imshow(image, cmap=cmap, alpha=alpha)
if return_output:
return gradients, max_gradients
def process_single_image_1(fname,
model,
target_class=None,
lr=9e-4,
output_path=None,
ilen=5,
jlen=201):
img = load_img(fname)
img_mean = img.mean()
img_std = img.std()
tensor = img.cuda()
model.cuda()
prev_tensor = tensor
model.eval()
optimizer = None
flag = False
## Load image one time
# print(torch.argmax(model(tensor)))
if target_class is None:
target_class = int(torch.argmax(model(tensor)))
for i in range(ilen):
with torch.no_grad():
if not optimizer is None:
tensor = optimizer.param_groups[0]['params'][0]
tensor = tensor.sub(tensor.mean()).div(
tensor.std()).mul(img_std).add(img_mean)
# tensor = tensor.clamp(0, 1)
tensor.requires_grad_(True)
optimizer = torch.optim.Adam([tensor], lr)
for j in range(jlen):
model_output = model(tensor)
target = torch.zeros_like(model_output)
target[0, target_class] = 1
optimizer.zero_grad()
model_output.backward(target)
optimizer.step()
if j == 0:
print(F.softmax(model_output[0])[target_class])
if torch.argmax(model_output[0]) != target_class:
print(j)
print(F.softmax(model_output[0])[target_class])
flag = True
break
if flag:
break
with torch.no_grad():
if not optimizer is None:
tensor = optimizer.param_groups[0]['params'][0]
tensor = tensor.sub(tensor.mean()).div(
tensor.std()).mul(img_std).add(img_mean)
# tensor = tensor.clamp(0, 1)
print(tensor.max())
tensor = do_inverse_IMNET(tensor)
plt.imshow(format_for_plotting(tensor.cpu()[0]))
model.load_state_dict(torch.load(r'C:\Users\johan\iCloudDrive\DTU\KID\4. semester\Fagprojekt\Resultater\CNN_b\model1_l5_b.pt'))
model.eval()
def get_image(image,num):
img1 = image[num, :, :, :].unsqueeze(0).requires_grad_(requires_grad=True)
return img1
img1 = get_image(images,0)
img2 = get_image(images,1)
img3 = get_image(images,2)
img4 = get_image(images,3)
img5 = get_image(images,4)
img6 = get_image(images,5)
img7 = get_image(images,6)
img8 = get_image(images,7)
plt.imshow(format_for_plotting(denormalize(img2)))
plt.show()
def visualize_saliency(model, tensor, k=1, guide=True):
backprop = Backprop(model)
backprop.visualize(tensor, k, guided=guide)
plt.show()
def get_image(image,num):
img1 = image[num, :, :, :].unsqueeze(0).requires_grad_(requires_grad=True)
return img1
start = 0
def multiple_saliency_maps(model, tensors, start, num, k=0, guide = True):
backprop = Backprop(model)
for i in range(start,start+num):
cur_image = tensors[i,:,:,:].unsqueeze(0).requires_grad_(requires_grad=True)
backprop.visualize(cur_image, k, guided=guide)
