def generate(self, target_class):
initial_learning_rate = 6
created_image = np.uint8(np.random.uniform(0, 255, (224, 224, 3)))
for i in range(1, 150):
# Process image and return variable
self.processed_image = preprocess_image(created_image, False)
# Define optimizer for the image
optimizer = SGD([self.processed_image], lr=initial_learning_rate)
# Forward
output = self.model(self.processed_image)
# Target specific class
class_loss = -output[0, target_class]
print('Iteration:', str(i), 'Loss',
"{0:.2f}".format(class_loss.data.numpy()))
# Zero grads
self.model.zero_grad()
# Backward
class_loss.backward()
# Update image
optimizer.step()
# Recreate image
created_image = recreate_image(self.processed_image)
if i % 10 == 0:
# Save image
im_path = os.path.join(
self.dst, 'c_specific_iteration_' + str(i) + '.jpg')
save_image(created_image, im_path)
return self.processed_image
def guide_grad_cam_test(model, img, target_cls, target_layer, dst, desc):
img = Image.open(img).convert('RGB')
prep_img = preprocess_image(img)
# Grad cam
gcv2 = GradCam(model, target_layer=target_layer)
# Generate cam mask
cam = gcv2.generate_cam(prep_img, target_cls)
print('Grad cam completed')
# Guided backprop
GBP = GuidedBackprop(model)
# Get gradients
guided_grads = GBP.generate_gradients(prep_img, target_cls)
print('Guided backpropagation completed')
# Guided Grad cam
cam_gb = guided_grad_cam(cam, guided_grads)
filename = os.path.join(dst, desc + '_ggrad_cam.jpg')
save_gradient_images(cam_gb, filename)
grayscale_cam_gb = convert_to_grayscale(cam_gb)
filename = os.path.join(dst, desc + 'ggrad_cam_gray.jpg')
save_gradient_images(grayscale_cam_gb, filename)
print('Guided grad cam completed')
def smooth_grad_test(model, img, target_cls, dst, param_n, param_sigma_mult,
desc):
VBP = VanillaBackprop(model)
# GBP = GuidedBackprop(pretrained_model) # if you want to use GBP dont forget to
# change the parametre in generate_smooth_grad
from PIL import Image
from utils.misc import preprocess_image
img = Image.open(img).convert("RGB")
prep_img = preprocess_image(img)
param_n = param_n
param_sigma_multiplier = param_sigma_mult
smooth_grad = generate_smooth_grad(
VBP, # ^This parameter
prep_img,
target_cls,
param_n,
param_sigma_multiplier)
import os
# Save colored gradients
filename = os.path.join(dst, desc + '_smooth_grad_color.jpg')
save_gradient_images(smooth_grad, filename)
# Convert to grayscale
grayscale_smooth_grad = convert_to_grayscale(smooth_grad)
# Save grayscale gradients
filename = os.path.join(dst, desc + '_smooth_grad_gray.jpg')
save_gradient_images(grayscale_smooth_grad, filename)
print('Smooth grad completed')
def dream(self, filename):
created_image = Image.open(filename).convert('RGB')
# Process image and return variable
self.processed_image = preprocess_image(created_image, True)
# Define optimizer for the image
# Earlier layers need higher learning rates to visualize whereas layer layers need less
optimizer = SGD([self.processed_image], lr=12, weight_decay=1e-4)
for i in range(1, 251):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = self.processed_image
for index, layer in enumerate(self.features):
# Forward
x = layer(x)
# Only need to forward until we the selected layer is reached
if index == self.selected_layer:
break
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image)
# Save image every 20 iteration
if i % 10 == 0:
print(created_image.size)
im_path = os.path.join(self.dst, 'ddream_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg')
save_image(self.created_image, im_path)
def layer_activation_guided_backprop_test(model, img, target_layer, target_pos,
target_class, dst, desc):
img = Image.open(img).convert("RGB")
img = preprocess_image(img)
# File export name
filename = os.path.join(
dst, '%s_layer_%d_filter_%d.jpg' % (desc, target_layer, target_pos))
# Guided backprop
GBP = GuidedBackprop(model)
# Get gradients
guided_grads = GBP.generate_gradients(img, target_class, target_layer,
target_pos)
# Save colored gradients
filename = os.path.join(dst, desc + '_guided_bp_color.jpg')
save_gradient_images(guided_grads, filename)
# Convert to grayscale
grayscale_guided_grads = convert_to_grayscale(guided_grads)
# Save grayscale gradients
filename = os.path.join(dst, desc + '_guided_bp_gray.jpg')
save_gradient_images(grayscale_guided_grads, filename)
# Positive and negative saliency maps
pos_sal, neg_sal = get_positive_negative_saliency(guided_grads)
filename = os.path.join(dst, desc + '_pos_sal.jpg')
save_gradient_images(pos_sal, filename)
filename = os.path.join(dst, desc + '_neg_sal.jpg')
save_gradient_images(neg_sal, filename)
print('Layer Guided backprop completed')
def get_example_params(example_index):
"""
Gets used variables for almost all visualizations, like the image, model etc.
Args:
example_index (int): Image id to use from examples
returns:
original_image (numpy arr): Original image read from the file
prep_img (numpy_arr): Processed image
target_class (int): Target class for the image
file_name_to_export (string): File name to export the visualizations
pretrained_model(Pytorch model): Model to use for the operations
"""
# Pick one of the examples
example_list = (('../input_images/snake.jpg',
56), ('../input_images/cat_dog.png', 243),
('../input_images/spider.png', 72))
img_path = example_list[example_index][0]
target_class = example_list[example_index][1]
file_name_to_export = img_path[img_path.rfind('/') + 1:img_path.rfind('.')]
# Read image
original_image = Image.open(img_path).convert('RGB')
# Process image
prep_img = preprocess_image(original_image)
# Define model
pretrained_model = models.alexnet(pretrained=True)
return (original_image, prep_img, target_class, file_name_to_export,
pretrained_model)
def grad_cam_save(model, origin_image, target_class, target_layer, dst, desc):
origin_image = Image.open(origin_image).convert('RGB')
prep_img = preprocess_image(origin_image)
grad_cam = GradCam(model, target_layer=target_layer)
# Generate cam mask
cam = grad_cam.generate_cam(prep_img, target_class)
# Save mask
save_class_activation_images(origin_image, cam, dst, desc)
def test_inverted_prepresentation(model, img, target_cls, target_layer, dst):
from PIL import Image
from utils.misc import preprocess_image
img = Image.open(img).convert("RGB")
dst = os.path.join(dst, 'inverted_representation')
img = preprocess_image(img)
inverted_re = inverted_representation.InvertedRepresentation(model, dst)
inverted_re.generate_inverted_image_specific_layer(img,
img.size()[-1],
target_layer)
def grad_times_image(model, origin_image, target_cls, dst, desc):
origin_image = Image.open(origin_image).convert('RGB')
prep_img = preprocess_image(origin_image)
vbp = VanillaBackprop(model)
vanilla_grads = vbp.generate_gradients(prep_img, target_cls)
grad_times_image = vanilla_grads[0] * prep_img.detach().numpy()[0]
# Convert to grayscale
grayscale_vanilla_grads = convert_to_grayscale(grad_times_image)
# Save grayscale gradients
file_utils.makesure_path(dst)
filename = os.path.join(dst, desc + '_Vanilla_grad_times_image_gray.jpg')
save_gradient_images(grayscale_vanilla_grads, filename)
def vannilla_back(model, origin_image, target_cls, dst, desc):
origin_image = Image.open(origin_image).convert('RGB')
prep_img = misc.preprocess_image(origin_image)
vbp = VanillaBackprop(model)
# Generate gradients
vanilla_grads = vbp.generate_gradients(prep_img, target_cls)
# Save colored gradients
file_utils.makesure_path(dst)
filename = os.path.join(dst, desc + '_Vanilla_BP_color.png')
save_gradient_images(vanilla_grads, filename)
# Convert to grayscale
grayscale_vanilla_grads = convert_to_grayscale(vanilla_grads)
# Save grayscale gradients
filename = os.path.join(dst, desc + '_Vanilla_BP_GRAY.png')
save_gradient_images(grayscale_vanilla_grads, filename)
def integraded_gradients_test(model, img, target_cls, dst, desc):
from PIL import Image
img = Image.open(img).convert("RGB")
prep_img = preprocess_image(img)
# Vanilla backprop
IG = IntegratedGradients(model)
# Generate gradients
integrated_grads = IG.generate_integrated_gradients(
prep_img, target_cls, 100)
# Convert to grayscale
grayscale_integrated_grads = convert_to_grayscale(integrated_grads)
# Save grayscale gradients
import os
filename = os.path.join(dst, desc + "intergraded_gray.jpg")
save_gradient_images(grayscale_integrated_grads, filename)
print('Integrated gradients completed.')
def visualise_layer_without_hooks(self):
# Process image and return variable
# Generate a random image
random_image = np.uint8(np.random.uniform(150, 180, (224, 224, 3)))
# Process image and return variable
processed_image = preprocess_image(random_image, False)
# Define optimizer for the image
optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6)
for i in range(1, 31):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image
for index, layer in enumerate(self.features):
# Forward pass layer by layer
x = layer(x)
if index == self.selected_layer:
# Only need to forward until the selected layer is reached
# Now, x is the output of the selected layer
break
# Here, we get the specific filter from the output of the convolution operation
# x is a tensor of shape 1x512x28x28.(For layer 17)
# So there are 512 unique filter outputs
# Following line selects a filter from 512 filters so self.conv_output will become
# a tensor of shape 28x28
self.conv_output = x[0, self.selected_filter]
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:', "{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(processed_image)
# Save image
if i % 5 == 0:
filename = os.path.join(self.path, 'layer_visual_%d_%d_iter_%d.jpg' + self.selected_layer, self.selected_filter, i)
save_image(self.created_image, filename)
def visualise_layer_with_hooks(self):
'''
img
'''
# Hook the selected layer
self.hook_layer()
# Generate a random image
random_image = np.uint8(np.random.uniform(150, 180, (224, 224, 3)))
# Process image and return variable
processed_image = preprocess_image(random_image, False)
# Define optimizer for the image
optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6)
for i in range(1, 31):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image
for index, layer in enumerate(self.features):
# Forward pass layer by layer
# x is not used after this point because it is only needed to trigger
# the forward hook function
x = layer(x)
# Only need to forward until the selected layer is reached
if index == self.selected_layer:
# (forward hook function triggered)
break
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:', "{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(processed_image)
# Save image
if i % 5 == 0:
filename = os.path.join(self.path, 'layer_visual_%d_%d_iter_%d.jpg' % (self.selected_layer, self.selected_filter, i))
save_image(self.created_image, filename)
def guided_backprop(model, original_image, target_class, dst, desc):
original_image = Image.open(original_image).convert('RGB')
prep_img = preprocess_image(original_image)
# Guided backprop
GBP = GuidedBackprop(model)
# Get gradients
guided_grads = GBP.generate_gradients(prep_img, target_class)
# Save colored gradients
filename = os.path.join(dst, desc + '_guided_bp_color.jpg')
save_gradient_images(guided_grads, filename)
# Convert to grayscale
grayscale_guided_grads = convert_to_grayscale(guided_grads)
# Save grayscale gradients
filename = os.path.join(dst, desc + '_guided_bp_gray.jpg')
save_gradient_images(grayscale_guided_grads, filename)
# Positive and negative saliency maps
pos_sal, neg_sal = get_positive_negative_saliency(guided_grads)
filename = os.path.join(dst, desc + '_pos_sal.jpg')
save_gradient_images(pos_sal, filename)
filename = os.path.join(dst, desc + '_neg_sal.jpg')
save_gradient_images(neg_sal, filename)