def run():
# Get params
target_example = 0 # Snake
(original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\
get_params(target_example)
# Grad cam
gcv2 = GradCam(pretrained_model, target_layer=10)
# Generate cam mask
cam = gcv2.generate_cam(prep_img, target_class)
print('Grad cam completed')
# Guided backprop
GBP = GuidedBackprop(pretrained_model, prep_img, target_class)
# Get gradients
guided_grads = GBP.generate_gradients()
print('Guided backpropagation completed')
# Guided Grad cam
cam_gb = guided_grad_cam(cam, guided_grads)
save_gradient_images(cam_gb, file_name_to_export + '_GGrad_Cam')
grayscale_cam_gb = convert_to_grayscale(cam_gb)
save_gradient_images(grayscale_cam_gb,
file_name_to_export + '_GGrad_Cam_gray')
print('Guided grad cam completed')
def save_gradient_image(gradient_data, class_label, greyscale=False):
"""
save_gradient_image: Saves the raw gradient data as either an RGB or greyscale image.
:param gradient_data: A torch.cuda.FloatTensor or (if CUDA disabled a torch.FloatTensor) representing the gradient
of the weights with respect to the input image.
:param greyscale: A boolean flag indicating whether to save a grayscale gradient or a colored gradient image.
:return:
"""
output_path = os.path.join(str.replace(args.data, 'data', 'results'))
output_path = os.path.join(output_path, str(class_label))
if not os.path.exists(output_path):
os.mkdir(output_path)
if greyscale:
gradient_data = convert_to_grayscale(gradient_data)
output_path = os.path.join(output_path, 'vanilla_bp_gray.png')
else:
output_path = os.path.join(output_path, 'vanilla_bp_color.png')
# Rescale from Tensor's range [-1, 1] to image range [0, 1]:
gradient_data = gradient_data - gradient_data.min()
gradient_data /= gradient_data.max()
# Convert RGB to GBR (8 bit image) because this is how OpenCV does it:
gradient_data = np.uint8(gradient_data * 255).transpose(1, 2, 0)
gradient_data = gradient_data[..., ::-1]
# Use OpenCV to save the image:
cv2.imwrite(output_path, gradient_data)
def save_gradient_frame(gradient,
target_class_label,
file_name,
grayscale=False):
"""
save_gradient_frame: Saves the gradient snapshot as an image with the provided file_name (preferrably ending in an
epoch number e#).
:param gradient: A torch.cuda.FloatTensor or (if CUDA disabled a torch.FloatTensor) representing the gradient
of the weights with respect to the input image.
:param target_label: The target class label which determines which directory the image should be saved under.
:param file_name: The name to save the file under. This filename preferably ends with _e# where # is the epoch
number during training in which the gradient was captured.
:param grayscale: A boolean flag indicating whether to save a grayscale gradient or a colored gradient image.
:return:
"""
output_path = os.path.join(str.replace(args.data, 'data', 'results'))
output_path = os.path.join(output_path, str(target_class_label))
# This time we add a folder just for the gradient animations:
output_path = os.path.join(output_path, 'anim')
if not os.path.exists(output_path):
os.mkdir(output_path)
if grayscale:
gradient = convert_to_grayscale(gradient)
output_path = os.path.join(output_path, file_name)
# Convert NaN's to zeros (see: https://discuss.pytorch.org/t/how-to-set-nan-in-tensor-to-0/3918/6)
# grads = gradient.clone()
# grads[grads != grads] = 0
# num_nonzero = np.count_nonzero(grads.cpu().numpy())
# print('type(grads)', type(grads))
# print('num_nonzero:', np.count_nonzero(grads.cpu().numpy()))
# if num_nonzero > 0:
# Rescale from Tensor's range [-1, 1] to image range [0, 1]:
gradient = gradient - gradient.min()
gradient /= gradient.max()
# Convert RGB to GBR (8 bit image) because this is how OpenCV does it:
gradient = np.uint8(gradient * 255).transpose(1, 2, 0)
gradient = gradient[..., ::-1]
# Use OpenCV to save the image:
cv2.imwrite(output_path, gradient)
def save_transformed_image(image_data, class_label, greyscale=False):
"""
save_transformed_image: Saves the original input image transformed via the image transformation pipeline.
:param image_data:
:param class_label:
:param greyscale:
:return:
"""
output_path = os.path.join(str.replace(args.data, 'data', 'results'))
output_path = os.path.join(output_path, str(class_label))
if not os.path.exists(output_path):
os.mkdir(output_path)
if greyscale:
image_data = convert_to_grayscale(image_data)
output_path = os.path.join(output_path,
'transformed_gray_original.jpg')
else:
output_path = os.path.join(output_path, 'transformed_original.jpg')
# normalize:
image_data = image_data - image_data.min()
image_data /= image_data.max()
image_data = np.uint8(image_data * 255).transpose(1, 2, 0)
image_data = image_data[..., ::-1]
cv2.imwrite(output_path, image_data)
paths = [vis_args.DATASET.path]
# Poining Game Constructor
if vis_args.RESULTS.POINTING_GAME.state:
from src.pointing_game import PointingGame
tolerance = vis_args.RESULTS.POINTING_GAME.tolerance
pg = PointingGame(vis_args.MODEL.n_classes, tolerance=tolerance)
for f, path in tqdm.tqdm(zip(files, paths)):
img = open_image(path)
prep_img = preprocess_image(img, h=h, w=w)
guided_grads = GBP.generate_gradients(prep_img,
vis_args.DATASET.target_class)
pos_sal, neg_sal = get_positive_negative_saliency(guided_grads)
bw_pos_sal = convert_to_grayscale(pos_sal)
bw_neg_sal = convert_to_grayscale(neg_sal)
if vis_args.RESULTS.POINTING_GAME.state:
boxes = get_boxes(vis_args.RESULTS.DRAW_GT_BBOX.gt_src, f, img)
hit = pg.evaluate(boxes, bw_pos_sal.squeeze())
_ = pg.accumulate(hit[0], 1)
prep_img = prep_img[0].mean(dim=0, keepdim=True)
r_pos = alpha * bw_pos_sal + (1 - alpha) * prep_img.detach().numpy()
r_neg = alpha * bw_neg_sal + (1 - alpha) * prep_img.detach().numpy()
custom_save_gradient_images(bw_pos_sal,
vis_args.RESULTS.dir,
f,
obj="bw_gbp_pos")
custom_save_gradient_images(bw_neg_sal,
# Generate xbar images
xbar_list = self.generate_images_on_linear_path(input_image, steps)
# Initialize an iamge composed of zeros
integrated_grads = np.zeros(input_image.size())
for xbar_image in xbar_list:
# Generate gradients from xbar images
single_integrated_grad = self.generate_gradients(
xbar_image, target_class)
# Add rescaled grads from xbar images
integrated_grads = integrated_grads + single_integrated_grad / steps
# [0] to get rid of the first channel (1,3,224,224)
return integrated_grads[0]
if __name__ == '__main__':
# Get params
target_example = 0 # Snake
(original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\
get_example_params(target_example)
# Vanilla backprop
IG = IntegratedGradients(pretrained_model)
# Generate gradients
integrated_grads = IG.generate_integrated_gradients(
prep_img, target_class, 100)
# Convert to grayscale
grayscale_integrated_grads = convert_to_grayscale(integrated_grads)
# Save grayscale gradients
save_gradient_images(grayscale_integrated_grads,
file_name_to_export + '_Integrated_G_gray')
print('Integrated gradients completed.')
# Poining Game Constructor
if vis_args.RESULTS.POINTING_GAME.state:
from src.pointing_game import PointingGame
tolerance = vis_args.RESULTS.POINTING_GAME.tolerance
pg = PointingGame(vis_args.MODEL.n_classes, tolerance=tolerance)
for f, path in tqdm.tqdm(zip(files, paths)):
img = open_image(path)
prep_img = preprocess_image(img, h=h, w=w)
vanilla_grads = VBP.generate_gradients(prep_img,
vis_args.DATASET.target_class)
#custom_save_gradient_images(vanilla_grads, vis_args.RESULTS.dir,
# f, obj="vanillabp")
grad_x_image = vanilla_grads[0] * prep_img.detach().numpy()[0]
bw_inputxgrad = convert_to_grayscale(grad_x_image)
if vis_args.RESULTS.POINTING_GAME.state:
boxes = get_boxes(vis_args.RESULTS.DRAW_GT_BBOX.gt_src, f, img)
hit = pg.evaluate(boxes, bw_inputxgrad.squeeze())
_ = pg.accumulate(hit[0], 1)
prep_img = prep_img[0].mean(dim=0, keepdim=True)
r = alpha * bw_inputxgrad + (1 - alpha) * prep_img.detach().numpy()
custom_save_gradient_images(bw_inputxgrad,
vis_args.RESULTS.dir,
f,
obj="bw_gradximage")
if vis_args.RESULTS.POINTING_GAME.state:
print(pg.print_stats())
"""
Created on Wed Jun 19 17:12:04 2019
@author: Utku Ozbulak - github.com/utkuozbulak
"""
from src.misc_functions import (get_example_params, convert_to_grayscale,
save_gradient_images)
from src.vanilla_backprop import VanillaBackprop
# from guided_backprop import GuidedBackprop # To use with guided backprop
# from integrated_gradients import IntegratedGradients # To use with integrated grads
if __name__ == '__main__':
# Get params
target_example = 0 # Snake
(original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\
get_example_params(target_example)
# Vanilla backprop
VBP = VanillaBackprop(pretrained_model)
# Generate gradients
vanilla_grads = VBP.generate_gradients(prep_img, target_class)
grad_times_image = vanilla_grads[0] * prep_img.detach().numpy()[0]
# Convert to grayscale
grayscale_vanilla_grads = convert_to_grayscale(grad_times_image)
# Save grayscale gradients
save_gradient_images(
grayscale_vanilla_grads,
file_name_to_export + '_Vanilla_grad_times_image_gray')
print('Grad times image completed.')
"""
cam_gb = np.multiply(grad_cam_mask, guided_backprop_mask)
return cam_gb
if __name__ == '__main__':
# Get params
target_example = 0 # Snake
(original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\
get_example_params(target_example)
# Grad cam
gcv2 = GradCam(pretrained_model, target_layer=11)
# Generate cam mask
cam = gcv2.generate_cam(prep_img, target_class)
print('Grad cam completed')
# Guided backprop
GBP = GuidedBackprop(pretrained_model)
# Get gradients
guided_grads = GBP.generate_gradients(prep_img, target_class)
print('Guided backpropagation completed')
# Guided Grad cam
cam_gb = guided_grad_cam(cam, guided_grads)
save_gradient_images(cam_gb, file_name_to_export + '_GGrad_Cam')
grayscale_cam_gb = convert_to_grayscale(cam_gb)
save_gradient_images(grayscale_cam_gb,
file_name_to_export + '_GGrad_Cam_gray')
print('Guided grad cam completed')
# Backward pass
model_output.backward(gradient=one_hot_output)
# Convert Pytorch variable to numpy array
# [0] to get rid of the first channel (1,3,224,224)
gradients_as_arr = self.gradients.data.numpy()[0]
return gradients_as_arr
if __name__ == '__main__':
target_example = 0 # Snake
(original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\
get_example_params(target_example)
# Guided backprop
GBP = GuidedBackprop(pretrained_model)
# Get gradients
guided_grads = GBP.generate_gradients(prep_img, target_class)
# Save colored gradients
save_gradient_images(guided_grads,
file_name_to_export + '_Guided_BP_color')
# Convert to grayscale
grayscale_guided_grads = convert_to_grayscale(guided_grads)
# Save grayscale gradients
save_gradient_images(grayscale_guided_grads,
file_name_to_export + '_Guided_BP_gray')
# Positive and negative saliency maps
pos_sal, neg_sal = get_positive_negative_saliency(guided_grads)
save_gradient_images(pos_sal, file_name_to_export + '_pos_sal')
save_gradient_images(neg_sal, file_name_to_export + '_neg_sal')
print('Guided backprop completed')
model_output = self.model(input_image)
# Zero grads
self.model.zero_grad()
# Target for backprop
one_hot_output = torch.FloatTensor(1, model_output.size()[-1]).zero_()
one_hot_output[0][target_class] = 1
# Backward pass
model_output.backward(gradient=one_hot_output)
# Convert Pytorch variable to numpy array
# [0] to get rid of the first channel (1,3,224,224)
gradients_as_arr = self.gradients.data.numpy()[0]
return gradients_as_arr
if __name__ == '__main__':
# Get params
target_example = 1 # Snake
(original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\
get_example_params(target_example)
# Vanilla backprop
VBP = VanillaBackprop(pretrained_model)
# Generate gradients
vanilla_grads = VBP.generate_gradients(prep_img, target_class)
# Save colored gradients
save_gradient_images(vanilla_grads, file_name_to_export + '_Vanilla_BP_color')
# Convert to grayscale
grayscale_vanilla_grads = convert_to_grayscale(vanilla_grads)
# Save grayscale gradients
save_gradient_images(grayscale_vanilla_grads, file_name_to_export + '_Vanilla_BP_gray')
print('Vanilla backprop completed')
# Average it out
smooth_grad = smooth_grad / param_n
return smooth_grad
if __name__ == '__main__':
# Get params
target_example = 0 # Snake
(original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\
get_example_params(target_example)
VBP = VanillaBackprop(pretrained_model)
# GBP = GuidedBackprop(pretrained_model) # if you want to use GBP dont forget to
# change the parametre in generate_smooth_grad
param_n = 50
param_sigma_multiplier = 4
smooth_grad = generate_smooth_grad(VBP, # ^This parameter
prep_img,
target_class,
param_n,
param_sigma_multiplier)
# Save colored gradients
save_gradient_images(smooth_grad, file_name_to_export + '_SmoothGrad_color')
# Convert to grayscale
grayscale_smooth_grad = convert_to_grayscale(smooth_grad)
# Save grayscale gradients
save_gradient_images(grayscale_smooth_grad, file_name_to_export + '_SmoothGrad_gray')
print('Smooth grad completed')