def compute_tcav_with_losses(input_tensor, losses, seed_input, wrt_tensor=None, grad_modifier='absolute'):
"""
Args:
input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
losses: List of ([Loss](vis.losses#Loss), weight) tuples.
seed_input: The model input for which activation map needs to be visualized.
wrt_tensor: Short for, with respect to. The gradients of losses are computed with respect to this tensor.
When None, this is assumed to be the same as `input_tensor` (Default value: None)
### NB. Here we can introduce our fl(x). The gradients will be computed wrt that tensor.
grad_modifier: gradient modifier to use. See [grad_modifiers](vis.grad_modifiers.md). By default `absolute`
value of gradients are used. To visualize positive or negative gradients, use `relu` and `negate`
respectively. (Default value = 'absolute')
Returns:
The normalized gradients of `seed_input` with respect to weighted `losses`.
### NB. Here you will have to add the dot product with the normalized direction
of the concept vector.
"""
#print ('wrt_tensor', wrt_tensor)
from keras.layers import Reshape
#wrt_tensor = Reshape((14,14,1024,))(wrt_tensor)
#print 'wrt_tensor', wrt_tensor
#return
opt = Optimizer(input_tensor, losses, wrt_tensor=wrt_tensor, norm_grads=False)
grads = opt.minimize(seed_input=seed_input, max_iter=1, grad_modifier=grad_modifier, verbose=False)[1]
channel_idx = 1 if K.image_data_format() == 'channels_first' else -1
#grads = np.max(grads, axis=channel_idx)
return utils.normalize(grads)[0]
def plot_multiple_saliency(images,
model,
layer,
filter_idx=None,
backprop_modifier=None,
grad_modifier=None):
fig, ax = plt.subplots(2, len(images), figsize=(4 * len(images), 4))
ax = ax.flatten()
for i, filename in enumerate(images):
image = load_img(filename,
target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
ax[i].imshow(image)
ax[i].axis('off')
for i, filename in enumerate(images):
grad = visualize_saliency(model,
find_layer_idx(model, layer),
filter_idx,
normalize(image),
backprop_modifier=backprop_modifier,
grad_modifier=grad_modifier)
image = load_img(filename,
target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
ax[i + len(images)].imshow(overlay(grad, image))
ax[i + len(images)].axis('off')
return fig
def compute_visualisation_mask(self, img):
grad_modifier = 'absolute'
grads = self.opt.minimize(seed_input=img, max_iter=1, grad_modifier=grad_modifier, verbose=False)[1]
channel_idx = 1 if K.image_data_format() == 'channels_first' else -1
grads = np.max(grads, axis=channel_idx)
res = utils.normalize(grads)[0]
return res
def visualize_saliency_with_losses(input_tensor, losses, seed_input,original_img, grad_modifier='absolute',save_path=''):
if not os.path.exists(save_path):
os.mkdir(save_path)
opt = Optimizer(input_tensor, losses, norm_grads=False)
grads = opt.minimize(seed_input=seed_input, max_iter=1, grad_modifier=grad_modifier, verbose=False)[1]
# print (np.shape(grads))
channel_idx = 1 if K.image_data_format() == 'channels_first' else -1
grads = np.max(grads, axis=channel_idx)
# if not os.path.exists('./image'):
# os.mkdir('./images')
print (np.shape(grads))
for i in range(np.shape(grads)[1]):
temp_grads = utils.normalize(grads[:,i,...])
# print ('temp_grads',np.shape(temp_grads))
heatmap = np.uint8(cm.jet(temp_grads)[..., :3] * 255)[0]
img = original_img[i,...]
temp = image.array_to_img(overlay(img, heatmap,alpha=0.5))
pil.Image.save(temp,save_path+'overlay{}.jpg'.format(i))
temp = image.array_to_img(heatmap)
pil.Image.save(temp,save_path+'heatmap{}.jpg'.format(i))
temp = image.array_to_img(img)
pil.Image.save(temp,save_path+'original{}.jpg'.format(i))
def plot_multiple_grad_cam(
images,
model,
layer,
penultimate_layer=None,
filter_idx=None,
backprop_modifier=None,
grad_modifier=None,
experts=None,
expert_spacing=0.1,
):
rows = 2
if experts is not None:
rows = 3
fig, ax = plt.subplots(
rows, len(images), figsize=(4 * len(images), 4 * rows))
ax = ax.flatten()
penultimate_layer_idx = None
if penultimate_layer:
penultimate_layer_idx = find_layer_idx(model, penultimate_layer)
for i, filename in enumerate(images):
image = load_img(
filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
ax[i].imshow(image)
ax[i].axis('off')
for i, filename in enumerate(images):
image = load_img(
filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
grad = visualize_cam(
model,
find_layer_idx(model, layer),
filter_idx,
normalize(image),
penultimate_layer_idx=penultimate_layer_idx,
backprop_modifier=backprop_modifier,
grad_modifier=grad_modifier)
ax[i + len(images)].imshow(overlay(grad, image))
ax[i + len(images)].axis('off')
if experts:
for i, filename in enumerate(images):
for j, expert in enumerate(experts):
if i == 0:
message = "expert {}: {}".format(j + 1, expert[i])
ax[i + 2 * len(images)].text(
0.3,
1 - (expert_spacing * j),
message,
horizontalalignment='left',
verticalalignment='center')
else:
message = "{}".format(expert[i])
ax[i + 2 * len(images)].text(
0.5,
1 - (expert_spacing * j),
message,
horizontalalignment='center',
verticalalignment='center')
ax[i + 2 * len(images)].axis('off')
return fig, ax
def plot_grad_cam(image_file,
model,
layer,
filter_idx=None,
backprop_modifier="relu"):
image = load_img(image_file,
target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
grad = visualize_cam(model,
find_layer_idx(model, layer),
filter_idx,
normalize(image),
backprop_modifier=backprop_modifier)
fig, ax = plt.subplots(1, 2)
ax[0].imshow(overlay(grad, image))
ax[0].axis('off')
ax[1].imshow(image)
ax[1].axis('off')
return fig
def visualize_cam_with_losses(input_tensor,
losses,
seed_input,
penultimate_layer,
grad_modifier=None):
"""Generates a gradient based class activation map (CAM) by using positive gradients of `input_tensor`
with respect to weighted `losses`.
For details on grad-CAM, see the paper:
[Grad-CAM: Why did you say that? Visual Explanations from Deep Networks via Gradient-based Localization]
(https://arxiv.org/pdf/1610.02391v1.pdf).
Unlike [class activation mapping](https://arxiv.org/pdf/1512.04150v1.pdf), which requires minor changes to
network architecture in some instances, grad-CAM has a more general applicability.
Compared to saliency maps, grad-CAM is class discriminative; i.e., the 'cat' explanation exclusively highlights
cat regions and not the 'dog' region and vice-versa.
Args:
input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
losses: List of ([Loss](vis.losses#Loss), weight) tuples.
seed_input: The model input for which activation map needs to be visualized.
penultimate_layer: The pre-layer to `layer_idx` whose feature maps should be used to compute gradients
with respect to filter output.
grad_modifier: gradient modifier to use. See [grad_modifiers](vis.grad_modifiers.md). If you don't
specify anything, gradients are unchanged (Default value = None)
Returns:
The normalized gradients of `seed_input` with respect to weighted `losses`.
"""
penultimate_output = penultimate_layer.output
opt = Optimizer(input_tensor,
losses,
wrt_tensor=penultimate_output,
norm_grads=False)
_, grads, penultimate_output_value = opt.minimize(
seed_input, max_iter=1, grad_modifier=grad_modifier, verbose=False)
# For numerical stability. Very small grad values along with small penultimate_output_value can cause
# w * penultimate_output_value to zero out, even for reasonable fp precision of float32.
grads = grads / (np.max(grads) + K.epsilon())
# Average pooling across all feature maps.
# This captures the importance of feature map (channel) idx to the output.
channel_idx = 1 if K.image_data_format() == 'channels_first' else -1
other_axis = np.delete(np.arange(len(grads.shape)), channel_idx)
weights = np.mean(grads, axis=tuple(other_axis))
# Generate heatmap by computing weight * output over feature maps
output_dims = utils.get_img_shape(penultimate_output)[2:]
heatmap = np.zeros(shape=output_dims, dtype=K.floatx())
for i, w in enumerate(weights):
if channel_idx == -1:
heatmap += w * penultimate_output_value[0, ..., i]
else:
heatmap += w * penultimate_output_value[0, i, ...]
# ReLU thresholding to exclude pattern mismatch information (negative gradients).
heatmap = np.maximum(heatmap, 0)
# The penultimate feature map size is definitely smaller than input image.
input_dims = utils.get_img_shape(input_tensor)[2:]
# Figure out the zoom factor.
zoom_factor = [
i / (j * 1.0) for i, j in iter(zip(input_dims, output_dims))
]
heatmap = zoom(heatmap, zoom_factor)
return utils.normalize(heatmap)