def test_visualize_saliency_with_losses_with_unkeepdims(model, data):
losses = [(ActivationMaximization(model.layers[-1], 0), -1)]
grads = visualize_saliency_with_losses(model.input,
losses,
data,
keepdims=True)
assert grads.shape == (28, 28, 3)
def generate_heatmap(model, layer_idx, filter_indices, seed_img):
losses = [(ActivationMaximization(model.layers[layer_idx],
filter_indices), 1)]
opt = Optimizer(model.input, losses)
grads = opt.minimize(max_iter=1, verbose=False, seed_img=seed_img)[1]
# We are minimizing loss as opposed to maximizing output as with the paper.
# So, negative gradients here mean that they reduce loss, maximizing class probability.
grads *= -1
grads = np.max(np.abs(grads), axis=3, keepdims=True)
grads = deprocess_image(grads[0]).astype(
'float32') # Smoothen activation map
grads = grads / np.max(grads) * 255
# Convert to heatmap and zero out low probabilities for a cleaner output.
heatmap = cv2.applyColorMap(cv2.GaussianBlur(grads, (3, 3), 0),
cv2.COLORMAP_JET)
heatmap = heatmap.reshape(list(heatmap.shape) + [1])
heatmap[heatmap <= np.mean(heatmap)] = 0
heatmap = np.maximum(heatmap, 0)
heatmap = heatmap / np.max(heatmap) * 255
return heatmap
def init_salient(self, model):
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
first_output_name = None
for i, layer in enumerate(model.layers):
if first_output_name is None and "dropout" not in layer.name.lower() and "out" in layer.name.lower():
first_output_name = layer.name
layer_idx = i
if first_output_name is None:
print("Failed to find the model layer named with 'out'. Skipping salient.")
return False
print("####################")
print("Visualizing activations on layer:", first_output_name)
print("####################")
# ensure we have linear activation
model.layers[layer_idx].activation = activations.linear
# build salient model and optimizer
sal_model = utils.apply_modifications(model)
modifier_fn = get('guided')
sal_model_mod = modifier_fn(sal_model)
losses = [
(ActivationMaximization(sal_model_mod.layers[layer_idx], None), -1)
]
self.opt = Optimizer(sal_model_mod.input, losses, norm_grads=False)
return True
def visualize_activation_ternary_dynamic(model, layer_idx,alpha=1e-6,filter_indices=None, wrt_tensor=None,
seed_input=None, input_range=(-1, 1),
backprop_modifier=None, grad_modifier=None,
act_max_weight=1, lp_norm_weight=10, tv_weight=10,
**optimizer_params):
"""Generates the model input that maximizes the output of all `filter_indices` in the given `layer_idx`, and
put it in ternary representation
Args:
model: The `keras.models.Model` instance. The model input shape must be: `(samples, channels, image_dims...)`
if `image_data_format=channels_first` or `(samples, image_dims..., channels)` if
`image_data_format=channels_last`.
layer_idx: The layer index within `model.layers` whose filters needs to be visualized.
filter_indices: filter indices within the layer to be maximized.
If None, all filters are visualized. (Default value = None)
For `keras.layers.Dense` layer, `filter_idx` is interpreted as the output index.
If you are visualizing final `keras.layers.Dense` layer, consider switching 'softmax' activation for
'linear' using [utils.apply_modifications](vis.utils.utils#apply_modifications) for better results.
wrt_tensor: Short for, with respect to. The gradients of losses are computed with respect to this tensor.
When None, this is assumed to be the same as `input_tensor` (Default value: None)
seed_input: Seeds the optimization with a starting input. Initialized with a random value when set to None.
(Default value = None)
input_range: Specifies the input range as a `(min, max)` tuple. This is used to rescale the
final optimized input to the given range. (Default value=(0, 255))
backprop_modifier: backprop modifier to use. See [backprop_modifiers](vis.backprop_modifiers.md). If you don't
specify anything, no backprop modification is applied. (Default value = None)
grad_modifier: gradient modifier to use. See [grad_modifiers](vis.grad_modifiers.md). If you don't
specify anything, gradients are unchanged (Default value = None)
act_max_weight: The weight param for `ActivationMaximization` loss. Not used if 0 or None. (Default value = 1)
lp_norm_weight: The weight param for `LPNorm` regularization loss. Not used if 0 or None. (Default value = 10)
tv_weight: The weight param for `TotalVariation` regularization loss. Not used if 0 or None. (Default value = 10)
alpha : regularization parameter for the ternarization
optimizer_params: The **kwargs for optimizer [params](vis.optimizer#optimizerminimize). Will default to
reasonable values when required keys are not found.
Example:
If you wanted to visualize the input image that would maximize the output index 22, say on
final `keras.layers.Dense` layer, then, `filter_indices = [22]`, `layer_idx = dense_layer_idx`.
If `filter_indices = [22, 23]`, then it should generate an input image that shows features of both classes.
Returns:
The model input that maximizes the output of `filter_indices` in the given `layer_idx`.
"""
if backprop_modifier is not None:
modifier_fn = get(backprop_modifier)
model = modifier_fn(model)
losses = [
(ActivationMaximization(model.layers[layer_idx], filter_indices), act_max_weight),
(LPNorm(model.input,1), lp_norm_weight),
(TotalVariation(model.input), tv_weight),
(EstimateTernaryInput(model.input), alpha)
]
# Add grad_filter to optimizer_params.
optimizer_params = utils.add_defaults_to_kwargs({
'grad_modifier': grad_modifier,
'input_modifiers' : [binarizer,],
}, **optimizer_params)
return visualize_activation_with_losses_dynamic(model.input, losses, wrt_tensor,alpha,
seed_input, input_range, **optimizer_params)
def visualize_saliency(model, layer_idx, filter_indices, seed_img, alpha=0.5):
"""Generates an attention heatmap over the `seed_img` for maximizing `filter_indices` output in the given `layer`.
For a full description of saliency, see the paper:
[Deep Inside Convolutional Networks: Visualising Image Classification Models and Saliency Maps](https://arxiv.org/pdf/1312.6034v2.pdf)
Args:
model: The `keras.models.Model` instance. Model input is expected to be a 4D image input of shape:
`(samples, channels, rows, cols)` if data_format='channels_first' or `(samples, rows, cols, channels)` if data_format='channels_last'.
layer_idx: The layer index within `model.layers` whose filters needs to be visualized.
filter_indices: filter indices within the layer to be maximized.
For `keras.layers.Dense` layer, `filter_idx` is interpreted as the output index.
If you are visualizing final `keras.layers.Dense` layer, you tend to get
better results with 'linear' activation as opposed to 'softmax'. This is because 'softmax'
output can be maximized by minimizing scores for other classes.
seed_img: The input image for which activation map needs to be visualized.
alpha: The alpha value of image as overlayed onto the heatmap.
This value needs to be between [0, 1], with 0 being heatmap only to 1 being image only (Default value = 0.5)
Example:
If you wanted to visualize attention over 'bird' category, say output index 22 on the
final `keras.layers.Dense` layer, then, `filter_indices = [22]`, `layer = dense_layer`.
One could also set filter indices to more than one value. For example, `filter_indices = [22, 23]` should
(hopefully) show attention map that corresponds to both 22, 23 output categories.
Returns:
The heatmap image, overlayed with `seed_img` using `alpha`, indicating image regions that, when changed,
would contribute the most towards maximizing the output of `filter_indices`.
"""
if alpha < 0. or alpha > 1.:
raise ValueError("`alpha` needs to be between [0, 1]")
filter_indices = utils.listify(filter_indices)
print("Working on filters: {}".format(pprint.pformat(filter_indices)))
losses = [(ActivationMaximization(model.layers[layer_idx],
filter_indices), 1)]
opt = Optimizer(model.input, losses)
grads = opt.minimize(max_iter=1, verbose=False, seed_img=seed_img)[1]
# We are minimizing loss as opposed to maximizing output as with the paper.
# So, negative gradients here mean that they reduce loss, maximizing class probability.
grads *= -1
s, c, row, col = utils.get_img_indices()
grads = np.max(np.abs(grads), axis=c)
# Normalize and zero out low probabilities for a cleaner output.
grads /= np.max(grads)
heatmap = np.uint8(cm.jet(grads)[..., :3] * 255)
heatmap[np.where(grads < 0.2)] = 0
heatmap = np.uint8(seed_img * alpha + heatmap * (1. - alpha))
return grads
def get_saliency_grads(model, layer_idx, filter_indices, seed_input):
# define loss to maximize pixels that need to be changed the least to affect activations the most
losses = [
(ActivationMaximization(model.layers[layer_idx], filter_indices), -1)
]
input_tensor = model.input[0]
# run optimization for input image
opt = Optimizer(input_tensor, losses, wrt_tensor=None, norm_grads=False)
grads = opt.minimize(seed_input=seed_input, max_iter=1, grad_modifier='absolute', verbose=False)[1]
return grads
def compute_tcav(model,
layer_idx,
filter_indices,
seed_input,
wrt_tensor=None,
backprop_modifier=None,
grad_modifier='absolute'):
"""Computes a Conceptual Sensitivity score `.
Args:
model: The `keras.models.Model` instance. The model input shape must be: `(samples, channels, image_dims...)`
if `image_data_format=channels_first` or `(samples, image_dims..., channels)` if
`image_data_format=channels_last`.
layer_idx: The layer index within `model.layers` whose filters needs to be visualized.
filter_indices: filter indices within the layer to be maximized.
If None, all filters are visualized. (Default value = None)
For `keras.layers.Dense` layer, `filter_idx` is interpreted as the output index.
If you are visualizing final `keras.layers.Dense` layer, consider switching 'softmax' activation for
'linear' using [utils.apply_modifications](vis.utils.utils#apply_modifications) for better results.
seed_input: The model input for which activation map needs to be visualized.
wrt_tensor: Short for, with respect to. The gradients of losses are computed with respect to this tensor.
When None, this is assumed to be the same as `input_tensor` (Default value: None)
### NB. This will become the output of the
layer at which Sensitivity is computed
backprop_modifier: backprop modifier to use. See [backprop_modifiers](vis.backprop_modifiers.md). If you don't
specify anything, no backprop modification is applied. (Default value = None)
grad_modifier: gradient modifier to use. See [grad_modifiers](vis.grad_modifiers.md). By default `absolute`
value of gradients are used. To visualize positive or negative gradients, use `relu` and `negate`
respectively. (Default value = 'absolute')
Example:
If you wanted to visualize attention over 'bird' category, say output index 22 on the
final `keras.layers.Dense` layer, then, `filter_indices = [22]`, `layer = dense_layer`.
One could also set filter indices to more than one value. For example, `filter_indices = [22, 23]` should
(hopefully) show attention map that corresponds to both 22, 23 output categories.
Returns:
Not sure yet.
"""
if backprop_modifier is not None:
modifier_fn = get(backprop_modifier)
model = modifier_fn(model)
# `ActivationMaximization` loss reduces as outputs get large, hence negative gradients indicate the direction
# for increasing activations. Multiply with -1 so that positive gradients indicate increase instead.
losses = [(ActivationMaximization(model.layers[layer_idx],
filter_indices), -1)]
return compute_tcav_with_losses(model.input, losses, seed_input,
wrt_tensor, grad_modifier)
def visualize_cam(model,
layer_idx,
filter_indices,
seed_input,
penultimate_layer_idx=None,
backprop_modifier=None,
grad_modifier=None):
"""Generates a gradient based class activation map (grad-CAM) that maximizes the outputs of
`filter_indices` in `layer_idx`.
Args:
model: The `keras.models.Model` instance. The model input shape must be: `(samples, channels, image_dims...)`
if `image_data_format=channels_first` or `(samples, image_dims..., channels)` if
`image_data_format=channels_last`.
layer_idx: The layer index within `model.layers` whose filters needs to be visualized.
filter_indices: filter indices within the layer to be maximized.
If None, all filters are visualized. (Default value = None)
For `keras.layers.Dense` layer, `filter_idx` is interpreted as the output index.
If you are visualizing final `keras.layers.Dense` layer, consider switching 'softmax' activation for
'linear' using [utils.apply_modifications](vis.utils.utils#apply_modifications) for better results.
seed_input: The input image for which activation map needs to be visualized.
penultimate_layer_idx: The pre-layer to `layer_idx` whose feature maps should be used to compute gradients
wrt filter output. If not provided, it is set to the nearest penultimate `Conv` or `Pooling` layer.
backprop_modifier: backprop modifier to use. See [backprop_modifiers](vis.backprop_modifiers.md). If you don't
specify anything, no backprop modification is applied. (Default value = None)
grad_modifier: gradient modifier to use. See [grad_modifiers](vis.grad_modifiers.md). If you don't
specify anything, gradients are unchanged (Default value = None)
Example:
If you wanted to visualize attention over 'bird' category, say output index 22 on the
final `keras.layers.Dense` layer, then, `filter_indices = [22]`, `layer = dense_layer`.
One could also set filter indices to more than one value. For example, `filter_indices = [22, 23]` should
(hopefully) show attention map that corresponds to both 22, 23 output categories.
Returns:
The heatmap image indicating the input regions whose change would most contribute towards
maximizing the output of `filter_indices`.
"""
if backprop_modifier is not None:
modifier_fn = get(backprop_modifier)
model = modifier_fn(model)
penultimate_layer = _find_penultimate_layer(model, layer_idx,
penultimate_layer_idx)
# `ActivationMaximization` outputs negative gradient values for increase in activations. Multiply with -1
# so that positive gradients indicate increase instead.
losses = [(ActivationMaximization(model.layers[layer_idx],
filter_indices), -1)]
return visualize_cam_with_losses(model.input, losses, seed_input,
penultimate_layer, grad_modifier)
def visualize_saliency_3Dcnn(model, layer_idx, filter_indices, seed_input,original_img,
backprop_modifier=None, grad_modifier='absolute',save_pathname='images'):
if backprop_modifier is not None:
modifier_fn = backprop_modifiers.get(backprop_modifier)
# model = backend.modify_model_backprop(model, 'guided')
model = modifier_fn(model)
# `ActivationMaximization` loss reduces as outputs get large, hence negative gradients indicate the direction
# for increasing activations. Multiply with -1 so that positive gradients indicate increase instead.
losses = [
(ActivationMaximization(model.layers[layer_idx], filter_indices), -1)
]
if not os.path.exists('{}/{}'.format(save_pathname,filter_indices+1)):
os.makedirs('{}/{}'.format(save_pathname,filter_indices+1))
visualize_saliency_with_losses(model.input, losses, seed_input,original_img, grad_modifier,save_path='{}/{}/'.format(save_pathname,filter_indices+1))
def generate_opt_gif():
"""Example to show how to generate the gif of optimization progress.
This example also shows how to use the optimizer directly with losses.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
output_class = [20]
losses = [(ActivationMaximization(layer_dict[layer_name],
output_class), 2),
(LPNorm(model.input), 10), (TotalVariation(model.input), 10)]
opt = Optimizer(model.input, losses)
opt.minimize(max_iter=500,
verbose=True,
callbacks=[GifGenerator('opt_progress')])
def generate_opt_gif():
"""Example to show how to generate the gif of optimization progress.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
output_class = [20]
losses = [(ActivationMaximization(layer_dict[layer_name],
output_class), 1), (LPNorm(), 10),
(TotalVariation(), 1)]
opt = Optimizer(model.input, losses)
# Jitter is used as a regularizer to create crisper images, but it makes gif animation ugly.
opt.minimize(max_iter=500,
verbose=True,
jitter=0,
progress_gif_path='opt_progress')
model.layers[layer_idx].activation = tf.keras.activations.linear
model = utils.apply_modifications(model)
model.compile(loss=loss_function)
#optimizer = tf.train.RMSPropOptimizer(learning_rate_var).minimize(loss)
#sess = tf.InteractiveSession()
#saver = tf.train.Saver()
#try:
# saver.restore(sess, model_add+'/best_model')
# the_print('Model is restored!',style='bold',tc='blue',bgc='black')
#except:
# the_print('Something is wrong, model can not be restored!',style='bold',tc='red',bgc='black')
# exit()
losses = [(ActivationMaximization(model.layers[-1], 0), -1)]
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
ch_mkdir(model_add + '/saliencies')
iii = 0
for data, rfi in dp:
num = data.shape[0]
for i in range(num):
def visualize_activation(model,
layer_idx,
filter_indices=None,
seed_img=None,
text=None,
act_max_weight=1,
lp_norm_weight=10,
tv_weight=10,
**optimizer_params):
"""Generates stitched input image(s) over all `filter_indices` in the given `layer` that maximize
the filter output activation.
Args:
model: The `keras.models.Model` instance. Model input is expected to be a 4D image input of shape:
`(samples, channels, rows, cols)` if data_format='channels_first' or `(samples, rows, cols, channels)` if data_format='channels_last'.
layer_idx: The layer index within `model.layers` whose filters needs to be visualized.
filter_indices: filter indices within the layer to be maximized.
For `keras.layers.Dense` layer, `filter_idx` is interpreted as the output index.
If you are visualizing final `keras.layers.Dense` layer, you tend to get
better results with 'linear' activation as opposed to 'softmax'. This is because 'softmax'
output can be maximized by minimizing scores for other classes.
filter indices within the layer to be maximized.
If None, all filters are visualized. (Default value = None)
An input image is generated for each entry in `filter_indices`. The entry can also be an array.
For example, `filter_indices = [[1, 2], 3, [4, 5, 6]]` would generate three input images. The first one
would maximize output of filters 1, 2, 3 jointly. A fun use of this might be to generate a dog-fish
image by maximizing 'dog' and 'fish' output in final `Dense` layer.
For `keras.layers.Dense` layers, `filter_idx` is interpreted as the output index.
If you are visualizing final `keras.layers.Dense` layer, you tend to get
better results with 'linear' activation as opposed to 'softmax'. This is because 'softmax'
output can be maximized by minimizing scores for other classes.
seed_img: Seeds the optimization with a starting image. Initialized with a random value when set to None.
(Default value = None)
text: The text to overlay on top of the generated image. (Default Value = None)
act_max_weight: The weight param for `ActivationMaximization` loss. Not used if 0 or None. (Default value = 1)
lp_norm_weight: The weight param for `LPNorm` regularization loss. Not used if 0 or None. (Default value = 10)
tv_weight: The weight param for `TotalVariation` regularization loss. Not used if 0 or None. (Default value = 10)
optimizer_params: The **kwargs for optimizer [params](vis.optimizer.md##optimizerminimize). Will default to
reasonable values when required keys are not found.
Example:
If you wanted to visualize the input image that would maximize the output index 22, say on
final `keras.layers.Dense` layer, then, `filter_indices = [22]`, `layer = dense_layer`.
If `filter_indices = [22, 23]`, then it should generate an input image that shows features of both classes.
Returns:
Stitched image output visualizing input images that maximize the filter output(s). (Default value = 10)
"""
filter_indices = utils.listify(filter_indices)
print("Working on filters: {}".format(pprint.pformat(filter_indices)))
# Default optimizer kwargs.
optimizer_params_default = {
'seed_img': seed_img,
'max_iter': 200,
'verbose': False,
'image_modifiers': _DEFAULT_IMG_MODIFIERS
}
optimizer_params_default.update(optimizer_params)
optimizer_params = optimizer_params_default
losses = [(ActivationMaximization(model.layers[layer_idx],
filter_indices), act_max_weight),
(LPNorm(model.input), lp_norm_weight),
(TotalVariation(model.input), tv_weight)]
opt = Optimizer(model.input, losses, norm_grads=False)
img = opt.minimize(**optimizer_params)[0]
if text:
img = utils.draw_text(img, text)
return img
def visualize_cam(model,
layer_idx,
filter_indices,
seed_img,
penultimate_layer_idx=None,
alpha=0):
"""Generates a gradient based class activation map (CAM) as described in paper
[Grad-CAM: Why did you say that? Visual Explanations from Deep Networks via Gradient-based Localization](https://arxiv.org/pdf/1610.02391v1.pdf).
Unlike [class activation mapping](https://arxiv.org/pdf/1512.04150v1.pdf), which requires minor changes to
network architecture in some instances, grad-CAM has a more general applicability.
Compared to saliency maps, grad-CAM is class discriminative; i.e., the 'cat' explanation exclusively highlights
cat regions and not the 'dog' region and vice-versa.
Args:
model: The `keras.models.Model` instance. Model input is expected to be a 4D image input of shape:
`(samples, channels, rows, cols)` if data_format='channels_first' or `(samples, rows, cols, channels)` if data_format='channels_last'.
layer_idx: The layer index within `model.layers` whose filters needs to be visualized.
filter_indices: filter indices within the layer to be maximized.
For `keras.layers.Dense` layer, `filter_idx` is interpreted as the output index.
If you are visualizing final `keras.layers.Dense` layer, you tend to get
better results with 'linear' activation as opposed to 'softmax'. This is because 'softmax'
output can be maximized by minimizing scores for other classes.
seed_img: The input image for which activation map needs to be visualized.
penultimate_layer_idx: The pre-layer to `layer_idx` whose feature maps should be used to compute gradients
wrt filter output. If not provided, it is set to the nearest penultimate `Convolutional` or `Pooling` layer.
alpha: The alpha value of image as overlayed onto the heatmap.
This value needs to be between [0, 1], with 0 being heatmap only to 1 being image only (Default value = 0.5)
Example:
If you wanted to visualize attention over 'bird' category, say output index 22 on the
final `keras.layers.Dense` layer, then, `filter_indices = [22]`, `layer = dense_layer`.
One could also set filter indices to more than one value. For example, `filter_indices = [22, 23]` should
(hopefully) show attention map that corresponds to both 22, 23 output categories.
Notes:
This technique deprecates occlusion maps as it gives similar results, but with one-pass gradient computation
as opposed inefficient sliding window approach.
Returns:
The heatmap image, overlayed with `seed_img` using `alpha`, indicating image regions that, when changed,
would contribute the most towards maximizing the output of `filter_indices`.
"""
if alpha < 0. or alpha > 1.:
raise ValueError("`alpha` needs to be between [0, 1]")
filter_indices = utils.listify(filter_indices)
print("Working on filters: {}".format(pprint.pformat(filter_indices)))
# Search for the nearest penultimate `Convolutional` or `Pooling` layer.
if penultimate_layer_idx is None:
for idx, layer in utils.reverse_enumerate(model.layers[:layer_idx -
1]):
if isinstance(layer, (Convolution2D, _Pooling2D)):
penultimate_layer_idx = idx
break
if penultimate_layer_idx is None:
raise ValueError(
'Unable to determine penultimate `Convolution2D` or `Pooling2D` '
'layer for layer_idx: {}'.format(layer_idx))
assert penultimate_layer_idx < layer_idx
losses = [(ActivationMaximization(model.layers[layer_idx],
filter_indices), 1)]
penultimate_output = model.layers[penultimate_layer_idx].output
opt = Optimizer(model.input, losses, wrt=penultimate_output)
_, grads, penultimate_output_value = opt.minimize(seed_img,
max_iter=1,
verbose=False)
# We are minimizing loss as opposed to maximizing output as with the paper.
# So, negative gradients here mean that they reduce loss, maximizing class probability.
grads *= -1
# Average pooling across all feature maps.
# This captures the importance of feature map (channel) idx to the output
s_idx, c_idx, row_idx, col_idx = utils.get_img_indices()
weights = np.mean(grads, axis=(s_idx, row_idx, col_idx))
# Generate heatmap by computing weight * output over feature maps
s, ch, rows, cols = utils.get_img_shape(penultimate_output)
heatmap = np.ones(shape=(rows, cols), dtype=np.float32)
for i, w in enumerate(weights):
heatmap += w * penultimate_output_value[utils.slicer[0, i, :, :]]
# The penultimate feature map size is definitely smaller than input image.
s, ch, rows, cols = utils.get_img_shape(model.input)
# TODO: Figure out a way to get rid of open cv dependency.
# skimage doesn't deal with arbitrary floating point ranges.
heatmap = cv2.resize(heatmap, (cols, rows), interpolation=cv2.INTER_CUBIC)
# ReLU thresholding, normalize between (0, 1)
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
heatmap *= 10
heatmap_colored = heatmap
# Convert to heatmap and zero out low probabilities for a cleaner output.
#heatmap_colored = np.uint8(cm.jet(heatmap)[..., :3] * 10)
#heatmap_colored[np.where(heatmap < 0.2)] = 0
#heatmap_colored = np.uint8(seed_img * alpha + heatmap_colored * (1. - alpha))
return heatmap_colored
from vis.losses import ActivationMaximization
from vis.regularizers import TotalVariation, LPNorm
filter_indices = [1, 2, 3]
# Tuple consists of (loss_function, weight)
# Add regularizers as needed.
losses = [
(ActivationMaximization(keras_layer, filter_indices), 1),
(LPNorm(model.input), 10),
(TotalVariation(model.input), 10)
]
from vis.losses import ActivationMaximization
from vis.regularizers import TotalVariation, LPNorm
from model3d import model
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'dense_2'
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
output_class = [20]
losses = [(ActivationMaximization(layer_dict[layer_name], output_class), 2),
(LPNorm(model.input), 10), (TotalVariation(model.input), 10)]
# opt = Optimizer(model.input, losses)
# opt.minimize(max_iter=500, verbose=True, callbacks=[GifGenerator('opt_progress')])
from vis.losses import ActivationMaximization
from vis.regularizers import TotalVariation, LPNorm
from vis.input_modifiers import Jitter
from vis.optimizer import Optimizer
from vis.callbacks import GifGenerator
if __name__ == "__main__":
lucky_num = 50756711264384381850616619995309447969109689825336919605444730053665222018857 % (2 ** 32)
np.random.seed(lucky_num)
set_random_seed(lucky_num)
os.environ["CUDA_VISIBLE_DEVICES"] = "2"
modelH5 = sys.argv[1]
# outputDir = sys.argv[2]
model = load_model(modelH5)
layerDict = dict([(layer.name, layer) for layer in model.layers[1:]])
layerName = "dense_5"
for idx in range(7):
outputClass = [0]
losses = [(ActivationMaximization(layerDict[layerName], outputClass), 2),
(LPNorm(model.input), 10),
(TotalVariation(model.input), 10)]
opt = Optimizer(model.input, losses)
opt.minimize(max_iter=500, verbose=True, input_modifiers=[Jitter()], callbacks=[GifGenerator('OptProgress_%d' % (idx))])
from vis.losses import ActivationMaximization
from vis.regularizers import TotalVariation, LPNorm
from vis.optimizer import Optimizer
from model import model
from keras import activations
layer_idx = -1
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
losses = [(ActivationMaximization(model.layers[-2], 3), 2),
(LPNorm(model.input), 10), (TotalVariation(model.input), 10)]
opt = Optimizer(model.input, losses)
opt.minimize(max_iter=500,
verbose=True,
callbacks=[GifGenerator('opt_progress')])
def salience_visualization(model,
save_directory,
conn_name,
output_x,
output_y,
output_f,
verbose=True,
clas=None,
perc_output=1):
from vis.visualization import visualize_saliency
from vis.losses import ActivationMaximization
from vis.optimizer import Optimizer
from vis.utils import utils
from vis.backprop_modifiers import get
from keras import activations
if clas != None:
output_y = np.zeros(output_y.shape) + clas
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
#layer_idx = utils.find_layer_idx(model, 'preds')
layer_idx = -1
# Swap softmax with linear
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
###
modifier = 'guided' # can be None (AKA vanilla) or 'relu'
save_grads_path = os.path.join(save_directory, 'salience', conn_name)
if not os.path.isdir(os.path.join(save_directory, 'salience')):
os.mkdir(os.path.join(save_directory, 'salience'))
if not os.path.isdir(save_grads_path):
os.mkdir(save_grads_path)
print("Outputting saliency maps")
if False:
for the_file in os.listdir(save_grads_path):
file_path = os.path.join(save_grads_path, the_file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
except Exception as e:
print(e)
modifier_fn = get(modifier)
model = modifier_fn(model)
for idx in range(output_x.shape[0]):
if verbose:
update_progress(float(idx) / output_x.shape[0])
if float(idx) / output_x.shape[0] > perc_output:
break
#savename = os.path.join(save_grads_path,output_f[idx])
#if os.path.isfile(savename):
# continue
losses = [(ActivationMaximization(model.layers[layer_idx],
int(output_y[idx][0])), -1)]
opt = Optimizer(model.input, losses, wrt_tensor=None, norm_grads=False)
grads = opt.minimize(seed_input=output_x[idx],
max_iter=1,
grad_modifier='absolute',
verbose=False)[1]
for i in range(grads.shape[3]):
wc_subfolder = os.path.join(save_grads_path, "wc%d" % i)
if not os.path.isdir(wc_subfolder):
os.mkdir(wc_subfolder)
np.savetxt(os.path.join(wc_subfolder, output_f[idx]),
np.squeeze(grads[:, :, :, i]))
# Pre-trained model
#model = ResNet50(weights='imagenet')
###---------------------------------------------------------------------------------------------------
layer_names = [
"activation_10", "activation_22", "activation_34", "activation_46"
]
count = 1
for layer_name in layer_names:
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
layer_idx = utils.find_layer_idx(model, layer_name)
# Select 50 filters for each layer; either first 50 or randomly selected.
filters = np.random.permutation(get_num_filters(
model.layers[layer_idx]))[:50]
for i in filters:
losses = [(ActivationMaximization(layer_dict[layer_name], i), 2),
(LPNorm(model.input), 6), (TotalVariation(model.input), 1)]
opt = Optimizer(model.input, losses)
a, b, c = opt.minimize(max_iter=200,
verbose=False,
input_modifiers=[Jitter(0.05)])
print(str(count) + '/200 DONE')
count += 1
a = Image.fromarray(a.astype("uint8"))
a.save('act_max_output/' + layer_name + '_finetuned_' + str(i) +
'.png') # change this when pre-trained model is used