def visualize_multiple_categories(show=True):
"""Example to show how to visualize images that activate multiple categories
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
# Visualize [20] (ouzel) and [20, 71] (An ouzel-scorpion :D)
indices = [20, [20, 71]]
images = []
for idx in indices:
img = visualize_class_activation(model,
layer_idx,
filter_indices=idx,
max_iter=500)
img = utils.draw_text(img, utils.get_imagenet_label(idx))
images.append(img)
# Easily stitch images via `utils.stitch_images`
stitched = utils.stitch_images(images)
if show:
plt.axis('off')
plt.imshow(stitched)
plt.title('Multiple category visualization')
plt.show()
def generate_cam(show=True):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in [
'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg'
]:
seed_img = utils.load_img(path, target_size=(224, 224))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(seed_img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img)
if show:
plt.axis('off')
plt.imshow(heatmap)
plt.title('Attention - {}'.format(
utils.get_imagenet_label(pred_class)))
plt.show()
def visualize_multiple_categories():
"""Example to show how to visualize images that activate multiple categories
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
# Visualize [20] (ouzel) and [20, 71] (An ouzel-scorpion :D)
indices = [20, [20, 71]]
images = [
visualize_activation(model,
layer_idx,
filter_indices=idx,
text=utils.get_imagenet_label(idx),
max_iter=500) for idx in indices
]
cv2.imshow('Multiple category visualization', utils.stitch_images(images))
cv2.waitKey(0)
def generate_saliceny_map(show=True):
"""Generates a heatmap indicating the pixels that contributed the most towards
maximizing the filter output. First, the class prediction is determined, then we generate heatmap
to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in ['../resources/ouzel.jpg', '../resources/ouzel_1.jpg']:
seed_img = utils.load_img(path, target_size=(224, 224))
pred_class = np.argmax(
model.predict(np.array([img_to_array(seed_img)])))
heatmap = visualize_saliency(model, layer_idx, [pred_class], seed_img)
if show:
cv2.imshow(
'Saliency - {}'.format(utils.get_imagenet_label(pred_class)),
heatmap)
cv2.waitKey(0)
def generate_saliceny_map(show=True):
"""Generates a heatmap indicating the pixels that contributed the most towards
maximizing the filter output. First, the class prediction is determined, then we generate heatmap
to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in ['../resources/ouzel.jpg', '../resources/ouzel_1.jpg']:
seed_img = utils.load_img(path, target_size=(224, 224))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(seed_img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_saliency(model, layer_idx, [pred_class], seed_img)
if show:
plt.axis('off')
plt.imshow(heatmap)
plt.title('Saliency - {}'.format(
utils.get_imagenet_label(pred_class)))
plt.show()
def generate_cam(show=True):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in [
'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg'
]:
seed_img = utils.load_img(path, target_size=(224, 224))
pred_class = np.argmax(
model.predict(np.array([img_to_array(seed_img)])))
heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img)
if show:
cv2.imshow(
'Attention - {}'.format(utils.get_imagenet_label(pred_class)),
heatmap)
cv2.waitKey(0)
def visualize_multiple_same_filter(num_runs=3, show=True):
"""Example to show how to visualize same filter multiple times via different runs.
Args:
num_runs: The number of times the same filter is visualized
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [idx for idx, layer in enumerate(model.layers) if layer.name == layer_name][0]
# 20 is the imagenet category for 'ouzel'
indices = [20] * num_runs
images = []
for idx in indices:
img = visualize_activation(model, layer_idx, filter_indices=idx, max_iter=500)
img = utils.draw_text(img, utils.get_imagenet_label(idx))
images.append(img)
# Easily stitch images via `utils.stitch_images`
stitched = utils.stitch_images(images)
if show:
plt.axis('off')
plt.imshow(stitched)
plt.title('Multiple runs of ouzel')
plt.show()
def visualize_random(num_categories=10, show=True):
"""Example to show how to visualize multiple filters via activation maximization.
Args:
num_categories: The number of random categories to visualize. (Default Value = 5)
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [idx for idx, layer in enumerate(model.layers) if layer.name == layer_name][0]
# Visualize couple random categories from imagenet.
indices = np.random.permutation(1000)[:num_categories]
images = []
for idx in indices:
img = visualize_activation(model, layer_idx, filter_indices=idx, max_iter=500)
img = utils.draw_text(img, utils.get_imagenet_label(idx))
images.append(img)
# Easily stitch images via `utils.stitch_images`
stitched = utils.stitch_images(images)
if show:
plt.axis('off')
plt.imshow(stitched)
plt.title('Random imagenet categories')
plt.show()
def visualize_random(num_categories=10):
"""Example to show how to visualize multiple filters via activation maximization.
Args:
num_categories: The number of random categories to visualize. (Default Value = 5)
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
# Visualize couple random categories from imagenet.
indices = np.random.permutation(1000)[:num_categories]
images = [
visualize_activation(model,
layer_idx,
filter_indices=idx,
text=utils.get_imagenet_label(idx),
max_iter=500) for idx in indices
]
# Easily stitch images via `utils.stitch_images`
cv2.imshow('Random imagenet categories', utils.stitch_images(images))
cv2.waitKey(0)
def visualize_multiple_same_filter(num_runs=3):
"""Example to show how to visualize same filter multiple times via different runs.
Args:
num_runs: The number of times the same filter is visualized
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
# 20 is the imagenet category for 'ouzel'
indices = [20] * num_runs
images = [
visualize_activation(model,
layer_idx,
filter_indices=idx,
text=utils.get_imagenet_label(idx),
max_iter=500) for idx in indices
]
cv2.imshow('Multiple runs of ouzel', utils.stitch_images(images))
cv2.waitKey(0)
def generate_cam_from_image(image, model=None, returnAsImage=True, layer=None):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
if model is None:
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
if layer is None:
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
else:
layer_idx = layer
#seed_img = Image.open(image)
seed_img = Image.fromarray(image)
seed_img = seed_img.resize((configs.img_width, configs.img_height))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(np.asarray(seed_img))
img_input = np.expand_dims(bgr_img, axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_cam(model, layer_idx, [pred_class],
np.asarray(seed_img))
if returnAsImage is True:
return Image.fromarray(heatmap)
else:
return heatmap
def generate_opt_gif():
"""Example to show how to generate the gif of optimization progress.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [idx for idx, layer in enumerate(model.layers) if layer.name == layer_name][0]
output_class = [20]
optimizer_params = {
'max_iter': 500,
'verbose': True,
'callbacks': [GifGenerator('opt_progress')]
}
visualize_class_activation(model, layer_idx, filter_indices=output_class, **optimizer_params)
def visualize_multiple_same_filter():
"""Example to show how to visualize same filter multiple times via different runs.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
# 20 is the imagenet category for 'ouzel'
indices = [20, 20, 20]
idx_label_map = dict((idx, utils.get_imagenet_label(idx)) for idx in indices)
vis_img = visualize_activation(model.input, layer_dict[layer_name], max_iter=500,
filter_indices=indices, idx_label_map=idx_label_map)
cv2.imshow('Multiple runs of ouzel', vis_img)
cv2.waitKey(0)
def visualize_random():
"""Example to show how to visualize multiple filters via activation maximization
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
# Visualize couple random categories from imagenet.
indices = np.random.permutation(1000)[:15]
idx_label_map = dict((idx, utils.get_imagenet_label(idx)) for idx in indices)
vis_img = visualize_activation(model.input, layer_dict[layer_name], max_iter=500,
filter_indices=indices, idx_label_map=idx_label_map)
cv2.imshow('Random imagenet output categories', vis_img)
cv2.waitKey(0)
def visualize_multiple_categories():
"""Example to show how to visualize images that activate multiple categories
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
# Visualize [20] (ouzel) and [20, 71] (An ouzel-scorpion :D)
indices = [20, [20, 71]]
idx_label_map = dict((idx, utils.get_imagenet_label(idx)) for idx in [20, 71])
vis_img = visualize_activation(model.input, layer_dict[layer_name], max_iter=500,
filter_indices=indices, idx_label_map=idx_label_map)
cv2.imshow('Multiple category visualization', vis_img)
cv2.waitKey(0)
def main():
"""Generates a heatmap indicating the pixels that contributed the most towards
maximizing the filter output.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
for path in ['../resources/ouzel.jpg', '../resources/ouzel_1.jpg']:
seed_img = utils.load_img(path, target_size=(224, 224))
# 20 is the imagenet category for 'ouzel'
heatmap = visualize_saliency(model.input, layer_dict[layer_name], [20],
seed_img)
cv2.imshow('Importance map', heatmap)
cv2.waitKey(0)
def generate_opt_gif():
"""Example to show how to generate the gif of optimization progress.
This example also shows how to use the optimizer directly with losses.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
output_class = [20]
losses = [(ActivationMaximization(layer_dict[layer_name],
output_class), 2),
(LPNorm(model.input), 10), (TotalVariation(model.input), 10)]
opt = Optimizer(model.input, losses)
opt.minimize(max_iter=500,
verbose=True,
callbacks=[GifGenerator('opt_progress')])
def generate_opt_gif():
"""Example to show how to generate the gif of optimization progress.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
output_class = [20]
losses = [(ActivationMaximization(layer_dict[layer_name],
output_class), 1), (LPNorm(), 10),
(TotalVariation(), 1)]
opt = Optimizer(model.input, losses)
# Jitter is used as a regularizer to create crisper images, but it makes gif animation ugly.
opt.minimize(max_iter=500,
verbose=True,
jitter=0,
progress_gif_path='opt_progress')
from keras.preprocessing.image import img_to_array
from keras.applications.imagenet_utils import decode_predictions, preprocess_input
from vis.utils import utils
from vis.utils.vggnet import VGG16
from vis.utils.inception_v3 import InceptionV3, conv2d_bn
from vis.visualization import visualize_saliency, visualize_cam, visualize_activation, get_num_filters
def preprocess_input(x):
x /= 255.
x -= 0.5
x *= 2.
return x
# Build the VGG16 network with ImageNet weights
model_vgg = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
model = model_vgg
# Build the InceptionV3 network with ImageNet weights
# model = InceptionV3(weights='imagenet', include_top=True)
# print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py or inception_v3.py)
layer_name = 'predictions'
layer_idx = [idx for idx, layer in enumerate(model.layers) if layer.name == layer_name][0]
# Images corresponding to tiger, penguin, dumbbell, speedboat, spider
image_paths = [
"http://www.tigerfdn.com/wp-content/uploads/2016/05/How-Much-Does-A-Tiger-Weigh.jpg",
