def show_gradient_map(graph,
sess,
y,
x,
img,
is_integrated=False,
is_smooth=True,
feed_dict=None,
is_cluster=False):
if not is_integrated and not is_smooth:
gradient_saliency = saliency.GradientSaliency(graph, sess, y, x)
vanilla_mask_3d = gradient_saliency.GetMask(img, feed_dict=feed_dict)
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_mask_3d)
if not is_cluster:
ShowGrayscaleImage(vanilla_mask_grayscale,
title='Vanilla Gradient')
return vanilla_mask_3d, vanilla_mask_grayscale
if not is_integrated and is_smooth:
gradient_saliency = saliency.GradientSaliency(graph, sess, y, x)
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
img, feed_dict=feed_dict)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
if not is_cluster:
ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad')
return smoothgrad_mask_3d, smoothgrad_mask_grayscale
if is_integrated and not is_smooth:
baseline = np.zeros(img.shape)
baseline.fill(-1)
gradient_saliency = saliency.IntegratedGradients(graph, sess, y, x)
vanilla_mask_3d = gradient_saliency.GetMask(img,
feed_dict=feed_dict,
x_steps=5,
x_baseline=baseline)
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_mask_3d)
if not is_cluster:
ShowGrayscaleImage(vanilla_mask_grayscale,
title='Vanilla Gradient')
return vanilla_mask_3d, vanilla_mask_grayscale
if is_integrated and is_smooth:
baseline = np.zeros(img.shape)
baseline.fill(-1)
gradient_saliency = saliency.IntegratedGradients(graph, sess, y, x)
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
img, feed_dict=feed_dict, x_steps=5, x_baseline=baseline)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
if not is_cluster:
ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad')
return smoothgrad_mask_3d, smoothgrad_mask_grayscale
def get_saliency_image(graph, sess, y, image, saliency_method):
"""generates saliency image.
Args:
graph: tensor flow graph.
sess: the current session.
y: the pre-softmax activation we want to assess attribution with respect to.
image: float32 image tensor with size [1, None, None].
saliency_method: string indicating saliency map type to generate.
Returns:
a saliency map and a smoothed saliency map.
Raises:
ValueError: if the saliency_method string does not match any included method
"""
if saliency_method == 'integrated_gradients':
integrated_placeholder = saliency.IntegratedGradients(graph, sess, y, image)
return integrated_placeholder
elif saliency_method == 'gradient':
gradient_placeholder = saliency.GradientSaliency(graph, sess, y, image)
return gradient_placeholder
elif saliency_method == 'guided_backprop':
gb_placeholder = saliency.GuidedBackprop(graph, sess, y, image)
return gb_placeholder
else:
raise ValueError('No saliency method method matched. Verification of'
'input needed')
def Integrated_Gradients_and_SmoothGrad(graph, sess, y, images):
print('Integrated_Gradients_and_SmoothGrad')
integrated_gradients = saliency.IntegratedGradients(graph, sess, y, images)
# Baseline is a black image.
baseline = np.zeros(im.shape)
baseline.fill(-1)
# Compute the vanilla mask and the smoothed mask.
vanilla_integrated_gradients_mask_3d = integrated_gradients.GetMask(
im,
feed_dict={neuron_selector: prediction_class},
x_steps=25,
x_baseline=baseline)
# Smoothed mask for integrated gradients will take a while since we are doing nsamples * nsamples computations.
smoothgrad_integrated_gradients_mask_3d = integrated_gradients.GetSmoothedMask(
im,
feed_dict={neuron_selector: prediction_class},
x_steps=25,
x_baseline=baseline)
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_integrated_gradients_mask_3d)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_integrated_gradients_mask_3d)
image_grad.append(vanilla_mask_grayscale)
image_grad.append(smoothgrad_mask_grayscale)
def extract_saliency(image, method, images, sess, logits, y, neuron_selector):
prediction = tf.argmax(logits, 1)
integrated_gradients = saliency.IntegratedGradients(GRAPH, sess, y, images)
image = LoadImage(image)
prediction_class = sess.run(prediction, feed_dict = {images:[image]})[0]
baseline = np.zeros(image.shape)
baseline.fill(-1)
gradients_mask_3d = None
# Vanilla
if method == 0:
gradients_mask_3d = integrated_gradients.GetMask(image,
feed_dict = {neuron_selector: prediction_class},
x_steps = 25,
x_baseline = baseline)
# Smooth
elif method == 1:
gradients_mask_3d = integrated_gradients.GetSmoothedMask(image,
feed_dict = {neuron_selector: prediction_class},
x_steps = 25,
x_baseline = baseline)
grayscale_mask = saliency.VisualizeImageGrayscale(gradients_mask_3d)
grayscale_mask *= (255 / gradients_mask_3d.max())
grayscale_mask = np.uint8(grayscale_mask)
return grayscale_mask
def reconstruct(X, model, classs, modelpath, outpath, do=0.3, bs=64):
graph = tf.Graph()
with graph.as_default():
x_in_, y_in_, logits_, nett_, ww_, pred_, neuron_selector_, ny_ = infer(
model=model, classes=classs, dropout=do)
with tf.Session(graph=graph,
config=tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=True)) as sess:
tf.global_variables_initializer().run()
saver = tf.train.import_meta_graph(str(modelpath + '.meta'))
saver.restore(sess, modelpath)
itr, file, ph = X.data(train=False)
next_element = itr.get_next()
with tf.Session() as sessa:
sessa.run(itr.initializer, feed_dict={ph: file})
ct = 0
while True:
try:
x, y = sessa.run(next_element)
for mm in range(np.shape(x)[0]):
grad = saliency.IntegratedGradients(
graph, sess, y, x_in_)
img = x[mm, :, :, :]
# Baseline is a white image.
baseline = np.zeros(img.shape)
baseline.fill(255)
smoothgrad_mask_3d = grad.GetSmoothedMask(
x,
feed_dict={neuron_selector_: 1},
x_steps=25,
x_baseline=baseline)
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
smoothgrad_mask_grayscale = im2double(
smoothgrad_mask_grayscale)
smoothgrad_mask_grayscale = py_map2jpg(
smoothgrad_mask_grayscale)
sa = im2double(img) * 255
sb = im2double(smoothgrad_mask_grayscale) * 255
scurHeatMap = sa * 0.5 + sb * 0.5
sab = np.hstack((sa, sb))
sfull = np.hstack((scurHeatMap, sab))
cv2.imwrite(str(outpath + str(ct) + '.png'), sfull)
ct += 1
except tf.errors.OutOfRangeError:
print("Done!")
break
def get_saliency_constructors(model_graph,
model_session,
logit_tensor,
input_tensor,
gradcam=False,
conv_layer_gradcam=None):
"""Initialize mask functions in saliency package.
Args:
model_graph: tf graph of the model.
model_session: tf session with trained model loaded.
logit_tensor: tensor corresponding to the model logit output.
input_tensor: tensor coresponding to the input data.
gradcam: boolean to indicate whether to include gradcam.
conv_layer_gradcam: tensor corresponding to activations
from a conv layer, from the trained model.
Authors recommend last layer.
Returns:
saliency_constructor: dictionary (name of method, and value is
function to each saliency method.
neuron_selector: tensor of specific output to explain.
"""
assert (type(tf.Graph()) == type(model_graph)),\
("Model graph should be of type {}".format(type(tf.Graph())))
if gradcam and conv_layer_gradcam is None:
raise ValueError("If gradcam is True, then conv_layer_gradcam"
"is be provided.")
with model_graph.as_default():
with tf.name_scope("saliency"):
neuron_selector = tf.placeholder(tf.int32)
y_salient = logit_tensor[neuron_selector]
gradient_saliency = saliency.GradientSaliency(model_graph, model_session,
y_salient, input_tensor)
guided_backprop = saliency.GuidedBackprop(model_graph, model_session,
y_salient, input_tensor)
integrated_gradients = saliency.IntegratedGradients(
model_graph, model_session, y_salient, input_tensor)
saliency_constructor = {
'vng': gradient_saliency,
'gbp': guided_backprop,
'ig': integrated_gradients
}
if gradcam:
gradcam = saliency.GradCam(model_graph, model_session, y_salient,
input_tensor, conv_layer_gradcam)
saliency_constructor['gc'] = gradcam
return saliency_constructor, neuron_selector
def Blur_IG(graph, sess, y, images):
integrated_gradients = saliency.IntegratedGradients(graph, sess, y, images)
blur_ig = saliency.BlurIG(graph, sess, y, images)
# Baseline is a black image for vanilla integrated gradients.
baseline = np.zeros(im.shape)
baseline.fill(-1)
# Compute the vanilla mask and the Blur IG mask.
vanilla_integrated_gradients_mask_3d = integrated_gradients.GetMask(
im,
feed_dict={neuron_selector: prediction_class},
x_steps=25,
x_baseline=baseline)
blur_ig_mask_3d = blur_ig.GetMask(
im, feed_dict={neuron_selector: prediction_class})
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_integrated_gradients_mask_3d)
blur_ig_mask_grayscale = saliency.VisualizeImageGrayscale(blur_ig_mask_3d)
image_grad.append(vanilla_mask_grayscale)
image_grad.append(blur_ig_mask_grayscale)
feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
np.reshape(img, (320, 320, 3)),
feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_mask_3d)
scores['grad'].append(
saliency_ttest(vanilla_mask_grayscale, bb_coords,
prediction_class))
scores['sg'].append(
saliency_ttest(smoothgrad_mask_grayscale, bb_coords,
prediction_class))
integrated_gradients = saliency.IntegratedGradients(
sess.graph, sess, y, inp)
baseline = np.zeros(img.shape)
baseline.fill(-1)
vanilla_integrated_gradients_mask_3d = integrated_gradients.GetMask(
img,
feed_dict={neuron_selector: prediction_class},
x_steps=10,
x_baseline=baseline)
smoothgrad_integrated_gradients_mask_3d = integrated_gradients.GetSmoothedMask(
img,
feed_dict={neuron_selector: prediction_class},
x_steps=10,
x_baseline=baseline)
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_integrated_gradients_mask_3d)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
# Load the image
print(sys.argv[1])
im = LoadImage(sys.argv[1])
im = cv2.resize(im, (299, 299))
start = time.time()
# Make a prediction.
prediction_class = sess.run(prediction, feed_dict={images: [im]})[0]
print("Prediction class: " + str(prediction_class))
# Construct the saliency object. This doesn't yet compute the saliency mask,
# it just sets up the necessary ops.
integrated_gradients = saliency.IntegratedGradients(graph, sess, y, images)
# Baseline is a black image.
baseline = np.zeros(im.shape)
baseline.fill(-1)
# Smoothed mask for integrated gradients will take a while since we are doing nsamples * nsamples computations.
smoothgrad_integrated_gradients_mask_3d = integrated_gradients.GetSmoothedMask(
im, feed_dict={neuron_selector: prediction_class}, x_steps=20, x_baseline=baseline)
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_integrated_gradients_mask_3d)
im = smoothgrad_mask_grayscale * 255
im = cv2.resize(im, (224, 224))
def compute_and_save_attr(model, data, indices, num_threads):
"""Given the name of a model and a set of data, select a set of images based on provided indices, and compute and save their saliency maps and object attributions."""
base_dir = os.getcwd()
data_dir = os.path.join(base_dir, 'data', data, 'val')
model_dir = os.path.join(base_dir, 'models', model)
sal_output_dir = os.path.join(base_dir, SAL_DIR, model + '-' + data)
attr_output_dir = os.path.join(base_dir, ATTR_DIR, model + '-' + data)
if not tf.gfile.Exists(sal_output_dir):
tf.gfile.MakeDirs(sal_output_dir)
if not tf.gfile.Exists(attr_output_dir):
tf.gfile.MakeDirs(attr_output_dir)
img_names = [sorted(tf.gfile.ListDirectory(data_dir))[i] for i in indices]
img_paths = [os.path.join(data_dir, img_name) for img_name in img_names]
imgs = load_imgs(img_paths, num_threads)
input_name = 'input_tensor:0'
logit_name = 'resnet_model/final_dense:0'
conv_name = 'resnet_model/block_layer4:0'
with tf.Session(graph=tf.Graph()) as sess:
tf.saved_model.loader.load(sess, ['serve'], model_dir)
graph = tf.get_default_graph()
input_tensor = graph.get_tensor_by_name(input_name)
logit_tensor = graph.get_tensor_by_name(logit_name)
neuron_selector = tf.placeholder(tf.int32)
y = logit_tensor[:, neuron_selector]
pred_tensor = tf.argmax(logit_tensor, 1)
vg = saliency.GradientSaliency(graph, sess, y, input_tensor)
gb = saliency.GuidedBackprop(graph, sess, y, input_tensor)
ig = saliency.IntegratedGradients(graph, sess, y, input_tensor)
gc = saliency.GradCam(graph, sess, y, input_tensor,
graph.get_tensor_by_name(conv_name))
def single_map(img, img_name):
pred = sess.run(pred_tensor, feed_dict={input_tensor: [img]})[0]
vg_mask = vg.GetMask(img, feed_dict={neuron_selector: pred})
# *s is SmoothGrad
vgs_mask = vg.GetSmoothedMask(img,
feed_dict={neuron_selector: pred})
gb_mask = gb.GetMask(img, feed_dict={neuron_selector: pred})
gbs_mask = gb.GetSmoothedMask(img,
feed_dict={neuron_selector: pred})
baseline = np.zeros(img.shape) - np.expand_dims(
np.expand_dims(_CHANNEL_MEANS, 0), 0)
ig_mask = ig.GetMask(img,
feed_dict={neuron_selector: pred},
x_baseline=baseline)
igs_mask = ig.GetSmoothedMask(img,
feed_dict={neuron_selector: pred},
x_baseline=baseline)
gc_mask = gc.GetMask(img, feed_dict={neuron_selector: pred})
gcs_mask = gc.GetSmoothedMask(img,
feed_dict={neuron_selector: pred})
# gbgc is guided GradCam
gbgc_mask = gb_mask * gc_mask
gbgcs_mask = gbs_mask * gcs_mask
# Also include gradient x input
masks = np.array([
vg_mask, vgs_mask, gb_mask, gbs_mask, ig_mask, igs_mask,
gc_mask, gcs_mask, gbgc_mask, gbgcs_mask, vg_mask * img,
vgs_mask * img
])
return masks, pred
sal_maps = []
preds = []
for img, img_name in zip(imgs, img_names):
sal_path = tf.gfile.Glob(
os.path.join(sal_output_dir, img_name[:-4] + '*'))
if len(sal_path) > 0:
sal_maps.append(np.load(tf.gfile.GFile(sal_path[0], 'rb')))
preds.append(sal_path[0].split('_')[-1])
tf.logging.info(
'Loaded saliency maps for {}.'.format(img_name))
else:
masks, pred = single_map(img, img_name)
sal_maps.append(masks)
preds.append(pred)
out_path = os.path.join(sal_output_dir,
img_name[:-4] + '_' + str(pred))
np.save(tf.gfile.GFile(out_path, 'w'), masks)
tf.logging.info('Saved saliency maps for {}.'.format(img_name))
# Locate the objects, convert 3D saliency maps to 2D, and compute
# the attributions of the object segments by averaging over the
# per-pixel attributions of the objects.
loc_fpath = os.path.join(base_dir, 'data', data, 'val_loc.txt')
lines = [tf.gfile.Open(loc_fpath).readlines()[i] for i in indices]
locs = np.array([[
int(int(l) * float(RESNET_SHAPE[0]) / IMG_SHAPE[0])
for l in line.rstrip('\n').split(' ')[-1].split(',')
] for line in lines])
pool = multiprocessing.Pool(num_threads)
maps_3d = np.array(sal_maps).reshape(-1, RESNET_SHAPE[0], RESNET_SHAPE[1],
3)
maps_2d = np.array(pool.map(visualize_pos_attr, maps_3d))
maps_2d = maps_2d.reshape(len(indices),
int(maps_2d.shape[0] // len(indices)),
RESNET_SHAPE[0], RESNET_SHAPE[1])
mask_fpath = os.path.join(base_dir, 'data', data, 'val_mask')
if data in ['obj', 'scene', 'scene_only']:
# MCS and IDR are evaluated on 10000 images and masks are 10x100.
# Find the right mask.
obj_dict = {
'backpack': 0,
'bird': 1,
'dog': 2,
'elephant': 3,
'kite': 4,
'pizza': 5,
'stop_sign': 6,
'toilet': 7,
'truck': 8,
'zebra': 9,
}
# Loading val_mask from the data directory
masks_mat = np.load(tf.gfile.GFile(mask_fpath, 'rb'),
allow_pickle=True)
# Getting obj indices
obj_inds = [obj_dict[i.split('.')[0].split('-')[0]] for i in img_names]
# getting indices for a particular object class
temp_inds = [int(i.split('.')[0][-2:]) for i in img_names]
obj_masks = [
masks_mat[obj_inds[i] * 100 + temp_inds[i]]
for i, _ in enumerate(img_names)
]
else:
obj_masks = [
np.load(tf.gfile.GFile(mask_fpath, 'rb'), allow_pickle=True)[i]
for i in indices
]
attrs = []
for i in range(len(indices)):
attr = single_attr(maps_2d[i], locs[i], obj_masks[i])
attrs.append(attr)
out_path = os.path.join(attr_output_dir,
img_names[i][:-4] + '_' + str(preds[i]))
np.save(tf.gfile.GFile(out_path, 'w'), attr)
def simple_example():
#--------------------
mnist = input_data.read_data_sets('./MNIST_data', one_hot=True)
#--------------------
# Define a model.
num_classes = 10
input_shape = (None, 28, 28, 1) # 784 = 28 * 28.
output_shape = (None, num_classes)
input_ph = tf.placeholder(tf.float32, shape=input_shape, name='input_ph')
output_ph = tf.placeholder(tf.float32,
shape=output_shape,
name='output_ph')
with tf.variable_scope('conv1', reuse=tf.AUTO_REUSE):
conv1 = tf.layers.conv2d(input_ph,
32,
5,
activation=tf.nn.relu,
name='conv')
conv1 = tf.layers.max_pooling2d(conv1, 2, 2, name='maxpool')
with tf.variable_scope('conv2', reuse=tf.AUTO_REUSE):
conv2 = tf.layers.conv2d(conv1,
64,
3,
activation=tf.nn.relu,
name='conv')
conv2 = tf.layers.max_pooling2d(conv2, 2, 2, name='maxpool')
with tf.variable_scope('fc1', reuse=tf.AUTO_REUSE):
fc1 = tf.layers.flatten(conv2, name='flatten')
fc1 = tf.layers.dense(fc1, 1024, activation=tf.nn.relu, name='dense')
with tf.variable_scope('fc2', reuse=tf.AUTO_REUSE):
model_output = tf.layers.dense(fc1,
num_classes,
activation=tf.nn.softmax,
name='dense')
#--------------------
# Train.
loss = tf.reduce_mean(-tf.reduce_sum(output_ph * tf.log(model_output),
reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
print('Start training...')
start_time = time.time()
for _ in range(2000):
batch_xs, batch_ys = mnist.train.next_batch(512)
batch_xs = np.reshape(batch_xs, (-1, ) + input_shape[1:])
sess.run(train_step,
feed_dict={
input_ph: batch_xs,
output_ph: batch_ys
})
if 0 == idx % 100: print('.', end='', flush=True)
print()
print('End training: {} secs.'.format(time.time() - start_time))
#--------------------
# Evaluate.
correct_prediction = tf.equal(tf.argmax(model_output, 1),
tf.argmax(output_ph, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print('Start testing...')
acc = sess.run(accuracy,
feed_dict={
input_ph:
np.reshape(mnist.test.images, (-1, ) + input_shape[1:]),
output_ph:
mnist.test.labels
})
print('Test accuracy = {}.'.format(acc))
print('End testing: {} secs.'.format(time.time() - start_time))
if acc < 0.95:
print('Failed to train...')
return
#--------------------
# Visualize.
images = np.reshape(mnist.test.images, (-1, ) + input_shape[1:])
img = images[0]
minval, maxval = np.min(img), np.max(img)
img_scaled = np.squeeze((img - minval) / (maxval - minval), axis=-1)
# Construct the scalar neuron tensor.
logits = model_output
neuron_selector = tf.placeholder(tf.int32)
y = logits[0][neuron_selector]
# Construct a tensor for predictions.
prediction = tf.argmax(logits, 1)
# Make a prediction.
prediction_class = sess.run(prediction, feed_dict={input_ph: [img]})[0]
#--------------------
start_time = time.time()
saliency_obj = saliency.Occlusion(sess.graph, sess, y, input_ph)
print('Occlusion: {} secs.'.format(time.time() - start_time))
# NOTE [info] >> An error exists in GetMask() of ${Saliency_HOME}/saliency/occlusion.py.
# <before>
# occlusion_window = np.array([size, size, x_value.shape[2]])
# occlusion_window.fill(value)
# <after>
# occlusion_window = np.full([size, size, x_value.shape[2]], value)
mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
mask_gray = saliency.VisualizeImageGrayscale(mask_3d)
mask_div = saliency.VisualizeImageDiverging(mask_3d)
fig = plt.figure()
ax = plt.subplot(1, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(1, 3, 2)
ax.imshow(mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Grayscale')
ax = plt.subplot(1, 3, 3)
ax.imshow(mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Diverging')
fig.suptitle('Occlusion', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_occlusion.png')
plt.show()
#--------------------
start_time = time.time()
conv_layer = sess.graph.get_tensor_by_name('conv2/conv/BiasAdd:0')
saliency_obj = saliency.GradCam(sess.graph, sess, y, input_ph, conv_layer)
print('GradCam: {} secs.'.format(time.time() - start_time))
mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
mask_gray = saliency.VisualizeImageGrayscale(mask_3d)
mask_div = saliency.VisualizeImageDiverging(mask_3d)
fig = plt.figure()
ax = plt.subplot(1, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(1, 3, 2)
ax.imshow(mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Grayscale')
ax = plt.subplot(1, 3, 3)
ax.imshow(mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Diverging')
fig.suptitle('Grad-CAM', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_gradcam.png')
plt.show()
#--------------------
start_time = time.time()
saliency_obj = saliency.GradientSaliency(sess.graph, sess, y, input_ph)
print('GradientSaliency: {} secs.'.format(time.time() - start_time))
vanilla_mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = saliency_obj.GetSmoothedMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d)
smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d)
fig = plt.figure()
ax = plt.subplot(2, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(2, 3, 2)
ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Grayscale')
ax = plt.subplot(2, 3, 3)
ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Grayscale')
ax = plt.subplot(2, 3, 5)
ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Diverging')
ax = plt.subplot(2, 3, 6)
ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Diverging')
fig.suptitle('Gradient Saliency', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_gradientsaliency.png')
plt.show()
#--------------------
start_time = time.time()
saliency_obj = saliency.GuidedBackprop(sess.graph, sess, y, input_ph)
print('GuidedBackprop: {} secs.'.format(time.time() - start_time))
vanilla_mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = saliency_obj.GetSmoothedMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d)
smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d)
fig = plt.figure()
ax = plt.subplot(2, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(2, 3, 2)
ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Grayscale')
ax = plt.subplot(2, 3, 3)
ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Grayscale')
ax = plt.subplot(2, 3, 4)
ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Diverging')
ax = plt.subplot(2, 3, 5)
ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Diverging')
fig.suptitle('Guided Backprop', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_guidedbackprop.png')
plt.show()
#--------------------
start_time = time.time()
saliency_obj = saliency.IntegratedGradients(sess.graph, sess, y, input_ph)
print('IntegratedGradients: {} secs.'.format(time.time() - start_time))
vanilla_mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = saliency_obj.GetSmoothedMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d)
smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d)
fig = plt.figure()
ax = plt.subplot(2, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(2, 3, 2)
ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Grayscale')
ax = plt.subplot(2, 3, 3)
ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Grayscale')
ax = plt.subplot(2, 3, 4)
ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Diverging')
ax = plt.subplot(2, 3, 5)
ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Diverging')
fig.suptitle('Integrated Gradients', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_integratedgradients.png')
plt.show()
#--------------------
start_time = time.time()
xrai_obj = saliency.XRAI(sess.graph, sess, y, input_ph)
print('XRAI: {} secs.'.format(time.time() - start_time))
if True:
xrai_attributions = xrai_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
else:
# Create XRAIParameters and set the algorithm to fast mode which will produce an approximate result.
xrai_params = saliency.XRAIParameters()
xrai_params.algorithm = 'fast'
xrai_attributions_fast = xrai_obj.GetMask(
img,
feed_dict={neuron_selector: prediction_class},
extra_parameters=xrai_params)
# Show most salient 30% of the image.
mask = xrai_attributions > np.percentile(xrai_attributions, 70)
img_masked = img_scaled.copy()
img_masked[~mask] = 0
fig = plt.figure()
ax = plt.subplot(1, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(1, 3, 2)
ax.imshow(xrai_attributions, cmap=plt.cm.inferno)
ax.axis('off')
ax.set_title('XRAI Attributions')
ax = plt.subplot(1, 3, 3)
ax.imshow(img_masked, cmap=plt.cm.gray)
ax.axis('off')
ax.set_title('Masked Input')
fig.suptitle('XRAI', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_xrai.png')
plt.show()
