def Integrated_Gradients_and_SmoothGrad(graph, sess, y, images):
print('Integrated_Gradients_and_SmoothGrad')
integrated_gradients = saliency.IntegratedGradients(graph, sess, y, images)
# Baseline is a black image.
baseline = np.zeros(im.shape)
baseline.fill(-1)
# Compute the vanilla mask and the smoothed mask.
vanilla_integrated_gradients_mask_3d = integrated_gradients.GetMask(
im,
feed_dict={neuron_selector: prediction_class},
x_steps=25,
x_baseline=baseline)
# Smoothed mask for integrated gradients will take a while since we are doing nsamples * nsamples computations.
smoothgrad_integrated_gradients_mask_3d = integrated_gradients.GetSmoothedMask(
im,
feed_dict={neuron_selector: prediction_class},
x_steps=25,
x_baseline=baseline)
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_integrated_gradients_mask_3d)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_integrated_gradients_mask_3d)
image_grad.append(vanilla_mask_grayscale)
image_grad.append(smoothgrad_mask_grayscale)
def show_gradient_map(graph,
sess,
y,
x,
img,
is_integrated=False,
is_smooth=True,
feed_dict=None,
is_cluster=False):
if not is_integrated and not is_smooth:
gradient_saliency = saliency.GradientSaliency(graph, sess, y, x)
vanilla_mask_3d = gradient_saliency.GetMask(img, feed_dict=feed_dict)
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_mask_3d)
if not is_cluster:
ShowGrayscaleImage(vanilla_mask_grayscale,
title='Vanilla Gradient')
return vanilla_mask_3d, vanilla_mask_grayscale
if not is_integrated and is_smooth:
gradient_saliency = saliency.GradientSaliency(graph, sess, y, x)
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
img, feed_dict=feed_dict)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
if not is_cluster:
ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad')
return smoothgrad_mask_3d, smoothgrad_mask_grayscale
if is_integrated and not is_smooth:
baseline = np.zeros(img.shape)
baseline.fill(-1)
gradient_saliency = saliency.IntegratedGradients(graph, sess, y, x)
vanilla_mask_3d = gradient_saliency.GetMask(img,
feed_dict=feed_dict,
x_steps=5,
x_baseline=baseline)
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_mask_3d)
if not is_cluster:
ShowGrayscaleImage(vanilla_mask_grayscale,
title='Vanilla Gradient')
return vanilla_mask_3d, vanilla_mask_grayscale
if is_integrated and is_smooth:
baseline = np.zeros(img.shape)
baseline.fill(-1)
gradient_saliency = saliency.IntegratedGradients(graph, sess, y, x)
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
img, feed_dict=feed_dict, x_steps=5, x_baseline=baseline)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
if not is_cluster:
ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad')
return smoothgrad_mask_3d, smoothgrad_mask_grayscale
def guided_vanilla(image):
image = image / 127.5 - 1.0
prediction_class = sess.run(prediction, feed_dict = {images: [image]})[0]
vanilla_guided_backprop_mask_3d = guided_backprop.GetMask(
image, feed_dict = {neuron_selector: prediction_class})
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_guided_backprop_mask_3d)
return vanilla_mask_grayscale.tolist()
def extract_saliency(image, method, images, sess, logits, y, neuron_selector):
prediction = tf.argmax(logits, 1)
integrated_gradients = saliency.IntegratedGradients(GRAPH, sess, y, images)
image = LoadImage(image)
prediction_class = sess.run(prediction, feed_dict = {images:[image]})[0]
baseline = np.zeros(image.shape)
baseline.fill(-1)
gradients_mask_3d = None
# Vanilla
if method == 0:
gradients_mask_3d = integrated_gradients.GetMask(image,
feed_dict = {neuron_selector: prediction_class},
x_steps = 25,
x_baseline = baseline)
# Smooth
elif method == 1:
gradients_mask_3d = integrated_gradients.GetSmoothedMask(image,
feed_dict = {neuron_selector: prediction_class},
x_steps = 25,
x_baseline = baseline)
grayscale_mask = saliency.VisualizeImageGrayscale(gradients_mask_3d)
grayscale_mask *= (255 / gradients_mask_3d.max())
grayscale_mask = np.uint8(grayscale_mask)
return grayscale_mask
def vanilla_gradient_smoothgrad(graph, sess, y, images):
print('Vanilla_adn_Smoothgrad')
gradient_saliency = saliency.GradientSaliency(graph, sess, y, images)
# Compute the vanilla mask and the smoothed mask.
vanilla_mask_3d = gradient_saliency.GetMask(
im, feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
im, feed_dict={neuron_selector: prediction_class})
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
image_grad.append(vanilla_mask_grayscale)
image_grad.append(smoothgrad_mask_grayscale)
def visualize(sess, images_ph, im, y, prediction_class, neuron_selector,
base_name):
import saliency
graph = tf.get_default_graph()
# Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops.
gradient_saliency = saliency.GradientSaliency(graph, sess, y, images_ph)
guided_backprop = saliency.GuidedBackprop(graph, sess, y, images_ph)
algo = guided_backprop
# Compute the vanilla mask and the smoothed mask.
for i, image in enumerate(im):
prediction_class = 1 #np.argmax(prediction_class[i])
vanilla_mask_3d = algo.GetMask(
image, feed_dict={neuron_selector: prediction_class})
#smoothgrad_mask_3d = algo.GetSmoothedMask(im, feed_dict = {neuron_selector: prediction_class})
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_mask_3d)
#smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
vanilla_mask_grayscale = ((vanilla_mask_grayscale + 1) * 127.5).astype(
np.uint8)
#smoothgrad_mask_grayscale = ((smoothgrad_mask_grayscale + 1) * 127.5).astype(np.uint8)
result = Image.fromarray(vanilla_mask_grayscale, mode='L')
#smoothed_result = Image.fromarray(smoothgrad_mask_grayscale, mode='L')
#print 'sal name', base_name[i]
print 'sal_image_dest_dir', sal_image_dest_dir
sal_name = sal_image_dest_dir + str(base_name[i]).replace(
'.jpg', '_sal.jpg')
print 'saving {} ...'.format(sal_name)
result.save(sal_name)
def Guided_Backprop_SmoothGrad(graph, sess, y, images):
print('Guided_Backprop_SmoothGrad')
guided_backprop = saliency.GuidedBackprop(graph, sess, y, images)
# Compute the vanilla mask and the smoothed mask.
vanilla_guided_backprop_mask_3d = guided_backprop.GetMask(
im, feed_dict={neuron_selector: prediction_class})
smoothgrad_guided_backprop_mask_3d = guided_backprop.GetSmoothedMask(
im, feed_dict={neuron_selector: prediction_class})
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_guided_backprop_mask_3d)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_guided_backprop_mask_3d)
image_grad.append(vanilla_mask_grayscale)
image_grad.append(smoothgrad_mask_grayscale)
def Blur_Smootth_Grad(graph, sess, y, images):
blur_ig = saliency.BlurIG(graph, sess, y, images)
# Compute the Blur IG mask and Smoothgrad+BlurIG mask.
blur_ig_mask_3d = blur_ig.GetMask(
im, feed_dict={neuron_selector: prediction_class})
# Smoothed mask for BlurIG will take a while since we are doing nsamples * nsamples computations.
smooth_blur_ig_mask_3d = blur_ig.GetSmoothedMask(
im, feed_dict={neuron_selector: prediction_class})
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
blur_ig_mask_grayscale = saliency.VisualizeImageGrayscale(blur_ig_mask_3d)
smooth_blur_ig_mask_grayscale = saliency.VisualizeImageGrayscale(
smooth_blur_ig_mask_3d)
image_grad.append(blur_ig_mask_grayscale)
image_grad.append(smooth_blur_ig_mask_grayscale)
def abs_grayscale_norm(img):
"""Returns absolute value normalized image 2D."""
assert isinstance(img, np.ndarray), "img should be a numpy array"
shp = img.shape
if len(shp) < 2:
raise ValueError("Array should have 2 or 3 dims!")
if len(shp) == 2:
img = np.absolute(img)
img = img / float(img.max())
else:
img = saliency.VisualizeImageGrayscale(img)
return img
def reconstruct(X, model, classs, modelpath, outpath, do=0.3, bs=64):
graph = tf.Graph()
with graph.as_default():
x_in_, y_in_, logits_, nett_, ww_, pred_, neuron_selector_, ny_ = infer(
model=model, classes=classs, dropout=do)
with tf.Session(graph=graph,
config=tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=True)) as sess:
tf.global_variables_initializer().run()
saver = tf.train.import_meta_graph(str(modelpath + '.meta'))
saver.restore(sess, modelpath)
itr, file, ph = X.data(train=False)
next_element = itr.get_next()
with tf.Session() as sessa:
sessa.run(itr.initializer, feed_dict={ph: file})
ct = 0
while True:
try:
x, y = sessa.run(next_element)
for mm in range(np.shape(x)[0]):
grad = saliency.IntegratedGradients(
graph, sess, y, x_in_)
img = x[mm, :, :, :]
# Baseline is a white image.
baseline = np.zeros(img.shape)
baseline.fill(255)
smoothgrad_mask_3d = grad.GetSmoothedMask(
x,
feed_dict={neuron_selector_: 1},
x_steps=25,
x_baseline=baseline)
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
smoothgrad_mask_grayscale = im2double(
smoothgrad_mask_grayscale)
smoothgrad_mask_grayscale = py_map2jpg(
smoothgrad_mask_grayscale)
sa = im2double(img) * 255
sb = im2double(smoothgrad_mask_grayscale) * 255
scurHeatMap = sa * 0.5 + sb * 0.5
sab = np.hstack((sa, sb))
sfull = np.hstack((scurHeatMap, sab))
cv2.imwrite(str(outpath + str(ct) + '.png'), sfull)
ct += 1
except tf.errors.OutOfRangeError:
print("Done!")
break
def getSaliency(self, images):
# Images: [batch, height, width, channel]
images = images / 127.5 - 1.0
predicted_class = self.saliency_sess.run(
self.pred, feed_dict={self.inputs: [images]})
gradient_saliency = saliency.GradientSaliency(self.graph,
self.saliency_sess,
self.y, self.inputs)
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
images, feed_dict={self.neuron_selector: predicted_class[0]})
return saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
def Blur_IG(graph, sess, y, images):
integrated_gradients = saliency.IntegratedGradients(graph, sess, y, images)
blur_ig = saliency.BlurIG(graph, sess, y, images)
# Baseline is a black image for vanilla integrated gradients.
baseline = np.zeros(im.shape)
baseline.fill(-1)
# Compute the vanilla mask and the Blur IG mask.
vanilla_integrated_gradients_mask_3d = integrated_gradients.GetMask(
im,
feed_dict={neuron_selector: prediction_class},
x_steps=25,
x_baseline=baseline)
blur_ig_mask_3d = blur_ig.GetMask(
im, feed_dict={neuron_selector: prediction_class})
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_integrated_gradients_mask_3d)
blur_ig_mask_grayscale = saliency.VisualizeImageGrayscale(blur_ig_mask_3d)
image_grad.append(vanilla_mask_grayscale)
image_grad.append(blur_ig_mask_grayscale)
def vis_guided_backprop(model, gradient_saliency, neuron_selector, img_path,
label):
im = load_image(img_path)
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
im.reshape(48, 48, 1),
feed_dict={
neuron_selector: label,
model.is_training: False
})
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
smoothgrad_mask_grayscale = cv2.resize(smoothgrad_mask_grayscale,
None,
fx=4.0,
fy=4.0,
interpolation=cv2.INTER_AREA)
cv2.imwrite(
os.path.join(FLAGS.output_dir,
"saliency_map_{}".format(img_path.split('/')[-1])),
scale_values(smoothgrad_mask_grayscale))
def GetSalientImage(imagePath):
im = LoadImage(imagePath)
# Show the image
# ShowImage(im)
# Make a prediction.
prediction_class = sess.run(prediction, feed_dict={images: [im]})[0]
#print("Prediction class: " + str(prediction_class))
# Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops.
gradient_saliency = saliency.GradientSaliency(graph, sess, y, images)
# Compute the vanilla mask and the smoothed mask.
# vanilla_mask_3d = gradient_saliency.GetMask(im, feed_dict = {neuron_selector: prediction_class})
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(im, feed_dict={neuron_selector: prediction_class})
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
# vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
output = GetBoundingBox(smoothgrad_mask_grayscale, im)
#ShowImage(output, title='output')
return output
def build_i3d_model(video_tensor):
# model_name = "/home/ar/Experiment/ucf-101/rgb_backup01/models/rgb_scratch_10000_6_64_0.0001_decay/i3d_ucf_model-19999" # Note: I3D trained model
model_name = "./models/rgb_imagenet_10000_6_64_0.0001_decay/i3d_ucf_model-9999"
print("load model succeed")
graph = tf.Graph()
with graph.as_default():
images_placeholder = tf.placeholder(tf.float32, [FLAGS.batch_size, FLAGS.n_frames, FLAGS.crop_size, FLAGS.crop_size, FLAGS.rgb_channels])
#is_training = tf.placeholder(tf.bool)
with tf.variable_scope('RGB'):
logits, _ = InceptionI3d(
num_classes=FLAGS.classics,
spatial_squeeze=True,
final_endpoint='Logits',
name='inception_i3d'
)(images_placeholder, is_training=False)
# Create a saver for writing training checkpoints
saver = tf.train.Saver()
init = tf.global_variables_initializer()
# Create a session for running Ops on the Graph
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
sess.run(init)
# Restore trained model
saver.restore(sess, model_name)
neuron_selector = tf.placeholder(tf.int32)
y = logits[0][neuron_selector]
prediction = tf.argmax(logits, 1)
out_feature = sess.run(logits,
feed_dict={images_placeholder: video_tensor})
prediction_class = sess.run(prediction,
feed_dict={images_placeholder: video_tensor})[0]
#print(prediction_class)
###############################################################################################
#gradient_saliency = saliency.GradientSaliency(graph, sess, y, images_placeholder)
# Compute the vanilla mask and the smoothed mask.
#vanilla_mask_3d = gradient_saliency.GetMask(video_tensor[0], feed_dict = {neuron_selector: prediction_class})
#print(vanilla_mask_3d.shape)
#smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(video_tensor[0], feed_dict = {neuron_selector: prediction_class})
#vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
#print(vanilla_mask_grayscale.shape)
#smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
###############################################################################################
guided_backprop = saliency.GuidedBackprop(graph, sess, y, images_placeholder)
# Compute the vanilla mask and the smoothed mask.
vanilla_guided_backprop_mask_3d = guided_backprop.GetMask(video_tensor[0], feed_dict = {neuron_selector: prediction_class})
smoothgrad_guided_backprop_mask_3d = guided_backprop.GetSmoothedMask(video_tensor[0], feed_dict = {neuron_selector: prediction_class})
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_guided_backprop_mask_3d)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_guided_backprop_mask_3d)
###############################################################################################
return vanilla_mask_grayscale, smoothgrad_mask_grayscale
def enjoy_env_sess():
# utils.setup_mpi_gpus()
# setup_utils.setup_and_load({'restore_id': collecting_model})
directory = './images/'
directory_saliency = "./images_saliency"
def create_saliency(model_idx, sess):
graph = tf.get_default_graph()
env = utils.make_general_env(1)
env = wrappers.add_final_wrappers(env)
agent = create_act_model(sess, env, 1)
action_selector = tf.placeholder(tf.int32)
gradient_saliency = saliency.GradientSaliency(
graph, sess, agent.pd.logits[0][action_selector], agent.X)
sess.run(tf.global_variables_initializer())
# setup_utils.restore_file(models[model_idx])
try:
loaded_params = utils.load_params_for_scope(sess, 'model')
if not loaded_params:
print('NO SAVED PARAMS LOADED')
except AssertionError as e:
models[model_idx] = None
return [None] * 3
return agent, gradient_saliency, action_selector
orig_images_low = []
orig_images_high = []
filenames = []
print("Loading files...")
for idx, filename in enumerate(os.listdir(directory)):
if len(filename) > 15 or os.path.isdir(
os.path.join(directory, filename)):
continue
print('.', end='')
img = imageio.imread(os.path.join(directory, filename))
img = img.astype(np.float32)
if filename.startswith('img_') and len(filename) < 15:
filenames.append(filename)
list_to_append = orig_images_low
if filename.startswith('imgL_') and len(filename) < 15:
list_to_append = orig_images_high
list_to_append.append(img)
list_of_images_lists = [] # First one for 0
list_of_vmax_lists = []
for idx, model_name in enumerate(models):
if model_name is None:
list_of_images_lists.append(None)
list_of_vmax_lists.append(None)
continue
model_images = []
vmaxs = []
config.Config = config.ConfigSingle()
setup_utils.setup_and_load(use_cmd_line_args=False,
restore_id=model_name,
replay=True)
print("\nComputing saliency for Model {}\{}: {}...".format(
idx,
len(models) - 1, names[model_name]))
with tf.compat.v1.Session() as sess:
agent, gradient_saliency, action_selector = create_saliency(
idx, sess)
for img in orig_images_low:
print('.', end='')
sys.stdout.flush()
action, values, state, _ = agent.step(np.expand_dims(img, 0),
agent.initial_state,
False)
s_vanilla_mask_3d = gradient_saliency.GetSmoothedMask(
img,
feed_dict={
'model/is_training_:0': False,
action_selector: action[0]
})
s_vanilla_mask_grayscale, vmax = saliency.VisualizeImageGrayscale(
s_vanilla_mask_3d)
model_images.append(s_vanilla_mask_grayscale)
vmaxs.append(vmax)
list_of_images_lists.append(model_images)
list_of_vmax_lists.append(vmaxs)
print("\nMaking pretty images..")
for idx, filename in enumerate(filenames):
print('.', end='')
sys.stdout.flush()
P.figure(figsize=(COLS * UPSCALE_FACTOR, ROWS * UPSCALE_FACTOR))
ShowImage(orig_images_high[idx] / 255,
title="Original",
ax=P.subplot(ROWS, COLS, 1))
for row in range(ROWS):
for col in range(COLS):
model_idx = col + row * COLS
if models[model_idx] is None:
continue
ShowGrayscaleImage(list_of_images_lists[model_idx][idx],
title=names[models[model_idx]] +
" Vmax: {:.2E}".format(
list_of_vmax_lists[model_idx][idx]),
ax=P.subplot(ROWS, COLS, model_idx + 1))
P.savefig(
os.path.join(directory_saliency, filename[:-4] + "_saliency.png"))
P.close()
print("\nDone")
xi=xi,
axis=axj,
dilation=.5,
percentile=99,
alpha=.2).set_title(a)
## show the saliency and smoothgradient map of img to class 208 and 258
label_logits = logits[0, y_label]
gradient_saliency = saliency.GradientSaliency(graph, sess, label_logits,
images_pl) # 1951/1874
true_class = 208
vanilla_mask_3d = gradient_saliency.GetMask(img,
feed_dict={y_label:
true_class}) # better
vanilla_mask_grayscale_208 = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
#
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
img, feed_dict={y_label: true_class}) # much clear, 2204/2192
smoothgrad_mask_grayscale_208 = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
true_class = 258
vanilla_mask_3d = gradient_saliency.GetMask(img,
feed_dict={y_label:
true_class}) # better
vanilla_mask_grayscale_258 = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
#
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
img, feed_dict={y_label: true_class}) # much clear, 2204/2192
smoothgrad_mask_grayscale_258 = saliency.VisualizeImageGrayscale(
cnt = 0
for j, imgid in enumerate(x_test[int(args.start):int(args.end)]):
i = j + int(args.start)
if y_test[i] == 1:
img = LoadImage(root_dir + 'train/' + imgid + '.png')
bb_coords = LoadImage2(root_dir + 'mask/' + imgid + '.png')
l = sess.run(logits, feed_dict={inp: [img]})[0]
prediction_class = sess.run(prediction, feed_dict={inp: [img]})[0]
gbp = saliency.GuidedBackprop(sess.graph, sess, y, inp)
gbp_mask = gbp.GetMask(img,
feed_dict={neuron_selector: prediction_class})
gbp_mask = saliency.VisualizeImageGrayscale(gbp_mask)
scores['gbp'].append(
saliency_ttest(gbp_mask, bb_coords, prediction_class))
gradient_saliency = saliency.GradientSaliency(sess.graph, sess, y, inp)
vanilla_mask_3d = gradient_saliency.GetMask(
np.reshape(img, (320, 320, 3)),
feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
np.reshape(img, (320, 320, 3)),
feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_mask_3d)
scores['grad'].append(
def main(args):
"""------------------ parse input--------------------"""
task = args.task
"""---------------------------------------------------"""
with tf.Graph().as_default() as graph:
lr = 2 * 1e-4
batch_size = 1
n_neurons = [5, 5]
n_steps = 5
n_layers = 2
n_outputs = 2
init_a = 100
init_b = 0
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
net = Network.Net(batch_size=1,
n_steps=n_steps,
n_layers=n_layers,
n_neurons=n_neurons,
n_outputs=n_outputs,
init_lr=lr)
net.trainable = True
input = tf.placeholder(tf.float32, (batch_size, n_steps, 224, 224, 3))
GT_label = tf.placeholder(
tf.int64, (batch_size, n_steps)) # size = [batch, n_steps
label_attention_map = tf.placeholder(
tf.float32, (batch_size, n_steps, 112, 112, 1))
label_polar_map = tf.placeholder(tf.float32,
(batch_size, n_steps, 224, 224, 3))
delta_year = tf.placeholder(tf.float32, (batch_size, n_steps))
label_predict_op, cnn_out = net.inference_model_10(
input, label_attention_map
) # label_predict_op=(batch, n_steps, n_outputs)
lr = tf.placeholder(tf.float32, shape=[])
loss_per_batch = net._loss_per_batch(label_predict_op, GT_label)
loss_op, loss_label_op, loss_weight_op = net._loss_liliu(
label_predict_op, GT_label) # [batch,n_steps]
acc_op = net._top_k_accuracy(label_predict_op, GT_label)
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(extra_update_ops):
opt = tf.train.AdamOptimizer(lr,
beta1=0.9,
beta2=0.999,
epsilon=1e-08)
gradients = opt.compute_gradients(
loss_op) # all variables are trainable
apply_gradient_op = opt.apply_gradients(
gradients) # , global_step=self.global_step
train_op = apply_gradient_op
with tf.Session(config=tf_config, graph=graph) as sess:
sess.run(tf.global_variables_initializer())
list_img_path_test = os.listdir('./data/test/image/all')
list_img_path_test.sort()
dataloader_test = DataLoader_atten_polar(
batch_size=batch_size,
list_img_path=list_img_path_test,
state='test')
for i in range(6):
image, year, GTmap, Polar, GTlabel = dataloader_test.get_batch_single(
)
"""--------------------guidedbp----------------------------"""
saver = tf.train.Saver()
saver.restore(sess, model_file_cls)
guided_backprop = saliency.GradientSaliency(
graph, sess, label_predict_op[0, :, 1], input)
mask0 = guided_backprop.GetMask(input[0],
feed_dict={
input: image,
GT_label: GTlabel,
label_attention_map: GTmap,
label_polar_map: Polar,
delta_year: year
})
for img_index in range(5):
mask = mask0[img_index][0][img_index]
mask = saliency.VisualizeImageGrayscale(mask)
mask = gate(mask, gate=0)
Image.imsave('visualization_result/cls_cvpr/guidedbp/img' +
str(i + 1) + '_' + str(img_index + 1) +
'_gbp.jpg',
mask,
cmap=plt.get_cmap('gray'))
Image.imsave(
'visualization_result/cls_cvpr/guidedbp/img' +
str(i + 1) + '_' + str(img_index + 1) + '.jpg',
image[0][img_index])
print(task + '_' + str(i + 1) + '_' + str(img_index + 1))
print("Prediction class: " + str(prediction_class))
# Construct the saliency object. This doesn't yet compute the saliency mask,
# it just sets up the necessary ops.
integrated_gradients = saliency.IntegratedGradients(graph, sess, y, images)
# Baseline is a black image.
baseline = np.zeros(im.shape)
baseline.fill(-1)
# Smoothed mask for integrated gradients will take a while since we are doing nsamples * nsamples computations.
smoothgrad_integrated_gradients_mask_3d = integrated_gradients.GetSmoothedMask(
im, feed_dict={neuron_selector: prediction_class}, x_steps=20, x_baseline=baseline)
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_integrated_gradients_mask_3d)
im = smoothgrad_mask_grayscale * 255
im = cv2.resize(im, (224, 224))
end = time.time()
# Fetches only the number and extension of the file name
file_name = sys.argv[1].split('_')[-1]
file_name = 'saliency_' + file_name
# Fetches everything in the path except the file name
folder = sys.argv[1].split('/')[0:-1]
# Concatenates the path back together
path = ''
for i in folder:
def main(args):
for arg in vars(args):
print(arg, getattr(args, arg))
model_name = args.model_name
img_path = args.img_path
img_label_path = 'imagenet.json'
true_class = args.true_label
adversarial_label = args.adv_label
label_num = args.label_num
lambda_up, lambda_down, lambda_label_loss = args.lambda_up, args.lambda_down, args.lambda_label_loss
# model_name = 'inception_v3'
# img_path = './picture/dog_cat.jpg'
# img_label_path = 'imagenet.json'
# true_class = 208
sess, graph, img_size, images_pl, logits = load_pretrain_model(
model_name, is_explain=True)
y_label = tf.placeholder(dtype=tf.int32, shape=())
label_logits = logits[0, y_label]
if len(args.imp) > 0:
img = np.load(args.imp)
init_epoch = int(args.imp[:-4].split('_')[-1])
loss_list = list(np.load('loss_' + args.imp))
else:
img = PIL.Image.open(img_path)
img = preprocess_img(img, img_size)
init_epoch = 0
loss_list = []
old_img = np.array(img)
batch_img = np.expand_dims(img, 0)
#new_img = np.load('vgg16_30_0.0004_1000_0.001_0.03_4000.npy')
#new_batch_img = np.concatenate((np.expand_dims(new_img,0),batch_img),axis=0)
#new_batch_img = np.expand_dims(new_img,0)
#all_img = np.concatenate((batch_img,new_batch_img))
imagenet_label = load_imagenet_label(img_label_path)
prob = tf.nn.softmax(logits)
_prob = sess.run(prob, feed_dict={images_pl: batch_img})[0]
#classify(img,_prob,imagenet_label,1,1)
####
#deep explain
# from deepexplain.tensorflow import DeepExplain
# label_logits = logits[0,208]
# with DeepExplain(session=sess) as de:
# attributions = {
# # Gradient-based
# # NOTE: reduce_max is used to select the output unit for the class predicted by the classifier
# # For an example of how to use the ground-truth labels instead, see mnist_cnn_keras notebook
# 'Saliency maps': de.explain('saliency', label_logits, images_pl, batch_img),
# 'Gradient * Input': de.explain('grad*input', label_logits, images_pl, batch_img),
# # 'Integrated Gradients': de.explain('intgrad', label_logits, images_pl, new_batch_img),
# 'Epsilon-LRP': de.explain('elrp', label_logits, images_pl, batch_img),
# 'DeepLIFT (Rescale)': de.explain('deeplift', label_logits, images_pl, batch_img),
# # Perturbation-based (comment out to evaluate, but this will take a while!)
# #'Occlusion [15x15]': de.explain('occlusion', label_logits, images_pl, batch_img, window_shape=(15,15,3), step=4)
# } ####
# new_attributions = {
# # Gradient-based
# # NOTE: reduce_max is used to select the output unit for the class predicted by the classifier
# # For an example of how to use the ground-truth labels instead, see mnist_cnn_keras notebook
# 'Saliency maps': de.explain('saliency', label_logits, images_pl, new_batch_img),
# 'Gradient * Input': de.explain('grad*input', label_logits, images_pl, new_batch_img),
# # 'Integrated Gradients': de.explain('intgrad', label_logits, images_pl, new_batch_img),
# 'Epsilon-LRP': de.explain('elrp', label_logits, images_pl, new_batch_img),
# 'DeepLIFT (Rescale)': de.explain('deeplift', label_logits, images_pl, new_batch_img),
# # Perturbation-based (comment out to evaluate, but this will take a while!)
# #'Occlusion [15x15]': de.explain('occlusion', label_logits, images_pl, batch_img, window_shape=(15,15,3), step=4)
# } ####
# attributions['Saliency maps'] = np.concatenate((attributions['Saliency maps'],new_attributions['Saliency maps']),axis=0)
# attributions['Gradient * Input'] = np.concatenate((attributions['Gradient * Input'],new_attributions['Gradient * Input']),axis=0)
# attributions['Epsilon-LRP'] = np.concatenate((attributions['Epsilon-LRP'],new_attributions['Epsilon-LRP']),axis=0)
# attributions['DeepLIFT (Rescale)'] = np.concatenate((attributions['DeepLIFT (Rescale)'],new_attributions['DeepLIFT (Rescale)']),axis=0)
#
# n_cols = int(len(attributions)) + 1
# n_rows = 2
# fig, axes = plt.subplots(nrows=n_rows, ncols=n_cols, figsize=(3 * n_cols, 3 * n_rows))
#
# for i, xi in enumerate(all_img):
# # xi = (xi - np.min(xi))
# # xi /= np.max(xi)
# ax = axes.flatten()[i * n_cols]
# ax.imshow(xi)
# ax.set_title('Original')
# ax.axis('off')
# for j, a in enumerate(attributions):
# axj = axes.flatten()[i * n_cols + j + 1]
# plot(attributions[a][i], xi=xi, axis=axj, dilation=.5, percentile=99, alpha=.2).set_title(a)
######
label_logits = logits[0, 208]
with DeepExplain(session=sess) as de:
dlift = de.explain('deeplift', label_logits, images_pl, batch_img)
grad_map_tensor = tf.gradients(label_logits, images_pl)[0]
grad_map = sess.run(grad_map_tensor,
feed_dict={
images_pl: np.expand_dims(img, 0),
y_label: true_class
})
gradient_saliency = saliency.GradientSaliency(graph, sess, label_logits,
images_pl) # 1951/1874
vanilla_mask_3d = gradient_saliency.GetMask(
img, feed_dict={y_label: true_class}) # better
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
# smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(img, feed_dict={y_label:true_class}) # much clear, 2204/2192
# smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
#
# new_img = np.load('vgg16_60_70_35_45_30_0.0001_800_0.0_0.0_9000.npy')
# new_grad_map = sess.run(grad_map_tensor,feed_dict={images_pl:np.expand_dims(new_img,0),y_label:true_class})
# new_vanilla_mask_3d = gradient_saliency.GetMask(new_img, feed_dict={y_label:true_class}) # better
# new_vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(new_vanilla_mask_3d)
# new_smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(new_img, feed_dict={y_label:true_class}) # much clear, 2204/2192
# new_smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(new_smoothgrad_mask_3d)
#to_dec_center = (60,70)
to_dec_center = (100, 65)
#to_dec_radius = (35,45)
to_dec_radius = (80, 60)
to_inc_center = (120, 170)
to_inc_radius = (40, 30)
_map = vanilla_mask_grayscale
print(calculate_region_importance(_map, to_dec_center, to_dec_radius))
print(calculate_region_importance(_map, to_inc_center, to_inc_radius))
# construct to_inc_region and to_dec_region
to_dec_region = calculate_img_region_importance(grad_map_tensor,
to_dec_center,
to_dec_radius)
to_inc_region = calculate_img_region_importance(grad_map_tensor,
to_inc_center,
to_inc_radius)
# try NES (Natural evolutionary strategies)
N = args.N
sigma = args.sigma
epsilon = round(args.eps, 2)
epoch = args.epoch
eta = args.lr
#loss = to_dec_region/to_inc_region
#old_loss = sess.run(loss,feed_dict={images_pl: np.expand_dims(img, 0), y_label: true_class})
old_loss = calculate_deeplift_loss(dlift, to_dec_center, to_dec_radius,
to_inc_center, to_inc_radius)
num_list = '_'.join([
'big', model_name,
str(N),
str(eta),
str(epoch),
str(sigma),
str(epsilon)
])
print(num_list)
for i in range(epoch):
delta = np.random.randn(int(N / 2), img_size * img_size * 3)
delta = np.concatenate((delta, -delta), axis=0)
grad_sum = 0
f_value_list = []
for idelta in delta:
img_plus = np.clip(
img + sigma * idelta.reshape(img_size, img_size, 3), 0, 1)
#f_value = sess.run(loss,feed_dict={images_pl:np.expand_dims(img_plus,0),y_label:true_class})
with DeepExplain(session=sess) as de:
dlift = de.explain('deeplift', label_logits, images_pl,
np.expand_dims(img_plus, 0))
f_value = calculate_deeplift_loss(dlift, to_dec_center,
to_dec_radius, to_inc_center,
to_inc_radius)
f_value_list.append(f_value)
grad_sum += f_value * idelta.reshape(img_size, img_size, 3)
grad_sum = grad_sum / (N * sigma)
new_img = np.clip(
np.clip(img - eta * grad_sum, old_img - epsilon,
old_img + epsilon), 0, 1)
#new_loss, new_logits = sess.run([loss, logits],
# feed_dict={images_pl: np.expand_dims(new_img, 0), y_label: true_class})
with DeepExplain(session=sess) as de:
dlift = de.explain('deeplift', label_logits, images_pl,
np.expand_dims(new_img, 0))
new_loss = calculate_deeplift_loss(dlift, to_dec_center, to_dec_radius,
to_inc_center, to_inc_radius)
loss_list.append(new_loss)
print("epoch:{} new:{}, old:{}, {}".format(i, new_loss, old_loss,
np.argmax(_prob)))
sys.stdout.flush()
img = np.array(new_img)
if i % args.image_interval == 0:
temp_name = num_list + '_' + str(i + init_epoch)
np.save(temp_name, new_img)
if i % args.image_interval == 0:
np.save('loss_' + temp_name, loss_list)
np.save(num_list + '_' + str(epoch + init_epoch), new_img)
np.save('loss_' + num_list + '_' + str(epoch + init_epoch), loss_list)
def simple_example():
#--------------------
mnist = input_data.read_data_sets('./MNIST_data', one_hot=True)
#--------------------
# Define a model.
num_classes = 10
input_shape = (None, 28, 28, 1) # 784 = 28 * 28.
output_shape = (None, num_classes)
input_ph = tf.placeholder(tf.float32, shape=input_shape, name='input_ph')
output_ph = tf.placeholder(tf.float32,
shape=output_shape,
name='output_ph')
with tf.variable_scope('conv1', reuse=tf.AUTO_REUSE):
conv1 = tf.layers.conv2d(input_ph,
32,
5,
activation=tf.nn.relu,
name='conv')
conv1 = tf.layers.max_pooling2d(conv1, 2, 2, name='maxpool')
with tf.variable_scope('conv2', reuse=tf.AUTO_REUSE):
conv2 = tf.layers.conv2d(conv1,
64,
3,
activation=tf.nn.relu,
name='conv')
conv2 = tf.layers.max_pooling2d(conv2, 2, 2, name='maxpool')
with tf.variable_scope('fc1', reuse=tf.AUTO_REUSE):
fc1 = tf.layers.flatten(conv2, name='flatten')
fc1 = tf.layers.dense(fc1, 1024, activation=tf.nn.relu, name='dense')
with tf.variable_scope('fc2', reuse=tf.AUTO_REUSE):
model_output = tf.layers.dense(fc1,
num_classes,
activation=tf.nn.softmax,
name='dense')
#--------------------
# Train.
loss = tf.reduce_mean(-tf.reduce_sum(output_ph * tf.log(model_output),
reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
print('Start training...')
start_time = time.time()
for _ in range(2000):
batch_xs, batch_ys = mnist.train.next_batch(512)
batch_xs = np.reshape(batch_xs, (-1, ) + input_shape[1:])
sess.run(train_step,
feed_dict={
input_ph: batch_xs,
output_ph: batch_ys
})
if 0 == idx % 100: print('.', end='', flush=True)
print()
print('End training: {} secs.'.format(time.time() - start_time))
#--------------------
# Evaluate.
correct_prediction = tf.equal(tf.argmax(model_output, 1),
tf.argmax(output_ph, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print('Start testing...')
acc = sess.run(accuracy,
feed_dict={
input_ph:
np.reshape(mnist.test.images, (-1, ) + input_shape[1:]),
output_ph:
mnist.test.labels
})
print('Test accuracy = {}.'.format(acc))
print('End testing: {} secs.'.format(time.time() - start_time))
if acc < 0.95:
print('Failed to train...')
return
#--------------------
# Visualize.
images = np.reshape(mnist.test.images, (-1, ) + input_shape[1:])
img = images[0]
minval, maxval = np.min(img), np.max(img)
img_scaled = np.squeeze((img - minval) / (maxval - minval), axis=-1)
# Construct the scalar neuron tensor.
logits = model_output
neuron_selector = tf.placeholder(tf.int32)
y = logits[0][neuron_selector]
# Construct a tensor for predictions.
prediction = tf.argmax(logits, 1)
# Make a prediction.
prediction_class = sess.run(prediction, feed_dict={input_ph: [img]})[0]
#--------------------
start_time = time.time()
saliency_obj = saliency.Occlusion(sess.graph, sess, y, input_ph)
print('Occlusion: {} secs.'.format(time.time() - start_time))
# NOTE [info] >> An error exists in GetMask() of ${Saliency_HOME}/saliency/occlusion.py.
# <before>
# occlusion_window = np.array([size, size, x_value.shape[2]])
# occlusion_window.fill(value)
# <after>
# occlusion_window = np.full([size, size, x_value.shape[2]], value)
mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
mask_gray = saliency.VisualizeImageGrayscale(mask_3d)
mask_div = saliency.VisualizeImageDiverging(mask_3d)
fig = plt.figure()
ax = plt.subplot(1, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(1, 3, 2)
ax.imshow(mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Grayscale')
ax = plt.subplot(1, 3, 3)
ax.imshow(mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Diverging')
fig.suptitle('Occlusion', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_occlusion.png')
plt.show()
#--------------------
start_time = time.time()
conv_layer = sess.graph.get_tensor_by_name('conv2/conv/BiasAdd:0')
saliency_obj = saliency.GradCam(sess.graph, sess, y, input_ph, conv_layer)
print('GradCam: {} secs.'.format(time.time() - start_time))
mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
mask_gray = saliency.VisualizeImageGrayscale(mask_3d)
mask_div = saliency.VisualizeImageDiverging(mask_3d)
fig = plt.figure()
ax = plt.subplot(1, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(1, 3, 2)
ax.imshow(mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Grayscale')
ax = plt.subplot(1, 3, 3)
ax.imshow(mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Diverging')
fig.suptitle('Grad-CAM', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_gradcam.png')
plt.show()
#--------------------
start_time = time.time()
saliency_obj = saliency.GradientSaliency(sess.graph, sess, y, input_ph)
print('GradientSaliency: {} secs.'.format(time.time() - start_time))
vanilla_mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = saliency_obj.GetSmoothedMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d)
smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d)
fig = plt.figure()
ax = plt.subplot(2, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(2, 3, 2)
ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Grayscale')
ax = plt.subplot(2, 3, 3)
ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Grayscale')
ax = plt.subplot(2, 3, 5)
ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Diverging')
ax = plt.subplot(2, 3, 6)
ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Diverging')
fig.suptitle('Gradient Saliency', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_gradientsaliency.png')
plt.show()
#--------------------
start_time = time.time()
saliency_obj = saliency.GuidedBackprop(sess.graph, sess, y, input_ph)
print('GuidedBackprop: {} secs.'.format(time.time() - start_time))
vanilla_mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = saliency_obj.GetSmoothedMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d)
smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d)
fig = plt.figure()
ax = plt.subplot(2, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(2, 3, 2)
ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Grayscale')
ax = plt.subplot(2, 3, 3)
ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Grayscale')
ax = plt.subplot(2, 3, 4)
ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Diverging')
ax = plt.subplot(2, 3, 5)
ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Diverging')
fig.suptitle('Guided Backprop', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_guidedbackprop.png')
plt.show()
#--------------------
start_time = time.time()
saliency_obj = saliency.IntegratedGradients(sess.graph, sess, y, input_ph)
print('IntegratedGradients: {} secs.'.format(time.time() - start_time))
vanilla_mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = saliency_obj.GetSmoothedMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d)
smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d)
fig = plt.figure()
ax = plt.subplot(2, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(2, 3, 2)
ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Grayscale')
ax = plt.subplot(2, 3, 3)
ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Grayscale')
ax = plt.subplot(2, 3, 4)
ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Diverging')
ax = plt.subplot(2, 3, 5)
ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Diverging')
fig.suptitle('Integrated Gradients', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_integratedgradients.png')
plt.show()
#--------------------
start_time = time.time()
xrai_obj = saliency.XRAI(sess.graph, sess, y, input_ph)
print('XRAI: {} secs.'.format(time.time() - start_time))
if True:
xrai_attributions = xrai_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
else:
# Create XRAIParameters and set the algorithm to fast mode which will produce an approximate result.
xrai_params = saliency.XRAIParameters()
xrai_params.algorithm = 'fast'
xrai_attributions_fast = xrai_obj.GetMask(
img,
feed_dict={neuron_selector: prediction_class},
extra_parameters=xrai_params)
# Show most salient 30% of the image.
mask = xrai_attributions > np.percentile(xrai_attributions, 70)
img_masked = img_scaled.copy()
img_masked[~mask] = 0
fig = plt.figure()
ax = plt.subplot(1, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(1, 3, 2)
ax.imshow(xrai_attributions, cmap=plt.cm.inferno)
ax.axis('off')
ax.set_title('XRAI Attributions')
ax = plt.subplot(1, 3, 3)
ax.imshow(img_masked, cmap=plt.cm.gray)
ax.axis('off')
ax.set_title('Masked Input')
fig.suptitle('XRAI', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_xrai.png')
plt.show()
COLS = 2
UPSCALE_FACTOR = 10
P.figure(figsize=(ROWS * UPSCALE_FACTOR, COLS * UPSCALE_FACTOR))
# Render the saliency masks.
ShowGrayscaleImage(vanilla_mask_grayscale, title='Vanilla Gradient', ax=P.subplot(ROWS, COLS, 1))
ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad', ax=P.subplot(ROWS, COLS, 2))
"""
guided_backprop = saliency.GuidedBackprop(graph, sess, y, images)
# Compute the vanilla mask and the smoothed mask.
for im in [im1, im2]:
vanilla_guided_backprop_mask_3d = guided_backprop.GetMask(
im, feed_dict = {neuron_selector: prediction_class})
smoothgrad_guided_backprop_mask_3d = guided_backprop.GetSmoothedMask(
im, feed_dict = {neuron_selector: prediction_class})
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_guided_backprop_mask_3d)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_guided_backprop_mask_3d)
# Set up matplot lib figures.
ROWS = 1
COLS = 2
UPSCALE_FACTOR = 10
P.figure(figsize=(ROWS * UPSCALE_FACTOR, COLS * UPSCALE_FACTOR))
# Render the saliency masks.
ShowGrayscaleImage(vanilla_mask_grayscale, title='Vanilla Guided Backprop', ax=P.subplot(ROWS, COLS, 1))
ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad Guided Backprop', ax=P.subplot(ROWS, COLS, 2))
