def show_gradient_map(graph,
sess,
y,
x,
img,
is_integrated=False,
is_smooth=True,
feed_dict=None,
is_cluster=False):
if not is_integrated and not is_smooth:
gradient_saliency = saliency.GradientSaliency(graph, sess, y, x)
vanilla_mask_3d = gradient_saliency.GetMask(img, feed_dict=feed_dict)
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_mask_3d)
if not is_cluster:
ShowGrayscaleImage(vanilla_mask_grayscale,
title='Vanilla Gradient')
return vanilla_mask_3d, vanilla_mask_grayscale
if not is_integrated and is_smooth:
gradient_saliency = saliency.GradientSaliency(graph, sess, y, x)
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
img, feed_dict=feed_dict)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
if not is_cluster:
ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad')
return smoothgrad_mask_3d, smoothgrad_mask_grayscale
if is_integrated and not is_smooth:
baseline = np.zeros(img.shape)
baseline.fill(-1)
gradient_saliency = saliency.IntegratedGradients(graph, sess, y, x)
vanilla_mask_3d = gradient_saliency.GetMask(img,
feed_dict=feed_dict,
x_steps=5,
x_baseline=baseline)
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_mask_3d)
if not is_cluster:
ShowGrayscaleImage(vanilla_mask_grayscale,
title='Vanilla Gradient')
return vanilla_mask_3d, vanilla_mask_grayscale
if is_integrated and is_smooth:
baseline = np.zeros(img.shape)
baseline.fill(-1)
gradient_saliency = saliency.IntegratedGradients(graph, sess, y, x)
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
img, feed_dict=feed_dict, x_steps=5, x_baseline=baseline)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
if not is_cluster:
ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad')
return smoothgrad_mask_3d, smoothgrad_mask_grayscale
def visualize(sess, images_ph, im, y, prediction_class, neuron_selector,
base_name):
import saliency
graph = tf.get_default_graph()
# Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops.
gradient_saliency = saliency.GradientSaliency(graph, sess, y, images_ph)
guided_backprop = saliency.GuidedBackprop(graph, sess, y, images_ph)
algo = guided_backprop
# Compute the vanilla mask and the smoothed mask.
for i, image in enumerate(im):
prediction_class = 1 #np.argmax(prediction_class[i])
vanilla_mask_3d = algo.GetMask(
image, feed_dict={neuron_selector: prediction_class})
#smoothgrad_mask_3d = algo.GetSmoothedMask(im, feed_dict = {neuron_selector: prediction_class})
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_mask_3d)
#smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
vanilla_mask_grayscale = ((vanilla_mask_grayscale + 1) * 127.5).astype(
np.uint8)
#smoothgrad_mask_grayscale = ((smoothgrad_mask_grayscale + 1) * 127.5).astype(np.uint8)
result = Image.fromarray(vanilla_mask_grayscale, mode='L')
#smoothed_result = Image.fromarray(smoothgrad_mask_grayscale, mode='L')
#print 'sal name', base_name[i]
print 'sal_image_dest_dir', sal_image_dest_dir
sal_name = sal_image_dest_dir + str(base_name[i]).replace(
'.jpg', '_sal.jpg')
print 'saving {} ...'.format(sal_name)
result.save(sal_name)
def get_saliency_image(graph, sess, y, image, saliency_method):
"""generates saliency image.
Args:
graph: tensor flow graph.
sess: the current session.
y: the pre-softmax activation we want to assess attribution with respect to.
image: float32 image tensor with size [1, None, None].
saliency_method: string indicating saliency map type to generate.
Returns:
a saliency map and a smoothed saliency map.
Raises:
ValueError: if the saliency_method string does not match any included method
"""
if saliency_method == 'integrated_gradients':
integrated_placeholder = saliency.IntegratedGradients(graph, sess, y, image)
return integrated_placeholder
elif saliency_method == 'gradient':
gradient_placeholder = saliency.GradientSaliency(graph, sess, y, image)
return gradient_placeholder
elif saliency_method == 'guided_backprop':
gb_placeholder = saliency.GuidedBackprop(graph, sess, y, image)
return gb_placeholder
else:
raise ValueError('No saliency method method matched. Verification of'
'input needed')
def getSaliency(self, images):
# Images: [batch, height, width, channel]
images = images / 127.5 - 1.0
predicted_class = self.saliency_sess.run(
self.pred, feed_dict={self.inputs: [images]})
gradient_saliency = saliency.GradientSaliency(self.graph,
self.saliency_sess,
self.y, self.inputs)
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
images, feed_dict={self.neuron_selector: predicted_class[0]})
return saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
def get_saliency_constructors(model_graph,
model_session,
logit_tensor,
input_tensor,
gradcam=False,
conv_layer_gradcam=None):
"""Initialize mask functions in saliency package.
Args:
model_graph: tf graph of the model.
model_session: tf session with trained model loaded.
logit_tensor: tensor corresponding to the model logit output.
input_tensor: tensor coresponding to the input data.
gradcam: boolean to indicate whether to include gradcam.
conv_layer_gradcam: tensor corresponding to activations
from a conv layer, from the trained model.
Authors recommend last layer.
Returns:
saliency_constructor: dictionary (name of method, and value is
function to each saliency method.
neuron_selector: tensor of specific output to explain.
"""
assert (type(tf.Graph()) == type(model_graph)),\
("Model graph should be of type {}".format(type(tf.Graph())))
if gradcam and conv_layer_gradcam is None:
raise ValueError("If gradcam is True, then conv_layer_gradcam"
"is be provided.")
with model_graph.as_default():
with tf.name_scope("saliency"):
neuron_selector = tf.placeholder(tf.int32)
y_salient = logit_tensor[neuron_selector]
gradient_saliency = saliency.GradientSaliency(model_graph, model_session,
y_salient, input_tensor)
guided_backprop = saliency.GuidedBackprop(model_graph, model_session,
y_salient, input_tensor)
integrated_gradients = saliency.IntegratedGradients(
model_graph, model_session, y_salient, input_tensor)
saliency_constructor = {
'vng': gradient_saliency,
'gbp': guided_backprop,
'ig': integrated_gradients
}
if gradcam:
gradcam = saliency.GradCam(model_graph, model_session, y_salient,
input_tensor, conv_layer_gradcam)
saliency_constructor['gc'] = gradcam
return saliency_constructor, neuron_selector
def vanilla_gradient_smoothgrad(graph, sess, y, images):
print('Vanilla_adn_Smoothgrad')
gradient_saliency = saliency.GradientSaliency(graph, sess, y, images)
# Compute the vanilla mask and the smoothed mask.
vanilla_mask_3d = gradient_saliency.GetMask(
im, feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
im, feed_dict={neuron_selector: prediction_class})
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
image_grad.append(vanilla_mask_grayscale)
image_grad.append(smoothgrad_mask_grayscale)
def main(_):
# Build Graph
graph = tf.Graph()
with graph.as_default():
model = Deep_CNN()
# Construct the scalar neuron tensor.
neuron_selector = tf.placeholder(tf.int32)
y = model.logits[0][neuron_selector]
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
print('\nLoading model from {}\n'.format(FLAGS.checkpoint_dir))
# Create a session
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement)
sess = tf.Session(config=session_conf)
# Restore trained model
saver.restore(sess, tf.train.latest_checkpoint(FLAGS.checkpoint_dir))
print("Model loaded!")
# Visualize first convolutional layer filters
vis_conv1_filters(sess)
# Visualize activation maps from conv4 layer
vis_conv4(sess, model, 'data/images/0/130.jpg')
vis_conv4(sess, model, 'data/images/0/607.jpg')
vis_conv4(sess, model, 'data/images/1/82.jpg')
vis_conv4(sess, model, 'data/images/1/791.jpg')
# Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops.
grad_saliency = saliency.GradientSaliency(graph, sess, y, model.x)
# Visualize using guided back-propagation
vis_guided_backprop(model, grad_saliency, neuron_selector,
'data/images/0/130.jpg', 0)
vis_guided_backprop(model, grad_saliency, neuron_selector,
'data/images/0/607.jpg', 0)
vis_guided_backprop(model, grad_saliency, neuron_selector,
'data/images/1/82.jpg', 1)
vis_guided_backprop(model, grad_saliency, neuron_selector,
'data/images/1/791.jpg', 1)
def create_saliency(model_idx, sess):
graph = tf.get_default_graph()
env = utils.make_general_env(1)
env = wrappers.add_final_wrappers(env)
agent = create_act_model(sess, env, 1)
action_selector = tf.placeholder(tf.int32)
gradient_saliency = saliency.GradientSaliency(graph, sess, agent.pd.logits[0][action_selector], agent.X)
sess.run(tf.compat.v1.global_variables_initializer())
# setup_utils.restore_file(models[model_idx])
try:
loaded_params = utils.load_params_for_scope(sess, 'model')
if not loaded_params:
print('NO SAVED PARAMS LOADED')
except AssertionError as e:
models[model_idx] = None
return [None]*3
return agent, gradient_saliency, action_selector
def GetSalientImage(imagePath):
im = LoadImage(imagePath)
# Show the image
# ShowImage(im)
# Make a prediction.
prediction_class = sess.run(prediction, feed_dict={images: [im]})[0]
#print("Prediction class: " + str(prediction_class))
# Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops.
gradient_saliency = saliency.GradientSaliency(graph, sess, y, images)
# Compute the vanilla mask and the smoothed mask.
# vanilla_mask_3d = gradient_saliency.GetMask(im, feed_dict = {neuron_selector: prediction_class})
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(im, feed_dict={neuron_selector: prediction_class})
# Call the visualization methods to convert the 3D tensors to 2D grayscale.
# vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
output = GetBoundingBox(smoothgrad_mask_grayscale, im)
#ShowImage(output, title='output')
return output
sess, graph, img_size, images_pl, logits = load_pretrain_model(model_name,
is_explain=True)
y_label = tf.placeholder(dtype=tf.int32, shape=())
img_label = load_imagenet_label(img_label_path)
img = PIL.Image.open(img_path)
img = preprocess_img(img, img_size)
#new_img = np.load('big_vgg16_30_0.0001_1000_0.001_0.03_3000.npy') # 258
new_img = np.load('vgg16_60_70_35_45_30_0.0001_800_0.0_0.0_9000.npy') # 208
batch_img = np.expand_dims(img, 0)
new_batch_img = np.expand_dims(new_img, 0)
true_class = 208
label_logits = logits[0, true_class]
gradient_saliency = saliency.GradientSaliency(graph, sess, label_logits,
images_pl) # 1951/1874
attributions = OrderedDict()
with DeepExplain(session=sess) as de:
ori_attributions = {
# Gradient-based
# NOTE: reduce_max is used to select the output unit for the class predicted by the classifier
# For an example of how to use the ground-truth labels instead, see mnist_cnn_keras notebook
'Saliency maps':
de.explain('saliency', label_logits, images_pl, batch_img),
'Gradient * Input':
de.explain('grad*input', label_logits, images_pl, batch_img),
'Epsilon-LRP':
de.explain('elrp', label_logits, images_pl, batch_img),
'Integrated Gradients':
de.explain('intgrad', label_logits, images_pl, new_batch_img,
i = j + int(args.start)
if y_test[i] == 1:
img = LoadImage(root_dir + 'train/' + imgid + '.png')
bb_coords = LoadImage2(root_dir + 'mask/' + imgid + '.png')
l = sess.run(logits, feed_dict={inp: [img]})[0]
prediction_class = sess.run(prediction, feed_dict={inp: [img]})[0]
gbp = saliency.GuidedBackprop(sess.graph, sess, y, inp)
gbp_mask = gbp.GetMask(img,
feed_dict={neuron_selector: prediction_class})
gbp_mask = saliency.VisualizeImageGrayscale(gbp_mask)
scores['gbp'].append(
saliency_ttest(gbp_mask, bb_coords, prediction_class))
gradient_saliency = saliency.GradientSaliency(sess.graph, sess, y, inp)
vanilla_mask_3d = gradient_saliency.GetMask(
np.reshape(img, (320, 320, 3)),
feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(
np.reshape(img, (320, 320, 3)),
feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(
smoothgrad_mask_3d)
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(
vanilla_mask_3d)
scores['grad'].append(
saliency_ttest(vanilla_mask_grayscale, bb_coords,
prediction_class))
scores['sg'].append(
saliency_ttest(smoothgrad_mask_grayscale, bb_coords,
def train():
print('[Dataset Configuration]')
print('\tImageNet test root: %s' % FLAGS.test_image_root)
print('\tImageNet test list: %s' % FLAGS.test_dataset)
print('\tNumber of classes: %d' % FLAGS.num_classes)
print('\tNumber of test images: %d' % FLAGS.num_test_instance)
print('[Network Configuration]')
print('\tBatch size: %d' % FLAGS.batch_size)
print('\tCheckpoint file: %s' % FLAGS.checkpoint)
print('[Optimization Configuration]')
print('\tL2 loss weight: %f' % FLAGS.l2_weight)
print('\tThe momentum optimizer: %f' % FLAGS.momentum)
print('\tInitial learning rate: %f' % FLAGS.initial_lr)
print('\tEpochs per lr step: %s' % FLAGS.lr_step_epoch)
print('\tLearning rate decay: %f' % FLAGS.lr_decay)
print('[Evaluation Configuration]')
print('\tOutput file path: %s' % FLAGS.output_file)
print('\tTest iterations: %d' % FLAGS.test_iter)
print('\tSteps per displaying info: %d' % FLAGS.display)
print('\tGPU memory fraction: %f' % FLAGS.gpu_fraction)
print('\tLog device placement: %d' % FLAGS.log_device_placement)
graph = tf.Graph()
with graph.as_default() as g:
global_step = tf.Variable(0,
trainable=False,
name='global_step',
dtype=tf.int64)
# Get images and labels of ImageNet
print('Load ImageNet dataset')
with tf.device('/cpu:0'):
print('\tLoading test data from %s' % FLAGS.test_dataset)
with tf.variable_scope('test_image'):
test_images, test_labels = data_input.inputs(
FLAGS.test_image_root,
FLAGS.test_dataset,
FLAGS.batch_size,
False,
num_threads=1,
center_crop=True)
# Build a Graph that computes the predictions from the inference model.
imagenet_mean = np.array([0.485, 0.456, 0.406], dtype=np.float32)
imagenet_std = np.array([0.229, 0.224, 0.225], dtype=np.float32)
images = tf.placeholder(tf.float32, [
FLAGS.batch_size, data_input.IMAGE_HEIGHT, data_input.IMAGE_WIDTH,
3
])
labels = tf.placeholder(tf.int64, [FLAGS.batch_size])
def build_network():
network = resnet_model.resnet_v1(resnet_depth=50,
num_classes=1000,
dropblock_size=None,
dropblock_keep_probs=[None] * 4,
data_format='channels_last')
return network(inputs=images, is_training=False)
logits = build_network()
sess = tf.Session(graph=g,
config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=True))
# Build an initialization operation to run below.
init = tf.initialize_all_variables()
sess.run(init)
# Create a saver.
saver = tf.train.Saver(tf.all_variables(), max_to_keep=10000)
if FLAGS.checkpoint is not None:
saver.restore(sess, FLAGS.checkpoint)
print('Load checkpoint %s' % FLAGS.checkpoint)
else:
print(
'No checkpoint file of basemodel found. Start from the scratch.'
)
# Start queue runners & summary_writer
tf.train.start_queue_runners(sess=sess)
# Test!
test_loss = 0.0
test_acc = 0.0
test_time = 0.0
confusion_matrix = np.zeros((FLAGS.num_classes, FLAGS.num_classes),
dtype=np.int32)
path = int(FLAGS.checkpoint.split('-')[1])
neuron_selector = tf.placeholder(tf.int32)
y = logits[0][neuron_selector]
gradient_saliency = saliency.GradientSaliency(g, sess, y, images)
os.system('mkdir -p ./{}/weight_{:05d}'.format(FLAGS.save_path, path))
classified_flag = []
ground_truth = []
pred_label = []
count = 0
for i in range(FLAGS.test_iter):
test_images_val, test_labels_val = sess.run(
[test_images[0], test_labels[0]])
start_time = time.time()
# Evaluate metrics
# Replace True with "test_labels_val[0] == FLAGS.class_ind" for analyzing the specified
# class_ind
if True:
predictions = np.argmax(logits.eval(
session=sess, feed_dict={images: test_images_val}),
axis=1)
ones = np.ones([FLAGS.batch_size])
zeros = np.zeros([FLAGS.batch_size])
correct = np.where(np.equal(predictions, test_labels_val),
ones, zeros)
acc = np.mean(correct)
duration = time.time() - start_time
test_acc += acc
test_time += duration
classified_flag.append([i, acc])
ground_truth.append(test_labels_val[0])
pred_label.append(predictions[0])
# Get gradients
grad = gradient_saliency.GetMask(
test_images_val[0, :],
feed_dict={neuron_selector: test_labels_val[0]})
imsave(
'./{}/weight_{:05d}/img_{:05d}.jpg'.format(
FLAGS.save_path, path, i),
(test_images_val[0, :] * imagenet_std + imagenet_mean))
np.save(
'./{}/weight_{:05d}/grad_{:05d}.npy'.format(
FLAGS.save_path, path, i), np.mean(grad, axis=-1))
count += 1
test_acc /= FLAGS.test_iter
np.save(
'./{}/weight_{:05d}/classified_flag.npy'.format(
FLAGS.save_path, int(path)), np.array(classified_flag))
np.save(
'./{}/weight_{:05d}/ground_truth.npy'.format(
FLAGS.save_path, int(path)), np.array(ground_truth))
np.save(
'./{}/weight_{:05d}/pred_label.npy'.format(FLAGS.save_path,
int(path)),
np.array(pred_label))
# Print and save results
sec_per_image = test_time / FLAGS.test_iter / FLAGS.batch_size
print('Done! Acc: %.6f, Test time: %.3f sec, %.7f sec/example' %
(test_acc, test_time, sec_per_image))
print('done!')
logits = graph.get_tensor_by_name('logits/BiasAdd:0')
neuron_selector = tf.placeholder(tf.int32)
y = logits[0][neuron_selector]
# Construct tensor for predictions
prediction = tf.argmax(logits, 1)
# Load an image
im = cv2.imread('all_db_fire/isolated_superpixels/test/310_rgb_sp25.png')
# Cast im as float32
im = np.float32(im)
# Make a prediction
prediction_class = sess.run(prediction, feed_dict={images: [im]})[0]
print("Prediction class: " + str(prediction_class))
# Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops.
gradient_saliency = saliency.GradientSaliency(graph, sess, y, images)
# Compute the vanilla mask and the smoothed mask.
vanilla_mask_3d = gradient_saliency.GetMask(
im, feed_dict={neuron_selector: prediction_class})
# Generate saliency map and save
abs_saliency = np.abs(vanilla_mask_3d).max(axis=-1)
abs_saliency = viz.intensity_to_rgb(
abs_saliency, normalize=True)[:, :, ::-1] # cv2 loads as BGR
cv2.imwrite("abs-saliency.jpg", abs_saliency)
def main(args):
"""------------------ parse input--------------------"""
task = args.task
"""---------------------------------------------------"""
with tf.Graph().as_default() as graph:
lr = 2 * 1e-4
batch_size = 1
n_neurons = [5, 5]
n_steps = 5
n_layers = 2
n_outputs = 2
init_a = 100
init_b = 0
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
net = Network.Net(batch_size=1,
n_steps=n_steps,
n_layers=n_layers,
n_neurons=n_neurons,
n_outputs=n_outputs,
init_lr=lr)
net.trainable = True
input = tf.placeholder(tf.float32, (batch_size, n_steps, 224, 224, 3))
GT_label = tf.placeholder(
tf.int64, (batch_size, n_steps)) # size = [batch, n_steps
label_attention_map = tf.placeholder(
tf.float32, (batch_size, n_steps, 112, 112, 1))
label_polar_map = tf.placeholder(tf.float32,
(batch_size, n_steps, 224, 224, 3))
delta_year = tf.placeholder(tf.float32, (batch_size, n_steps))
label_predict_op, cnn_out = net.inference_model_10(
input, label_attention_map
) # label_predict_op=(batch, n_steps, n_outputs)
lr = tf.placeholder(tf.float32, shape=[])
loss_per_batch = net._loss_per_batch(label_predict_op, GT_label)
loss_op, loss_label_op, loss_weight_op = net._loss_liliu(
label_predict_op, GT_label) # [batch,n_steps]
acc_op = net._top_k_accuracy(label_predict_op, GT_label)
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(extra_update_ops):
opt = tf.train.AdamOptimizer(lr,
beta1=0.9,
beta2=0.999,
epsilon=1e-08)
gradients = opt.compute_gradients(
loss_op) # all variables are trainable
apply_gradient_op = opt.apply_gradients(
gradients) # , global_step=self.global_step
train_op = apply_gradient_op
with tf.Session(config=tf_config, graph=graph) as sess:
sess.run(tf.global_variables_initializer())
list_img_path_test = os.listdir('./data/test/image/all')
list_img_path_test.sort()
dataloader_test = DataLoader_atten_polar(
batch_size=batch_size,
list_img_path=list_img_path_test,
state='test')
for i in range(6):
image, year, GTmap, Polar, GTlabel = dataloader_test.get_batch_single(
)
"""--------------------guidedbp----------------------------"""
saver = tf.train.Saver()
saver.restore(sess, model_file_cls)
guided_backprop = saliency.GradientSaliency(
graph, sess, label_predict_op[0, :, 1], input)
mask0 = guided_backprop.GetMask(input[0],
feed_dict={
input: image,
GT_label: GTlabel,
label_attention_map: GTmap,
label_polar_map: Polar,
delta_year: year
})
for img_index in range(5):
mask = mask0[img_index][0][img_index]
mask = saliency.VisualizeImageGrayscale(mask)
mask = gate(mask, gate=0)
Image.imsave('visualization_result/cls_cvpr/guidedbp/img' +
str(i + 1) + '_' + str(img_index + 1) +
'_gbp.jpg',
mask,
cmap=plt.get_cmap('gray'))
Image.imsave(
'visualization_result/cls_cvpr/guidedbp/img' +
str(i + 1) + '_' + str(img_index + 1) + '.jpg',
image[0][img_index])
print(task + '_' + str(i + 1) + '_' + str(img_index + 1))
def compute_and_save_attr(model, data, indices, num_threads):
"""Given the name of a model and a set of data, select a set of images based on provided indices, and compute and save their saliency maps and object attributions."""
base_dir = os.getcwd()
data_dir = os.path.join(base_dir, 'data', data, 'val')
model_dir = os.path.join(base_dir, 'models', model)
sal_output_dir = os.path.join(base_dir, SAL_DIR, model + '-' + data)
attr_output_dir = os.path.join(base_dir, ATTR_DIR, model + '-' + data)
if not tf.gfile.Exists(sal_output_dir):
tf.gfile.MakeDirs(sal_output_dir)
if not tf.gfile.Exists(attr_output_dir):
tf.gfile.MakeDirs(attr_output_dir)
img_names = [sorted(tf.gfile.ListDirectory(data_dir))[i] for i in indices]
img_paths = [os.path.join(data_dir, img_name) for img_name in img_names]
imgs = load_imgs(img_paths, num_threads)
input_name = 'input_tensor:0'
logit_name = 'resnet_model/final_dense:0'
conv_name = 'resnet_model/block_layer4:0'
with tf.Session(graph=tf.Graph()) as sess:
tf.saved_model.loader.load(sess, ['serve'], model_dir)
graph = tf.get_default_graph()
input_tensor = graph.get_tensor_by_name(input_name)
logit_tensor = graph.get_tensor_by_name(logit_name)
neuron_selector = tf.placeholder(tf.int32)
y = logit_tensor[:, neuron_selector]
pred_tensor = tf.argmax(logit_tensor, 1)
vg = saliency.GradientSaliency(graph, sess, y, input_tensor)
gb = saliency.GuidedBackprop(graph, sess, y, input_tensor)
ig = saliency.IntegratedGradients(graph, sess, y, input_tensor)
gc = saliency.GradCam(graph, sess, y, input_tensor,
graph.get_tensor_by_name(conv_name))
def single_map(img, img_name):
pred = sess.run(pred_tensor, feed_dict={input_tensor: [img]})[0]
vg_mask = vg.GetMask(img, feed_dict={neuron_selector: pred})
# *s is SmoothGrad
vgs_mask = vg.GetSmoothedMask(img,
feed_dict={neuron_selector: pred})
gb_mask = gb.GetMask(img, feed_dict={neuron_selector: pred})
gbs_mask = gb.GetSmoothedMask(img,
feed_dict={neuron_selector: pred})
baseline = np.zeros(img.shape) - np.expand_dims(
np.expand_dims(_CHANNEL_MEANS, 0), 0)
ig_mask = ig.GetMask(img,
feed_dict={neuron_selector: pred},
x_baseline=baseline)
igs_mask = ig.GetSmoothedMask(img,
feed_dict={neuron_selector: pred},
x_baseline=baseline)
gc_mask = gc.GetMask(img, feed_dict={neuron_selector: pred})
gcs_mask = gc.GetSmoothedMask(img,
feed_dict={neuron_selector: pred})
# gbgc is guided GradCam
gbgc_mask = gb_mask * gc_mask
gbgcs_mask = gbs_mask * gcs_mask
# Also include gradient x input
masks = np.array([
vg_mask, vgs_mask, gb_mask, gbs_mask, ig_mask, igs_mask,
gc_mask, gcs_mask, gbgc_mask, gbgcs_mask, vg_mask * img,
vgs_mask * img
])
return masks, pred
sal_maps = []
preds = []
for img, img_name in zip(imgs, img_names):
sal_path = tf.gfile.Glob(
os.path.join(sal_output_dir, img_name[:-4] + '*'))
if len(sal_path) > 0:
sal_maps.append(np.load(tf.gfile.GFile(sal_path[0], 'rb')))
preds.append(sal_path[0].split('_')[-1])
tf.logging.info(
'Loaded saliency maps for {}.'.format(img_name))
else:
masks, pred = single_map(img, img_name)
sal_maps.append(masks)
preds.append(pred)
out_path = os.path.join(sal_output_dir,
img_name[:-4] + '_' + str(pred))
np.save(tf.gfile.GFile(out_path, 'w'), masks)
tf.logging.info('Saved saliency maps for {}.'.format(img_name))
# Locate the objects, convert 3D saliency maps to 2D, and compute
# the attributions of the object segments by averaging over the
# per-pixel attributions of the objects.
loc_fpath = os.path.join(base_dir, 'data', data, 'val_loc.txt')
lines = [tf.gfile.Open(loc_fpath).readlines()[i] for i in indices]
locs = np.array([[
int(int(l) * float(RESNET_SHAPE[0]) / IMG_SHAPE[0])
for l in line.rstrip('\n').split(' ')[-1].split(',')
] for line in lines])
pool = multiprocessing.Pool(num_threads)
maps_3d = np.array(sal_maps).reshape(-1, RESNET_SHAPE[0], RESNET_SHAPE[1],
3)
maps_2d = np.array(pool.map(visualize_pos_attr, maps_3d))
maps_2d = maps_2d.reshape(len(indices),
int(maps_2d.shape[0] // len(indices)),
RESNET_SHAPE[0], RESNET_SHAPE[1])
mask_fpath = os.path.join(base_dir, 'data', data, 'val_mask')
if data in ['obj', 'scene', 'scene_only']:
# MCS and IDR are evaluated on 10000 images and masks are 10x100.
# Find the right mask.
obj_dict = {
'backpack': 0,
'bird': 1,
'dog': 2,
'elephant': 3,
'kite': 4,
'pizza': 5,
'stop_sign': 6,
'toilet': 7,
'truck': 8,
'zebra': 9,
}
# Loading val_mask from the data directory
masks_mat = np.load(tf.gfile.GFile(mask_fpath, 'rb'),
allow_pickle=True)
# Getting obj indices
obj_inds = [obj_dict[i.split('.')[0].split('-')[0]] for i in img_names]
# getting indices for a particular object class
temp_inds = [int(i.split('.')[0][-2:]) for i in img_names]
obj_masks = [
masks_mat[obj_inds[i] * 100 + temp_inds[i]]
for i, _ in enumerate(img_names)
]
else:
obj_masks = [
np.load(tf.gfile.GFile(mask_fpath, 'rb'), allow_pickle=True)[i]
for i in indices
]
attrs = []
for i in range(len(indices)):
attr = single_attr(maps_2d[i], locs[i], obj_masks[i])
attrs.append(attr)
out_path = os.path.join(attr_output_dir,
img_names[i][:-4] + '_' + str(preds[i]))
np.save(tf.gfile.GFile(out_path, 'w'), attr)
def simple_example():
#--------------------
mnist = input_data.read_data_sets('./MNIST_data', one_hot=True)
#--------------------
# Define a model.
num_classes = 10
input_shape = (None, 28, 28, 1) # 784 = 28 * 28.
output_shape = (None, num_classes)
input_ph = tf.placeholder(tf.float32, shape=input_shape, name='input_ph')
output_ph = tf.placeholder(tf.float32,
shape=output_shape,
name='output_ph')
with tf.variable_scope('conv1', reuse=tf.AUTO_REUSE):
conv1 = tf.layers.conv2d(input_ph,
32,
5,
activation=tf.nn.relu,
name='conv')
conv1 = tf.layers.max_pooling2d(conv1, 2, 2, name='maxpool')
with tf.variable_scope('conv2', reuse=tf.AUTO_REUSE):
conv2 = tf.layers.conv2d(conv1,
64,
3,
activation=tf.nn.relu,
name='conv')
conv2 = tf.layers.max_pooling2d(conv2, 2, 2, name='maxpool')
with tf.variable_scope('fc1', reuse=tf.AUTO_REUSE):
fc1 = tf.layers.flatten(conv2, name='flatten')
fc1 = tf.layers.dense(fc1, 1024, activation=tf.nn.relu, name='dense')
with tf.variable_scope('fc2', reuse=tf.AUTO_REUSE):
model_output = tf.layers.dense(fc1,
num_classes,
activation=tf.nn.softmax,
name='dense')
#--------------------
# Train.
loss = tf.reduce_mean(-tf.reduce_sum(output_ph * tf.log(model_output),
reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
print('Start training...')
start_time = time.time()
for _ in range(2000):
batch_xs, batch_ys = mnist.train.next_batch(512)
batch_xs = np.reshape(batch_xs, (-1, ) + input_shape[1:])
sess.run(train_step,
feed_dict={
input_ph: batch_xs,
output_ph: batch_ys
})
if 0 == idx % 100: print('.', end='', flush=True)
print()
print('End training: {} secs.'.format(time.time() - start_time))
#--------------------
# Evaluate.
correct_prediction = tf.equal(tf.argmax(model_output, 1),
tf.argmax(output_ph, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print('Start testing...')
acc = sess.run(accuracy,
feed_dict={
input_ph:
np.reshape(mnist.test.images, (-1, ) + input_shape[1:]),
output_ph:
mnist.test.labels
})
print('Test accuracy = {}.'.format(acc))
print('End testing: {} secs.'.format(time.time() - start_time))
if acc < 0.95:
print('Failed to train...')
return
#--------------------
# Visualize.
images = np.reshape(mnist.test.images, (-1, ) + input_shape[1:])
img = images[0]
minval, maxval = np.min(img), np.max(img)
img_scaled = np.squeeze((img - minval) / (maxval - minval), axis=-1)
# Construct the scalar neuron tensor.
logits = model_output
neuron_selector = tf.placeholder(tf.int32)
y = logits[0][neuron_selector]
# Construct a tensor for predictions.
prediction = tf.argmax(logits, 1)
# Make a prediction.
prediction_class = sess.run(prediction, feed_dict={input_ph: [img]})[0]
#--------------------
start_time = time.time()
saliency_obj = saliency.Occlusion(sess.graph, sess, y, input_ph)
print('Occlusion: {} secs.'.format(time.time() - start_time))
# NOTE [info] >> An error exists in GetMask() of ${Saliency_HOME}/saliency/occlusion.py.
# <before>
# occlusion_window = np.array([size, size, x_value.shape[2]])
# occlusion_window.fill(value)
# <after>
# occlusion_window = np.full([size, size, x_value.shape[2]], value)
mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
mask_gray = saliency.VisualizeImageGrayscale(mask_3d)
mask_div = saliency.VisualizeImageDiverging(mask_3d)
fig = plt.figure()
ax = plt.subplot(1, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(1, 3, 2)
ax.imshow(mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Grayscale')
ax = plt.subplot(1, 3, 3)
ax.imshow(mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Diverging')
fig.suptitle('Occlusion', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_occlusion.png')
plt.show()
#--------------------
start_time = time.time()
conv_layer = sess.graph.get_tensor_by_name('conv2/conv/BiasAdd:0')
saliency_obj = saliency.GradCam(sess.graph, sess, y, input_ph, conv_layer)
print('GradCam: {} secs.'.format(time.time() - start_time))
mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
mask_gray = saliency.VisualizeImageGrayscale(mask_3d)
mask_div = saliency.VisualizeImageDiverging(mask_3d)
fig = plt.figure()
ax = plt.subplot(1, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(1, 3, 2)
ax.imshow(mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Grayscale')
ax = plt.subplot(1, 3, 3)
ax.imshow(mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Diverging')
fig.suptitle('Grad-CAM', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_gradcam.png')
plt.show()
#--------------------
start_time = time.time()
saliency_obj = saliency.GradientSaliency(sess.graph, sess, y, input_ph)
print('GradientSaliency: {} secs.'.format(time.time() - start_time))
vanilla_mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = saliency_obj.GetSmoothedMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d)
smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d)
fig = plt.figure()
ax = plt.subplot(2, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(2, 3, 2)
ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Grayscale')
ax = plt.subplot(2, 3, 3)
ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Grayscale')
ax = plt.subplot(2, 3, 5)
ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Diverging')
ax = plt.subplot(2, 3, 6)
ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Diverging')
fig.suptitle('Gradient Saliency', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_gradientsaliency.png')
plt.show()
#--------------------
start_time = time.time()
saliency_obj = saliency.GuidedBackprop(sess.graph, sess, y, input_ph)
print('GuidedBackprop: {} secs.'.format(time.time() - start_time))
vanilla_mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = saliency_obj.GetSmoothedMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d)
smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d)
fig = plt.figure()
ax = plt.subplot(2, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(2, 3, 2)
ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Grayscale')
ax = plt.subplot(2, 3, 3)
ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Grayscale')
ax = plt.subplot(2, 3, 4)
ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Diverging')
ax = plt.subplot(2, 3, 5)
ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Diverging')
fig.suptitle('Guided Backprop', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_guidedbackprop.png')
plt.show()
#--------------------
start_time = time.time()
saliency_obj = saliency.IntegratedGradients(sess.graph, sess, y, input_ph)
print('IntegratedGradients: {} secs.'.format(time.time() - start_time))
vanilla_mask_3d = saliency_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
smoothgrad_mask_3d = saliency_obj.GetSmoothedMask(
img, feed_dict={neuron_selector: prediction_class})
# Compute a 2D tensor for visualization.
vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d)
smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d)
fig = plt.figure()
ax = plt.subplot(2, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(2, 3, 2)
ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Grayscale')
ax = plt.subplot(2, 3, 3)
ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Grayscale')
ax = plt.subplot(2, 3, 4)
ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Vanilla Diverging')
ax = plt.subplot(2, 3, 5)
ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('SmoothGrad Diverging')
fig.suptitle('Integrated Gradients', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_integratedgradients.png')
plt.show()
#--------------------
start_time = time.time()
xrai_obj = saliency.XRAI(sess.graph, sess, y, input_ph)
print('XRAI: {} secs.'.format(time.time() - start_time))
if True:
xrai_attributions = xrai_obj.GetMask(
img, feed_dict={neuron_selector: prediction_class})
else:
# Create XRAIParameters and set the algorithm to fast mode which will produce an approximate result.
xrai_params = saliency.XRAIParameters()
xrai_params.algorithm = 'fast'
xrai_attributions_fast = xrai_obj.GetMask(
img,
feed_dict={neuron_selector: prediction_class},
extra_parameters=xrai_params)
# Show most salient 30% of the image.
mask = xrai_attributions > np.percentile(xrai_attributions, 70)
img_masked = img_scaled.copy()
img_masked[~mask] = 0
fig = plt.figure()
ax = plt.subplot(1, 3, 1)
ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1)
ax.axis('off')
ax.set_title('Input')
ax = plt.subplot(1, 3, 2)
ax.imshow(xrai_attributions, cmap=plt.cm.inferno)
ax.axis('off')
ax.set_title('XRAI Attributions')
ax = plt.subplot(1, 3, 3)
ax.imshow(img_masked, cmap=plt.cm.gray)
ax.axis('off')
ax.set_title('Masked Input')
fig.suptitle('XRAI', fontsize=16)
fig.tight_layout()
#plt.savefig('./saliency_xrai.png')
plt.show()
def main(args):
for arg in vars(args):
print(arg, getattr(args, arg))
model_name = args.model_name
img_path = args.img_path
img_label_path = 'imagenet.json'
true_class = args.true_label
adversarial_label = args.adv_label
label_num = args.label_num
lambda_up, lambda_down, lambda_label_loss = args.lambda_up, args.lambda_down, args.lambda_label_loss
# model_name = 'inception_v3'
# img_path = './picture/dog_cat.jpg'
# img_label_path = 'imagenet.json'
# true_class = 208
sess, graph, img_size, images_pl, logits = load_pretrain_model(
model_name, is_explain=True)
y_label = tf.placeholder(dtype=tf.int32, shape=())
label_logits = logits[0, y_label]
if len(args.imp) > 0:
img = np.load(args.imp)
init_epoch = int(args.imp[:-4].split('_')[-1])
loss_list = list(np.load('loss_' + args.imp))
else:
img = PIL.Image.open(img_path)
img = preprocess_img(img, img_size)
init_epoch = 0
loss_list = []
old_img = np.array(img)
batch_img = np.expand_dims(img, 0)
#new_img = np.load('vgg16_30_0.0004_1000_0.001_0.03_4000.npy')
#new_batch_img = np.concatenate((np.expand_dims(new_img,0),batch_img),axis=0)
#new_batch_img = np.expand_dims(new_img,0)
#all_img = np.concatenate((batch_img,new_batch_img))
imagenet_label = load_imagenet_label(img_label_path)
prob = tf.nn.softmax(logits)
_prob = sess.run(prob, feed_dict={images_pl: batch_img})[0]
#classify(img,_prob,imagenet_label,1,1)
####
#deep explain
# from deepexplain.tensorflow import DeepExplain
# label_logits = logits[0,208]
# with DeepExplain(session=sess) as de:
# attributions = {
# # Gradient-based
# # NOTE: reduce_max is used to select the output unit for the class predicted by the classifier
# # For an example of how to use the ground-truth labels instead, see mnist_cnn_keras notebook
# 'Saliency maps': de.explain('saliency', label_logits, images_pl, batch_img),
# 'Gradient * Input': de.explain('grad*input', label_logits, images_pl, batch_img),
# # 'Integrated Gradients': de.explain('intgrad', label_logits, images_pl, new_batch_img),
# 'Epsilon-LRP': de.explain('elrp', label_logits, images_pl, batch_img),
# 'DeepLIFT (Rescale)': de.explain('deeplift', label_logits, images_pl, batch_img),
# # Perturbation-based (comment out to evaluate, but this will take a while!)
# #'Occlusion [15x15]': de.explain('occlusion', label_logits, images_pl, batch_img, window_shape=(15,15,3), step=4)
# } ####
# new_attributions = {
# # Gradient-based
# # NOTE: reduce_max is used to select the output unit for the class predicted by the classifier
# # For an example of how to use the ground-truth labels instead, see mnist_cnn_keras notebook
# 'Saliency maps': de.explain('saliency', label_logits, images_pl, new_batch_img),
# 'Gradient * Input': de.explain('grad*input', label_logits, images_pl, new_batch_img),
# # 'Integrated Gradients': de.explain('intgrad', label_logits, images_pl, new_batch_img),
# 'Epsilon-LRP': de.explain('elrp', label_logits, images_pl, new_batch_img),
# 'DeepLIFT (Rescale)': de.explain('deeplift', label_logits, images_pl, new_batch_img),
# # Perturbation-based (comment out to evaluate, but this will take a while!)
# #'Occlusion [15x15]': de.explain('occlusion', label_logits, images_pl, batch_img, window_shape=(15,15,3), step=4)
# } ####
# attributions['Saliency maps'] = np.concatenate((attributions['Saliency maps'],new_attributions['Saliency maps']),axis=0)
# attributions['Gradient * Input'] = np.concatenate((attributions['Gradient * Input'],new_attributions['Gradient * Input']),axis=0)
# attributions['Epsilon-LRP'] = np.concatenate((attributions['Epsilon-LRP'],new_attributions['Epsilon-LRP']),axis=0)
# attributions['DeepLIFT (Rescale)'] = np.concatenate((attributions['DeepLIFT (Rescale)'],new_attributions['DeepLIFT (Rescale)']),axis=0)
#
# n_cols = int(len(attributions)) + 1
# n_rows = 2
# fig, axes = plt.subplots(nrows=n_rows, ncols=n_cols, figsize=(3 * n_cols, 3 * n_rows))
#
# for i, xi in enumerate(all_img):
# # xi = (xi - np.min(xi))
# # xi /= np.max(xi)
# ax = axes.flatten()[i * n_cols]
# ax.imshow(xi)
# ax.set_title('Original')
# ax.axis('off')
# for j, a in enumerate(attributions):
# axj = axes.flatten()[i * n_cols + j + 1]
# plot(attributions[a][i], xi=xi, axis=axj, dilation=.5, percentile=99, alpha=.2).set_title(a)
######
label_logits = logits[0, 208]
with DeepExplain(session=sess) as de:
dlift = de.explain('deeplift', label_logits, images_pl, batch_img)
grad_map_tensor = tf.gradients(label_logits, images_pl)[0]
grad_map = sess.run(grad_map_tensor,
feed_dict={
images_pl: np.expand_dims(img, 0),
y_label: true_class
})
gradient_saliency = saliency.GradientSaliency(graph, sess, label_logits,
images_pl) # 1951/1874
vanilla_mask_3d = gradient_saliency.GetMask(
img, feed_dict={y_label: true_class}) # better
vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d)
# smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(img, feed_dict={y_label:true_class}) # much clear, 2204/2192
# smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
#
# new_img = np.load('vgg16_60_70_35_45_30_0.0001_800_0.0_0.0_9000.npy')
# new_grad_map = sess.run(grad_map_tensor,feed_dict={images_pl:np.expand_dims(new_img,0),y_label:true_class})
# new_vanilla_mask_3d = gradient_saliency.GetMask(new_img, feed_dict={y_label:true_class}) # better
# new_vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(new_vanilla_mask_3d)
# new_smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(new_img, feed_dict={y_label:true_class}) # much clear, 2204/2192
# new_smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(new_smoothgrad_mask_3d)
#to_dec_center = (60,70)
to_dec_center = (100, 65)
#to_dec_radius = (35,45)
to_dec_radius = (80, 60)
to_inc_center = (120, 170)
to_inc_radius = (40, 30)
_map = vanilla_mask_grayscale
print(calculate_region_importance(_map, to_dec_center, to_dec_radius))
print(calculate_region_importance(_map, to_inc_center, to_inc_radius))
# construct to_inc_region and to_dec_region
to_dec_region = calculate_img_region_importance(grad_map_tensor,
to_dec_center,
to_dec_radius)
to_inc_region = calculate_img_region_importance(grad_map_tensor,
to_inc_center,
to_inc_radius)
# try NES (Natural evolutionary strategies)
N = args.N
sigma = args.sigma
epsilon = round(args.eps, 2)
epoch = args.epoch
eta = args.lr
#loss = to_dec_region/to_inc_region
#old_loss = sess.run(loss,feed_dict={images_pl: np.expand_dims(img, 0), y_label: true_class})
old_loss = calculate_deeplift_loss(dlift, to_dec_center, to_dec_radius,
to_inc_center, to_inc_radius)
num_list = '_'.join([
'big', model_name,
str(N),
str(eta),
str(epoch),
str(sigma),
str(epsilon)
])
print(num_list)
for i in range(epoch):
delta = np.random.randn(int(N / 2), img_size * img_size * 3)
delta = np.concatenate((delta, -delta), axis=0)
grad_sum = 0
f_value_list = []
for idelta in delta:
img_plus = np.clip(
img + sigma * idelta.reshape(img_size, img_size, 3), 0, 1)
#f_value = sess.run(loss,feed_dict={images_pl:np.expand_dims(img_plus,0),y_label:true_class})
with DeepExplain(session=sess) as de:
dlift = de.explain('deeplift', label_logits, images_pl,
np.expand_dims(img_plus, 0))
f_value = calculate_deeplift_loss(dlift, to_dec_center,
to_dec_radius, to_inc_center,
to_inc_radius)
f_value_list.append(f_value)
grad_sum += f_value * idelta.reshape(img_size, img_size, 3)
grad_sum = grad_sum / (N * sigma)
new_img = np.clip(
np.clip(img - eta * grad_sum, old_img - epsilon,
old_img + epsilon), 0, 1)
#new_loss, new_logits = sess.run([loss, logits],
# feed_dict={images_pl: np.expand_dims(new_img, 0), y_label: true_class})
with DeepExplain(session=sess) as de:
dlift = de.explain('deeplift', label_logits, images_pl,
np.expand_dims(new_img, 0))
new_loss = calculate_deeplift_loss(dlift, to_dec_center, to_dec_radius,
to_inc_center, to_inc_radius)
loss_list.append(new_loss)
print("epoch:{} new:{}, old:{}, {}".format(i, new_loss, old_loss,
np.argmax(_prob)))
sys.stdout.flush()
img = np.array(new_img)
if i % args.image_interval == 0:
temp_name = num_list + '_' + str(i + init_epoch)
np.save(temp_name, new_img)
if i % args.image_interval == 0:
np.save('loss_' + temp_name, loss_list)
np.save(num_list + '_' + str(epoch + init_epoch), new_img)
np.save('loss_' + num_list + '_' + str(epoch + init_epoch), loss_list)
