def Integrated_Gradients_and_SmoothGrad(graph, sess, y, images): print('Integrated_Gradients_and_SmoothGrad') integrated_gradients = saliency.IntegratedGradients(graph, sess, y, images) # Baseline is a black image. baseline = np.zeros(im.shape) baseline.fill(-1) # Compute the vanilla mask and the smoothed mask. vanilla_integrated_gradients_mask_3d = integrated_gradients.GetMask( im, feed_dict={neuron_selector: prediction_class}, x_steps=25, x_baseline=baseline) # Smoothed mask for integrated gradients will take a while since we are doing nsamples * nsamples computations. smoothgrad_integrated_gradients_mask_3d = integrated_gradients.GetSmoothedMask( im, feed_dict={neuron_selector: prediction_class}, x_steps=25, x_baseline=baseline) # Call the visualization methods to convert the 3D tensors to 2D grayscale. vanilla_mask_grayscale = saliency.VisualizeImageGrayscale( vanilla_integrated_gradients_mask_3d) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale( smoothgrad_integrated_gradients_mask_3d) image_grad.append(vanilla_mask_grayscale) image_grad.append(smoothgrad_mask_grayscale)
def show_gradient_map(graph, sess, y, x, img, is_integrated=False, is_smooth=True, feed_dict=None, is_cluster=False): if not is_integrated and not is_smooth: gradient_saliency = saliency.GradientSaliency(graph, sess, y, x) vanilla_mask_3d = gradient_saliency.GetMask(img, feed_dict=feed_dict) vanilla_mask_grayscale = saliency.VisualizeImageGrayscale( vanilla_mask_3d) if not is_cluster: ShowGrayscaleImage(vanilla_mask_grayscale, title='Vanilla Gradient') return vanilla_mask_3d, vanilla_mask_grayscale if not is_integrated and is_smooth: gradient_saliency = saliency.GradientSaliency(graph, sess, y, x) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( img, feed_dict=feed_dict) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale( smoothgrad_mask_3d) if not is_cluster: ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad') return smoothgrad_mask_3d, smoothgrad_mask_grayscale if is_integrated and not is_smooth: baseline = np.zeros(img.shape) baseline.fill(-1) gradient_saliency = saliency.IntegratedGradients(graph, sess, y, x) vanilla_mask_3d = gradient_saliency.GetMask(img, feed_dict=feed_dict, x_steps=5, x_baseline=baseline) vanilla_mask_grayscale = saliency.VisualizeImageGrayscale( vanilla_mask_3d) if not is_cluster: ShowGrayscaleImage(vanilla_mask_grayscale, title='Vanilla Gradient') return vanilla_mask_3d, vanilla_mask_grayscale if is_integrated and is_smooth: baseline = np.zeros(img.shape) baseline.fill(-1) gradient_saliency = saliency.IntegratedGradients(graph, sess, y, x) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( img, feed_dict=feed_dict, x_steps=5, x_baseline=baseline) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale( smoothgrad_mask_3d) if not is_cluster: ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad') return smoothgrad_mask_3d, smoothgrad_mask_grayscale
def guided_vanilla(image): image = image / 127.5 - 1.0 prediction_class = sess.run(prediction, feed_dict = {images: [image]})[0] vanilla_guided_backprop_mask_3d = guided_backprop.GetMask( image, feed_dict = {neuron_selector: prediction_class}) vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_guided_backprop_mask_3d) return vanilla_mask_grayscale.tolist()
def extract_saliency(image, method, images, sess, logits, y, neuron_selector): prediction = tf.argmax(logits, 1) integrated_gradients = saliency.IntegratedGradients(GRAPH, sess, y, images) image = LoadImage(image) prediction_class = sess.run(prediction, feed_dict = {images:[image]})[0] baseline = np.zeros(image.shape) baseline.fill(-1) gradients_mask_3d = None # Vanilla if method == 0: gradients_mask_3d = integrated_gradients.GetMask(image, feed_dict = {neuron_selector: prediction_class}, x_steps = 25, x_baseline = baseline) # Smooth elif method == 1: gradients_mask_3d = integrated_gradients.GetSmoothedMask(image, feed_dict = {neuron_selector: prediction_class}, x_steps = 25, x_baseline = baseline) grayscale_mask = saliency.VisualizeImageGrayscale(gradients_mask_3d) grayscale_mask *= (255 / gradients_mask_3d.max()) grayscale_mask = np.uint8(grayscale_mask) return grayscale_mask
def vanilla_gradient_smoothgrad(graph, sess, y, images): print('Vanilla_adn_Smoothgrad') gradient_saliency = saliency.GradientSaliency(graph, sess, y, images) # Compute the vanilla mask and the smoothed mask. vanilla_mask_3d = gradient_saliency.GetMask( im, feed_dict={neuron_selector: prediction_class}) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( im, feed_dict={neuron_selector: prediction_class}) # Call the visualization methods to convert the 3D tensors to 2D grayscale. vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale( smoothgrad_mask_3d) image_grad.append(vanilla_mask_grayscale) image_grad.append(smoothgrad_mask_grayscale)
def visualize(sess, images_ph, im, y, prediction_class, neuron_selector, base_name): import saliency graph = tf.get_default_graph() # Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops. gradient_saliency = saliency.GradientSaliency(graph, sess, y, images_ph) guided_backprop = saliency.GuidedBackprop(graph, sess, y, images_ph) algo = guided_backprop # Compute the vanilla mask and the smoothed mask. for i, image in enumerate(im): prediction_class = 1 #np.argmax(prediction_class[i]) vanilla_mask_3d = algo.GetMask( image, feed_dict={neuron_selector: prediction_class}) #smoothgrad_mask_3d = algo.GetSmoothedMask(im, feed_dict = {neuron_selector: prediction_class}) # Call the visualization methods to convert the 3D tensors to 2D grayscale. vanilla_mask_grayscale = saliency.VisualizeImageGrayscale( vanilla_mask_3d) #smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) vanilla_mask_grayscale = ((vanilla_mask_grayscale + 1) * 127.5).astype( np.uint8) #smoothgrad_mask_grayscale = ((smoothgrad_mask_grayscale + 1) * 127.5).astype(np.uint8) result = Image.fromarray(vanilla_mask_grayscale, mode='L') #smoothed_result = Image.fromarray(smoothgrad_mask_grayscale, mode='L') #print 'sal name', base_name[i] print 'sal_image_dest_dir', sal_image_dest_dir sal_name = sal_image_dest_dir + str(base_name[i]).replace( '.jpg', '_sal.jpg') print 'saving {} ...'.format(sal_name) result.save(sal_name)
def Guided_Backprop_SmoothGrad(graph, sess, y, images): print('Guided_Backprop_SmoothGrad') guided_backprop = saliency.GuidedBackprop(graph, sess, y, images) # Compute the vanilla mask and the smoothed mask. vanilla_guided_backprop_mask_3d = guided_backprop.GetMask( im, feed_dict={neuron_selector: prediction_class}) smoothgrad_guided_backprop_mask_3d = guided_backprop.GetSmoothedMask( im, feed_dict={neuron_selector: prediction_class}) # Call the visualization methods to convert the 3D tensors to 2D grayscale. vanilla_mask_grayscale = saliency.VisualizeImageGrayscale( vanilla_guided_backprop_mask_3d) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale( smoothgrad_guided_backprop_mask_3d) image_grad.append(vanilla_mask_grayscale) image_grad.append(smoothgrad_mask_grayscale)
def Blur_Smootth_Grad(graph, sess, y, images): blur_ig = saliency.BlurIG(graph, sess, y, images) # Compute the Blur IG mask and Smoothgrad+BlurIG mask. blur_ig_mask_3d = blur_ig.GetMask( im, feed_dict={neuron_selector: prediction_class}) # Smoothed mask for BlurIG will take a while since we are doing nsamples * nsamples computations. smooth_blur_ig_mask_3d = blur_ig.GetSmoothedMask( im, feed_dict={neuron_selector: prediction_class}) # Call the visualization methods to convert the 3D tensors to 2D grayscale. blur_ig_mask_grayscale = saliency.VisualizeImageGrayscale(blur_ig_mask_3d) smooth_blur_ig_mask_grayscale = saliency.VisualizeImageGrayscale( smooth_blur_ig_mask_3d) image_grad.append(blur_ig_mask_grayscale) image_grad.append(smooth_blur_ig_mask_grayscale)
def abs_grayscale_norm(img): """Returns absolute value normalized image 2D.""" assert isinstance(img, np.ndarray), "img should be a numpy array" shp = img.shape if len(shp) < 2: raise ValueError("Array should have 2 or 3 dims!") if len(shp) == 2: img = np.absolute(img) img = img / float(img.max()) else: img = saliency.VisualizeImageGrayscale(img) return img
def reconstruct(X, model, classs, modelpath, outpath, do=0.3, bs=64): graph = tf.Graph() with graph.as_default(): x_in_, y_in_, logits_, nett_, ww_, pred_, neuron_selector_, ny_ = infer( model=model, classes=classs, dropout=do) with tf.Session(graph=graph, config=tf.ConfigProto( allow_soft_placement=True, log_device_placement=True)) as sess: tf.global_variables_initializer().run() saver = tf.train.import_meta_graph(str(modelpath + '.meta')) saver.restore(sess, modelpath) itr, file, ph = X.data(train=False) next_element = itr.get_next() with tf.Session() as sessa: sessa.run(itr.initializer, feed_dict={ph: file}) ct = 0 while True: try: x, y = sessa.run(next_element) for mm in range(np.shape(x)[0]): grad = saliency.IntegratedGradients( graph, sess, y, x_in_) img = x[mm, :, :, :] # Baseline is a white image. baseline = np.zeros(img.shape) baseline.fill(255) smoothgrad_mask_3d = grad.GetSmoothedMask( x, feed_dict={neuron_selector_: 1}, x_steps=25, x_baseline=baseline) # Call the visualization methods to convert the 3D tensors to 2D grayscale. smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale( smoothgrad_mask_3d) smoothgrad_mask_grayscale = im2double( smoothgrad_mask_grayscale) smoothgrad_mask_grayscale = py_map2jpg( smoothgrad_mask_grayscale) sa = im2double(img) * 255 sb = im2double(smoothgrad_mask_grayscale) * 255 scurHeatMap = sa * 0.5 + sb * 0.5 sab = np.hstack((sa, sb)) sfull = np.hstack((scurHeatMap, sab)) cv2.imwrite(str(outpath + str(ct) + '.png'), sfull) ct += 1 except tf.errors.OutOfRangeError: print("Done!") break
def getSaliency(self, images): # Images: [batch, height, width, channel] images = images / 127.5 - 1.0 predicted_class = self.saliency_sess.run( self.pred, feed_dict={self.inputs: [images]}) gradient_saliency = saliency.GradientSaliency(self.graph, self.saliency_sess, self.y, self.inputs) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( images, feed_dict={self.neuron_selector: predicted_class[0]}) return saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
def Blur_IG(graph, sess, y, images): integrated_gradients = saliency.IntegratedGradients(graph, sess, y, images) blur_ig = saliency.BlurIG(graph, sess, y, images) # Baseline is a black image for vanilla integrated gradients. baseline = np.zeros(im.shape) baseline.fill(-1) # Compute the vanilla mask and the Blur IG mask. vanilla_integrated_gradients_mask_3d = integrated_gradients.GetMask( im, feed_dict={neuron_selector: prediction_class}, x_steps=25, x_baseline=baseline) blur_ig_mask_3d = blur_ig.GetMask( im, feed_dict={neuron_selector: prediction_class}) # Call the visualization methods to convert the 3D tensors to 2D grayscale. vanilla_mask_grayscale = saliency.VisualizeImageGrayscale( vanilla_integrated_gradients_mask_3d) blur_ig_mask_grayscale = saliency.VisualizeImageGrayscale(blur_ig_mask_3d) image_grad.append(vanilla_mask_grayscale) image_grad.append(blur_ig_mask_grayscale)
def vis_guided_backprop(model, gradient_saliency, neuron_selector, img_path, label): im = load_image(img_path) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( im.reshape(48, 48, 1), feed_dict={ neuron_selector: label, model.is_training: False }) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale( smoothgrad_mask_3d) smoothgrad_mask_grayscale = cv2.resize(smoothgrad_mask_grayscale, None, fx=4.0, fy=4.0, interpolation=cv2.INTER_AREA) cv2.imwrite( os.path.join(FLAGS.output_dir, "saliency_map_{}".format(img_path.split('/')[-1])), scale_values(smoothgrad_mask_grayscale))
def GetSalientImage(imagePath): im = LoadImage(imagePath) # Show the image # ShowImage(im) # Make a prediction. prediction_class = sess.run(prediction, feed_dict={images: [im]})[0] #print("Prediction class: " + str(prediction_class)) # Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops. gradient_saliency = saliency.GradientSaliency(graph, sess, y, images) # Compute the vanilla mask and the smoothed mask. # vanilla_mask_3d = gradient_saliency.GetMask(im, feed_dict = {neuron_selector: prediction_class}) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(im, feed_dict={neuron_selector: prediction_class}) # Call the visualization methods to convert the 3D tensors to 2D grayscale. # vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) output = GetBoundingBox(smoothgrad_mask_grayscale, im) #ShowImage(output, title='output') return output
def build_i3d_model(video_tensor): # model_name = "/home/ar/Experiment/ucf-101/rgb_backup01/models/rgb_scratch_10000_6_64_0.0001_decay/i3d_ucf_model-19999" # Note: I3D trained model model_name = "./models/rgb_imagenet_10000_6_64_0.0001_decay/i3d_ucf_model-9999" print("load model succeed") graph = tf.Graph() with graph.as_default(): images_placeholder = tf.placeholder(tf.float32, [FLAGS.batch_size, FLAGS.n_frames, FLAGS.crop_size, FLAGS.crop_size, FLAGS.rgb_channels]) #is_training = tf.placeholder(tf.bool) with tf.variable_scope('RGB'): logits, _ = InceptionI3d( num_classes=FLAGS.classics, spatial_squeeze=True, final_endpoint='Logits', name='inception_i3d' )(images_placeholder, is_training=False) # Create a saver for writing training checkpoints saver = tf.train.Saver() init = tf.global_variables_initializer() # Create a session for running Ops on the Graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) sess.run(init) # Restore trained model saver.restore(sess, model_name) neuron_selector = tf.placeholder(tf.int32) y = logits[0][neuron_selector] prediction = tf.argmax(logits, 1) out_feature = sess.run(logits, feed_dict={images_placeholder: video_tensor}) prediction_class = sess.run(prediction, feed_dict={images_placeholder: video_tensor})[0] #print(prediction_class) ############################################################################################### #gradient_saliency = saliency.GradientSaliency(graph, sess, y, images_placeholder) # Compute the vanilla mask and the smoothed mask. #vanilla_mask_3d = gradient_saliency.GetMask(video_tensor[0], feed_dict = {neuron_selector: prediction_class}) #print(vanilla_mask_3d.shape) #smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(video_tensor[0], feed_dict = {neuron_selector: prediction_class}) #vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d) #print(vanilla_mask_grayscale.shape) #smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) ############################################################################################### guided_backprop = saliency.GuidedBackprop(graph, sess, y, images_placeholder) # Compute the vanilla mask and the smoothed mask. vanilla_guided_backprop_mask_3d = guided_backprop.GetMask(video_tensor[0], feed_dict = {neuron_selector: prediction_class}) smoothgrad_guided_backprop_mask_3d = guided_backprop.GetSmoothedMask(video_tensor[0], feed_dict = {neuron_selector: prediction_class}) vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_guided_backprop_mask_3d) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_guided_backprop_mask_3d) ############################################################################################### return vanilla_mask_grayscale, smoothgrad_mask_grayscale
def enjoy_env_sess(): # utils.setup_mpi_gpus() # setup_utils.setup_and_load({'restore_id': collecting_model}) directory = './images/' directory_saliency = "./images_saliency" def create_saliency(model_idx, sess): graph = tf.get_default_graph() env = utils.make_general_env(1) env = wrappers.add_final_wrappers(env) agent = create_act_model(sess, env, 1) action_selector = tf.placeholder(tf.int32) gradient_saliency = saliency.GradientSaliency( graph, sess, agent.pd.logits[0][action_selector], agent.X) sess.run(tf.global_variables_initializer()) # setup_utils.restore_file(models[model_idx]) try: loaded_params = utils.load_params_for_scope(sess, 'model') if not loaded_params: print('NO SAVED PARAMS LOADED') except AssertionError as e: models[model_idx] = None return [None] * 3 return agent, gradient_saliency, action_selector orig_images_low = [] orig_images_high = [] filenames = [] print("Loading files...") for idx, filename in enumerate(os.listdir(directory)): if len(filename) > 15 or os.path.isdir( os.path.join(directory, filename)): continue print('.', end='') img = imageio.imread(os.path.join(directory, filename)) img = img.astype(np.float32) if filename.startswith('img_') and len(filename) < 15: filenames.append(filename) list_to_append = orig_images_low if filename.startswith('imgL_') and len(filename) < 15: list_to_append = orig_images_high list_to_append.append(img) list_of_images_lists = [] # First one for 0 list_of_vmax_lists = [] for idx, model_name in enumerate(models): if model_name is None: list_of_images_lists.append(None) list_of_vmax_lists.append(None) continue model_images = [] vmaxs = [] config.Config = config.ConfigSingle() setup_utils.setup_and_load(use_cmd_line_args=False, restore_id=model_name, replay=True) print("\nComputing saliency for Model {}\{}: {}...".format( idx, len(models) - 1, names[model_name])) with tf.compat.v1.Session() as sess: agent, gradient_saliency, action_selector = create_saliency( idx, sess) for img in orig_images_low: print('.', end='') sys.stdout.flush() action, values, state, _ = agent.step(np.expand_dims(img, 0), agent.initial_state, False) s_vanilla_mask_3d = gradient_saliency.GetSmoothedMask( img, feed_dict={ 'model/is_training_:0': False, action_selector: action[0] }) s_vanilla_mask_grayscale, vmax = saliency.VisualizeImageGrayscale( s_vanilla_mask_3d) model_images.append(s_vanilla_mask_grayscale) vmaxs.append(vmax) list_of_images_lists.append(model_images) list_of_vmax_lists.append(vmaxs) print("\nMaking pretty images..") for idx, filename in enumerate(filenames): print('.', end='') sys.stdout.flush() P.figure(figsize=(COLS * UPSCALE_FACTOR, ROWS * UPSCALE_FACTOR)) ShowImage(orig_images_high[idx] / 255, title="Original", ax=P.subplot(ROWS, COLS, 1)) for row in range(ROWS): for col in range(COLS): model_idx = col + row * COLS if models[model_idx] is None: continue ShowGrayscaleImage(list_of_images_lists[model_idx][idx], title=names[models[model_idx]] + " Vmax: {:.2E}".format( list_of_vmax_lists[model_idx][idx]), ax=P.subplot(ROWS, COLS, model_idx + 1)) P.savefig( os.path.join(directory_saliency, filename[:-4] + "_saliency.png")) P.close() print("\nDone")
xi=xi, axis=axj, dilation=.5, percentile=99, alpha=.2).set_title(a) ## show the saliency and smoothgradient map of img to class 208 and 258 label_logits = logits[0, y_label] gradient_saliency = saliency.GradientSaliency(graph, sess, label_logits, images_pl) # 1951/1874 true_class = 208 vanilla_mask_3d = gradient_saliency.GetMask(img, feed_dict={y_label: true_class}) # better vanilla_mask_grayscale_208 = saliency.VisualizeImageGrayscale(vanilla_mask_3d) # smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( img, feed_dict={y_label: true_class}) # much clear, 2204/2192 smoothgrad_mask_grayscale_208 = saliency.VisualizeImageGrayscale( smoothgrad_mask_3d) true_class = 258 vanilla_mask_3d = gradient_saliency.GetMask(img, feed_dict={y_label: true_class}) # better vanilla_mask_grayscale_258 = saliency.VisualizeImageGrayscale(vanilla_mask_3d) # smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( img, feed_dict={y_label: true_class}) # much clear, 2204/2192 smoothgrad_mask_grayscale_258 = saliency.VisualizeImageGrayscale(
cnt = 0 for j, imgid in enumerate(x_test[int(args.start):int(args.end)]): i = j + int(args.start) if y_test[i] == 1: img = LoadImage(root_dir + 'train/' + imgid + '.png') bb_coords = LoadImage2(root_dir + 'mask/' + imgid + '.png') l = sess.run(logits, feed_dict={inp: [img]})[0] prediction_class = sess.run(prediction, feed_dict={inp: [img]})[0] gbp = saliency.GuidedBackprop(sess.graph, sess, y, inp) gbp_mask = gbp.GetMask(img, feed_dict={neuron_selector: prediction_class}) gbp_mask = saliency.VisualizeImageGrayscale(gbp_mask) scores['gbp'].append( saliency_ttest(gbp_mask, bb_coords, prediction_class)) gradient_saliency = saliency.GradientSaliency(sess.graph, sess, y, inp) vanilla_mask_3d = gradient_saliency.GetMask( np.reshape(img, (320, 320, 3)), feed_dict={neuron_selector: prediction_class}) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( np.reshape(img, (320, 320, 3)), feed_dict={neuron_selector: prediction_class}) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale( smoothgrad_mask_3d) vanilla_mask_grayscale = saliency.VisualizeImageGrayscale( vanilla_mask_3d) scores['grad'].append(
def main(args): """------------------ parse input--------------------""" task = args.task """---------------------------------------------------""" with tf.Graph().as_default() as graph: lr = 2 * 1e-4 batch_size = 1 n_neurons = [5, 5] n_steps = 5 n_layers = 2 n_outputs = 2 init_a = 100 init_b = 0 tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True net = Network.Net(batch_size=1, n_steps=n_steps, n_layers=n_layers, n_neurons=n_neurons, n_outputs=n_outputs, init_lr=lr) net.trainable = True input = tf.placeholder(tf.float32, (batch_size, n_steps, 224, 224, 3)) GT_label = tf.placeholder( tf.int64, (batch_size, n_steps)) # size = [batch, n_steps label_attention_map = tf.placeholder( tf.float32, (batch_size, n_steps, 112, 112, 1)) label_polar_map = tf.placeholder(tf.float32, (batch_size, n_steps, 224, 224, 3)) delta_year = tf.placeholder(tf.float32, (batch_size, n_steps)) label_predict_op, cnn_out = net.inference_model_10( input, label_attention_map ) # label_predict_op=(batch, n_steps, n_outputs) lr = tf.placeholder(tf.float32, shape=[]) loss_per_batch = net._loss_per_batch(label_predict_op, GT_label) loss_op, loss_label_op, loss_weight_op = net._loss_liliu( label_predict_op, GT_label) # [batch,n_steps] acc_op = net._top_k_accuracy(label_predict_op, GT_label) extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(extra_update_ops): opt = tf.train.AdamOptimizer(lr, beta1=0.9, beta2=0.999, epsilon=1e-08) gradients = opt.compute_gradients( loss_op) # all variables are trainable apply_gradient_op = opt.apply_gradients( gradients) # , global_step=self.global_step train_op = apply_gradient_op with tf.Session(config=tf_config, graph=graph) as sess: sess.run(tf.global_variables_initializer()) list_img_path_test = os.listdir('./data/test/image/all') list_img_path_test.sort() dataloader_test = DataLoader_atten_polar( batch_size=batch_size, list_img_path=list_img_path_test, state='test') for i in range(6): image, year, GTmap, Polar, GTlabel = dataloader_test.get_batch_single( ) """--------------------guidedbp----------------------------""" saver = tf.train.Saver() saver.restore(sess, model_file_cls) guided_backprop = saliency.GradientSaliency( graph, sess, label_predict_op[0, :, 1], input) mask0 = guided_backprop.GetMask(input[0], feed_dict={ input: image, GT_label: GTlabel, label_attention_map: GTmap, label_polar_map: Polar, delta_year: year }) for img_index in range(5): mask = mask0[img_index][0][img_index] mask = saliency.VisualizeImageGrayscale(mask) mask = gate(mask, gate=0) Image.imsave('visualization_result/cls_cvpr/guidedbp/img' + str(i + 1) + '_' + str(img_index + 1) + '_gbp.jpg', mask, cmap=plt.get_cmap('gray')) Image.imsave( 'visualization_result/cls_cvpr/guidedbp/img' + str(i + 1) + '_' + str(img_index + 1) + '.jpg', image[0][img_index]) print(task + '_' + str(i + 1) + '_' + str(img_index + 1))
print("Prediction class: " + str(prediction_class)) # Construct the saliency object. This doesn't yet compute the saliency mask, # it just sets up the necessary ops. integrated_gradients = saliency.IntegratedGradients(graph, sess, y, images) # Baseline is a black image. baseline = np.zeros(im.shape) baseline.fill(-1) # Smoothed mask for integrated gradients will take a while since we are doing nsamples * nsamples computations. smoothgrad_integrated_gradients_mask_3d = integrated_gradients.GetSmoothedMask( im, feed_dict={neuron_selector: prediction_class}, x_steps=20, x_baseline=baseline) # Call the visualization methods to convert the 3D tensors to 2D grayscale. smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_integrated_gradients_mask_3d) im = smoothgrad_mask_grayscale * 255 im = cv2.resize(im, (224, 224)) end = time.time() # Fetches only the number and extension of the file name file_name = sys.argv[1].split('_')[-1] file_name = 'saliency_' + file_name # Fetches everything in the path except the file name folder = sys.argv[1].split('/')[0:-1] # Concatenates the path back together path = '' for i in folder:
def main(args): for arg in vars(args): print(arg, getattr(args, arg)) model_name = args.model_name img_path = args.img_path img_label_path = 'imagenet.json' true_class = args.true_label adversarial_label = args.adv_label label_num = args.label_num lambda_up, lambda_down, lambda_label_loss = args.lambda_up, args.lambda_down, args.lambda_label_loss # model_name = 'inception_v3' # img_path = './picture/dog_cat.jpg' # img_label_path = 'imagenet.json' # true_class = 208 sess, graph, img_size, images_pl, logits = load_pretrain_model( model_name, is_explain=True) y_label = tf.placeholder(dtype=tf.int32, shape=()) label_logits = logits[0, y_label] if len(args.imp) > 0: img = np.load(args.imp) init_epoch = int(args.imp[:-4].split('_')[-1]) loss_list = list(np.load('loss_' + args.imp)) else: img = PIL.Image.open(img_path) img = preprocess_img(img, img_size) init_epoch = 0 loss_list = [] old_img = np.array(img) batch_img = np.expand_dims(img, 0) #new_img = np.load('vgg16_30_0.0004_1000_0.001_0.03_4000.npy') #new_batch_img = np.concatenate((np.expand_dims(new_img,0),batch_img),axis=0) #new_batch_img = np.expand_dims(new_img,0) #all_img = np.concatenate((batch_img,new_batch_img)) imagenet_label = load_imagenet_label(img_label_path) prob = tf.nn.softmax(logits) _prob = sess.run(prob, feed_dict={images_pl: batch_img})[0] #classify(img,_prob,imagenet_label,1,1) #### #deep explain # from deepexplain.tensorflow import DeepExplain # label_logits = logits[0,208] # with DeepExplain(session=sess) as de: # attributions = { # # Gradient-based # # NOTE: reduce_max is used to select the output unit for the class predicted by the classifier # # For an example of how to use the ground-truth labels instead, see mnist_cnn_keras notebook # 'Saliency maps': de.explain('saliency', label_logits, images_pl, batch_img), # 'Gradient * Input': de.explain('grad*input', label_logits, images_pl, batch_img), # # 'Integrated Gradients': de.explain('intgrad', label_logits, images_pl, new_batch_img), # 'Epsilon-LRP': de.explain('elrp', label_logits, images_pl, batch_img), # 'DeepLIFT (Rescale)': de.explain('deeplift', label_logits, images_pl, batch_img), # # Perturbation-based (comment out to evaluate, but this will take a while!) # #'Occlusion [15x15]': de.explain('occlusion', label_logits, images_pl, batch_img, window_shape=(15,15,3), step=4) # } #### # new_attributions = { # # Gradient-based # # NOTE: reduce_max is used to select the output unit for the class predicted by the classifier # # For an example of how to use the ground-truth labels instead, see mnist_cnn_keras notebook # 'Saliency maps': de.explain('saliency', label_logits, images_pl, new_batch_img), # 'Gradient * Input': de.explain('grad*input', label_logits, images_pl, new_batch_img), # # 'Integrated Gradients': de.explain('intgrad', label_logits, images_pl, new_batch_img), # 'Epsilon-LRP': de.explain('elrp', label_logits, images_pl, new_batch_img), # 'DeepLIFT (Rescale)': de.explain('deeplift', label_logits, images_pl, new_batch_img), # # Perturbation-based (comment out to evaluate, but this will take a while!) # #'Occlusion [15x15]': de.explain('occlusion', label_logits, images_pl, batch_img, window_shape=(15,15,3), step=4) # } #### # attributions['Saliency maps'] = np.concatenate((attributions['Saliency maps'],new_attributions['Saliency maps']),axis=0) # attributions['Gradient * Input'] = np.concatenate((attributions['Gradient * Input'],new_attributions['Gradient * Input']),axis=0) # attributions['Epsilon-LRP'] = np.concatenate((attributions['Epsilon-LRP'],new_attributions['Epsilon-LRP']),axis=0) # attributions['DeepLIFT (Rescale)'] = np.concatenate((attributions['DeepLIFT (Rescale)'],new_attributions['DeepLIFT (Rescale)']),axis=0) # # n_cols = int(len(attributions)) + 1 # n_rows = 2 # fig, axes = plt.subplots(nrows=n_rows, ncols=n_cols, figsize=(3 * n_cols, 3 * n_rows)) # # for i, xi in enumerate(all_img): # # xi = (xi - np.min(xi)) # # xi /= np.max(xi) # ax = axes.flatten()[i * n_cols] # ax.imshow(xi) # ax.set_title('Original') # ax.axis('off') # for j, a in enumerate(attributions): # axj = axes.flatten()[i * n_cols + j + 1] # plot(attributions[a][i], xi=xi, axis=axj, dilation=.5, percentile=99, alpha=.2).set_title(a) ###### label_logits = logits[0, 208] with DeepExplain(session=sess) as de: dlift = de.explain('deeplift', label_logits, images_pl, batch_img) grad_map_tensor = tf.gradients(label_logits, images_pl)[0] grad_map = sess.run(grad_map_tensor, feed_dict={ images_pl: np.expand_dims(img, 0), y_label: true_class }) gradient_saliency = saliency.GradientSaliency(graph, sess, label_logits, images_pl) # 1951/1874 vanilla_mask_3d = gradient_saliency.GetMask( img, feed_dict={y_label: true_class}) # better vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d) # smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(img, feed_dict={y_label:true_class}) # much clear, 2204/2192 # smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) # # new_img = np.load('vgg16_60_70_35_45_30_0.0001_800_0.0_0.0_9000.npy') # new_grad_map = sess.run(grad_map_tensor,feed_dict={images_pl:np.expand_dims(new_img,0),y_label:true_class}) # new_vanilla_mask_3d = gradient_saliency.GetMask(new_img, feed_dict={y_label:true_class}) # better # new_vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(new_vanilla_mask_3d) # new_smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(new_img, feed_dict={y_label:true_class}) # much clear, 2204/2192 # new_smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(new_smoothgrad_mask_3d) #to_dec_center = (60,70) to_dec_center = (100, 65) #to_dec_radius = (35,45) to_dec_radius = (80, 60) to_inc_center = (120, 170) to_inc_radius = (40, 30) _map = vanilla_mask_grayscale print(calculate_region_importance(_map, to_dec_center, to_dec_radius)) print(calculate_region_importance(_map, to_inc_center, to_inc_radius)) # construct to_inc_region and to_dec_region to_dec_region = calculate_img_region_importance(grad_map_tensor, to_dec_center, to_dec_radius) to_inc_region = calculate_img_region_importance(grad_map_tensor, to_inc_center, to_inc_radius) # try NES (Natural evolutionary strategies) N = args.N sigma = args.sigma epsilon = round(args.eps, 2) epoch = args.epoch eta = args.lr #loss = to_dec_region/to_inc_region #old_loss = sess.run(loss,feed_dict={images_pl: np.expand_dims(img, 0), y_label: true_class}) old_loss = calculate_deeplift_loss(dlift, to_dec_center, to_dec_radius, to_inc_center, to_inc_radius) num_list = '_'.join([ 'big', model_name, str(N), str(eta), str(epoch), str(sigma), str(epsilon) ]) print(num_list) for i in range(epoch): delta = np.random.randn(int(N / 2), img_size * img_size * 3) delta = np.concatenate((delta, -delta), axis=0) grad_sum = 0 f_value_list = [] for idelta in delta: img_plus = np.clip( img + sigma * idelta.reshape(img_size, img_size, 3), 0, 1) #f_value = sess.run(loss,feed_dict={images_pl:np.expand_dims(img_plus,0),y_label:true_class}) with DeepExplain(session=sess) as de: dlift = de.explain('deeplift', label_logits, images_pl, np.expand_dims(img_plus, 0)) f_value = calculate_deeplift_loss(dlift, to_dec_center, to_dec_radius, to_inc_center, to_inc_radius) f_value_list.append(f_value) grad_sum += f_value * idelta.reshape(img_size, img_size, 3) grad_sum = grad_sum / (N * sigma) new_img = np.clip( np.clip(img - eta * grad_sum, old_img - epsilon, old_img + epsilon), 0, 1) #new_loss, new_logits = sess.run([loss, logits], # feed_dict={images_pl: np.expand_dims(new_img, 0), y_label: true_class}) with DeepExplain(session=sess) as de: dlift = de.explain('deeplift', label_logits, images_pl, np.expand_dims(new_img, 0)) new_loss = calculate_deeplift_loss(dlift, to_dec_center, to_dec_radius, to_inc_center, to_inc_radius) loss_list.append(new_loss) print("epoch:{} new:{}, old:{}, {}".format(i, new_loss, old_loss, np.argmax(_prob))) sys.stdout.flush() img = np.array(new_img) if i % args.image_interval == 0: temp_name = num_list + '_' + str(i + init_epoch) np.save(temp_name, new_img) if i % args.image_interval == 0: np.save('loss_' + temp_name, loss_list) np.save(num_list + '_' + str(epoch + init_epoch), new_img) np.save('loss_' + num_list + '_' + str(epoch + init_epoch), loss_list)
def simple_example(): #-------------------- mnist = input_data.read_data_sets('./MNIST_data', one_hot=True) #-------------------- # Define a model. num_classes = 10 input_shape = (None, 28, 28, 1) # 784 = 28 * 28. output_shape = (None, num_classes) input_ph = tf.placeholder(tf.float32, shape=input_shape, name='input_ph') output_ph = tf.placeholder(tf.float32, shape=output_shape, name='output_ph') with tf.variable_scope('conv1', reuse=tf.AUTO_REUSE): conv1 = tf.layers.conv2d(input_ph, 32, 5, activation=tf.nn.relu, name='conv') conv1 = tf.layers.max_pooling2d(conv1, 2, 2, name='maxpool') with tf.variable_scope('conv2', reuse=tf.AUTO_REUSE): conv2 = tf.layers.conv2d(conv1, 64, 3, activation=tf.nn.relu, name='conv') conv2 = tf.layers.max_pooling2d(conv2, 2, 2, name='maxpool') with tf.variable_scope('fc1', reuse=tf.AUTO_REUSE): fc1 = tf.layers.flatten(conv2, name='flatten') fc1 = tf.layers.dense(fc1, 1024, activation=tf.nn.relu, name='dense') with tf.variable_scope('fc2', reuse=tf.AUTO_REUSE): model_output = tf.layers.dense(fc1, num_classes, activation=tf.nn.softmax, name='dense') #-------------------- # Train. loss = tf.reduce_mean(-tf.reduce_sum(output_ph * tf.log(model_output), reduction_indices=[1])) train_step = tf.train.GradientDescentOptimizer(0.01).minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) print('Start training...') start_time = time.time() for _ in range(2000): batch_xs, batch_ys = mnist.train.next_batch(512) batch_xs = np.reshape(batch_xs, (-1, ) + input_shape[1:]) sess.run(train_step, feed_dict={ input_ph: batch_xs, output_ph: batch_ys }) if 0 == idx % 100: print('.', end='', flush=True) print() print('End training: {} secs.'.format(time.time() - start_time)) #-------------------- # Evaluate. correct_prediction = tf.equal(tf.argmax(model_output, 1), tf.argmax(output_ph, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) print('Start testing...') acc = sess.run(accuracy, feed_dict={ input_ph: np.reshape(mnist.test.images, (-1, ) + input_shape[1:]), output_ph: mnist.test.labels }) print('Test accuracy = {}.'.format(acc)) print('End testing: {} secs.'.format(time.time() - start_time)) if acc < 0.95: print('Failed to train...') return #-------------------- # Visualize. images = np.reshape(mnist.test.images, (-1, ) + input_shape[1:]) img = images[0] minval, maxval = np.min(img), np.max(img) img_scaled = np.squeeze((img - minval) / (maxval - minval), axis=-1) # Construct the scalar neuron tensor. logits = model_output neuron_selector = tf.placeholder(tf.int32) y = logits[0][neuron_selector] # Construct a tensor for predictions. prediction = tf.argmax(logits, 1) # Make a prediction. prediction_class = sess.run(prediction, feed_dict={input_ph: [img]})[0] #-------------------- start_time = time.time() saliency_obj = saliency.Occlusion(sess.graph, sess, y, input_ph) print('Occlusion: {} secs.'.format(time.time() - start_time)) # NOTE [info] >> An error exists in GetMask() of ${Saliency_HOME}/saliency/occlusion.py. # <before> # occlusion_window = np.array([size, size, x_value.shape[2]]) # occlusion_window.fill(value) # <after> # occlusion_window = np.full([size, size, x_value.shape[2]], value) mask_3d = saliency_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}) # Compute a 2D tensor for visualization. mask_gray = saliency.VisualizeImageGrayscale(mask_3d) mask_div = saliency.VisualizeImageDiverging(mask_3d) fig = plt.figure() ax = plt.subplot(1, 3, 1) ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Input') ax = plt.subplot(1, 3, 2) ax.imshow(mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Grayscale') ax = plt.subplot(1, 3, 3) ax.imshow(mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Diverging') fig.suptitle('Occlusion', fontsize=16) fig.tight_layout() #plt.savefig('./saliency_occlusion.png') plt.show() #-------------------- start_time = time.time() conv_layer = sess.graph.get_tensor_by_name('conv2/conv/BiasAdd:0') saliency_obj = saliency.GradCam(sess.graph, sess, y, input_ph, conv_layer) print('GradCam: {} secs.'.format(time.time() - start_time)) mask_3d = saliency_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}) # Compute a 2D tensor for visualization. mask_gray = saliency.VisualizeImageGrayscale(mask_3d) mask_div = saliency.VisualizeImageDiverging(mask_3d) fig = plt.figure() ax = plt.subplot(1, 3, 1) ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Input') ax = plt.subplot(1, 3, 2) ax.imshow(mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Grayscale') ax = plt.subplot(1, 3, 3) ax.imshow(mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Diverging') fig.suptitle('Grad-CAM', fontsize=16) fig.tight_layout() #plt.savefig('./saliency_gradcam.png') plt.show() #-------------------- start_time = time.time() saliency_obj = saliency.GradientSaliency(sess.graph, sess, y, input_ph) print('GradientSaliency: {} secs.'.format(time.time() - start_time)) vanilla_mask_3d = saliency_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}) smoothgrad_mask_3d = saliency_obj.GetSmoothedMask( img, feed_dict={neuron_selector: prediction_class}) # Compute a 2D tensor for visualization. vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d) smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d) smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d) fig = plt.figure() ax = plt.subplot(2, 3, 1) ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Input') ax = plt.subplot(2, 3, 2) ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Vanilla Grayscale') ax = plt.subplot(2, 3, 3) ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('SmoothGrad Grayscale') ax = plt.subplot(2, 3, 5) ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Vanilla Diverging') ax = plt.subplot(2, 3, 6) ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('SmoothGrad Diverging') fig.suptitle('Gradient Saliency', fontsize=16) fig.tight_layout() #plt.savefig('./saliency_gradientsaliency.png') plt.show() #-------------------- start_time = time.time() saliency_obj = saliency.GuidedBackprop(sess.graph, sess, y, input_ph) print('GuidedBackprop: {} secs.'.format(time.time() - start_time)) vanilla_mask_3d = saliency_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}) smoothgrad_mask_3d = saliency_obj.GetSmoothedMask( img, feed_dict={neuron_selector: prediction_class}) # Compute a 2D tensor for visualization. vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d) smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d) smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d) fig = plt.figure() ax = plt.subplot(2, 3, 1) ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Input') ax = plt.subplot(2, 3, 2) ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Vanilla Grayscale') ax = plt.subplot(2, 3, 3) ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('SmoothGrad Grayscale') ax = plt.subplot(2, 3, 4) ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Vanilla Diverging') ax = plt.subplot(2, 3, 5) ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('SmoothGrad Diverging') fig.suptitle('Guided Backprop', fontsize=16) fig.tight_layout() #plt.savefig('./saliency_guidedbackprop.png') plt.show() #-------------------- start_time = time.time() saliency_obj = saliency.IntegratedGradients(sess.graph, sess, y, input_ph) print('IntegratedGradients: {} secs.'.format(time.time() - start_time)) vanilla_mask_3d = saliency_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}) smoothgrad_mask_3d = saliency_obj.GetSmoothedMask( img, feed_dict={neuron_selector: prediction_class}) # Compute a 2D tensor for visualization. vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d) smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d) smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d) fig = plt.figure() ax = plt.subplot(2, 3, 1) ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Input') ax = plt.subplot(2, 3, 2) ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Vanilla Grayscale') ax = plt.subplot(2, 3, 3) ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('SmoothGrad Grayscale') ax = plt.subplot(2, 3, 4) ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Vanilla Diverging') ax = plt.subplot(2, 3, 5) ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('SmoothGrad Diverging') fig.suptitle('Integrated Gradients', fontsize=16) fig.tight_layout() #plt.savefig('./saliency_integratedgradients.png') plt.show() #-------------------- start_time = time.time() xrai_obj = saliency.XRAI(sess.graph, sess, y, input_ph) print('XRAI: {} secs.'.format(time.time() - start_time)) if True: xrai_attributions = xrai_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}) else: # Create XRAIParameters and set the algorithm to fast mode which will produce an approximate result. xrai_params = saliency.XRAIParameters() xrai_params.algorithm = 'fast' xrai_attributions_fast = xrai_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}, extra_parameters=xrai_params) # Show most salient 30% of the image. mask = xrai_attributions > np.percentile(xrai_attributions, 70) img_masked = img_scaled.copy() img_masked[~mask] = 0 fig = plt.figure() ax = plt.subplot(1, 3, 1) ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Input') ax = plt.subplot(1, 3, 2) ax.imshow(xrai_attributions, cmap=plt.cm.inferno) ax.axis('off') ax.set_title('XRAI Attributions') ax = plt.subplot(1, 3, 3) ax.imshow(img_masked, cmap=plt.cm.gray) ax.axis('off') ax.set_title('Masked Input') fig.suptitle('XRAI', fontsize=16) fig.tight_layout() #plt.savefig('./saliency_xrai.png') plt.show()
COLS = 2 UPSCALE_FACTOR = 10 P.figure(figsize=(ROWS * UPSCALE_FACTOR, COLS * UPSCALE_FACTOR)) # Render the saliency masks. ShowGrayscaleImage(vanilla_mask_grayscale, title='Vanilla Gradient', ax=P.subplot(ROWS, COLS, 1)) ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad', ax=P.subplot(ROWS, COLS, 2)) """ guided_backprop = saliency.GuidedBackprop(graph, sess, y, images) # Compute the vanilla mask and the smoothed mask. for im in [im1, im2]: vanilla_guided_backprop_mask_3d = guided_backprop.GetMask( im, feed_dict = {neuron_selector: prediction_class}) smoothgrad_guided_backprop_mask_3d = guided_backprop.GetSmoothedMask( im, feed_dict = {neuron_selector: prediction_class}) # Call the visualization methods to convert the 3D tensors to 2D grayscale. vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_guided_backprop_mask_3d) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_guided_backprop_mask_3d) # Set up matplot lib figures. ROWS = 1 COLS = 2 UPSCALE_FACTOR = 10 P.figure(figsize=(ROWS * UPSCALE_FACTOR, COLS * UPSCALE_FACTOR)) # Render the saliency masks. ShowGrayscaleImage(vanilla_mask_grayscale, title='Vanilla Guided Backprop', ax=P.subplot(ROWS, COLS, 1)) ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad Guided Backprop', ax=P.subplot(ROWS, COLS, 2))