def show_gradient_map(graph, sess, y, x, img, is_integrated=False, is_smooth=True, feed_dict=None, is_cluster=False): if not is_integrated and not is_smooth: gradient_saliency = saliency.GradientSaliency(graph, sess, y, x) vanilla_mask_3d = gradient_saliency.GetMask(img, feed_dict=feed_dict) vanilla_mask_grayscale = saliency.VisualizeImageGrayscale( vanilla_mask_3d) if not is_cluster: ShowGrayscaleImage(vanilla_mask_grayscale, title='Vanilla Gradient') return vanilla_mask_3d, vanilla_mask_grayscale if not is_integrated and is_smooth: gradient_saliency = saliency.GradientSaliency(graph, sess, y, x) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( img, feed_dict=feed_dict) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale( smoothgrad_mask_3d) if not is_cluster: ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad') return smoothgrad_mask_3d, smoothgrad_mask_grayscale if is_integrated and not is_smooth: baseline = np.zeros(img.shape) baseline.fill(-1) gradient_saliency = saliency.IntegratedGradients(graph, sess, y, x) vanilla_mask_3d = gradient_saliency.GetMask(img, feed_dict=feed_dict, x_steps=5, x_baseline=baseline) vanilla_mask_grayscale = saliency.VisualizeImageGrayscale( vanilla_mask_3d) if not is_cluster: ShowGrayscaleImage(vanilla_mask_grayscale, title='Vanilla Gradient') return vanilla_mask_3d, vanilla_mask_grayscale if is_integrated and is_smooth: baseline = np.zeros(img.shape) baseline.fill(-1) gradient_saliency = saliency.IntegratedGradients(graph, sess, y, x) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( img, feed_dict=feed_dict, x_steps=5, x_baseline=baseline) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale( smoothgrad_mask_3d) if not is_cluster: ShowGrayscaleImage(smoothgrad_mask_grayscale, title='SmoothGrad') return smoothgrad_mask_3d, smoothgrad_mask_grayscale
def visualize(sess, images_ph, im, y, prediction_class, neuron_selector, base_name): import saliency graph = tf.get_default_graph() # Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops. gradient_saliency = saliency.GradientSaliency(graph, sess, y, images_ph) guided_backprop = saliency.GuidedBackprop(graph, sess, y, images_ph) algo = guided_backprop # Compute the vanilla mask and the smoothed mask. for i, image in enumerate(im): prediction_class = 1 #np.argmax(prediction_class[i]) vanilla_mask_3d = algo.GetMask( image, feed_dict={neuron_selector: prediction_class}) #smoothgrad_mask_3d = algo.GetSmoothedMask(im, feed_dict = {neuron_selector: prediction_class}) # Call the visualization methods to convert the 3D tensors to 2D grayscale. vanilla_mask_grayscale = saliency.VisualizeImageGrayscale( vanilla_mask_3d) #smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) vanilla_mask_grayscale = ((vanilla_mask_grayscale + 1) * 127.5).astype( np.uint8) #smoothgrad_mask_grayscale = ((smoothgrad_mask_grayscale + 1) * 127.5).astype(np.uint8) result = Image.fromarray(vanilla_mask_grayscale, mode='L') #smoothed_result = Image.fromarray(smoothgrad_mask_grayscale, mode='L') #print 'sal name', base_name[i] print 'sal_image_dest_dir', sal_image_dest_dir sal_name = sal_image_dest_dir + str(base_name[i]).replace( '.jpg', '_sal.jpg') print 'saving {} ...'.format(sal_name) result.save(sal_name)
def get_saliency_image(graph, sess, y, image, saliency_method): """generates saliency image. Args: graph: tensor flow graph. sess: the current session. y: the pre-softmax activation we want to assess attribution with respect to. image: float32 image tensor with size [1, None, None]. saliency_method: string indicating saliency map type to generate. Returns: a saliency map and a smoothed saliency map. Raises: ValueError: if the saliency_method string does not match any included method """ if saliency_method == 'integrated_gradients': integrated_placeholder = saliency.IntegratedGradients(graph, sess, y, image) return integrated_placeholder elif saliency_method == 'gradient': gradient_placeholder = saliency.GradientSaliency(graph, sess, y, image) return gradient_placeholder elif saliency_method == 'guided_backprop': gb_placeholder = saliency.GuidedBackprop(graph, sess, y, image) return gb_placeholder else: raise ValueError('No saliency method method matched. Verification of' 'input needed')
def getSaliency(self, images): # Images: [batch, height, width, channel] images = images / 127.5 - 1.0 predicted_class = self.saliency_sess.run( self.pred, feed_dict={self.inputs: [images]}) gradient_saliency = saliency.GradientSaliency(self.graph, self.saliency_sess, self.y, self.inputs) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( images, feed_dict={self.neuron_selector: predicted_class[0]}) return saliency.VisualizeImageGrayscale(smoothgrad_mask_3d)
def get_saliency_constructors(model_graph, model_session, logit_tensor, input_tensor, gradcam=False, conv_layer_gradcam=None): """Initialize mask functions in saliency package. Args: model_graph: tf graph of the model. model_session: tf session with trained model loaded. logit_tensor: tensor corresponding to the model logit output. input_tensor: tensor coresponding to the input data. gradcam: boolean to indicate whether to include gradcam. conv_layer_gradcam: tensor corresponding to activations from a conv layer, from the trained model. Authors recommend last layer. Returns: saliency_constructor: dictionary (name of method, and value is function to each saliency method. neuron_selector: tensor of specific output to explain. """ assert (type(tf.Graph()) == type(model_graph)),\ ("Model graph should be of type {}".format(type(tf.Graph()))) if gradcam and conv_layer_gradcam is None: raise ValueError("If gradcam is True, then conv_layer_gradcam" "is be provided.") with model_graph.as_default(): with tf.name_scope("saliency"): neuron_selector = tf.placeholder(tf.int32) y_salient = logit_tensor[neuron_selector] gradient_saliency = saliency.GradientSaliency(model_graph, model_session, y_salient, input_tensor) guided_backprop = saliency.GuidedBackprop(model_graph, model_session, y_salient, input_tensor) integrated_gradients = saliency.IntegratedGradients( model_graph, model_session, y_salient, input_tensor) saliency_constructor = { 'vng': gradient_saliency, 'gbp': guided_backprop, 'ig': integrated_gradients } if gradcam: gradcam = saliency.GradCam(model_graph, model_session, y_salient, input_tensor, conv_layer_gradcam) saliency_constructor['gc'] = gradcam return saliency_constructor, neuron_selector
def vanilla_gradient_smoothgrad(graph, sess, y, images): print('Vanilla_adn_Smoothgrad') gradient_saliency = saliency.GradientSaliency(graph, sess, y, images) # Compute the vanilla mask and the smoothed mask. vanilla_mask_3d = gradient_saliency.GetMask( im, feed_dict={neuron_selector: prediction_class}) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( im, feed_dict={neuron_selector: prediction_class}) # Call the visualization methods to convert the 3D tensors to 2D grayscale. vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale( smoothgrad_mask_3d) image_grad.append(vanilla_mask_grayscale) image_grad.append(smoothgrad_mask_grayscale)
def main(_): # Build Graph graph = tf.Graph() with graph.as_default(): model = Deep_CNN() # Construct the scalar neuron tensor. neuron_selector = tf.placeholder(tf.int32) y = model.logits[0][neuron_selector] # Initialize an saver for store model checkpoints saver = tf.train.Saver() print('\nLoading model from {}\n'.format(FLAGS.checkpoint_dir)) # Create a session session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement) sess = tf.Session(config=session_conf) # Restore trained model saver.restore(sess, tf.train.latest_checkpoint(FLAGS.checkpoint_dir)) print("Model loaded!") # Visualize first convolutional layer filters vis_conv1_filters(sess) # Visualize activation maps from conv4 layer vis_conv4(sess, model, 'data/images/0/130.jpg') vis_conv4(sess, model, 'data/images/0/607.jpg') vis_conv4(sess, model, 'data/images/1/82.jpg') vis_conv4(sess, model, 'data/images/1/791.jpg') # Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops. grad_saliency = saliency.GradientSaliency(graph, sess, y, model.x) # Visualize using guided back-propagation vis_guided_backprop(model, grad_saliency, neuron_selector, 'data/images/0/130.jpg', 0) vis_guided_backprop(model, grad_saliency, neuron_selector, 'data/images/0/607.jpg', 0) vis_guided_backprop(model, grad_saliency, neuron_selector, 'data/images/1/82.jpg', 1) vis_guided_backprop(model, grad_saliency, neuron_selector, 'data/images/1/791.jpg', 1)
def create_saliency(model_idx, sess): graph = tf.get_default_graph() env = utils.make_general_env(1) env = wrappers.add_final_wrappers(env) agent = create_act_model(sess, env, 1) action_selector = tf.placeholder(tf.int32) gradient_saliency = saliency.GradientSaliency(graph, sess, agent.pd.logits[0][action_selector], agent.X) sess.run(tf.compat.v1.global_variables_initializer()) # setup_utils.restore_file(models[model_idx]) try: loaded_params = utils.load_params_for_scope(sess, 'model') if not loaded_params: print('NO SAVED PARAMS LOADED') except AssertionError as e: models[model_idx] = None return [None]*3 return agent, gradient_saliency, action_selector
def GetSalientImage(imagePath): im = LoadImage(imagePath) # Show the image # ShowImage(im) # Make a prediction. prediction_class = sess.run(prediction, feed_dict={images: [im]})[0] #print("Prediction class: " + str(prediction_class)) # Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops. gradient_saliency = saliency.GradientSaliency(graph, sess, y, images) # Compute the vanilla mask and the smoothed mask. # vanilla_mask_3d = gradient_saliency.GetMask(im, feed_dict = {neuron_selector: prediction_class}) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(im, feed_dict={neuron_selector: prediction_class}) # Call the visualization methods to convert the 3D tensors to 2D grayscale. # vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) output = GetBoundingBox(smoothgrad_mask_grayscale, im) #ShowImage(output, title='output') return output
sess, graph, img_size, images_pl, logits = load_pretrain_model(model_name, is_explain=True) y_label = tf.placeholder(dtype=tf.int32, shape=()) img_label = load_imagenet_label(img_label_path) img = PIL.Image.open(img_path) img = preprocess_img(img, img_size) #new_img = np.load('big_vgg16_30_0.0001_1000_0.001_0.03_3000.npy') # 258 new_img = np.load('vgg16_60_70_35_45_30_0.0001_800_0.0_0.0_9000.npy') # 208 batch_img = np.expand_dims(img, 0) new_batch_img = np.expand_dims(new_img, 0) true_class = 208 label_logits = logits[0, true_class] gradient_saliency = saliency.GradientSaliency(graph, sess, label_logits, images_pl) # 1951/1874 attributions = OrderedDict() with DeepExplain(session=sess) as de: ori_attributions = { # Gradient-based # NOTE: reduce_max is used to select the output unit for the class predicted by the classifier # For an example of how to use the ground-truth labels instead, see mnist_cnn_keras notebook 'Saliency maps': de.explain('saliency', label_logits, images_pl, batch_img), 'Gradient * Input': de.explain('grad*input', label_logits, images_pl, batch_img), 'Epsilon-LRP': de.explain('elrp', label_logits, images_pl, batch_img), 'Integrated Gradients': de.explain('intgrad', label_logits, images_pl, new_batch_img,
i = j + int(args.start) if y_test[i] == 1: img = LoadImage(root_dir + 'train/' + imgid + '.png') bb_coords = LoadImage2(root_dir + 'mask/' + imgid + '.png') l = sess.run(logits, feed_dict={inp: [img]})[0] prediction_class = sess.run(prediction, feed_dict={inp: [img]})[0] gbp = saliency.GuidedBackprop(sess.graph, sess, y, inp) gbp_mask = gbp.GetMask(img, feed_dict={neuron_selector: prediction_class}) gbp_mask = saliency.VisualizeImageGrayscale(gbp_mask) scores['gbp'].append( saliency_ttest(gbp_mask, bb_coords, prediction_class)) gradient_saliency = saliency.GradientSaliency(sess.graph, sess, y, inp) vanilla_mask_3d = gradient_saliency.GetMask( np.reshape(img, (320, 320, 3)), feed_dict={neuron_selector: prediction_class}) smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask( np.reshape(img, (320, 320, 3)), feed_dict={neuron_selector: prediction_class}) smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale( smoothgrad_mask_3d) vanilla_mask_grayscale = saliency.VisualizeImageGrayscale( vanilla_mask_3d) scores['grad'].append( saliency_ttest(vanilla_mask_grayscale, bb_coords, prediction_class)) scores['sg'].append( saliency_ttest(smoothgrad_mask_grayscale, bb_coords,
def train(): print('[Dataset Configuration]') print('\tImageNet test root: %s' % FLAGS.test_image_root) print('\tImageNet test list: %s' % FLAGS.test_dataset) print('\tNumber of classes: %d' % FLAGS.num_classes) print('\tNumber of test images: %d' % FLAGS.num_test_instance) print('[Network Configuration]') print('\tBatch size: %d' % FLAGS.batch_size) print('\tCheckpoint file: %s' % FLAGS.checkpoint) print('[Optimization Configuration]') print('\tL2 loss weight: %f' % FLAGS.l2_weight) print('\tThe momentum optimizer: %f' % FLAGS.momentum) print('\tInitial learning rate: %f' % FLAGS.initial_lr) print('\tEpochs per lr step: %s' % FLAGS.lr_step_epoch) print('\tLearning rate decay: %f' % FLAGS.lr_decay) print('[Evaluation Configuration]') print('\tOutput file path: %s' % FLAGS.output_file) print('\tTest iterations: %d' % FLAGS.test_iter) print('\tSteps per displaying info: %d' % FLAGS.display) print('\tGPU memory fraction: %f' % FLAGS.gpu_fraction) print('\tLog device placement: %d' % FLAGS.log_device_placement) graph = tf.Graph() with graph.as_default() as g: global_step = tf.Variable(0, trainable=False, name='global_step', dtype=tf.int64) # Get images and labels of ImageNet print('Load ImageNet dataset') with tf.device('/cpu:0'): print('\tLoading test data from %s' % FLAGS.test_dataset) with tf.variable_scope('test_image'): test_images, test_labels = data_input.inputs( FLAGS.test_image_root, FLAGS.test_dataset, FLAGS.batch_size, False, num_threads=1, center_crop=True) # Build a Graph that computes the predictions from the inference model. imagenet_mean = np.array([0.485, 0.456, 0.406], dtype=np.float32) imagenet_std = np.array([0.229, 0.224, 0.225], dtype=np.float32) images = tf.placeholder(tf.float32, [ FLAGS.batch_size, data_input.IMAGE_HEIGHT, data_input.IMAGE_WIDTH, 3 ]) labels = tf.placeholder(tf.int64, [FLAGS.batch_size]) def build_network(): network = resnet_model.resnet_v1(resnet_depth=50, num_classes=1000, dropblock_size=None, dropblock_keep_probs=[None] * 4, data_format='channels_last') return network(inputs=images, is_training=False) logits = build_network() sess = tf.Session(graph=g, config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)) # Build an initialization operation to run below. init = tf.initialize_all_variables() sess.run(init) # Create a saver. saver = tf.train.Saver(tf.all_variables(), max_to_keep=10000) if FLAGS.checkpoint is not None: saver.restore(sess, FLAGS.checkpoint) print('Load checkpoint %s' % FLAGS.checkpoint) else: print( 'No checkpoint file of basemodel found. Start from the scratch.' ) # Start queue runners & summary_writer tf.train.start_queue_runners(sess=sess) # Test! test_loss = 0.0 test_acc = 0.0 test_time = 0.0 confusion_matrix = np.zeros((FLAGS.num_classes, FLAGS.num_classes), dtype=np.int32) path = int(FLAGS.checkpoint.split('-')[1]) neuron_selector = tf.placeholder(tf.int32) y = logits[0][neuron_selector] gradient_saliency = saliency.GradientSaliency(g, sess, y, images) os.system('mkdir -p ./{}/weight_{:05d}'.format(FLAGS.save_path, path)) classified_flag = [] ground_truth = [] pred_label = [] count = 0 for i in range(FLAGS.test_iter): test_images_val, test_labels_val = sess.run( [test_images[0], test_labels[0]]) start_time = time.time() # Evaluate metrics # Replace True with "test_labels_val[0] == FLAGS.class_ind" for analyzing the specified # class_ind if True: predictions = np.argmax(logits.eval( session=sess, feed_dict={images: test_images_val}), axis=1) ones = np.ones([FLAGS.batch_size]) zeros = np.zeros([FLAGS.batch_size]) correct = np.where(np.equal(predictions, test_labels_val), ones, zeros) acc = np.mean(correct) duration = time.time() - start_time test_acc += acc test_time += duration classified_flag.append([i, acc]) ground_truth.append(test_labels_val[0]) pred_label.append(predictions[0]) # Get gradients grad = gradient_saliency.GetMask( test_images_val[0, :], feed_dict={neuron_selector: test_labels_val[0]}) imsave( './{}/weight_{:05d}/img_{:05d}.jpg'.format( FLAGS.save_path, path, i), (test_images_val[0, :] * imagenet_std + imagenet_mean)) np.save( './{}/weight_{:05d}/grad_{:05d}.npy'.format( FLAGS.save_path, path, i), np.mean(grad, axis=-1)) count += 1 test_acc /= FLAGS.test_iter np.save( './{}/weight_{:05d}/classified_flag.npy'.format( FLAGS.save_path, int(path)), np.array(classified_flag)) np.save( './{}/weight_{:05d}/ground_truth.npy'.format( FLAGS.save_path, int(path)), np.array(ground_truth)) np.save( './{}/weight_{:05d}/pred_label.npy'.format(FLAGS.save_path, int(path)), np.array(pred_label)) # Print and save results sec_per_image = test_time / FLAGS.test_iter / FLAGS.batch_size print('Done! Acc: %.6f, Test time: %.3f sec, %.7f sec/example' % (test_acc, test_time, sec_per_image)) print('done!')
logits = graph.get_tensor_by_name('logits/BiasAdd:0') neuron_selector = tf.placeholder(tf.int32) y = logits[0][neuron_selector] # Construct tensor for predictions prediction = tf.argmax(logits, 1) # Load an image im = cv2.imread('all_db_fire/isolated_superpixels/test/310_rgb_sp25.png') # Cast im as float32 im = np.float32(im) # Make a prediction prediction_class = sess.run(prediction, feed_dict={images: [im]})[0] print("Prediction class: " + str(prediction_class)) # Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops. gradient_saliency = saliency.GradientSaliency(graph, sess, y, images) # Compute the vanilla mask and the smoothed mask. vanilla_mask_3d = gradient_saliency.GetMask( im, feed_dict={neuron_selector: prediction_class}) # Generate saliency map and save abs_saliency = np.abs(vanilla_mask_3d).max(axis=-1) abs_saliency = viz.intensity_to_rgb( abs_saliency, normalize=True)[:, :, ::-1] # cv2 loads as BGR cv2.imwrite("abs-saliency.jpg", abs_saliency)
def main(args): """------------------ parse input--------------------""" task = args.task """---------------------------------------------------""" with tf.Graph().as_default() as graph: lr = 2 * 1e-4 batch_size = 1 n_neurons = [5, 5] n_steps = 5 n_layers = 2 n_outputs = 2 init_a = 100 init_b = 0 tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True net = Network.Net(batch_size=1, n_steps=n_steps, n_layers=n_layers, n_neurons=n_neurons, n_outputs=n_outputs, init_lr=lr) net.trainable = True input = tf.placeholder(tf.float32, (batch_size, n_steps, 224, 224, 3)) GT_label = tf.placeholder( tf.int64, (batch_size, n_steps)) # size = [batch, n_steps label_attention_map = tf.placeholder( tf.float32, (batch_size, n_steps, 112, 112, 1)) label_polar_map = tf.placeholder(tf.float32, (batch_size, n_steps, 224, 224, 3)) delta_year = tf.placeholder(tf.float32, (batch_size, n_steps)) label_predict_op, cnn_out = net.inference_model_10( input, label_attention_map ) # label_predict_op=(batch, n_steps, n_outputs) lr = tf.placeholder(tf.float32, shape=[]) loss_per_batch = net._loss_per_batch(label_predict_op, GT_label) loss_op, loss_label_op, loss_weight_op = net._loss_liliu( label_predict_op, GT_label) # [batch,n_steps] acc_op = net._top_k_accuracy(label_predict_op, GT_label) extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(extra_update_ops): opt = tf.train.AdamOptimizer(lr, beta1=0.9, beta2=0.999, epsilon=1e-08) gradients = opt.compute_gradients( loss_op) # all variables are trainable apply_gradient_op = opt.apply_gradients( gradients) # , global_step=self.global_step train_op = apply_gradient_op with tf.Session(config=tf_config, graph=graph) as sess: sess.run(tf.global_variables_initializer()) list_img_path_test = os.listdir('./data/test/image/all') list_img_path_test.sort() dataloader_test = DataLoader_atten_polar( batch_size=batch_size, list_img_path=list_img_path_test, state='test') for i in range(6): image, year, GTmap, Polar, GTlabel = dataloader_test.get_batch_single( ) """--------------------guidedbp----------------------------""" saver = tf.train.Saver() saver.restore(sess, model_file_cls) guided_backprop = saliency.GradientSaliency( graph, sess, label_predict_op[0, :, 1], input) mask0 = guided_backprop.GetMask(input[0], feed_dict={ input: image, GT_label: GTlabel, label_attention_map: GTmap, label_polar_map: Polar, delta_year: year }) for img_index in range(5): mask = mask0[img_index][0][img_index] mask = saliency.VisualizeImageGrayscale(mask) mask = gate(mask, gate=0) Image.imsave('visualization_result/cls_cvpr/guidedbp/img' + str(i + 1) + '_' + str(img_index + 1) + '_gbp.jpg', mask, cmap=plt.get_cmap('gray')) Image.imsave( 'visualization_result/cls_cvpr/guidedbp/img' + str(i + 1) + '_' + str(img_index + 1) + '.jpg', image[0][img_index]) print(task + '_' + str(i + 1) + '_' + str(img_index + 1))
def compute_and_save_attr(model, data, indices, num_threads): """Given the name of a model and a set of data, select a set of images based on provided indices, and compute and save their saliency maps and object attributions.""" base_dir = os.getcwd() data_dir = os.path.join(base_dir, 'data', data, 'val') model_dir = os.path.join(base_dir, 'models', model) sal_output_dir = os.path.join(base_dir, SAL_DIR, model + '-' + data) attr_output_dir = os.path.join(base_dir, ATTR_DIR, model + '-' + data) if not tf.gfile.Exists(sal_output_dir): tf.gfile.MakeDirs(sal_output_dir) if not tf.gfile.Exists(attr_output_dir): tf.gfile.MakeDirs(attr_output_dir) img_names = [sorted(tf.gfile.ListDirectory(data_dir))[i] for i in indices] img_paths = [os.path.join(data_dir, img_name) for img_name in img_names] imgs = load_imgs(img_paths, num_threads) input_name = 'input_tensor:0' logit_name = 'resnet_model/final_dense:0' conv_name = 'resnet_model/block_layer4:0' with tf.Session(graph=tf.Graph()) as sess: tf.saved_model.loader.load(sess, ['serve'], model_dir) graph = tf.get_default_graph() input_tensor = graph.get_tensor_by_name(input_name) logit_tensor = graph.get_tensor_by_name(logit_name) neuron_selector = tf.placeholder(tf.int32) y = logit_tensor[:, neuron_selector] pred_tensor = tf.argmax(logit_tensor, 1) vg = saliency.GradientSaliency(graph, sess, y, input_tensor) gb = saliency.GuidedBackprop(graph, sess, y, input_tensor) ig = saliency.IntegratedGradients(graph, sess, y, input_tensor) gc = saliency.GradCam(graph, sess, y, input_tensor, graph.get_tensor_by_name(conv_name)) def single_map(img, img_name): pred = sess.run(pred_tensor, feed_dict={input_tensor: [img]})[0] vg_mask = vg.GetMask(img, feed_dict={neuron_selector: pred}) # *s is SmoothGrad vgs_mask = vg.GetSmoothedMask(img, feed_dict={neuron_selector: pred}) gb_mask = gb.GetMask(img, feed_dict={neuron_selector: pred}) gbs_mask = gb.GetSmoothedMask(img, feed_dict={neuron_selector: pred}) baseline = np.zeros(img.shape) - np.expand_dims( np.expand_dims(_CHANNEL_MEANS, 0), 0) ig_mask = ig.GetMask(img, feed_dict={neuron_selector: pred}, x_baseline=baseline) igs_mask = ig.GetSmoothedMask(img, feed_dict={neuron_selector: pred}, x_baseline=baseline) gc_mask = gc.GetMask(img, feed_dict={neuron_selector: pred}) gcs_mask = gc.GetSmoothedMask(img, feed_dict={neuron_selector: pred}) # gbgc is guided GradCam gbgc_mask = gb_mask * gc_mask gbgcs_mask = gbs_mask * gcs_mask # Also include gradient x input masks = np.array([ vg_mask, vgs_mask, gb_mask, gbs_mask, ig_mask, igs_mask, gc_mask, gcs_mask, gbgc_mask, gbgcs_mask, vg_mask * img, vgs_mask * img ]) return masks, pred sal_maps = [] preds = [] for img, img_name in zip(imgs, img_names): sal_path = tf.gfile.Glob( os.path.join(sal_output_dir, img_name[:-4] + '*')) if len(sal_path) > 0: sal_maps.append(np.load(tf.gfile.GFile(sal_path[0], 'rb'))) preds.append(sal_path[0].split('_')[-1]) tf.logging.info( 'Loaded saliency maps for {}.'.format(img_name)) else: masks, pred = single_map(img, img_name) sal_maps.append(masks) preds.append(pred) out_path = os.path.join(sal_output_dir, img_name[:-4] + '_' + str(pred)) np.save(tf.gfile.GFile(out_path, 'w'), masks) tf.logging.info('Saved saliency maps for {}.'.format(img_name)) # Locate the objects, convert 3D saliency maps to 2D, and compute # the attributions of the object segments by averaging over the # per-pixel attributions of the objects. loc_fpath = os.path.join(base_dir, 'data', data, 'val_loc.txt') lines = [tf.gfile.Open(loc_fpath).readlines()[i] for i in indices] locs = np.array([[ int(int(l) * float(RESNET_SHAPE[0]) / IMG_SHAPE[0]) for l in line.rstrip('\n').split(' ')[-1].split(',') ] for line in lines]) pool = multiprocessing.Pool(num_threads) maps_3d = np.array(sal_maps).reshape(-1, RESNET_SHAPE[0], RESNET_SHAPE[1], 3) maps_2d = np.array(pool.map(visualize_pos_attr, maps_3d)) maps_2d = maps_2d.reshape(len(indices), int(maps_2d.shape[0] // len(indices)), RESNET_SHAPE[0], RESNET_SHAPE[1]) mask_fpath = os.path.join(base_dir, 'data', data, 'val_mask') if data in ['obj', 'scene', 'scene_only']: # MCS and IDR are evaluated on 10000 images and masks are 10x100. # Find the right mask. obj_dict = { 'backpack': 0, 'bird': 1, 'dog': 2, 'elephant': 3, 'kite': 4, 'pizza': 5, 'stop_sign': 6, 'toilet': 7, 'truck': 8, 'zebra': 9, } # Loading val_mask from the data directory masks_mat = np.load(tf.gfile.GFile(mask_fpath, 'rb'), allow_pickle=True) # Getting obj indices obj_inds = [obj_dict[i.split('.')[0].split('-')[0]] for i in img_names] # getting indices for a particular object class temp_inds = [int(i.split('.')[0][-2:]) for i in img_names] obj_masks = [ masks_mat[obj_inds[i] * 100 + temp_inds[i]] for i, _ in enumerate(img_names) ] else: obj_masks = [ np.load(tf.gfile.GFile(mask_fpath, 'rb'), allow_pickle=True)[i] for i in indices ] attrs = [] for i in range(len(indices)): attr = single_attr(maps_2d[i], locs[i], obj_masks[i]) attrs.append(attr) out_path = os.path.join(attr_output_dir, img_names[i][:-4] + '_' + str(preds[i])) np.save(tf.gfile.GFile(out_path, 'w'), attr)
def simple_example(): #-------------------- mnist = input_data.read_data_sets('./MNIST_data', one_hot=True) #-------------------- # Define a model. num_classes = 10 input_shape = (None, 28, 28, 1) # 784 = 28 * 28. output_shape = (None, num_classes) input_ph = tf.placeholder(tf.float32, shape=input_shape, name='input_ph') output_ph = tf.placeholder(tf.float32, shape=output_shape, name='output_ph') with tf.variable_scope('conv1', reuse=tf.AUTO_REUSE): conv1 = tf.layers.conv2d(input_ph, 32, 5, activation=tf.nn.relu, name='conv') conv1 = tf.layers.max_pooling2d(conv1, 2, 2, name='maxpool') with tf.variable_scope('conv2', reuse=tf.AUTO_REUSE): conv2 = tf.layers.conv2d(conv1, 64, 3, activation=tf.nn.relu, name='conv') conv2 = tf.layers.max_pooling2d(conv2, 2, 2, name='maxpool') with tf.variable_scope('fc1', reuse=tf.AUTO_REUSE): fc1 = tf.layers.flatten(conv2, name='flatten') fc1 = tf.layers.dense(fc1, 1024, activation=tf.nn.relu, name='dense') with tf.variable_scope('fc2', reuse=tf.AUTO_REUSE): model_output = tf.layers.dense(fc1, num_classes, activation=tf.nn.softmax, name='dense') #-------------------- # Train. loss = tf.reduce_mean(-tf.reduce_sum(output_ph * tf.log(model_output), reduction_indices=[1])) train_step = tf.train.GradientDescentOptimizer(0.01).minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) print('Start training...') start_time = time.time() for _ in range(2000): batch_xs, batch_ys = mnist.train.next_batch(512) batch_xs = np.reshape(batch_xs, (-1, ) + input_shape[1:]) sess.run(train_step, feed_dict={ input_ph: batch_xs, output_ph: batch_ys }) if 0 == idx % 100: print('.', end='', flush=True) print() print('End training: {} secs.'.format(time.time() - start_time)) #-------------------- # Evaluate. correct_prediction = tf.equal(tf.argmax(model_output, 1), tf.argmax(output_ph, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) print('Start testing...') acc = sess.run(accuracy, feed_dict={ input_ph: np.reshape(mnist.test.images, (-1, ) + input_shape[1:]), output_ph: mnist.test.labels }) print('Test accuracy = {}.'.format(acc)) print('End testing: {} secs.'.format(time.time() - start_time)) if acc < 0.95: print('Failed to train...') return #-------------------- # Visualize. images = np.reshape(mnist.test.images, (-1, ) + input_shape[1:]) img = images[0] minval, maxval = np.min(img), np.max(img) img_scaled = np.squeeze((img - minval) / (maxval - minval), axis=-1) # Construct the scalar neuron tensor. logits = model_output neuron_selector = tf.placeholder(tf.int32) y = logits[0][neuron_selector] # Construct a tensor for predictions. prediction = tf.argmax(logits, 1) # Make a prediction. prediction_class = sess.run(prediction, feed_dict={input_ph: [img]})[0] #-------------------- start_time = time.time() saliency_obj = saliency.Occlusion(sess.graph, sess, y, input_ph) print('Occlusion: {} secs.'.format(time.time() - start_time)) # NOTE [info] >> An error exists in GetMask() of ${Saliency_HOME}/saliency/occlusion.py. # <before> # occlusion_window = np.array([size, size, x_value.shape[2]]) # occlusion_window.fill(value) # <after> # occlusion_window = np.full([size, size, x_value.shape[2]], value) mask_3d = saliency_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}) # Compute a 2D tensor for visualization. mask_gray = saliency.VisualizeImageGrayscale(mask_3d) mask_div = saliency.VisualizeImageDiverging(mask_3d) fig = plt.figure() ax = plt.subplot(1, 3, 1) ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Input') ax = plt.subplot(1, 3, 2) ax.imshow(mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Grayscale') ax = plt.subplot(1, 3, 3) ax.imshow(mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Diverging') fig.suptitle('Occlusion', fontsize=16) fig.tight_layout() #plt.savefig('./saliency_occlusion.png') plt.show() #-------------------- start_time = time.time() conv_layer = sess.graph.get_tensor_by_name('conv2/conv/BiasAdd:0') saliency_obj = saliency.GradCam(sess.graph, sess, y, input_ph, conv_layer) print('GradCam: {} secs.'.format(time.time() - start_time)) mask_3d = saliency_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}) # Compute a 2D tensor for visualization. mask_gray = saliency.VisualizeImageGrayscale(mask_3d) mask_div = saliency.VisualizeImageDiverging(mask_3d) fig = plt.figure() ax = plt.subplot(1, 3, 1) ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Input') ax = plt.subplot(1, 3, 2) ax.imshow(mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Grayscale') ax = plt.subplot(1, 3, 3) ax.imshow(mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Diverging') fig.suptitle('Grad-CAM', fontsize=16) fig.tight_layout() #plt.savefig('./saliency_gradcam.png') plt.show() #-------------------- start_time = time.time() saliency_obj = saliency.GradientSaliency(sess.graph, sess, y, input_ph) print('GradientSaliency: {} secs.'.format(time.time() - start_time)) vanilla_mask_3d = saliency_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}) smoothgrad_mask_3d = saliency_obj.GetSmoothedMask( img, feed_dict={neuron_selector: prediction_class}) # Compute a 2D tensor for visualization. vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d) smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d) smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d) fig = plt.figure() ax = plt.subplot(2, 3, 1) ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Input') ax = plt.subplot(2, 3, 2) ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Vanilla Grayscale') ax = plt.subplot(2, 3, 3) ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('SmoothGrad Grayscale') ax = plt.subplot(2, 3, 5) ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Vanilla Diverging') ax = plt.subplot(2, 3, 6) ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('SmoothGrad Diverging') fig.suptitle('Gradient Saliency', fontsize=16) fig.tight_layout() #plt.savefig('./saliency_gradientsaliency.png') plt.show() #-------------------- start_time = time.time() saliency_obj = saliency.GuidedBackprop(sess.graph, sess, y, input_ph) print('GuidedBackprop: {} secs.'.format(time.time() - start_time)) vanilla_mask_3d = saliency_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}) smoothgrad_mask_3d = saliency_obj.GetSmoothedMask( img, feed_dict={neuron_selector: prediction_class}) # Compute a 2D tensor for visualization. vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d) smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d) smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d) fig = plt.figure() ax = plt.subplot(2, 3, 1) ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Input') ax = plt.subplot(2, 3, 2) ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Vanilla Grayscale') ax = plt.subplot(2, 3, 3) ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('SmoothGrad Grayscale') ax = plt.subplot(2, 3, 4) ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Vanilla Diverging') ax = plt.subplot(2, 3, 5) ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('SmoothGrad Diverging') fig.suptitle('Guided Backprop', fontsize=16) fig.tight_layout() #plt.savefig('./saliency_guidedbackprop.png') plt.show() #-------------------- start_time = time.time() saliency_obj = saliency.IntegratedGradients(sess.graph, sess, y, input_ph) print('IntegratedGradients: {} secs.'.format(time.time() - start_time)) vanilla_mask_3d = saliency_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}) smoothgrad_mask_3d = saliency_obj.GetSmoothedMask( img, feed_dict={neuron_selector: prediction_class}) # Compute a 2D tensor for visualization. vanilla_mask_gray = saliency.VisualizeImageGrayscale(vanilla_mask_3d) smoothgrad_mask_gray = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) vanilla_mask_div = saliency.VisualizeImageDiverging(vanilla_mask_3d) smoothgrad_mask_div = saliency.VisualizeImageDiverging(smoothgrad_mask_3d) fig = plt.figure() ax = plt.subplot(2, 3, 1) ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Input') ax = plt.subplot(2, 3, 2) ax.imshow(vanilla_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Vanilla Grayscale') ax = plt.subplot(2, 3, 3) ax.imshow(smoothgrad_mask_gray, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('SmoothGrad Grayscale') ax = plt.subplot(2, 3, 4) ax.imshow(vanilla_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Vanilla Diverging') ax = plt.subplot(2, 3, 5) ax.imshow(smoothgrad_mask_div, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('SmoothGrad Diverging') fig.suptitle('Integrated Gradients', fontsize=16) fig.tight_layout() #plt.savefig('./saliency_integratedgradients.png') plt.show() #-------------------- start_time = time.time() xrai_obj = saliency.XRAI(sess.graph, sess, y, input_ph) print('XRAI: {} secs.'.format(time.time() - start_time)) if True: xrai_attributions = xrai_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}) else: # Create XRAIParameters and set the algorithm to fast mode which will produce an approximate result. xrai_params = saliency.XRAIParameters() xrai_params.algorithm = 'fast' xrai_attributions_fast = xrai_obj.GetMask( img, feed_dict={neuron_selector: prediction_class}, extra_parameters=xrai_params) # Show most salient 30% of the image. mask = xrai_attributions > np.percentile(xrai_attributions, 70) img_masked = img_scaled.copy() img_masked[~mask] = 0 fig = plt.figure() ax = plt.subplot(1, 3, 1) ax.imshow(img_scaled, cmap=plt.cm.gray, vmin=0, vmax=1) ax.axis('off') ax.set_title('Input') ax = plt.subplot(1, 3, 2) ax.imshow(xrai_attributions, cmap=plt.cm.inferno) ax.axis('off') ax.set_title('XRAI Attributions') ax = plt.subplot(1, 3, 3) ax.imshow(img_masked, cmap=plt.cm.gray) ax.axis('off') ax.set_title('Masked Input') fig.suptitle('XRAI', fontsize=16) fig.tight_layout() #plt.savefig('./saliency_xrai.png') plt.show()
def main(args): for arg in vars(args): print(arg, getattr(args, arg)) model_name = args.model_name img_path = args.img_path img_label_path = 'imagenet.json' true_class = args.true_label adversarial_label = args.adv_label label_num = args.label_num lambda_up, lambda_down, lambda_label_loss = args.lambda_up, args.lambda_down, args.lambda_label_loss # model_name = 'inception_v3' # img_path = './picture/dog_cat.jpg' # img_label_path = 'imagenet.json' # true_class = 208 sess, graph, img_size, images_pl, logits = load_pretrain_model( model_name, is_explain=True) y_label = tf.placeholder(dtype=tf.int32, shape=()) label_logits = logits[0, y_label] if len(args.imp) > 0: img = np.load(args.imp) init_epoch = int(args.imp[:-4].split('_')[-1]) loss_list = list(np.load('loss_' + args.imp)) else: img = PIL.Image.open(img_path) img = preprocess_img(img, img_size) init_epoch = 0 loss_list = [] old_img = np.array(img) batch_img = np.expand_dims(img, 0) #new_img = np.load('vgg16_30_0.0004_1000_0.001_0.03_4000.npy') #new_batch_img = np.concatenate((np.expand_dims(new_img,0),batch_img),axis=0) #new_batch_img = np.expand_dims(new_img,0) #all_img = np.concatenate((batch_img,new_batch_img)) imagenet_label = load_imagenet_label(img_label_path) prob = tf.nn.softmax(logits) _prob = sess.run(prob, feed_dict={images_pl: batch_img})[0] #classify(img,_prob,imagenet_label,1,1) #### #deep explain # from deepexplain.tensorflow import DeepExplain # label_logits = logits[0,208] # with DeepExplain(session=sess) as de: # attributions = { # # Gradient-based # # NOTE: reduce_max is used to select the output unit for the class predicted by the classifier # # For an example of how to use the ground-truth labels instead, see mnist_cnn_keras notebook # 'Saliency maps': de.explain('saliency', label_logits, images_pl, batch_img), # 'Gradient * Input': de.explain('grad*input', label_logits, images_pl, batch_img), # # 'Integrated Gradients': de.explain('intgrad', label_logits, images_pl, new_batch_img), # 'Epsilon-LRP': de.explain('elrp', label_logits, images_pl, batch_img), # 'DeepLIFT (Rescale)': de.explain('deeplift', label_logits, images_pl, batch_img), # # Perturbation-based (comment out to evaluate, but this will take a while!) # #'Occlusion [15x15]': de.explain('occlusion', label_logits, images_pl, batch_img, window_shape=(15,15,3), step=4) # } #### # new_attributions = { # # Gradient-based # # NOTE: reduce_max is used to select the output unit for the class predicted by the classifier # # For an example of how to use the ground-truth labels instead, see mnist_cnn_keras notebook # 'Saliency maps': de.explain('saliency', label_logits, images_pl, new_batch_img), # 'Gradient * Input': de.explain('grad*input', label_logits, images_pl, new_batch_img), # # 'Integrated Gradients': de.explain('intgrad', label_logits, images_pl, new_batch_img), # 'Epsilon-LRP': de.explain('elrp', label_logits, images_pl, new_batch_img), # 'DeepLIFT (Rescale)': de.explain('deeplift', label_logits, images_pl, new_batch_img), # # Perturbation-based (comment out to evaluate, but this will take a while!) # #'Occlusion [15x15]': de.explain('occlusion', label_logits, images_pl, batch_img, window_shape=(15,15,3), step=4) # } #### # attributions['Saliency maps'] = np.concatenate((attributions['Saliency maps'],new_attributions['Saliency maps']),axis=0) # attributions['Gradient * Input'] = np.concatenate((attributions['Gradient * Input'],new_attributions['Gradient * Input']),axis=0) # attributions['Epsilon-LRP'] = np.concatenate((attributions['Epsilon-LRP'],new_attributions['Epsilon-LRP']),axis=0) # attributions['DeepLIFT (Rescale)'] = np.concatenate((attributions['DeepLIFT (Rescale)'],new_attributions['DeepLIFT (Rescale)']),axis=0) # # n_cols = int(len(attributions)) + 1 # n_rows = 2 # fig, axes = plt.subplots(nrows=n_rows, ncols=n_cols, figsize=(3 * n_cols, 3 * n_rows)) # # for i, xi in enumerate(all_img): # # xi = (xi - np.min(xi)) # # xi /= np.max(xi) # ax = axes.flatten()[i * n_cols] # ax.imshow(xi) # ax.set_title('Original') # ax.axis('off') # for j, a in enumerate(attributions): # axj = axes.flatten()[i * n_cols + j + 1] # plot(attributions[a][i], xi=xi, axis=axj, dilation=.5, percentile=99, alpha=.2).set_title(a) ###### label_logits = logits[0, 208] with DeepExplain(session=sess) as de: dlift = de.explain('deeplift', label_logits, images_pl, batch_img) grad_map_tensor = tf.gradients(label_logits, images_pl)[0] grad_map = sess.run(grad_map_tensor, feed_dict={ images_pl: np.expand_dims(img, 0), y_label: true_class }) gradient_saliency = saliency.GradientSaliency(graph, sess, label_logits, images_pl) # 1951/1874 vanilla_mask_3d = gradient_saliency.GetMask( img, feed_dict={y_label: true_class}) # better vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_mask_3d) # smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(img, feed_dict={y_label:true_class}) # much clear, 2204/2192 # smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(smoothgrad_mask_3d) # # new_img = np.load('vgg16_60_70_35_45_30_0.0001_800_0.0_0.0_9000.npy') # new_grad_map = sess.run(grad_map_tensor,feed_dict={images_pl:np.expand_dims(new_img,0),y_label:true_class}) # new_vanilla_mask_3d = gradient_saliency.GetMask(new_img, feed_dict={y_label:true_class}) # better # new_vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(new_vanilla_mask_3d) # new_smoothgrad_mask_3d = gradient_saliency.GetSmoothedMask(new_img, feed_dict={y_label:true_class}) # much clear, 2204/2192 # new_smoothgrad_mask_grayscale = saliency.VisualizeImageGrayscale(new_smoothgrad_mask_3d) #to_dec_center = (60,70) to_dec_center = (100, 65) #to_dec_radius = (35,45) to_dec_radius = (80, 60) to_inc_center = (120, 170) to_inc_radius = (40, 30) _map = vanilla_mask_grayscale print(calculate_region_importance(_map, to_dec_center, to_dec_radius)) print(calculate_region_importance(_map, to_inc_center, to_inc_radius)) # construct to_inc_region and to_dec_region to_dec_region = calculate_img_region_importance(grad_map_tensor, to_dec_center, to_dec_radius) to_inc_region = calculate_img_region_importance(grad_map_tensor, to_inc_center, to_inc_radius) # try NES (Natural evolutionary strategies) N = args.N sigma = args.sigma epsilon = round(args.eps, 2) epoch = args.epoch eta = args.lr #loss = to_dec_region/to_inc_region #old_loss = sess.run(loss,feed_dict={images_pl: np.expand_dims(img, 0), y_label: true_class}) old_loss = calculate_deeplift_loss(dlift, to_dec_center, to_dec_radius, to_inc_center, to_inc_radius) num_list = '_'.join([ 'big', model_name, str(N), str(eta), str(epoch), str(sigma), str(epsilon) ]) print(num_list) for i in range(epoch): delta = np.random.randn(int(N / 2), img_size * img_size * 3) delta = np.concatenate((delta, -delta), axis=0) grad_sum = 0 f_value_list = [] for idelta in delta: img_plus = np.clip( img + sigma * idelta.reshape(img_size, img_size, 3), 0, 1) #f_value = sess.run(loss,feed_dict={images_pl:np.expand_dims(img_plus,0),y_label:true_class}) with DeepExplain(session=sess) as de: dlift = de.explain('deeplift', label_logits, images_pl, np.expand_dims(img_plus, 0)) f_value = calculate_deeplift_loss(dlift, to_dec_center, to_dec_radius, to_inc_center, to_inc_radius) f_value_list.append(f_value) grad_sum += f_value * idelta.reshape(img_size, img_size, 3) grad_sum = grad_sum / (N * sigma) new_img = np.clip( np.clip(img - eta * grad_sum, old_img - epsilon, old_img + epsilon), 0, 1) #new_loss, new_logits = sess.run([loss, logits], # feed_dict={images_pl: np.expand_dims(new_img, 0), y_label: true_class}) with DeepExplain(session=sess) as de: dlift = de.explain('deeplift', label_logits, images_pl, np.expand_dims(new_img, 0)) new_loss = calculate_deeplift_loss(dlift, to_dec_center, to_dec_radius, to_inc_center, to_inc_radius) loss_list.append(new_loss) print("epoch:{} new:{}, old:{}, {}".format(i, new_loss, old_loss, np.argmax(_prob))) sys.stdout.flush() img = np.array(new_img) if i % args.image_interval == 0: temp_name = num_list + '_' + str(i + init_epoch) np.save(temp_name, new_img) if i % args.image_interval == 0: np.save('loss_' + temp_name, loss_list) np.save(num_list + '_' + str(epoch + init_epoch), new_img) np.save('loss_' + num_list + '_' + str(epoch + init_epoch), loss_list)