# ## Visualize Activations # # In some cases it helps to look at the output from different layers and visualize them to catch issues and odd patterns. # In[32]: # Get activations of a few sample layers activations = model.run_graph( [image], [ ("input_image", model.keras_model.get_layer("input_image").output), ("res2c_out", model.keras_model.get_layer("res2c_out").output), ("res3c_out", model.keras_model.get_layer("res3c_out").output), ("res4w_out", model.keras_model.get_layer("res4w_out").output), # for resnet100 ("rpn_bbox", model.keras_model.get_layer("rpn_bbox").output), ("roi", model.keras_model.get_layer("ROI").output), ]) # In[33]: # Input image (normalized) _ = plt.imshow(modellib.unmold_image(activations["input_image"][0], config)) # In[34]: # Backbone feature map display_images(np.transpose(activations["res2c_out"][0, :, :, :4], [2, 0, 1]), cols=4)
def train_adversarial_batch(model, datagenerator, target_attack=False, show_perturbation=False, use_mask=False, save_adversarials_to_logs=False): for inputs in datagenerator: images = inputs[0] image_metas = inputs[1] rpn_match = inputs[2] rpn_bbox = inputs[3] gt_class_ids = inputs[4] gt_boxes = inputs[5] gt_masks = inputs[6] # image_metas as numpy array image_metas = image_metas.numpy() # To GPU if model.config.GPU_COUNT: images = images.cuda() rpn_match = rpn_match.cuda() rpn_bbox = rpn_bbox.cuda() gt_class_ids = gt_class_ids.cuda() gt_boxes = gt_boxes.cuda() gt_masks = gt_masks.cuda() # SETTINGS steps = 20 max_perturbation = 15 # Wrap in variables images_orig = images.clone() images = Variable(images, requires_grad=True) rpn_match = Variable(rpn_match) rpn_bbox = Variable(rpn_bbox) gt_class_ids = Variable(gt_class_ids) gt_boxes = Variable(gt_boxes) gt_masks = Variable(gt_masks) # Create mask of mask = create_mask(images_orig.shape, gt_boxes[0][0]) for step in range(steps): model.zero_grad() zero_gradients(images) # Run object detection rpn_class_logits, rpn_pred_bbox, target_class_ids, mrcnn_class_logits, target_deltas, mrcnn_bbox, target_mask, mrcnn_mask = \ model.predict([images, image_metas, gt_class_ids, gt_boxes, gt_masks], mode='training') # Compute losses rpn_class_loss, rpn_bbox_loss, mrcnn_class_loss, mrcnn_bbox_loss, mrcnn_mask_loss = compute_losses( rpn_match, rpn_bbox, rpn_class_logits, rpn_pred_bbox, target_class_ids, mrcnn_class_logits, target_deltas, mrcnn_bbox, target_mask, mrcnn_mask) loss = rpn_class_loss + rpn_bbox_loss + mrcnn_class_loss + mrcnn_bbox_loss + mrcnn_mask_loss loss.backward() print("step {}: loss={}".format(step, loss.data.cpu().numpy()[0])) # Calculate gradient #grad = images.grad * 10000 grad = torch.sign(images.grad) # Change part of the image in mask only if enabled if use_mask: grad.data = grad.data * mask # Clamp max perturbation per step grad = torch.clamp(grad, -max_perturbation / steps, max_perturbation / steps) # Add/Subtract perturbation based on attack if target_attack: images_tmp = unmold_image_tensor(images.data - grad.data, model.config) else: images_tmp = unmold_image_tensor(images.data + grad.data, model.config) # Clamp to reasonable image values images_tmp = torch.clamp(images_tmp, 0, 255) images_data = mold_image_tensor(images_tmp, model.config) # Set adversarial image as new input images.data = images_data a = images[0].data.clone() class_names = [ 'BG', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush' ] # Run detection image_org = unmold_image(img_to_np(images_orig[0]), model.config) results = model.detect([image_org]) # Visualize original r = results[0] display_instances(image_org, r['rois'], r['masks'], r['class_ids'], class_names, r['scores']) # Run detection image_adv = unmold_image(img_to_np(a), model.config) results = model.detect([image_adv]) # Visualize adversarial r = results[0] display_instances(image_adv, r['rois'], r['masks'], r['class_ids'], class_names, r['scores']) if save_adversarials_to_logs: path = os.path.join(DEFAULT_LOGS_DIR, "adversarial_examples") pathlib.Path(path).mkdir(parents=True, exist_ok=True) path = os.path.join( path, "adversarial_example_" + str(int(time.time())) + ".jpg") skimage.io.imsave(path, image_adv) print("Adversarial exaple saved to: " + path) # Visualize perturbation if show_perturbation: image_tmp = image_adv.astype(np.int16) perturbation = image_tmp - image_org perturbation.clip(0, 255) scale_factor = 255 / perturbation.max() # Show Plot _, ax = plt.subplots(1, figsize=(16, 16)) height, width = perturbation.shape[:2] ax.set_ylim(height + 10, -10) ax.set_xlim(-10, width + 10) ax.axis('off') ax.set_title('Attacking noise (x{0:4.2f})'.format(scale_factor)) ax.imshow(perturbation * scale_factor) plt.show()
def train_adversarial_batch(model, datagenerator, target_attack=False, use_mask=False): for i, inputs in tqdm(enumerate(datagenerator), total=len(datagenerator)): images = inputs[0] image_metas = inputs[1] rpn_match = inputs[2] rpn_bbox = inputs[3] gt_class_ids = inputs[4] gt_boxes = inputs[5] gt_masks = inputs[6] # image_metas as numpy array image_metas = image_metas.numpy() # To GPU if model.config.GPU_COUNT: images = images.cuda() rpn_match = rpn_match.cuda() rpn_bbox = rpn_bbox.cuda() gt_class_ids = gt_class_ids.cuda() gt_boxes = gt_boxes.cuda() gt_masks = gt_masks.cuda() # SETTINGS steps = 30 max_perturbation = 15 # Wrap in variables images_orig = images.clone() images = Variable(images, requires_grad=True) rpn_match = Variable(rpn_match) rpn_bbox = Variable(rpn_bbox) gt_class_ids = Variable(gt_class_ids) gt_boxes = Variable(gt_boxes) gt_masks = Variable(gt_masks) # Create mask of mask = create_mask(images_orig.shape, gt_boxes[0][0]) for step in range(steps): model.zero_grad() zero_gradients(images) # Run object detection rpn_class_logits, rpn_pred_bbox, target_class_ids, mrcnn_class_logits, target_deltas, mrcnn_bbox, target_mask, mrcnn_mask = \ model.predict([images, image_metas, gt_class_ids, gt_boxes, gt_masks], mode='training') # Compute losses rpn_class_loss, rpn_bbox_loss, mrcnn_class_loss, mrcnn_bbox_loss, mrcnn_mask_loss = compute_losses( rpn_match, rpn_bbox, rpn_class_logits, rpn_pred_bbox, target_class_ids, mrcnn_class_logits, target_deltas, mrcnn_bbox, target_mask, mrcnn_mask) loss = rpn_class_loss + rpn_bbox_loss + mrcnn_class_loss + mrcnn_bbox_loss + mrcnn_mask_loss loss.backward() # Calculate gradient # grad = images.grad * 10000 grad = torch.sign(images.grad) # Change part of the image in mask only if enabled if use_mask: grad.data = grad.data * mask # Clamp max perturbation per step grad = torch.clamp(grad, -max_perturbation / steps, max_perturbation / steps) # Add/Subtract perturbation based on attack if target_attack: images_tmp = unmold_image_tensor(images.data - grad.data, model.config) else: images_tmp = unmold_image_tensor(images.data + grad.data, model.config) # Clamp to reasonable image values images_tmp = torch.clamp(images_tmp, 0, 255) images_data = mold_image_tensor(images_tmp, model.config) # Set adversarial image as new input images.data = images_data a = images[0].data.clone() """ # Run detection image = unmold_image(img_to_np(images_orig[0]), model.config) results = model.detect([image]) # Visualize results r = results[0] plt = display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores']) #plt.show() #plt.close() plt.savefig('./data/video/results/{}.png'.format(i)) """ # Run detection image = unmold_image(img_to_np(a), model.config) results = model.detect([image]) # Visualize results r = results[0] plt = display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores']) #plt.show() #plt.close() plt.savefig('./data/video/results_adversarial/{}.png'.format(i))
rpn_match, ) log("rpn_bbox", rpn_bbox) image_id = image_meta[0][0] print("image_id: ", image_id, dataset.image_reference(image_id)) # Remove the last dim in mrcnn_class_ids. It's only added # to satisfy Keras restriction on target shape. mrcnn_class_ids = mrcnn_class_ids[:, :, 0] # In[16]: b = 0 # Restore original image (reverse normalization) sample_image = modellib.unmold_image(normalized_images[b], config) # Compute anchor shifts. indices = np.where(rpn_match[b] == 1)[0] refined_anchors = utils.apply_box_deltas( anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV) log("anchors", anchors) log("refined_anchors", refined_anchors) # Get list of positive anchors positive_anchor_ids = np.where(rpn_match[b] == 1)[0] print("Positive anchors: {}".format(len(positive_anchor_ids))) negative_anchor_ids = np.where(rpn_match[b] == -1)[0] print("Negative anchors: {}".format(len(negative_anchor_ids))) neutral_anchor_ids = np.where(rpn_match[b] == 0)[0] print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
log("det_masks", det_masks) display_images(det_mask_specific[:4] * 255, cmap="Blues", interpolation="none") display_images(det_masks[:4] * 255, cmap="Blues", interpolation="none") # Get activations of a few sample layers activations = model.run_graph([image], [ ("input_image", model.keras_model.get_layer("input_image").output), ("res4w_out", model.keras_model.get_layer("res4w_out").output), # for resnet100 ("rpn_bbox", model.keras_model.get_layer("rpn_bbox").output), ("roi", model.keras_model.get_layer("ROI").output), ]) # Input image (normalized) _ = plt.imshow(modellib.unmold_image(activations["input_image"][0],config)) # Backbone feature map display_images(np.transpose(activations["res4w_out"][0,:,:,:4], [2, 0, 1])) # Histograms of RPN bounding box deltas plt.figure(figsize=(12, 3)) plt.subplot(1, 4, 1) plt.title("dy") _ = plt.hist(activations["rpn_bbox"][0,:,0], 50) plt.subplot(1, 4, 2) plt.title("dx") _ = plt.hist(activations["rpn_bbox"][0,:,1], 50) plt.subplot(1, 4, 3) plt.title("dw") _ = plt.hist(activations["rpn_bbox"][0,:,2], 50)