def init(self, dataset, criterion, sample_nums, path_graph, path_wnids, classes, Rules, tree_supervision_weight=1.): """ Extra init method makes clear which arguments are finally necessary for this class to function. The constructor for this class may generate some of these required arguments if initially missing. """ self.dataset = dataset self.num_classes = len(classes) self.nodes = Node.get_nodes(path_graph, path_wnids, classes) self.rules = Rules(dataset, path_graph, path_wnids, classes) self.tree_supervision_weight = tree_supervision_weight self.criterion = criterion self.sample_nums = np.array(sample_nums) self.node_depths = defaultdict(lambda: []) self.node_weights = defaultdict(lambda: []) effective_num = 1.0 - np.power(0.999, self.sample_nums) weights = (1.0 - 0.999) / np.array(effective_num) self.weights = weights for node in self.nodes: key = node.num_classes depth = node.get_depth() self.node_depths[key].append(depth) node_weight = [] for new_label in range(node.num_classes): node_weight.append(weights[node.new_to_old_classes[new_label]]) self.node_weights[key].append(node_weight)
def __init__(self, dataset, path_graph=None, path_wnids=None, classes=()): if not path_graph: path_graph = dataset_to_default_path_graph(dataset) if not path_wnids: path_wnids = dataset_to_default_path_wnids(dataset) if not classes: classes = dataset_to_dummy_classes(dataset) super().__init__() assert all([dataset, path_graph, path_wnids, classes]) self.classes = classes self.nodes = Node.get_nodes(path_graph, path_wnids, classes) self.G = self.nodes[0].G self.wnid_to_node = {node.wnid: node for node in self.nodes} self.wnids = get_wnids(path_wnids) self.wnid_to_class = { wnid: cls for wnid, cls in zip(self.wnids, self.classes) } self.correct = 0 self.total = 0 self.I = torch.eye(len(classes))
def forward_with_decisions(self, outputs): outputs = self.forward(outputs) _, predicted = outputs.max(1) decisions = [] node = self.nodes[0] leaf_to_path_nodes = Node.get_leaf_to_path(self.nodes) for index, prediction in enumerate(predicted): leaf = node.wnids[prediction] decision = leaf_to_path_nodes[leaf] for justification in decision: justification['prob'] = -1 # TODO(alvin): fill in prob decisions.append(decision) return outputs, decisions
def forward_with_decisions(self, outputs): outputs_ = outputs[:, 1:] outputs_ = self.forward(outputs_) _, predicted = outputs_.max(1) decisions = [] node = self.nodes[0] leaf_to_path_nodes = Node.get_leaf_to_path(self.nodes) for index, prediction in enumerate(predicted): leaf = node.wnids[prediction] decision = leaf_to_path_nodes[leaf] for justification in decision: justification['prob'] = -1 # TODO(alvin): fill in prob decisions.append(decision) outputs_score = F.softmax(outputs, dim=-1) outputs = torch.cat((outputs_score[:, 0].unsqueeze(1), outputs_), dim=1) return outputs, decisions
def init(self, dataset, criterion, path_graph, path_wnids, classes, Rules, tree_supervision_weight=1.): """ Extra init method makes clear which arguments are finally necessary for this class to function. The constructor for this class may generate some of these required arguments if initially missing. """ self.dataset = dataset self.num_classes = len(classes) self.nodes = Node.get_nodes(path_graph, path_wnids, classes) self.rules = Rules(dataset, path_graph, path_wnids, classes) self.tree_supervision_weight = tree_supervision_weight self.criterion = criterion
def main(): args = parse_args() logger, final_output_dir, _ = create_logger(config, args.cfg, 'vis_gradcam') logger.info(pprint.pformat(args)) logger.info(pprint.pformat(config)) # cudnn related setting cudnn.benchmark = config.CUDNN.BENCHMARK cudnn.deterministic = config.CUDNN.DETERMINISTIC cudnn.enabled = config.CUDNN.ENABLED # build model model = eval('models.' + config.MODEL.NAME + '.get_seg_model')(config) dump_input = torch.rand( (1, 3, config.TRAIN.IMAGE_SIZE[1], config.TRAIN.IMAGE_SIZE[0])) logger.info(get_model_summary(model.cuda(), dump_input.cuda())) if config.TEST.MODEL_FILE: model_state_file = config.TEST.MODEL_FILE else: model_state_file = os.path.join(final_output_dir, 'best.pth') logger.info('=> loading model from {}'.format(model_state_file)) pretrained_dict = torch.load(model_state_file) model_dict = model.state_dict() pretrained_dict = { k[6:]: v for k, v in pretrained_dict.items() if k[6:] in model_dict.keys() } for k, _ in pretrained_dict.items(): logger.info('=> loading {} from pretrained model'.format(k)) model_dict.update(pretrained_dict) model.load_state_dict(model_dict) # Wrap original model with NBDT if config.NBDT.USE_NBDT: from nbdt.model import SoftSegNBDT model = SoftSegNBDT(config.NBDT.DATASET, model, hierarchy=config.NBDT.HIERARCHY, classes=class_names) device = 'cuda' if torch.cuda.is_available() else 'cpu' model = model.to(device).eval() # Retrieve input image corresponding to args.image_index test_size = (config.TEST.IMAGE_SIZE[1], config.TEST.IMAGE_SIZE[0]) test_dataset = eval('datasets.' + config.DATASET.DATASET)( root=config.DATASET.ROOT, list_path=config.DATASET.TEST_SET, num_samples=None, num_classes=config.DATASET.NUM_CLASSES, multi_scale=False, flip=False, ignore_label=config.TRAIN.IGNORE_LABEL, base_size=config.TEST.BASE_SIZE, crop_size=test_size, downsample_rate=1) # Define target layer as final convolution layer if not specified if args.target_layers: target_layers = args.target_layers.split(',') else: for name, module in list(model.named_modules())[::-1]: if isinstance(module, nn.Conv2d): target_layers = [name] break logger.info('Target layers set to {}'.format(str(target_layers))) # Append model. to target layers if using nbdt if config.NBDT.USE_NBDT: target_layers = ['model.' + layer for layer in target_layers] def generate_and_save_saliency(image_index, pixel_i=None, pixel_j=None, crop_size=None, normalize=False): """too lazy to move out to global lol""" nonlocal maximum, minimum, label # Generate GradCAM + save heatmap heatmaps = [] raw_image = retrieve_raw_image(test_dataset, image_index) should_crop = crop_size is not None and pixel_i is not None and pixel_j is not None if should_crop: raw_image = crop(pixel_i, pixel_j, crop_size, raw_image, is_tensor=False) for layer in target_layers: gradcam_region = gradcam.generate(target_layer=layer, normalize=False) if should_crop: gradcam_region = crop(pixel_i, pixel_j, crop_size, gradcam_region, is_tensor=True) maximum = max(float(gradcam_region.max()), maximum) minimum = min(float(gradcam_region.min()), minimum) logger.info(f'=> Bounds: ({minimum}, {maximum})') heatmaps.append(gradcam_region) output_dir = generate_output_dir(final_output_dir, args.vis_mode, layer, config.NBDT.USE_NBDT, nbdt_node_wnid, args.crop_size, args.nbdt_node_wnids_for) save_path = generate_save_path(output_dir, gradcam_kwargs) logger.info('Saving {} heatmap at {}...'.format( args.vis_mode, save_path)) if normalize: gradcam_region = GradCAM.normalize(gradcam_region) save_gradcam(save_path, gradcam_region, raw_image, save_npy=not args.skip_save_npy) else: save_gradcam(save_path, gradcam_region, raw_image, minimum=minimum, maximum=maximum, save_npy=not args.skip_save_npy) output_dir_original = output_dir + '_original' os.makedirs(output_dir_original, exist_ok=True) save_path_original = generate_save_path(output_dir_original, gradcam_kwargs, ext='jpg') logger.info('Saving {} original at {}...'.format( args.vis_mode, save_path_original)) cv2.imwrite(save_path_original, raw_image) if crop_size and pixel_i and pixel_j: continue output_dir += '_overlap' os.makedirs(output_dir, exist_ok=True) save_path_overlap = generate_save_path(output_dir, gradcam_kwargs, ext='npy') save_path_plot = generate_save_path(output_dir, gradcam_kwargs, ext='jpg') if not args.skip_save_npy: logger.info('Saving {} overlap data at {}...'.format( args.vis_mode, save_path_overlap)) logger.info('Saving {} overlap plot at {}...'.format( args.vis_mode, save_path_plot)) save_overlap(save_path_overlap, save_path_plot, gradcam_region, label, save_npy=not args.skip_save_npy) if len(heatmaps) > 1: combined = torch.prod(torch.stack(heatmaps, dim=0), dim=0) combined /= combined.max() save_path = generate_save_path(final_output_dir, args.vis_mode, gradcam_kwargs, 'combined', config.NBDT.USE_NBDT, nbdt_node_wnid) logger.info('Saving combined {} heatmap at {}...'.format( args.vis_mode, save_path)) save_gradcam(save_path, combined, raw_image) nbdt_node_wnids = args.nbdt_node_wnid or [] cls = args.nbdt_node_wnids_for if cls: assert config.NBDT.USE_NBDT, 'Must be using NBDT' from nbdt.data.custom import Node assert hasattr(model, 'rules') and hasattr(model.rules, 'nodes'), \ 'NBDT must have rules with nodes' logger.info("Getting nodes leading up to class leaf {}...".format(cls)) leaf_to_path_nodes = Node.get_leaf_to_path(model.rules.nodes) cls_index = class_names.index(cls) leaf = model.rules.nodes[0].wnids[cls_index] path_nodes = leaf_to_path_nodes[leaf] nbdt_node_wnids = [ item['node'].wnid for item in path_nodes if item['node'] ] def run(): nonlocal maximum, minimum, label, gradcam_kwargs for image_index in get_image_indices(args.image_index, args.image_index_range): image, label, _, name = test_dataset[image_index] image = torch.from_numpy(image).unsqueeze(0).to(device) logger.info("Using image {}...".format(name)) pred_probs, pred_labels = gradcam.forward(image) maximum, minimum = -1000, 0 logger.info(f'=> Starting bounds: ({minimum}, {maximum})') if args.crop_for and class_names.index(args.crop_for) not in label: print( f'Skipping image {image_index} because no {args.crop_for} found' ) continue if getattr(Saliency, 'whole_image', False): assert not ( args.pixel_i or args.pixel_j or args.pixel_i_range or args.pixel_j_range), \ 'the "Whole" saliency method generates one map for the whole ' \ 'image, not for specific pixels' gradcam_kwargs = {'image': image_index} if args.suffix: gradcam_kwargs['suffix'] = args.suffix gradcam.backward(pred_labels[:, [0], :, :]) generate_and_save_saliency(image_index) if args.crop_size <= 0: continue if args.crop_for: cls_index = class_names.index(args.crop_for) label = torch.Tensor(label).to(pred_labels.device) # is_right_class = pred_labels[0,0,:,:] == cls_index # is_correct = pred_labels == label is_right_class = is_correct = label == cls_index #TODO:tmp pixels = (is_right_class * is_correct).nonzero() pixels = get_random_pixels(args.pixel_max_num_random, pixels, seed=cls_index) else: assert (args.pixel_i or args.pixel_i_range) and (args.pixel_j or args.pixel_j_range) pixels = get_pixels(args.pixel_i, args.pixel_j, args.pixel_i_range, args.pixel_j_range, args.pixel_cartesian_product) logger.info(f'Running on {len(pixels)} pixels.') for pixel_i, pixel_j in pixels: pixel_i, pixel_j = int(pixel_i), int(pixel_j) assert pixel_i < test_size[0] and pixel_j < test_size[1], \ "Pixel ({},{}) is out of bounds for image of size ({},{})".format( pixel_i,pixel_j,test_size[0],test_size[1]) # Run backward pass # Note: Computes backprop wrt most likely predicted class rather than gt class gradcam_kwargs = { 'image': image_index, 'pixel_i': pixel_i, 'pixel_j': pixel_j } if args.suffix: gradcam_kwargs['suffix'] = args.suffix logger.info( f'Running {args.vis_mode} on image {image_index} at pixel ({pixel_i},{pixel_j}). Using filename suffix: {args.suffix}' ) output_pixel_i, output_pixel_j = compute_output_coord( pixel_i, pixel_j, test_size, pred_probs.shape[2:]) if not getattr(Saliency, 'whole_image', False): gradcam.backward(pred_labels[:, [0], :, :], output_pixel_i, output_pixel_j) if args.crop_size <= 0: generate_and_save_saliency(image_index) else: generate_and_save_saliency(image_index, pixel_i, pixel_j, args.crop_size) logger.info(f'=> Final bounds are: ({minimum}, {maximum})') # Instantiate wrapper once, outside of loop Saliency = METHODS[args.vis_mode] gradcam = Saliency(model=model, candidate_layers=target_layers, use_nbdt=config.NBDT.USE_NBDT, nbdt_node_wnid=None) maximum, minimum, label, gradcam_kwargs = -1000, 0, None, {} for nbdt_node_wnid in nbdt_node_wnids: if config.NBDT.USE_NBDT: logger.info("Using logits from node with wnid {}...".format( nbdt_node_wnid)) gradcam.set_nbdt_node_wnid(nbdt_node_wnid) run() if not nbdt_node_wnids: nbdt_node_wnid = None run()
def main(): random.seed(11) args = parse_args() logger, final_output_dir, _ = create_logger(config, args.cfg, 'vis_gradcam') # logger.info(pprint.pformat(args)) # logger.info(pprint.pformat(config)) # cudnn related setting cudnn.benchmark = config.CUDNN.BENCHMARK cudnn.deterministic = config.CUDNN.DETERMINISTIC cudnn.enabled = config.CUDNN.ENABLED # build model model = eval('models.' + config.MODEL.NAME + '.get_seg_model')(config) dump_input = torch.rand( (1, 3, config.TRAIN.IMAGE_SIZE[1], config.TRAIN.IMAGE_SIZE[0])) # logger.info(get_model_summary(model.cuda(), dump_input.cuda())) if config.TEST.MODEL_FILE: model_state_file = config.TEST.MODEL_FILE else: model_state_file = os.path.join(final_output_dir, 'best.pth') model_state_file = 'pretrained_models/hrnet_w18_small_model_v1.pth' logger.info('=> loading model from {}'.format(model_state_file)) # __import__('ipdb').set_trace() pretrained_dict = torch.load(model_state_file) model_dict = model.state_dict() pretrained_dict = { k[6:]: v for k, v in pretrained_dict.items() if k[6:] in model_dict.keys() } # for k, _ in pretrained_dict.items(): # logger.info( # '=> loading {} from pretrained model'.format(k)) model_dict.update(pretrained_dict) model.load_state_dict(model_dict) # Wrap original model with NBDT if config.NBDT.USE_NBDT: from nbdt.model import SoftSegNBDT model = SoftSegNBDT(config.NBDT.DATASET, model, hierarchy=config.NBDT.HIERARCHY, classes=class_names) device = 'cuda' if torch.cuda.is_available() else 'cpu' # device = 'cpu' model = model.to(device).eval() # for x in model.named_modules(): # print(x) # Retrieve input image corresponding to args.image_index test_size = (config.TEST.IMAGE_SIZE[1], config.TEST.IMAGE_SIZE[0]) test_dataset = eval('datasets.' + config.DATASET.DATASET)( root=config.DATASET.ROOT, list_path=config.DATASET.TEST_SET, num_samples=None, num_classes=config.DATASET.NUM_CLASSES, multi_scale=False, flip=False, ignore_label=config.TRAIN.IGNORE_LABEL, base_size=config.TEST.BASE_SIZE, crop_size=test_size, downsample_rate=1) # Define target layer as final convolution layer if not specified if args.target_layers: target_layers = args.target_layers.split(',') else: for name, module in list(model.named_modules())[::-1]: if isinstance(module, nn.Conv2d): target_layers = [name] break logger.info('Target layers set to {}'.format(str(target_layers))) # Append model. to target layers if using nbdt if config.NBDT.USE_NBDT: target_layers = ['model.' + layer for layer in target_layers] dmg_by_cls = [[] for _ in class_names] def generate_and_save_saliency(image_index, pixel_i=None, pixel_j=None, crop_size=None, cls_index=0, normalize=False, previous_pred=1): """too lazy to move out to global lol""" nonlocal maximum, minimum, label # Generate GradCAM + save heatmap # raw_image = retrieve_raw_image(test_dataset, image_index) # should_crop = crop_size is not None and pixel_i is not None and pixel_j is not None # if should_crop: # raw_image = crop(pixel_i, pixel_j, crop_size, raw_image, is_tensor=False) for layer in target_layers: gradcam_region = gradcam.generate(target_layer=layer, normalize=False) # if should_crop: # gradcam_region = crop(pixel_i, pixel_j, crop_size, gradcam_region, is_tensor=True) maximum = max(float(gradcam_region.max()), maximum) minimum = min(float(gradcam_region.min()), minimum) logger.info(f'=> Bounds: ({minimum}, {maximum})') # print('start') image = test_dataset[image_index][0] # coords = gradcam_region.nonzero().cpu().numpy() # __import__('ipdb').set_trace() # centered_coords = coords - np.array([0, 0, pixel_i, pixel_j]) # centered_coords = np.abs(centered_coords) # # Filter function for determining whether pixel is within receptive field # f = lambda x: x[2] < 200 and x[3] < 200 # within_receptive_field = np.apply_along_axis(f, 1, centered_coords) # coords = coords[within_receptive_field] # gradcam_region = gradcam_region.cpu() # def v(x): # return gradcam_region[x[0], x[1], x[2], x[3]] # within_weight = np.apply_along_axis(v, 1, coords).sum() # count = 0 # inside_mass = 0 # total_mass = 0 # for coord in foo: # total_mass += gradcam_region[coord[0], coord[1], coord[2], coord[3]] # image[:, coord[2], coord[3]] = torch.zeros(1, 3) # # 400px receptive field # if abs(coord[2] - pixel_i) < 200 and abs(coord[3] - pixel_j) < 200: # count += 1 # inside_mass += gradcam_region[coord[0], coord[1], coord[2], coord[3]] # percent_mass_inside = within_weight / gradcam_region.sum() # percent_mass_outside = 1 - percent_mass_inside # percent_pixels_inside = np.sum(within_receptive_field) / len(within_receptive_field) # percent_pixels_outside = 1 - percent_pixels_inside # pct_mass_inside_list.append(percent_mass_inside) # pct_pixels_inside_list.append(percent_pixels_inside) image = torch.from_numpy(image).unsqueeze(0).to( gradcam_region.device) # Zero out pixels marked by the saliency map coords_to_zero = gradcam_region != 0 image[:, 0:1, :, :][coords_to_zero] = 0 image[:, 1:2, :, :][coords_to_zero] = 0 image[:, 2:3, :, :][coords_to_zero] = 0 pred_probs, pred_labels = gradcam.forward(image) # New location of the target class after occlusion new_index = np.where( pred_labels[0, :, pixel_i, pixel_j].cpu().numpy() == cls_index)[0][0] damage = previous_pred - pred_probs[0, new_index, pixel_i, pixel_j].item() # dmg_by_cls[cls_index].append(damage) dmg_by_cls[cls_index].append( (damage, image_index, [pixel_i, pixel_j])) del image del pred_probs del pred_labels torch.cuda.empty_cache() # bar = pred_probs[:,:,pixel_i,pixel_j] # baz = pred_labels[:,:,pixel_i,pixel_j] # print("percent_mass_inside ", percent_mass_inside) # print("percent_pixels_inside ", percent_pixels_inside) # __import__('ipdb').set_trace() # image = test_dataset[image_index][0] # image = torch.from_numpy(image).unsqueeze(0).to(gradcam_region.device) # pred_probs, pred_labels = gradcam.forward(image) # ordered_class_names = [class_names[i] for i in pred_labels[0,:,180,1000]] # target_index = ordered_class_names.index('building') # images = image # pred_labels[:, [target_index], :, :] # occlusion_map = occlusion_sensitivity(model, images, gradcam) # output_dir = generate_output_dir(final_output_dir, args.vis_mode, layer, config.NBDT.USE_NBDT, nbdt_node_wnid, args.crop_size, args.nbdt_node_wnids_for) # save_path = generate_save_path(output_dir, gradcam_kwargs) # logger.info('Saving {} heatmap at {}...'.format(args.vis_mode, save_path)) # if normalize: # gradcam_region = GradCAM.normalize(gradcam_region) # save_gradcam(save_path, gradcam_region, raw_image, save_npy=not args.skip_save_npy) # else: # save_gradcam(save_path, gradcam_region, raw_image, minimum=minimum, maximum=maximum, save_npy=not args.skip_save_npy) # output_dir_original = output_dir + '_original' # os.makedirs(output_dir_original, exist_ok=True) # save_path_original = generate_save_path(output_dir_original, gradcam_kwargs, ext='jpg') # logger.info('Saving {} original at {}...'.format(args.vis_mode, save_path_original)) # cv2.imwrite(save_path_original, raw_image) # if crop_size and pixel_i and pixel_j: # continue # output_dir += '_overlap' # os.makedirs(output_dir, exist_ok=True) # save_path_overlap = generate_save_path(output_dir, gradcam_kwargs, ext='npy') # save_path_plot = generate_save_path(output_dir, gradcam_kwargs, ext='jpg') # if not args.skip_save_npy: # logger.info('Saving {} overlap data at {}...'.format(args.vis_mode, save_path_overlap)) # logger.info('Saving {} overlap plot at {}...'.format(args.vis_mode, save_path_plot)) # save_overlap(save_path_overlap, save_path_plot, gradcam_region, label, save_npy=not args.skip_save_npy) # if len(heatmaps) > 1: # combined = torch.prod(torch.stack(heatmaps, dim=0), dim=0) # combined /= combined.max() # save_path = generate_save_path(final_output_dir, args.vis_mode, gradcam_kwargs, 'combined', config.NBDT.USE_NBDT, nbdt_node_wnid) # logger.info('Saving combined {} heatmap at {}...'.format(args.vis_mode, save_path)) # save_gradcam(save_path, combined, raw_image) nbdt_node_wnids = args.nbdt_node_wnid or [] cls = args.nbdt_node_wnids_for if cls: assert config.NBDT.USE_NBDT, 'Must be using NBDT' from nbdt.data.custom import Node assert hasattr(model, 'rules') and hasattr(model.rules, 'nodes'), \ 'NBDT must have rules with nodes' logger.info("Getting nodes leading up to class leaf {}...".format(cls)) leaf_to_path_nodes = Node.get_leaf_to_path(model.rules.nodes) cls_index = class_names.index(cls) leaf = model.rules.nodes[0].wnids[cls_index] path_nodes = leaf_to_path_nodes[leaf] nbdt_node_wnids = [ item['node'].wnid for item in path_nodes if item['node'] ] def run(): nonlocal maximum, minimum, label, gradcam_kwargs np.random.seed(13) for image_index in get_image_indices(args.image_index, args.image_index_range): image, label, _, name = test_dataset[image_index] image = torch.from_numpy(image).unsqueeze(0).to(device) logger.info("Using image {}...".format(name)) pred_probs, pred_labels = gradcam.forward(image) maximum, minimum = -1000, 0 logger.info(f'=> Starting bounds: ({minimum}, {maximum})') if args.crop_for and class_names.index(args.crop_for) not in label: print( f'Skipping image {image_index} because no {args.crop_for} found' ) continue if getattr(Saliency, 'whole_image', False): assert not ( args.pixel_i or args.pixel_j or args.pixel_i_range or args.pixel_j_range), \ 'the "Whole" saliency method generates one map for the whole ' \ 'image, not for specific pixels' gradcam_kwargs = {'image': image_index} if args.suffix: gradcam_kwargs['suffix'] = args.suffix gradcam.backward(pred_labels[:, [0], :, :]) generate_and_save_saliency(image_index) if args.crop_size <= 0: continue if args.crop_for: cls_index = class_names.index(args.crop_for) label = torch.Tensor(label).to(pred_labels.device) # is_right_class = pred_labels[0,0,:,:] == cls_index # is_correct = pred_labels == label is_right_class = is_correct = label == cls_index #TODO:tmp pixels = (is_right_class * is_correct).nonzero() pixels = get_random_pixels(args.pixel_max_num_random, pixels, seed=cls_index) else: assert (args.pixel_i or args.pixel_i_range) and (args.pixel_j or args.pixel_j_range) pixels = get_pixels(args.pixel_i, args.pixel_j, args.pixel_i_range, args.pixel_j_range, args.pixel_cartesian_product) logger.info(f'Running on {len(pixels)} pixels.') # for pixel_index in range(len(pixels)): # for cls_index in range(len(class_names)): for _ in range(1): cls_index = 17 # hardcode person # pixel_i, pixel_j = int(pixels[pixel_index][0]), int(pixels[pixel_index][1]) # pixel_i, pixel_j = int(args.pixel_i[0]), int(args.pixel_j[0]) # list of all coords where the label matches the current class # index. matching_coords = np.random.permutation( np.array(np.where(label == cls_index)).T) if matching_coords.shape[0] == 0: continue pixel_i, pixel_j = 0, 0 found_matching = False for coord in matching_coords: pixel_i, pixel_j = coord[0], coord[1] if pred_labels[0, 0, pixel_i, pixel_j].item() == cls_index: found_matching = True break if not found_matching: continue assert pred_labels[0, 0, pixel_i, pixel_j].item() == cls_index # pixel_i, pixel_j = ps[pixel_index] # top_class_index = pred_labels[0,0,pixel_i,pixel_j] # top_class_name = class_names[top_class_index] # print("Running on pixel ({}, {})".format(pixel_i, pixel_j)) assert pixel_i < test_size[0] and pixel_j < test_size[1], \ "Pixel ({},{}) is out of bounds for image of size ({},{})".format( pixel_i,pixel_j,test_size[0],test_size[1]) # Get the current prediction confidence for the top class. # Will use to compute occlusion damage later. curr_pred = pred_probs[0, 0, pixel_i, pixel_j].item() # Run backward pass # Note: Computes backprop wrt most likely predicted class rather than gt class # gradcam_kwargs = {'image': image_index, 'pixel_i': pixel_i, 'pixel_j': pixel_j, 'class_name': top_class_name} # if args.suffix: # gradcam_kwargs['suffix'] = args.suffix # logger.info(f'Running {args.vis_mode} on image {image_index} at pixel ({pixel_i},{pixel_j}). Using filename suffix: {args.suffix}') output_pixel_i, output_pixel_j = compute_output_coord( pixel_i, pixel_j, test_size, pred_probs.shape[2:]) target_index = 0 if not getattr(Saliency, 'whole_image', False): gradcam.backward(pred_labels[:, [target_index], :, :], output_pixel_i, output_pixel_j) # gradcam_region = gradcam.generate(target_layer=target_layers[0], normalize=False).cpu() # __import__('ipdb').set_trace() # now `gradcam_region` and `second_region` contain both saliency maps for two pixels of the # same class but at least 800px apart. # ordered_class_names = [class_names[i] for i in pred_labels[0,:,pixel_i,pixel_j]] # target_index = ordered_class_names.index('car') # print("TOP CLASS", pred_labels[0, 0, pixel_i, pixel_j], pred_probs[0,0,pixel_i,pixel_j], class_names[pred_labels[0, 0, pixel_i, pixel_j]]) # print("######################################") if args.crop_size <= 0: generate_and_save_saliency(image_index, pixel_i, pixel_j, cls_index=cls_index, previous_pred=curr_pred) else: generate_and_save_saliency(image_index, pixel_i, pixel_j, args.crop_size, cls_index=cls_index, previous_pred=curr_pred) # for i in range(len(dmg_by_cls)): # if len(dmg_by_cls[i]) > 0 and i == 16: # # print(i, np.mean(dmg_by_cls[i])) # # logger.info('{}, {}'.format(i, np.mean(dmg_by_cls[i]))) # logger.info('{}, {}'.format(i, dmg_by_cls[i])) # logger.info(f'=> Final bounds are: ({minimum}, {maximum})') # Instantiate wrapper once, outside of loop Saliency = METHODS[args.vis_mode] gradcam = Saliency(model=model, candidate_layers=target_layers, use_nbdt=config.NBDT.USE_NBDT, nbdt_node_wnid=None) maximum, minimum, label, gradcam_kwargs = -1000, 0, None, {} for nbdt_node_wnid in nbdt_node_wnids: if config.NBDT.USE_NBDT: logger.info("Using logits from node with wnid {}...".format( nbdt_node_wnid)) gradcam.set_nbdt_node_wnid(nbdt_node_wnid) run() if not nbdt_node_wnids: nbdt_node_wnid = None run() # for i in range(len(dmg_by_cls)): # if len(dmg_by_cls[i]) > 0: # logger.info('{}, {}'.format(i, np.mean(dmg_by_cls[i]))) for i in range(len(dmg_by_cls)): if len(dmg_by_cls[i]) > 0: # print(i, np.mean(dmg_by_cls[i])) # logger.info('{}, {}'.format(i, np.mean(dmg_by_cls[i]))) logger.info('{}, {}'.format(i, dmg_by_cls[i])) # __import__('ipdb').set_trace() print('done')