iteration = None run_name_prefix = '' if args.run_name_prefix: run_name_prefix = args.run_name_prefix.lower().replace(' ', '_') + '.' time = datetime.datetime.now() time = time.replace(microsecond=0) experiment_run_name = run_name_prefix + '{}.{}.{}'.format( args.dataset, platform.node(), time.isoformat()) best_accu = 0. current_push_best_accu = 0. print('Starting new experiment: {}'.format(experiment_run_name)) print('Saving code snapshot with git-experiments') snapshot_code(experiment_run_name) model_dir = os.path.join(SAVED_MODELS_PATH, experiment_run_name) makedir(model_dir) img_dir = os.path.join(model_dir, 'img') makedir(img_dir) log_writer = SummaryWriter(os.path.join(LOGS_DIR, experiment_run_name), purge_step=step + 1) weight_matrix_filename = 'outputL_weights' prototype_img_filename_prefix = 'prototype-img' prototype_self_act_filename_prefix = 'prototype-self-act' proto_bound_boxes_filename_prefix = 'bb' # noinspection PyTypeChecker log_writer.add_text( 'dataset_stats', 'training set size: {}, push set size: {}, valid set size: {}, test set size: {}'
def push_prototypes(dataloader, # pytorch dataloader (must be unnormalized in [0,1]) prototype_network_parallel, # pytorch network with prototype_vectors class_specific=True, preprocess_input_function=None, # normalize if needed prototype_layer_stride=1, root_dir_for_saving_prototypes=None, # if not None, prototypes will be saved here epoch_number=None, # if not provided, prototypes saved previously will be overwritten prototype_img_filename_prefix=None, prototype_self_act_filename_prefix=None, proto_bound_boxes_filename_prefix=None, save_prototype_class_identity=True, # which class the prototype image comes from log=print, prototype_activation_function_in_numpy=None): prototype_network_parallel.eval() log('\tpush') start = time.time() prototype_shape = prototype_network_parallel.module.prototype_shape n_prototypes = prototype_network_parallel.module.num_prototypes # saves the closest distance seen so far global_min_proto_dist = np.full(n_prototypes, np.inf) # saves the patch representation that gives the current smallest distance global_min_fmap_patches = np.zeros( [n_prototypes, prototype_shape[1], prototype_shape[2], prototype_shape[3]]) ''' proto_rf_boxes and proto_bound_boxes column: 0: image index in the entire dataset 1: height start index 2: height end index 3: width start index 4: width end index 5: (optional) class identity ''' if save_prototype_class_identity: proto_rf_boxes = np.full(shape=[n_prototypes, 6], fill_value=-1) proto_bound_boxes = np.full(shape=[n_prototypes, 6], fill_value=-1) else: proto_rf_boxes = np.full(shape=[n_prototypes, 5], fill_value=-1) proto_bound_boxes = np.full(shape=[n_prototypes, 5], fill_value=-1) if root_dir_for_saving_prototypes != None: if epoch_number != None: proto_epoch_dir = os.path.join(root_dir_for_saving_prototypes, 'epoch-'+str(epoch_number)) makedir(proto_epoch_dir) else: proto_epoch_dir = root_dir_for_saving_prototypes else: proto_epoch_dir = None search_batch_size = dataloader.batch_size num_classes = prototype_network_parallel.module.num_classes for push_iter, (search_batch_input, search_y) in enumerate(dataloader): ''' start_index_of_search keeps track of the index of the image assigned to serve as prototype ''' start_index_of_search_batch = push_iter * search_batch_size update_prototypes_on_batch(search_batch_input, start_index_of_search_batch, prototype_network_parallel, global_min_proto_dist, global_min_fmap_patches, proto_rf_boxes, proto_bound_boxes, class_specific=class_specific, search_y=search_y, num_classes=num_classes, preprocess_input_function=preprocess_input_function, prototype_layer_stride=prototype_layer_stride, dir_for_saving_prototypes=proto_epoch_dir, prototype_img_filename_prefix=prototype_img_filename_prefix, prototype_self_act_filename_prefix=prototype_self_act_filename_prefix, prototype_activation_function_in_numpy=prototype_activation_function_in_numpy) if proto_epoch_dir != None and proto_bound_boxes_filename_prefix != None: np.save(os.path.join(proto_epoch_dir, proto_bound_boxes_filename_prefix + '-receptive_field' + str(epoch_number) + '.npy'), proto_rf_boxes) np.save(os.path.join(proto_epoch_dir, proto_bound_boxes_filename_prefix + str(epoch_number) + '.npy'), proto_bound_boxes) log('\tExecuting push ...') prototype_update = np.reshape(global_min_fmap_patches, tuple(prototype_shape)) prototype_network_parallel.module.prototype_vectors.data.copy_(torch.tensor(prototype_update, dtype=torch.float32).cuda()) # prototype_network_parallel.cuda() end = time.time() log('\tpush time: \t{0}'.format(end - start))
def find_k_nearest_patches_to_prototypes(dataloader, # pytorch dataloader (must be unnormalized in [0,1]) ppnet, # pytorch network with prototype_vectors k=5, preprocess_input_function=None, # normalize if needed full_save=False, # save all the images root_dir_for_saving_images='./nearest', log=print, prototype_activation_function_in_numpy=None, only_n_most_activated=None): ppnet.eval() ''' full_save=False will only return the class identity of the closest patches, but it will not save anything. ''' print(' find nearest patches') start = time.time() n_prototypes = ppnet.num_prototypes prototype_shape = ppnet.prototype_shape max_dist = prototype_shape[1] * prototype_shape[2] * prototype_shape[3] protoL_rf_info = ppnet.proto_layer_rf_info heaps = [] # allocate an array of n_prototypes number of heaps for _ in range(n_prototypes): # a heap in python is just a maintained list heaps.append([]) with tqdm(total=len(dataloader.dataset), unit='bag') as pbar: for idx, (search_batch_raw, search_batch, search_y) in enumerate(dataloader): with torch.no_grad(): search_batch = search_batch.cuda() ppnet.forward(search_batch) proto_dist_torch = ppnet.distances attention = ppnet.A raw_sample = search_batch_raw[0] # protoL_input_ = np.copy(protoL_input_torch.detach().cpu().numpy()) proto_dist_ = np.copy(proto_dist_torch.detach().cpu().numpy()) if only_n_most_activated: most_activates = list( torch.topk(attention, min(only_n_most_activated, attention.shape[1]), dim=-1, largest=True)[1].detach().cpu().numpy()[0]) for img_idx, distance_map in enumerate(proto_dist_): if only_n_most_activated: if img_idx not in most_activates: continue for j in range(n_prototypes): # find the closest patches in this batch to prototype j closest_patch_distance_to_prototype_j = np.amin(distance_map[j]) if full_save: closest_patch = ImagePatchLazy( label=search_y[0], distance=closest_patch_distance_to_prototype_j, batch_idx=idx, image_idx=img_idx, dataset=dataloader.dataset, protoL_rf_info=protoL_rf_info, distance_map_j=distance_map[j], prototype_activation_function_in_numpy=prototype_activation_function_in_numpy, prototype_activation_function=ppnet.prototype_activation_function, max_dist=max_dist, epsilon=ppnet.epsilon ) else: closest_patch = ImagePatchInfo(label=search_y[0], distance=closest_patch_distance_to_prototype_j) # add to the j-th heap if len(heaps[j]) < k: heapq.heappush(heaps[j], closest_patch) else: # heappushpop runs more efficiently than heappush # followed by heappop heapq.heappushpop(heaps[j], closest_patch) pbar.update(1) # after looping through the dataset every heap will # have the k closest prototypes for j in range(n_prototypes): # finally sort the heap; the heap only contains the k closest # but they are not ranked yet heaps[j].sort() heaps[j] = heaps[j][::-1] if full_save: dir_for_saving_images = os.path.join(root_dir_for_saving_images, str(j)) makedir(dir_for_saving_images) labels = [] for i, patch in enumerate(heaps[j]): # save the activation pattern of the original image where the patch comes from np.save(os.path.join(dir_for_saving_images, 'nearest-' + str(i + 1) + '_act.npy'), patch.act_pattern) # save the original image where the patch comes from plt.imsave(fname=os.path.join(dir_for_saving_images, 'nearest-' + str(i + 1) + '_original.png'), arr=patch.original_img, vmin=0.0, vmax=1.0) # overlay (upsampled) activation on original image and save the result img_size = patch.original_img.shape[0] upsampled_act_pattern = cv2.resize(patch.act_pattern, dsize=(img_size, img_size), interpolation=cv2.INTER_CUBIC) rescaled_act_pattern = upsampled_act_pattern - np.amin(upsampled_act_pattern) rescaled_act_pattern = rescaled_act_pattern / np.amax(rescaled_act_pattern) heatmap = cv2.applyColorMap(np.uint8(255 * rescaled_act_pattern), cv2.COLORMAP_JET) heatmap = np.float32(heatmap) / 255 heatmap = heatmap[..., ::-1] overlayed_original_img = 0.5 * patch.original_img + 0.3 * heatmap plt.imsave(fname=os.path.join(dir_for_saving_images, 'nearest-' + str(i + 1) + '_original_with_heatmap.png'), arr=overlayed_original_img, vmin=0.0, vmax=1.0) # if different from original image, save the patch (i.e. receptive field) if patch.patch.shape[0] != img_size or patch.patch.shape[1] != img_size: np.save(os.path.join(dir_for_saving_images, 'nearest-' + str(i + 1) + '_receptive_field_indices.npy'), patch.patch_indices) plt.imsave(fname=os.path.join(dir_for_saving_images, 'nearest-' + str(i + 1) + '_receptive_field.png'), arr=patch.patch, vmin=0.0, vmax=1.0) # save the receptive field patch with heatmap overlayed_patch = overlayed_original_img[patch.patch_indices[0]:patch.patch_indices[1], patch.patch_indices[2]:patch.patch_indices[3], :] plt.imsave(fname=os.path.join(dir_for_saving_images, 'nearest-' + str(i + 1) + '_receptive_field_with_heatmap.png'), arr=overlayed_patch, vmin=0.0, vmax=1.0) # save the highly activated patch high_act_patch_indices = find_high_activation_crop(upsampled_act_pattern) high_act_patch = patch.original_img[high_act_patch_indices[0]:high_act_patch_indices[1], high_act_patch_indices[2]:high_act_patch_indices[3], :] np.save(os.path.join(dir_for_saving_images, 'nearest-' + str(i + 1) + '_high_act_patch_indices.npy'), high_act_patch_indices) plt.imsave(fname=os.path.join(dir_for_saving_images, 'nearest-' + str(i + 1) + '_high_act_patch.png'), arr=high_act_patch, vmin=0.0, vmax=1.0) # save the original image with bounding box showing high activation patch imsave_with_bbox(fname=os.path.join(dir_for_saving_images, 'nearest-' + str(i + 1) + '_high_act_patch_in_original_img.png'), img_rgb=patch.original_img, bbox_height_start=high_act_patch_indices[0], bbox_height_end=high_act_patch_indices[1], bbox_width_start=high_act_patch_indices[2], bbox_width_end=high_act_patch_indices[3], color=(0, 255, 255)) labels = np.array([patch.label for patch in heaps[j]]) np.save(os.path.join(dir_for_saving_images, 'class_id.npy'), labels) labels_all_prototype = np.array([[patch.label for patch in heaps[j]] for j in range(n_prototypes)]) if full_save: np.save(os.path.join(root_dir_for_saving_images, 'full_class_id.npy'), labels_all_prototype) end = time.time() log(' find nearest patches time: \t{0}'.format(end - start)) return labels_all_prototype
def push_orig(self, epoch_number): self.prototype_network_parallel.eval() self.log('\tpush') start = time.time() prototype_shape = self.prototype_network_parallel.module.prototype_shape n_prototypes = self.prototype_network_parallel.module.num_prototypes # saves the closest distance seen so far global_min_proto_dist = np.full(n_prototypes, np.inf) # saves the patch representation that gives the current smallest distance global_min_fmap_patches = np.zeros([ n_prototypes, prototype_shape[1], prototype_shape[2], prototype_shape[3] ]) ''' proto_rf_boxes and proto_bound_boxes column: 0: image index in the entire dataset 1: height start index 2: height end index 3: width start index 4: width end index 5: (optional) class identity ''' if self.save_prototype_class_identity: proto_rf_boxes = np.full(shape=[n_prototypes, 6], fill_value=-1) proto_bound_boxes = np.full(shape=[n_prototypes, 6], fill_value=-1) else: proto_rf_boxes = np.full(shape=[n_prototypes, 5], fill_value=-1) proto_bound_boxes = np.full(shape=[n_prototypes, 5], fill_value=-1) if self.dir_for_saving_prototypes != None: if epoch_number != None: proto_epoch_dir = os.path.join(self.dir_for_saving_prototypes, 'epoch-' + str(epoch_number)) makedir(proto_epoch_dir) else: # XXX I think dir_for_saving_proto and root_dir are actually # different variables and it wasnt a misnaming. Oh well # I'll come back to this later proto_epoch_dir = self.dir_for_saving_prototypes else: proto_epoch_dir = None search_batch_size = self.dataloader.batch_size num_classes = self.prototype_network_parallel.module.num_classes for push_iter, (search_batch_input, search_y) in enumerate(self.dataloader): ''' start_index_of_search keeps track of the index of the image assigned to serve as prototype ''' start_index_of_search_batch = push_iter * search_batch_size self.update_prototypes_on_batch(search_batch_input, start_index_of_search_batch, global_min_proto_dist, global_min_fmap_patches, proto_rf_boxes, proto_bound_boxes, search_y=search_y, num_classes=num_classes) if proto_epoch_dir != None and self.proto_bound_boxes_filename_prefix != None: np.save( os.path.join( proto_epoch_dir, self.proto_bound_boxes_filename_prefix + '-receptive_field' + str(epoch_number) + '.npy'), proto_rf_boxes) np.save( os.path.join( proto_epoch_dir, self.proto_bound_boxes_filename_prefix + str(epoch_number) + '.npy'), proto_bound_boxes) # XXX push here is different because we're choosing top K vectors. self.log('\tExecuting push ...') prototype_update = np.reshape(global_min_fmap_patches, tuple(prototype_shape)) self.prototype_network_parallel.module.prototype_vectors.data.copy_( torch.tensor(prototype_update, dtype=torch.float32).cuda()) # prototype_network_parallel.cuda() end = time.time() self.log('\tpush time: \t{0}'.format(end - start))
valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False) # test set test_dataset = datasets.ImageFolder(test_dir, transform_test) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False) # set up directories root_dir_for_saving_train_images = os.path.join(args.resume_path, 'nearest_train') root_dir_for_saving_test_images = os.path.join(args.resume_path, 'nearest_test') makedir(root_dir_for_saving_train_images) makedir(root_dir_for_saving_test_images) root = Node("root") root.add_children( ['animal', 'vehicle', 'everyday_object', 'weapon', 'scuba_diver']) root.add_children_to('animal', ['non_primate', 'primate']) root.add_children_to('non_primate', ['African_elephant', 'giant_panda', 'lion']) root.add_children_to('primate', ['capuchin', 'gibbon', 'orangutan']) root.add_children_to('vehicle', ['ambulance', 'pickup', 'sports_car']) root.add_children_to('everyday_object', ['laptop', 'sandal', 'wine_bottle']) root.add_children_to('weapon', ['assault_rifle', 'rifle']) root.assign_all_descendents() flat_root = Node("root")
need_push = ('nopush' in original_model_name) if need_push: assert (False) # pruning must happen after push else: epoch = original_model_name.split('push')[0] if '_' in epoch: epoch = int(epoch.split('_')[0]) else: epoch = int(epoch) model_dir = os.path.join( original_model_dir, 'pruned_prototypes_epoch{}_k{}_pt{}'.format(epoch, k, prune_threshold)) makedir(model_dir) shutil.copy(src=os.path.join(os.getcwd(), __file__), dst=model_dir) log, logclose = create_logger( log_filename=os.path.join(model_dir, 'prune.log')) ppnet = torch.load(original_model_dir + original_model_name) ppnet = ppnet.cuda() ppnet_multi = torch.nn.DataParallel(ppnet) class_specific = True # load the data from settings import train_dir, test_dir, train_push_dir, train_inst_dir train_batch_size = 80 test_batch_size = 100
def find_k_nearest_patches_to_prototypes( dataloader, # pytorch dataloader (must be unnormalized in [0,1]) prototype_network_parallel, # pytorch network with prototype_vectors k=5, preprocess_input_function=None, # normalize if needed full_save=False, # save all the images root_dir_for_saving_images='./nearest', log=print, prototype_activation_function_in_numpy=None): prototype_network_parallel.eval() ''' full_save=False will only return the class identity of the closest patches, but it will not save anything. ''' log('find nearest patches') start = time.time() n_prototypes = prototype_network_parallel.module.num_prototypes prototype_shape = prototype_network_parallel.module.prototype_shape max_dist = prototype_shape[1] * prototype_shape[2] * prototype_shape[3] protoL_rf_info = prototype_network_parallel.module.proto_layer_rf_info heaps = [] # allocate an array of n_prototypes number of heaps for _ in range(n_prototypes): # a heap in python is just a maintained list heaps.append([]) for idx, (search_batch_input, search_y) in enumerate(dataloader): print('batch {}'.format(idx)) if preprocess_input_function is not None: # print('preprocessing input for pushing ...') # search_batch = copy.deepcopy(search_batch_input) search_batch = preprocess_input_function(search_batch_input) else: search_batch = search_batch_input with torch.no_grad(): search_batch = search_batch.cuda() protoL_input_torch, proto_dist_torch = \ prototype_network_parallel.module.push_forward(search_batch) #protoL_input_ = np.copy(protoL_input_torch.detach().cpu().numpy()) proto_dist_ = np.copy(proto_dist_torch.detach().cpu().numpy()) for img_idx, distance_map in enumerate(proto_dist_): for j in range(n_prototypes): # find the closest patches in this batch to prototype j closest_patch_distance_to_prototype_j = np.amin( distance_map[j]) if full_save: closest_patch_indices_in_distance_map_j = \ list(np.unravel_index(np.argmin(distance_map[j],axis=None), distance_map[j].shape)) closest_patch_indices_in_distance_map_j = [ 0 ] + closest_patch_indices_in_distance_map_j closest_patch_indices_in_img = \ compute_rf_prototype(search_batch.size(2), closest_patch_indices_in_distance_map_j, protoL_rf_info) closest_patch = \ search_batch_input[img_idx, :, closest_patch_indices_in_img[1]:closest_patch_indices_in_img[2], closest_patch_indices_in_img[3]:closest_patch_indices_in_img[4]] closest_patch = closest_patch.numpy() closest_patch = np.transpose(closest_patch, (1, 2, 0)) original_img = search_batch_input[img_idx].numpy() original_img = np.transpose(original_img, (1, 2, 0)) if prototype_network_parallel.module.prototype_activation_function == 'log': act_pattern = np.log( (distance_map[j] + 1) / (distance_map[j] + prototype_network_parallel.module.epsilon)) elif prototype_network_parallel.module.prototype_activation_function == 'linear': act_pattern = max_dist - distance_map[j] else: act_pattern = prototype_activation_function_in_numpy( distance_map[j]) # 4 numbers: height_start, height_end, width_start, width_end patch_indices = closest_patch_indices_in_img[1:5] # construct the closest patch object closest_patch = ImagePatch( patch=closest_patch, label=search_y[img_idx], distance=closest_patch_distance_to_prototype_j, original_img=original_img, act_pattern=act_pattern, patch_indices=patch_indices) else: closest_patch = ImagePatchInfo( label=search_y[img_idx], distance=closest_patch_distance_to_prototype_j) # add to the j-th heap if len(heaps[j]) < k: heapq.heappush(heaps[j], closest_patch) else: # heappushpop runs more efficiently than heappush # followed by heappop heapq.heappushpop(heaps[j], closest_patch) # after looping through the dataset every heap will # have the k closest prototypes for j in range(n_prototypes): # finally sort the heap; the heap only contains the k closest # but they are not ranked yet heaps[j].sort() heaps[j] = heaps[j][::-1] if full_save: dir_for_saving_images = os.path.join(root_dir_for_saving_images, str(j)) makedir(dir_for_saving_images) labels = [] for i, patch in enumerate(heaps[j]): # save the activation pattern of the original image where the patch comes from np.save( os.path.join(dir_for_saving_images, 'nearest-' + str(i + 1) + '_act.npy'), patch.act_pattern) # save the original image where the patch comes from plt.imsave(fname=os.path.join( dir_for_saving_images, 'nearest-' + str(i + 1) + '_original.png'), arr=patch.original_img, vmin=0.0, vmax=1.0) # overlay (upsampled) activation on original image and save the result img_size = patch.original_img.shape[0] upsampled_act_pattern = cv2.resize( patch.act_pattern, dsize=(img_size, img_size), interpolation=cv2.INTER_CUBIC) rescaled_act_pattern = upsampled_act_pattern - np.amin( upsampled_act_pattern) rescaled_act_pattern = rescaled_act_pattern / np.amax( rescaled_act_pattern) heatmap = cv2.applyColorMap( np.uint8(255 * rescaled_act_pattern), cv2.COLORMAP_JET) heatmap = np.float32(heatmap) / 255 heatmap = heatmap[..., ::-1] overlayed_original_img = 0.5 * patch.original_img + 0.3 * heatmap plt.imsave(fname=os.path.join( dir_for_saving_images, 'nearest-' + str(i + 1) + '_original_with_heatmap.png'), arr=overlayed_original_img, vmin=0.0, vmax=1.0) # if different from original image, save the patch (i.e. receptive field) if patch.patch.shape[0] != img_size or patch.patch.shape[ 1] != img_size: np.save( os.path.join( dir_for_saving_images, 'nearest-' + str(i + 1) + '_receptive_field_indices.npy'), patch.patch_indices) plt.imsave(fname=os.path.join( dir_for_saving_images, 'nearest-' + str(i + 1) + '_receptive_field.png'), arr=patch.patch, vmin=0.0, vmax=1.0) # save the receptive field patch with heatmap overlayed_patch = overlayed_original_img[ patch.patch_indices[0]:patch.patch_indices[1], patch.patch_indices[2]:patch.patch_indices[3], :] plt.imsave(fname=os.path.join( dir_for_saving_images, 'nearest-' + str(i + 1) + '_receptive_field_with_heatmap.png'), arr=overlayed_patch, vmin=0.0, vmax=1.0) # save the highly activated patch high_act_patch_indices = find_high_activation_crop( upsampled_act_pattern) high_act_patch = patch.original_img[ high_act_patch_indices[0]:high_act_patch_indices[1], high_act_patch_indices[2]:high_act_patch_indices[3], :] np.save( os.path.join( dir_for_saving_images, 'nearest-' + str(i + 1) + '_high_act_patch_indices.npy'), high_act_patch_indices) plt.imsave(fname=os.path.join( dir_for_saving_images, 'nearest-' + str(i + 1) + '_high_act_patch.png'), arr=high_act_patch, vmin=0.0, vmax=1.0) # save the original image with bounding box showing high activation patch imsave_with_bbox(fname=os.path.join( dir_for_saving_images, 'nearest-' + str(i + 1) + '_high_act_patch_in_original_img.png'), img_rgb=patch.original_img, bbox_height_start=high_act_patch_indices[0], bbox_height_end=high_act_patch_indices[1], bbox_width_start=high_act_patch_indices[2], bbox_width_end=high_act_patch_indices[3], color=(0, 255, 255)) labels = np.array([patch.label for patch in heaps[j]]) np.save(os.path.join(dir_for_saving_images, 'class_id.npy'), labels) labels_all_prototype = np.array([[patch.label for patch in heaps[j]] for j in range(n_prototypes)]) if full_save: np.save(os.path.join(root_dir_for_saving_images, 'full_class_id.npy'), labels_all_prototype) end = time.time() log('\tfind nearest patches time: \t{0}'.format(end - start)) return labels_all_prototype
load_model_dir = './saved_models/vgg19/003/' #args.modeldir[0] load_model_name = '160push0.8167.pth' #args.model[0] '10_18push0.7822.pth' #if load_model_dir[-1] == '/': # model_base_architecture = load_model_dir.split('/')[-3] # experiment_run = load_model_dir.split('/')[-2] #else: # model_base_architecture = load_model_dir.split('/')[-2] # experiment_run = load_model_dir.split('/')[-1] model_base_architecture = load_model_dir.split('/')[2] experiment_run = '/'.join(load_model_dir.split('/')[3:]) save_analysis_path = os.path.join(test_image_dir, model_base_architecture, experiment_run, load_model_name) makedir(save_analysis_path) log, logclose = create_logger( log_filename=os.path.join(save_analysis_path, 'local_analysis.log')) load_model_path = os.path.join(load_model_dir, load_model_name) epoch_number_str = re.search(r'\d+', load_model_name).group(0) start_epoch_number = int(epoch_number_str) log('load model from ' + load_model_path) log('model base architecture: ' + model_base_architecture) log('experiment run: ' + experiment_run) ppnet = torch.load(load_model_path) ppnet = ppnet.cuda() ppnet_multi = torch.nn.DataParallel(ppnet)
transforms.ToTensor(), transforms.Lambda(lambda x: x.repeat(3, 1, 1)) ])) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=test_batch_size, shuffle=False, num_workers=4, pin_memory=False) root_dir_for_saving_train_images = os.path.join( load_model_dir, load_model_name.split('.pth')[0] + '_nearest_train') root_dir_for_saving_test_images = os.path.join( load_model_dir, load_model_name.split('.pth')[0] + '_nearest_test') makedir(root_dir_for_saving_train_images) makedir(root_dir_for_saving_test_images) # save prototypes in original images load_img_dir = os.path.join(load_model_dir, 'img') prototype_info = np.load( os.path.join(load_img_dir, 'epoch-' + str(start_epoch_number), 'bb' + str(start_epoch_number) + '.npy')) def save_prototype_original_img_with_bbox(fname, epoch, index, bbox_height_start, bbox_height_end, bbox_width_start,
def prune_prototypes(dataloader, ppnet, k, prune_threshold, preprocess_input_function, original_model_dir, epoch_number, log=print, copy_prototype_imgs=True, find_threshold_prune_n_patches=None, only_n_most_activated=None): ### run global analysis nearest_train_patch_class_ids = \ find_nearest.find_k_nearest_patches_to_prototypes(dataloader=dataloader, ppnet=ppnet, k=k, preprocess_input_function=preprocess_input_function, full_save=False, log=log, only_n_most_activated=only_n_most_activated) ### find prototypes to prune original_num_prototypes = ppnet.num_prototypes if find_threshold_prune_n_patches is None: prototypes_to_prune = find_prototypes_to_prune( ppnet, nearest_train_patch_class_ids, prune_threshold) else: low = 0.0001 high = 10 prototypes_to_prune = [] for _ in range(30): m = (low + high) / 2 current_prototypes_to_prune = find_prototypes_to_prune( ppnet, nearest_train_patch_class_ids, m) print(m, current_prototypes_to_prune) if len(current_prototypes_to_prune ) > find_threshold_prune_n_patches: high = m else: low = m if len(current_prototypes_to_prune ) <= find_threshold_prune_n_patches: prototypes_to_prune = current_prototypes_to_prune prune_threshold = m log('k = {}, prune_threshold = {}'.format(k, prune_threshold)) log('{} prototypes will be pruned'.format(len(prototypes_to_prune))) ### bookkeeping of prototypes to be pruned class_of_prototypes_to_prune = \ torch.argmax( ppnet.prototype_class_identity[prototypes_to_prune], dim=1).numpy().reshape(-1, 1) prototypes_to_prune_np = np.array(prototypes_to_prune).reshape(-1, 1) prune_info = np.hstack( (prototypes_to_prune_np, class_of_prototypes_to_prune)) makedir( os.path.join( original_model_dir, 'pruned_prototypes_epoch{}_k{}_pt{}'.format( epoch_number, k, prune_threshold))) np.save( os.path.join( original_model_dir, 'pruned_prototypes_epoch{}_k{}_pt{}'.format( epoch_number, k, prune_threshold), 'prune_info.npy'), prune_info) ### prune prototypes print('Prototypes to prune', prototypes_to_prune) ppnet.prune_prototypes(prototypes_to_prune) # torch.save(obj=ppnet, # f=os.path.join(original_model_dir, 'pruned_prototypes_epoch{}_k{}_pt{}'.format(epoch_number, # k, # prune_threshold), # model_name + '-pruned.pth')) if copy_prototype_imgs: original_img_dir = os.path.join(original_model_dir, 'img', 'epoch-%d' % epoch_number) dst_img_dir = os.path.join( original_model_dir, 'pruned_prototypes_epoch{}_k{}_pt{}'.format( epoch_number, k, prune_threshold), 'img', 'epoch-%d' % epoch_number) makedir(dst_img_dir) prototypes_to_keep = list( set(range(original_num_prototypes)) - set(prototypes_to_prune)) for idx in range(len(prototypes_to_keep)): shutil.copyfile(src=os.path.join( original_img_dir, 'prototype-img%d.png' % prototypes_to_keep[idx]), dst=os.path.join(dst_img_dir, 'prototype-img%d.png' % idx)) shutil.copyfile(src=os.path.join( original_img_dir, 'prototype-img-original%d.png' % prototypes_to_keep[idx]), dst=os.path.join( dst_img_dir, 'prototype-img-original%d.png' % idx)) shutil.copyfile( src=os.path.join( original_img_dir, 'prototype-img-original_with_self_act%d.png' % prototypes_to_keep[idx]), dst=os.path.join( dst_img_dir, 'prototype-img-original_with_self_act%d.png' % idx)) shutil.copyfile(src=os.path.join( original_img_dir, 'prototype-self-act%d.npy' % prototypes_to_keep[idx]), dst=os.path.join(dst_img_dir, 'prototype-self-act%d.npy' % idx)) bb = np.load( os.path.join(original_img_dir, 'bb%d.npy' % epoch_number)) bb = bb[prototypes_to_keep] np.save(os.path.join(dst_img_dir, 'bb%d.npy' % epoch_number), bb) bb_rf = np.load( os.path.join(original_img_dir, 'bb-receptive_field%d.npy' % epoch_number)) bb_rf = bb_rf[prototypes_to_keep] np.save( os.path.join(dst_img_dir, 'bb-receptive_field%d.npy' % epoch_number), bb_rf) return prune_info
def prune_prototypes( dataloader, prototype_network_parallel, k, prune_threshold, preprocess_input_function, original_model_dir, epoch_number, #model_name=None, log=print, copy_prototype_imgs=True): ### run global analysis nearest_train_patch_class_ids = \ find_nearest.find_k_nearest_patches_to_prototypes(dataloader=dataloader, prototype_network_parallel=prototype_network_parallel, k=k, preprocess_input_function=preprocess_input_function, full_save=False, log=log) ### find prototypes to prune original_num_prototypes = prototype_network_parallel.module.num_prototypes prototypes_to_prune = [] for j in range(prototype_network_parallel.module.num_prototypes): class_j = torch.argmax(prototype_network_parallel.module. prototype_class_identity[j]).item() nearest_train_patch_class_counts_j = Counter( nearest_train_patch_class_ids[j]) # if no such element is in Counter, it will return 0 if nearest_train_patch_class_counts_j[class_j] < prune_threshold: prototypes_to_prune.append(j) log('k = {}, prune_threshold = {}'.format(k, prune_threshold)) log('{} prototypes will be pruned'.format(len(prototypes_to_prune))) ### bookkeeping of prototypes to be pruned class_of_prototypes_to_prune = \ torch.argmax( prototype_network_parallel.module.prototype_class_identity[prototypes_to_prune], dim=1).numpy().reshape(-1, 1) prototypes_to_prune_np = np.array(prototypes_to_prune).reshape(-1, 1) prune_info = np.hstack( (prototypes_to_prune_np, class_of_prototypes_to_prune)) makedir( os.path.join( original_model_dir, 'pruned_prototypes_epoch{}_k{}_pt{}'.format( epoch_number, k, prune_threshold))) np.save( os.path.join( original_model_dir, 'pruned_prototypes_epoch{}_k{}_pt{}'.format( epoch_number, k, prune_threshold), 'prune_info.npy'), prune_info) ### prune prototypes prototype_network_parallel.module.prune_prototypes(prototypes_to_prune) #torch.save(obj=prototype_network_parallel.module, # f=os.path.join(original_model_dir, 'pruned_prototypes_epoch{}_k{}_pt{}'.format(epoch_number, # k, # prune_threshold), # model_name + '-pruned.pth')) if copy_prototype_imgs: original_img_dir = os.path.join(original_model_dir, 'img', 'epoch-%d' % epoch_number) dst_img_dir = os.path.join( original_model_dir, 'pruned_prototypes_epoch{}_k{}_pt{}'.format( epoch_number, k, prune_threshold), 'img', 'epoch-%d' % epoch_number) makedir(dst_img_dir) prototypes_to_keep = list( set(range(original_num_prototypes)) - set(prototypes_to_prune)) for idx in range(len(prototypes_to_keep)): shutil.copyfile(src=os.path.join( original_img_dir, 'prototype-img%d.png' % prototypes_to_keep[idx]), dst=os.path.join(dst_img_dir, 'prototype-img%d.png' % idx)) shutil.copyfile(src=os.path.join( original_img_dir, 'prototype-img-original%d.png' % prototypes_to_keep[idx]), dst=os.path.join( dst_img_dir, 'prototype-img-original%d.png' % idx)) shutil.copyfile( src=os.path.join( original_img_dir, 'prototype-img-original_with_self_act%d.png' % prototypes_to_keep[idx]), dst=os.path.join( dst_img_dir, 'prototype-img-original_with_self_act%d.png' % idx)) shutil.copyfile(src=os.path.join( original_img_dir, 'prototype-self-act%d.npy' % prototypes_to_keep[idx]), dst=os.path.join(dst_img_dir, 'prototype-self-act%d.npy' % idx)) bb = np.load( os.path.join(original_img_dir, 'bb%d.npy' % epoch_number)) bb = bb[prototypes_to_keep] np.save(os.path.join(dst_img_dir, 'bb%d.npy' % epoch_number), bb) bb_rf = np.load( os.path.join(original_img_dir, 'bb-receptive_field%d.npy' % epoch_number)) bb_rf = bb_rf[prototypes_to_keep] np.save( os.path.join(dst_img_dir, 'bb-receptive_field%d.npy' % epoch_number), bb_rf) return prune_info
def generate_prototype_activation_matrix(ppnet, test_push_dataloader, push_dataloader, epoch, model_dir, device, bag_class=0, N=10, do_nearest=True): print(' analysis for class', bag_class) epoch_number_str = str(epoch) load_img_dir = os.path.join(model_dir, 'img') prototype_info = np.load(os.path.join(load_img_dir, 'epoch-' + epoch_number_str, 'bb' + epoch_number_str + '.npy')) prototype_img_identity = prototype_info[:, -1] prototype_shape = ppnet.prototype_shape max_dist = prototype_shape[1] * prototype_shape[2] * prototype_shape[3] # print('Prototypes are chosen from ' + str(len(set(prototype_img_identity))) + ' number of classes.') # print('Their class identities are: ' + str(prototype_img_identity)) raw, bag, label = next( ((r, b, l) for r, b, l in iter(test_push_dataloader) if l.max().unsqueeze(0) == bag_class and len(b) >= N)) count_positive_patches = sum(label) if len(label) > 1: label = label.max().unsqueeze(0) bag = bag.squeeze(0) img_size = raw[0].shape[1] with torch.no_grad(): ppnet.eval() images_test = bag.to(device) labels_test = label.to(device) logits, min_distances, attention, vector_scores = ppnet.forward_( images_test) # function forward in model.py should return logits, min_distances, A, prototype_activations conv_output, distances = ppnet.push_forward(images_test) prototype_activation_patterns = ppnet.distance_2_similarity(distances) if ppnet.prototype_activation_function == 'linear': prototype_activation_patterns = prototype_activation_patterns + max_dist tables = [] for i in range(logits.size(0)): tables.append((torch.argmax(logits, dim=1)[i].item(), labels_test[i].item())) # print(str(i) + ' ' + str(tables[-1])) idx = 0 predicted_cls = tables[idx][0] correct_cls = tables[idx][1] # print('Predicted: ' + str(predicted_cls)) # print('Actual: ' + str(correct_cls)) # Take the N patches with the most attention at = attention.squeeze(0).detach().cpu().numpy() top_patches = at.argsort()[-N:][::-1] # print(f' patch indexes: {top_patches}') imgs = [raw[i].permute(1, 2, 0) for i in top_patches] # Take the most highly activated area of the image by prototype imgs_with_self_activation_by_prototype = [] for img, idx in zip(imgs, top_patches): original_img = img self_activation_for_img = [] # for every prototype for i in range(len(prototype_img_identity)): activation_pattern = prototype_activation_patterns[idx][i].detach().cpu().numpy() upsampled_activation_pattern = cv2.resize(activation_pattern, dsize=(original_img.shape[0], original_img.shape[1]), interpolation=cv2.INTER_CUBIC) rescaled_activation_pattern = upsampled_activation_pattern - np.amin(upsampled_activation_pattern) rescaled_activation_pattern = rescaled_activation_pattern / np.amax(rescaled_activation_pattern) heatmap = cv2.applyColorMap(np.uint8(255 * rescaled_activation_pattern), cv2.COLORMAP_JET) heatmap = np.float32(heatmap) / 255 heatmap = heatmap[..., ::-1] overlayed_img = 0.5 * original_img + 0.3 * heatmap self_activation_for_img.append(np.asarray(overlayed_img)) imgs_with_self_activation_by_prototype.append(np.asarray(self_activation_for_img)) ### Take prototypes prototype_dir = os.path.join(load_img_dir, 'epoch-' + epoch_number_str) prototypes = [] for i in range(len(prototype_img_identity)): prototypes.append(plt.imread(f'{prototype_dir}/prototype-img{i}.png')) # Take prototypes img original with self activation prototypes_img_with_act = [] for i in range(len(prototype_img_identity)): prototypes_img_with_act.append(plt.imread(f'{prototype_dir}/prototype-img-original_with_self_act{i}.png')) ### Find the k-nearest patches in the dataset to each prototype k = 5 root_dir_for_saving_train_images = os.path.join(model_dir) makedir(root_dir_for_saving_train_images) if do_nearest: find_nearest.find_k_nearest_patches_to_prototypes( dataloader=push_dataloader, # pytorch dataloader (must be unnormalized in [0,1]) ppnet=ppnet, # pytorch network with prototype_vectors k=k, full_save=True, root_dir_for_saving_images=root_dir_for_saving_train_images, log=print) k_nearest_patches = [] for i in range(len(prototype_img_identity)): tmp = [] for j in range(1, k + 1): tmp.append(plt.imread(f'{root_dir_for_saving_train_images}/{i}/nearest-{j}_original_with_heatmap.png')) k_nearest_patches.append(np.asarray(tmp)) ### Vector score for top patches arr = vector_scores.detach().cpu().numpy() arr = np.array([arr[i] for i in top_patches]) def get_colors(inp, colormap, vmin=None, vmax=None): norm = plt.Normalize(vmin, vmax) return colormap(norm(inp)) grid_score = np.around(arr.T, 2) colors = get_colors(grid_score, plt.cm.magma) len_proto = len(prototype_img_identity) # Set up the axes with gridspec fig = plt.figure(figsize=(2 * N + 2 + k, len_proto + 2)) fig.suptitle(f'patches in bag: {len(bag)}, positive patches: {count_positive_patches}, class label: {label.item()}', fontsize=40) grid = plt.GridSpec(len_proto + 2, 2 * N + 2 + k, hspace=0.04, wspace=0.04) # build a rectangle in axes coords left, width = .25, .5 bottom, height = .25, .5 right = left + width top = bottom + height # histogram l = 0 for j in range(3 + k, 2 * N + 3 + k, 2): for i in range(2, len_proto + 2): main_ax = fig.add_subplot(grid[i, j]) main_ax.set_facecolor(colors[i - 2][l]) main_ax.text(0.5 * (left + right), 0.5 * (bottom + top), grid_score[i - 2][l], horizontalalignment='center', verticalalignment='center', fontsize=15, color='white' if grid_score[i - 2][l] < grid_score.max() * 0.9 else 'black', transform=main_ax.transAxes) main_ax.get_xaxis().set_visible(False) main_ax.get_yaxis().set_visible(False) l = l + 1 # images with self activation by prototype l = 0 for j in range(2 + k, 2 * N + 2 + k, 2): for i in range(2, len_proto + 2): main_ax = fig.add_subplot(grid[i, j]) main_ax.set_xlim([0, img_size]) main_ax.set_ylim([0, img_size]) main_ax.invert_yaxis() main_ax.imshow(imgs_with_self_activation_by_prototype[l][i - 2], aspect='auto') main_ax.get_xaxis().set_visible(False) main_ax.get_yaxis().set_visible(False) l = l + 1 # prototypes for i in range(2, len_proto + 2): main_ax = fig.add_subplot(grid[i, 1 + k]) main_ax.invert_yaxis() main_ax.imshow(prototypes[i - 2]) main_ax.get_xaxis().set_visible(False) main_ax.get_yaxis().set_visible(False) # prototypes images with activation for i in range(2, len_proto + 2): main_ax = fig.add_subplot(grid[i, 0 + k]) main_ax.set_xlim([0, img_size]) main_ax.set_ylim([0, img_size]) main_ax.invert_yaxis() main_ax.imshow(prototypes_img_with_act[i - 2]) if i - 2 < len_proto // 2: main_ax.patch.set_edgecolor('red') else: main_ax.patch.set_edgecolor('green') main_ax.patch.set_linewidth('5') main_ax.get_xaxis().set_visible(False) main_ax.get_yaxis().set_visible(False) # k nearest patches for j in range(k): for i in range(2, len_proto + 2): main_ax = fig.add_subplot(grid[i, j]) main_ax.set_xlim([0, img_size]) main_ax.set_ylim([0, img_size]) main_ax.invert_yaxis() main_ax.imshow(k_nearest_patches[i - 2][j]) main_ax.get_xaxis().set_visible(False) main_ax.get_yaxis().set_visible(False) # patches l = 0 for i in range(2 + k, 2 * N + 2 + k, 2): main_ax = fig.add_subplot(grid[0:2, i:i + 2]) main_ax.set_xlim([0, img_size]) main_ax.set_ylim([0, img_size]) main_ax.invert_yaxis() main_ax.set_title(f'{at[top_patches[l]]:.3f}', fontsize=25) main_ax.imshow(imgs[l], aspect='auto') main_ax.get_xaxis().set_visible(False) main_ax.get_yaxis().set_visible(False) l = l + 1 plt.axis('off') return fig
def find_k_nearest_patches_to_prototypes( dataloader, # pytorch dataloader (must be unnormalized in [0,1]) prototype_network_parallel, # pytorch network with prototype_vectors k=5, preprocess_input_function=None, # normalize if needed) prototype_layer_stride=1, root_dir_for_saving_images='./nearest_', save_image_class_identity=True, log=print): log('find nearest patches') start = time.time() protoL_rf_info = prototype_network_parallel.module.proto_layer_rf_info parent_names = [ node.name for node in prototype_network_parallel.module.root.nodes_with_children() ] cmap = "jet" node2heaps = {name: [] for name in parent_names} # for each parent node, organize a heap for every prototype for node in prototype_network_parallel.module.root.nodes_with_children(): name = node.name num_prototypes = getattr(prototype_network_parallel.module, "num_" + node.name + "_prototypes") for j in range(num_prototypes): node2heaps[name].append([]) for (search_batch_input, search_y) in dataloader: if preprocess_input_function != None: #print('preprocessing input for pushing ...') search_batch = copy.deepcopy(search_batch_input) search_batch = preprocess_input_function(search_batch) else: search_batch = search_batch_input with torch.no_grad(): search_batch = search_batch.cuda() conv_features = prototype_network_parallel.module.conv_features( search_batch) for node in prototype_network_parallel.module.root.nodes_with_children( ): num_prototypes = getattr(prototype_network_parallel.module, "num_" + node.name + "_prototypes") with torch.no_grad(): proto_dist_torch = prototype_network_parallel.module.prototype_distances( conv_features, node.name) proto_dist_ = np.copy(proto_dist_torch.detach().cpu().numpy()) heaps = node2heaps[node.name] for img_idx, distance_map in enumerate(proto_dist_): for j in range(num_prototypes): # find the closest patch to prototype j closest_patch_distance_to_prototype_j = np.amin( distance_map[j]) closest_patch_indices_in_distance_map_j = \ list(np.unravel_index(np.argmin(distance_map[j],axis=None), distance_map[j].shape)) closest_patch_indices_in_distance_map_j = [ 0 ] + closest_patch_indices_in_distance_map_j closest_patch_indices_in_img = \ compute_rf_prototype(search_batch.size(2), closest_patch_indices_in_distance_map_j, protoL_rf_info) closest_patch = \ search_batch_input[img_idx, :, closest_patch_indices_in_img[1]:closest_patch_indices_in_img[2], closest_patch_indices_in_img[3]:closest_patch_indices_in_img[4]] closest_patch = closest_patch.numpy() closest_patch = np.transpose(closest_patch, (1, 2, 0)) original_img = search_batch_input[img_idx].numpy() original_img = np.transpose(original_img, (1, 2, 0)) act_pattern = np.log(1 + (1 / (distance_map[j] + 1e-4))) patch_indices = closest_patch_indices_in_img[1:5] # construct the closest patch object closest_patch = ImagePatch( patch=closest_patch, label=search_y[img_idx], distance=closest_patch_distance_to_prototype_j, original_img=original_img, act_pattern=act_pattern, patch_indices=patch_indices) # add to the j-th heap if len(heaps[j]) < k: heapq.heappush(heaps[j], closest_patch) else: heapq.heappushpop(heaps[j], closest_patch) del conv_features, search_batch for node in prototype_network_parallel.module.root.nodes_with_children(): heaps = node2heaps[node.name] num_prototypes = getattr(prototype_network_parallel.module, "num_" + node.name + "_prototypes") for j in range(num_prototypes): heaps[j].sort() heaps[j] = heaps[j][::-1] # reverses dir_for_saving_images = os.path.join(root_dir_for_saving_images, node.name, str(j)) makedir(dir_for_saving_images) labels = [] for i, patch in enumerate(heaps[j]): # save the patch itself plt.imsave(fname=os.path.join(dir_for_saving_images, 'nearest-' + str(i) + '.png'), arr=patch.patch, vmin=0.0, vmax=1.0) # save the original image where the patch comes from plt.imsave(fname=os.path.join( dir_for_saving_images, 'nearest-' + str(i) + '_original.png'), arr=patch.original_img, vmin=0.0, vmax=1.0) # save the activation pattern and the patch indices np.save( os.path.join(dir_for_saving_images, 'nearest-' + str(i) + '_act.npy'), patch.act_pattern) np.save( os.path.join(dir_for_saving_images, 'nearest-' + str(i) + '_indices.npy'), patch.patch_indices) # upsample the activation pattern img_size = patch.original_img.shape[0] original_img_gray = cv2.cvtColor(patch.original_img, cv2.COLOR_BGR2GRAY) upsampled_act_pattern = cv2.resize( patch.act_pattern, dsize=(img_size, img_size), interpolation=cv2.INTER_CUBIC) # overlay heatmap on the original image and save the result overlayed_original_img = 0.7 * original_img_gray + 0.3 * upsampled_act_pattern plt.imsave(fname=os.path.join( dir_for_saving_images, 'nearest-' + str(i) + '_original_with_heatmap.png'), arr=overlayed_original_img, cmap=cmap, vmin=0.0, vmax=1.0) # save the patch with heatmap overlayed_patch = overlayed_original_img[ patch.patch_indices[0]:patch.patch_indices[1], patch.patch_indices[2]:patch.patch_indices[3]] plt.imsave(fname=os.path.join( dir_for_saving_images, 'nearest-' + str(i) + '_patch_with_heatmap.png'), arr=overlayed_patch, cmap=cmap, vmin=0.0, vmax=1.0) if save_image_class_identity: labels = np.array([patch.label for patch in heaps[j]]) np.save(os.path.join(dir_for_saving_images, 'class_id.npy'), labels) end = time.time() log('\tfind nearest patches time: \t{0}'.format(end - start))