def get_mAP_results(config_names, configs_list, inference_configs_list): # Level 0 is coco validation set with no corruption, level 10 is open # images, level 11 is open images ood image_corruption_levels = [0, 1, 2, 3, 4, 5, 10] test_dataset_coco = "coco_2017_custom_val" test_dataset_open_images = "openimages_val" arg_parser = setup_arg_parser() args = arg_parser.parse_args() # Initiate dataframe dict mAP_results = defaultdict(list) for config_name, config, inference_config_name in zip( config_names, configs_list, inference_configs_list): # Setup config args.config_file = config args.inference_config = inference_config_name args.test_dataset = test_dataset_coco cfg = setup_config(args, random_seed=args.random_seed, is_testing=True) cfg.defrost() # Read coco dataset results cfg.ACTUAL_TEST_DATASET = args.test_dataset for image_corruption_level in image_corruption_levels: # Build path to gt instances and inference output args.image_corruption_level = image_corruption_level if image_corruption_level == 0: image_corruption_level = 'Val' elif image_corruption_level == 10: image_corruption_level = 'OpenIm' else: image_corruption_level = 'C' + str(image_corruption_level) if 'OpenIm' not in image_corruption_level: inference_output_dir = get_inference_output_dir( cfg['OUTPUT_DIR'], args.test_dataset, args.inference_config, args.image_corruption_level) else: args.image_corruption_level = 0 args.test_dataset = test_dataset_open_images inference_output_dir = get_inference_output_dir( cfg['OUTPUT_DIR'], args.test_dataset, args.inference_config, args.image_corruption_level) text_file_name = glob.glob( os.path.join(inference_output_dir, 'mAP_res.txt'))[0] with open(text_file_name, "r") as f: mAP = f.read().strip('][\n').split(', ')[0] mAP = float(mAP) * 100 mAP_results['Method Name'].append(config_name) mAP_results['Image Corruption Level'].append( image_corruption_level) mAP_results['mAP'].append(mAP) return mAP_results
trainer = Trainer(cfg) if args.eval_only: model = trainer.build_model(cfg) DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( cfg.MODEL.WEIGHTS, resume=args.resume) res = trainer.test(cfg, model) if comm.is_main_process(): verify_results(cfg, res) return res trainer.resume_or_load(resume=args.resume) return trainer.train() if __name__ == "__main__": # Create arg parser arg_parser = setup_arg_parser() args = arg_parser.parse_args() print("Command Line Args:", args) launch( main, args.num_gpus, num_machines=args.num_machines, machine_rank=args.machine_rank, dist_url=args.dist_url, args=(args, ), )
def get_matched_results_dicts(config_names, configs_list, inference_configs_list, iou_min=0.1, iou_correct=0.5): # Level 0 is coco validation set with no corruption, level 10 is open # images, level 11 is open images ood image_corruption_levels = [0, 10, 11] test_dataset_coco = "coco_2017_custom_val" test_dataset_open_images = "openimages_val" test_dataset_open_images_odd = "openimages_odd_val" arg_parser = setup_arg_parser() args = arg_parser.parse_args() # Initiate dataframe dict res_dict_clean = defaultdict( lambda: defaultdict(lambda: defaultdict(list))) for config_name, config, inference_config_name in zip( config_names, configs_list, inference_configs_list): # Setup config args.config_file = config args.inference_config = inference_config_name args.test_dataset = test_dataset_coco cfg = setup_config(args, random_seed=args.random_seed, is_testing=True) cfg.defrost() # Read coco dataset results cfg.ACTUAL_TEST_DATASET = args.test_dataset for image_corruption_level in image_corruption_levels: # Build path to gt instances and inference output args.image_corruption_level = image_corruption_level if image_corruption_level == 0: image_corruption_level = 'Val' elif image_corruption_level == 10: image_corruption_level = 'OpenIm' elif image_corruption_level == 11: image_corruption_level = 'OpenIm OOD' else: image_corruption_level = 'C' + str(image_corruption_level) if 'OpenIm' not in image_corruption_level: inference_output_dir = get_inference_output_dir( cfg['OUTPUT_DIR'], args.test_dataset, args.inference_config, args.image_corruption_level) # Get matched results by either generating them or loading from # file. dictionary_file_name = glob.glob( os.path.join( inference_output_dir, "matched_results_{}_{}_*.pth".format( iou_min, iou_correct)))[0] matched_results = torch.load(dictionary_file_name, map_location='cuda') elif image_corruption_level == 'OpenIm': args.image_corruption_level = 0 args.test_dataset = test_dataset_open_images if image_corruption_level == 'OpenIm' else test_dataset_open_images_odd inference_output_dir = get_inference_output_dir( cfg['OUTPUT_DIR'], args.test_dataset, args.inference_config, args.image_corruption_level) dictionary_file_name = glob.glob( os.path.join( inference_output_dir, "matched_results_{}_{}_*.pth".format( iou_min, iou_correct)))[0] matched_results = torch.load(dictionary_file_name, map_location='cuda') else: args.image_corruption_level = 0 args.test_dataset = test_dataset_open_images if image_corruption_level == 'OpenIm' else test_dataset_open_images_odd inference_output_dir = get_inference_output_dir( cfg['OUTPUT_DIR'], args.test_dataset, args.inference_config, args.image_corruption_level) dictionary_file_name = glob.glob( os.path.join( inference_output_dir, "preprocessed_predicted_instances_odd_*.pth"))[0] preprocessed_predicted_instances = torch.load( dictionary_file_name, map_location='cuda') predicted_boxes = preprocessed_predicted_instances[ 'predicted_boxes'] predicted_cov_mats = preprocessed_predicted_instances[ 'predicted_covar_mats'] predicted_cls_probs = preprocessed_predicted_instances[ 'predicted_cls_probs'] predicted_boxes = list( itertools.chain.from_iterable([ predicted_boxes[key] for key in predicted_boxes.keys() ])) predicted_cov_mats = list( itertools.chain.from_iterable([ predicted_cov_mats[key] for key in predicted_cov_mats.keys() ])) predicted_cls_probs = list( itertools.chain.from_iterable([ predicted_cls_probs[key] for key in predicted_cls_probs.keys() ])) predicted_boxes = torch.stack(predicted_boxes, 1).transpose(0, 1) predicted_cov_mats = torch.stack(predicted_cov_mats, 1).transpose(0, 1) predicted_cls_probs = torch.stack(predicted_cls_probs, 1).transpose(0, 1) matched_results = { 'predicted_box_means': predicted_boxes, 'predicted_box_covariances': predicted_cov_mats, 'predicted_cls_probs': predicted_cls_probs } if image_corruption_level != 'OpenIm OOD': all_results_means = torch.cat(( matched_results['true_positives']['predicted_box_means'], matched_results['localization_errors'] ['predicted_box_means'], matched_results['duplicates']['predicted_box_means'], matched_results['false_positives']['predicted_box_means'])) all_results_covs = torch.cat(( matched_results['true_positives'] ['predicted_box_covariances'], matched_results['localization_errors'] ['predicted_box_covariances'], matched_results['duplicates']['predicted_box_covariances'], matched_results['false_positives'] ['predicted_box_covariances'])) all_gt_means = torch.cat(( matched_results['true_positives']['gt_box_means'], matched_results['localization_errors']['gt_box_means'], matched_results['duplicates']['gt_box_means'], matched_results['false_positives']['predicted_box_means'] * np.NaN)) predicted_multivariate_normal_dists = torch.distributions.multivariate_normal.MultivariateNormal( all_results_means.to('cpu'), all_results_covs.to('cpu') + 1e-2 * torch.eye(all_results_covs.shape[2]).to('cpu')) predicted_multivariate_normal_dists.loc = predicted_multivariate_normal_dists.loc.to( 'cuda') predicted_multivariate_normal_dists.scale_tril = predicted_multivariate_normal_dists.scale_tril.to( 'cuda') predicted_multivariate_normal_dists._unbroadcasted_scale_tril = predicted_multivariate_normal_dists._unbroadcasted_scale_tril.to( 'cuda') predicted_multivariate_normal_dists.covariance_matrix = predicted_multivariate_normal_dists.covariance_matrix.to( 'cuda') predicted_multivariate_normal_dists.precision_matrix = predicted_multivariate_normal_dists.precision_matrix.to( 'cuda') all_entropy = predicted_multivariate_normal_dists.entropy() all_log_prob = -predicted_multivariate_normal_dists.log_prob( all_gt_means) # Energy Score. sample_set = predicted_multivariate_normal_dists.sample( (3, )).to('cuda') sample_set_1 = sample_set[:-1] sample_set_2 = sample_set[1:] energy_score = torch.norm( (sample_set_1 - all_gt_means), dim=2).mean(0) - 0.5 * torch.norm( (sample_set_1 - sample_set_2), dim=2).mean(0) mse_loss = torch.nn.MSELoss(reduction='none') mse = mse_loss(all_gt_means, all_results_means).mean(1) res_dict_clean[config_name][image_corruption_level][ 'Entropy'].extend(all_entropy.cpu().numpy()) res_dict_clean[config_name][image_corruption_level][ 'MSE'].extend(mse.cpu().numpy()) res_dict_clean[config_name][image_corruption_level][ 'NLL'].extend(all_log_prob.cpu().numpy()) res_dict_clean[config_name][image_corruption_level][ 'ED'].extend(energy_score.cpu().numpy()) res_dict_clean[config_name][image_corruption_level][ 'IOU With GT'].extend( torch.cat( (matched_results['true_positives'] ['iou_with_ground_truth'], matched_results['localization_errors'] ['iou_with_ground_truth'][:, 0], matched_results['duplicates'] ['iou_with_ground_truth'], torch.zeros( matched_results['false_positives'] ['predicted_box_means'].shape[0]).to('cuda') * np.NaN)).cpu().numpy()) predicted_multivariate_normal_dists = torch.distributions.multivariate_normal.MultivariateNormal( matched_results['false_positives'] ['predicted_box_means'].to('cpu'), matched_results['false_positives'] ['predicted_box_covariances'].to('cpu') + 1e-2 * torch.eye(matched_results['false_positives'][ 'predicted_box_covariances'].shape[2]).to('cpu')) predicted_multivariate_normal_dists.loc = predicted_multivariate_normal_dists.loc.to( 'cuda') predicted_multivariate_normal_dists.scale_tril = predicted_multivariate_normal_dists.scale_tril.to( 'cuda') predicted_multivariate_normal_dists._unbroadcasted_scale_tril = predicted_multivariate_normal_dists._unbroadcasted_scale_tril.to( 'cuda') predicted_multivariate_normal_dists.covariance_matrix = predicted_multivariate_normal_dists.covariance_matrix.to( 'cuda') predicted_multivariate_normal_dists.precision_matrix = predicted_multivariate_normal_dists.precision_matrix.to( 'cuda') FP_Entropy = predicted_multivariate_normal_dists.entropy() res_dict_clean[config_name][image_corruption_level][ 'FP_Entropy'].extend(FP_Entropy.cpu().numpy()) predicted_cat_dists_fp = matched_results['false_positives'][ 'predicted_cls_probs'] if predicted_cat_dists_fp.shape[1] == 80: predicted_cat_dists_fp, _ = predicted_cat_dists_fp.max( dim=1) predicted_cat_dists_fp = 1 - predicted_cat_dists_fp predicted_categorical_dists = torch.distributions.Bernoulli( probs=predicted_cat_dists_fp) else: predicted_categorical_dists = torch.distributions.Categorical( probs=matched_results['false_positives'] ['predicted_cls_probs']) all_pred_ent = predicted_categorical_dists.entropy() res_dict_clean[config_name][image_corruption_level][ 'Cat_Entropy'].extend(all_pred_ent.cpu().numpy()) if image_corruption_level == 'OpenIm': res_dict_clean[config_name][image_corruption_level][ 'Truncated'].extend( torch.cat( (matched_results['true_positives'] ['is_truncated'], matched_results['localization_errors'] ['is_truncated'], matched_results['duplicates']['is_truncated'], torch.full( (matched_results['false_positives'] ['predicted_box_means'].shape[0], ), -1, dtype=torch.float32).to('cuda') * np.NaN)).cpu().numpy()) res_dict_clean[config_name][image_corruption_level][ 'Occluded'].extend( torch.cat( (matched_results['true_positives'] ['is_occluded'], matched_results['localization_errors'] ['is_occluded'], matched_results['duplicates']['is_occluded'], torch.full( (matched_results['false_positives'] ['predicted_box_means'].shape[0], ), -1, dtype=torch.float32).to('cuda') * np.NaN)).cpu().numpy()) else: res_dict_clean[config_name][image_corruption_level][ 'Truncated'].extend( torch.cat( (torch.full( (matched_results['true_positives'] ['predicted_box_means'].shape[0], ), -1, dtype=torch.float32).to('cuda') * np.NaN, torch.full( (matched_results['localization_errors'] ['predicted_box_means'].shape[0], ), -1, dtype=torch.float32).to('cuda'), torch.full( (matched_results['duplicates'] ['predicted_box_means'].shape[0], ), -1, dtype=torch.float32).to('cuda'), torch.full( (matched_results['false_positives'] ['predicted_box_means'].shape[0], ), -1, dtype=torch.float32).to('cuda') * np.NaN)).cpu().numpy()) res_dict_clean[config_name][image_corruption_level][ 'Occluded'].extend( torch.cat( (torch.full( (matched_results['true_positives'] ['predicted_box_means'].shape[0], ), -1, dtype=torch.float32).to('cuda') * np.NaN, torch.full( (matched_results['localization_errors'] ['predicted_box_means'].shape[0], ), -1, dtype=torch.float32).to('cuda') * np.NaN, torch.full( (matched_results['duplicates'] ['predicted_box_means'].shape[0], ), -1, dtype=torch.float32).to('cuda') * np.NaN, torch.full( (matched_results['false_positives'] ['predicted_box_means'].shape[0], ), -1, dtype=torch.float32).to('cuda') * np.NaN)).cpu().numpy()) else: predicted_multivariate_normal_dists = torch.distributions.multivariate_normal.MultivariateNormal( matched_results['predicted_box_means'].to('cpu'), matched_results['predicted_box_covariances'].to('cpu') + 1e-2 * torch.eye(matched_results['predicted_box_covariances']. shape[2]).to('cpu')) predicted_multivariate_normal_dists.loc = predicted_multivariate_normal_dists.loc.to( 'cuda') predicted_multivariate_normal_dists.scale_tril = predicted_multivariate_normal_dists.scale_tril.to( 'cuda') predicted_multivariate_normal_dists._unbroadcasted_scale_tril = predicted_multivariate_normal_dists._unbroadcasted_scale_tril.to( 'cuda') predicted_multivariate_normal_dists.covariance_matrix = predicted_multivariate_normal_dists.covariance_matrix.to( 'cuda') predicted_multivariate_normal_dists.precision_matrix = predicted_multivariate_normal_dists.precision_matrix.to( 'cuda') all_entropy = predicted_multivariate_normal_dists.entropy() res_dict_clean[config_name][image_corruption_level][ 'FP_Entropy'].extend(all_entropy.cpu().numpy()) res_dict_clean[config_name][image_corruption_level][ 'IOU With GT'].extend( torch.zeros(matched_results['predicted_box_means']. shape[0]).cpu().numpy()) res_dict_clean[config_name][image_corruption_level][ 'Truncated'].extend( torch.full(( matched_results['predicted_box_means'].shape[0], ), -1, dtype=torch.float32).cpu().numpy() * np.NaN) res_dict_clean[config_name][image_corruption_level][ 'Occluded'].extend( torch.full(( matched_results['predicted_box_means'].shape[0], ), -1, dtype=torch.float32).cpu().numpy() * np.NaN) all_results_cat = matched_results['predicted_cls_probs'] if all_results_cat.shape[1] == 80: predicted_cat_dists_fp, _ = all_results_cat.max(dim=1) predicted_cat_dists_fp = 1 - predicted_cat_dists_fp predicted_categorical_dists = torch.distributions.Bernoulli( probs=predicted_cat_dists_fp) else: predicted_categorical_dists = torch.distributions.Categorical( probs=all_results_cat) all_pred_ent = predicted_categorical_dists.entropy() res_dict_clean[config_name][image_corruption_level][ 'Cat_Entropy'].extend(all_pred_ent.cpu().numpy()) return res_dict_clean
def get_clean_results_dict(config_names, configs_list, inference_configs_list): # Level 0 is coco validation set with no corruption, level 10 is open # images, level 11 is open images ood image_corruption_levels = [0, 1, 3, 5, 10, 11] test_dataset_coco = "coco_2017_custom_val" test_dataset_open_images = "openimages_val" test_dataset_open_images_odd = "openimages_odd_val" arg_parser = setup_arg_parser() args = arg_parser.parse_args() # Initiate dataframe dict res_dict_clean = defaultdict(lambda: defaultdict(list)) for config_name, config, inference_config_name in zip( config_names, configs_list, inference_configs_list): # Setup config args.config_file = config args.inference_config = inference_config_name args.test_dataset = test_dataset_coco cfg = setup_config(args, random_seed=args.random_seed, is_testing=True) cfg.defrost() # Read coco dataset results cfg.ACTUAL_TEST_DATASET = args.test_dataset for image_corruption_level in image_corruption_levels: # Build path to gt instances and inference output args.image_corruption_level = image_corruption_level if image_corruption_level == 0: image_corruption_level = 'Val' elif image_corruption_level == 10: image_corruption_level = 'OpenIm' elif image_corruption_level == 11: image_corruption_level = 'OpenIm OOD' else: image_corruption_level = 'C' + str(image_corruption_level) if 'OpenIm' not in image_corruption_level: inference_output_dir = get_inference_output_dir( cfg['OUTPUT_DIR'], args.test_dataset, args.inference_config, args.image_corruption_level) dictionary_file_name = glob.glob( os.path.join( inference_output_dir, 'probabilistic_scoring_res_averaged_*.pkl'))[0] else: args.image_corruption_level = 0 args.test_dataset = test_dataset_open_images if image_corruption_level == 'OpenIm' else test_dataset_open_images_odd inference_output_dir = get_inference_output_dir( cfg['OUTPUT_DIR'], args.test_dataset, args.inference_config, args.image_corruption_level) prob_dict_name = 'probabilistic_scoring_res_averaged_*.pkl' if image_corruption_level == 'OpenIm' else 'probabilistic_scoring_res_odd_*.pkl' dictionary_file_name = glob.glob( os.path.join(inference_output_dir, prob_dict_name))[0] with open(dictionary_file_name, "rb") as pickle_file: res_dict = pickle.load(pickle_file) if image_corruption_level != 'OpenIm OOD': # True Positives Results res_dict_clean['True Positives'][ 'Negative Log Likelihood (Classification)'].extend( res_dict['true_positives_cls_analysis'] ['ignorance_score_mean']) res_dict_clean['True Positives']['Brier Score'].extend( res_dict['true_positives_cls_analysis'] ['brier_score_mean']) res_dict_clean['True Positives'][ 'Negative Log Likelihood (Regression)'].extend( res_dict['true_positives_reg_analysis'] ['ignorance_score_mean']) res_dict_clean['True Positives'][ 'Mean Squared Error'].extend( res_dict['true_positives_reg_analysis'] ['mean_squared_error']) res_dict_clean['True Positives']['Energy Score'].extend( res_dict['true_positives_reg_analysis'] ['energy_score_mean']) res_dict_clean['True Positives'][ 'Image Corruption Level'].extend( [image_corruption_level] * res_dict['true_positives_reg_analysis'] ['energy_score_mean'].shape[0]) res_dict_clean['True Positives']['Method Name'].extend( [config_name] * res_dict['true_positives_reg_analysis'] ['energy_score_mean'].shape[0]) # Duplicates Results res_dict_clean['Duplicates'][ 'Negative Log Likelihood (Classification)'].extend( res_dict['duplicates_cls_analysis'] ['ignorance_score_mean']) res_dict_clean['Duplicates']['Brier Score'].extend( res_dict['duplicates_cls_analysis'] ['brier_score_mean']) res_dict_clean['Duplicates'][ 'Negative Log Likelihood (Regression)'].extend( res_dict['duplicates_reg_analysis'] ['ignorance_score_mean']) res_dict_clean['Duplicates']['Mean Squared Error'].extend( res_dict['duplicates_reg_analysis'] ['mean_squared_error']) res_dict_clean['Duplicates']['Energy Score'].extend( res_dict['duplicates_reg_analysis'] ['energy_score_mean']) res_dict_clean['Duplicates'][ 'Image Corruption Level'].extend( [image_corruption_level] * res_dict['duplicates_reg_analysis'] ['energy_score_mean'].shape[0]) res_dict_clean['Duplicates']['Method Name'].extend( [config_name] * res_dict['duplicates_reg_analysis'] ['energy_score_mean'].shape[0]) # Localization Error Results res_dict_clean['Localization Errors'][ 'Negative Log Likelihood (Classification)'].extend( res_dict['localization_errors_cls_analysis'] ['ignorance_score_mean']) res_dict_clean['Localization Errors'][ 'Brier Score'].extend( res_dict['localization_errors_cls_analysis'] ['brier_score_mean']) res_dict_clean['Localization Errors'][ 'Negative Log Likelihood (Regression)'].extend( res_dict['localization_errors_reg_analysis'] ['ignorance_score_mean']) res_dict_clean['Localization Errors'][ 'Mean Squared Error'].extend( res_dict['localization_errors_reg_analysis'] ['mean_squared_error']) res_dict_clean['Localization Errors'][ 'Energy Score'].extend( res_dict['localization_errors_reg_analysis'] ['energy_score_mean']) res_dict_clean['Localization Errors'][ 'Image Corruption Level'].extend( [image_corruption_level] * res_dict['localization_errors_reg_analysis'] ['energy_score_mean'].shape[0]) res_dict_clean['Localization Errors'][ 'Method Name'].extend( [config_name] * res_dict['localization_errors_reg_analysis'] ['energy_score_mean'].shape[0]) # False Positives Results res_dict_clean['False Positives'][ 'Negative Log Likelihood (Classification)'].extend( res_dict['false_positives_cls_analysis'] ['ignorance_score_mean']) res_dict_clean['False Positives']['Brier Score'].extend( res_dict['false_positives_cls_analysis'] ['brier_score_mean']) res_dict_clean['False Positives']['Entropy'].extend( res_dict['false_positives_reg_analysis'] ['total_entropy_mean']) res_dict_clean['False Positives'][ 'Image Corruption Level'].extend( [image_corruption_level] * res_dict['false_positives_reg_analysis'] ['total_entropy_mean'].shape[0]) res_dict_clean['False Positives']['Method Name'].extend( [config_name] * res_dict['false_positives_reg_analysis'] ['total_entropy_mean'].shape[0]) else: # False Positives Results res_dict_clean['False Positives'][ 'Negative Log Likelihood (Classification)'].append( res_dict['ignorance_score_mean']) res_dict_clean['False Positives']['Brier Score'].append( res_dict['brier_score_mean']) res_dict_clean['False Positives']['Entropy'].append( res_dict['total_entropy_mean']) res_dict_clean['False Positives'][ 'Image Corruption Level'].append( image_corruption_level) res_dict_clean['False Positives']['Method Name'].append( config_name) return res_dict_clean