def inference( model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, skip_eval=False, ): # convert to a torch.device for efficiency device = torch.device(device) num_devices = (torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1) logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) start_time = time.time() predictions = compute_on_dataset(model, data_loader, device) # wait for all processes to complete before measuring the time synchronize() total_time = time.time() - start_time total_time_str = str(datetime.timedelta(seconds=total_time)) logger.info( "Total inference time: {} ({} s / img per device, on {} devices)". format(total_time_str, total_time * num_devices / len(dataset), num_devices)) predictions = _accumulate_predictions_from_multiple_gpus(predictions) if not is_main_process(): return if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) if skip_eval: logger.info("Skipping evaluation. Stored predictions to {}".format( os.path.join(output_folder, "predictions.pth"))) return extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def main(): config_file = './semi_test/retinanet_R-50-FPN_1x_semi.yaml' output_folder = './output_folder' str_pth = "./model_path" cfg.merge_from_file(config_file) data_loaders_val = make_data_loader(cfg, is_train=False, is_distributed=False) dataset = data_loaders_val[0].dataset range_start = 2 range_end = 3 predictions = torch.load(str_pth)[range_start:range_end] dataset_imgs = {} id_to_img_map = {} for _prediction in predictions: ind_sort = _prediction.get_field('scores').argsort(descending=True)[:12] _prediction.bbox = _prediction.bbox[ind_sort] _prediction.extra_fields['scores'] = _prediction.extra_fields['scores'][ind_sort] _prediction.extra_fields['labels'] = _prediction.extra_fields['labels'][ind_sort] _prediction = _prediction.resize([640,478]) iCount = 0 for i in range(range_start,range_end): _id = dataset.id_to_img_map[i] id_to_img_map[iCount] = _id dataset_imgs[_id] = dataset.coco.imgs[_id] iCount += 1 dataset.coco.imgs = dataset_imgs dataset.id_to_img_map = id_to_img_map extra_args = dict( box_only=False, iou_types=('bbox',), expected_results=[], expected_results_sigma_tol=4 ) if not os.path.exists(output_folder): os.mkdir(output_folder) r=evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args) check_expected_results(r, [], 4) print(r[0])
def inference( model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, ): # convert to a torch.device for efficiency device = torch.device(device) num_devices = (torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1) logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) start_time = time.time() predictions = compute_on_dataset(model, data_loader, device) # wait for all processes to complete before measuring the time synchronize() total_time = time.time() - start_time total_time_str = str(datetime.timedelta(seconds=total_time)) logger.info( "Total inference time: {} ({} s / img per device, on {} devices)". format(total_time_str, total_time * num_devices / len(dataset), num_devices)) # We have an optimised path for COCO which takes advantage of more parallelism. # If not using COCO, fall back to regular path: gather predictions from all ranks # and call evaluate on those results. if not isinstance(dataset, datasets.COCODataset): predictions = _accumulate_predictions_from_multiple_gpus(predictions) if not is_main_process(): return if output_folder and is_main_process(): torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def inference( model, data_loader, dataset_name, iou_types=("bbox",), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, ): # convert to a torch.device for efficiency device = torch.device(device) num_devices = ( torch.distributed.deprecated.get_world_size() if torch.distributed.deprecated.is_initialized() else 1 ) logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format(dataset_name, len(dataset))) start_time = time.time() predictions = compute_on_dataset(model, data_loader, device) # wait for all processes to complete before measuring the time synchronize() total_time = time.time() - start_time total_time_str = str(datetime.timedelta(seconds=total_time)) logger.info( "Total inference time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices ) ) predictions = _accumulate_predictions_from_multiple_gpus(predictions) if not is_main_process(): return if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def eval( predictions, data_loader, iou_types=("bbox", ), box_only=False, expected_results=(), expected_results_sigma_tol=4, output_folder=None, ): dataset = data_loader.dataset extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def test_while_train(cfg, model, distributed, logger, curr_iter, val_tags, data_loader, output_folder): torch.cuda.empty_cache() logger.info("start testing while training...") # only the first one for test model.eval() results_dict = {} device = torch.device('cuda') cpu_device = torch.device("cpu") meters = MetricLogger(delimiter=" ", ) for bid, (images, targets, image_ids, phrase_ids, sent_ids, sentence, precompute_bbox, precompute_score, feature_map, vocab_label_elmo, sent_sg, topN_box) in enumerate(tqdm(data_loader)): # if bid>3: # break vocab_label_elmo = [vocab.to(device) for vocab in vocab_label_elmo] features_list = [feat.to(device) for feat in feature_map] with torch.no_grad(): loss_dict, results = model(images, features_list, targets, phrase_ids, sentence, precompute_bbox, precompute_score, image_ids, vocab_label_elmo, sent_sg, topN_box) loss_dict_reduced = reduce_loss_dict(loss_dict) losses_reduced = sum(loss for loss in loss_dict_reduced.values()) meters.update(loss=losses_reduced, **loss_dict_reduced) # collect and move result to cpu memory moved_res = [] if cfg.MODEL.VG.TWO_STAGE: if cfg.MODEL.RELATION_ON and cfg.MODEL.RELATION.USE_RELATION_CONST: batch_gt_boxes, batch_pred_box, batch_pred_box_topN, batch_pred_box_det,\ batch_pred_similarity, batch_pred_similarity_topN, batch_rel_pred_similarity, batch_rel_gt_label, batch_topN_boxes, batch_reg_offset_topN, batch_rel_score_mat=results for idx, each_gt_boxes in enumerate(batch_gt_boxes): moved_res.append( (each_gt_boxes.to(cpu_device), batch_pred_box[idx].to(cpu_device), batch_pred_box_topN[idx].to(cpu_device), batch_pred_box_det[idx].to(cpu_device), batch_pred_similarity[idx].to(cpu_device), batch_pred_similarity_topN[idx].to(cpu_device), batch_rel_pred_similarity[idx].to(cpu_device), batch_rel_gt_label[idx].to(cpu_device), batch_topN_boxes[idx].to(cpu_device), batch_reg_offset_topN[idx].to(cpu_device), batch_rel_score_mat[idx])) else: batch_gt_boxes, batch_pred_box, batch_pred_box_topN, batch_pred_box_det, batch_pred_similarity = results for idx, each_gt_boxes in enumerate(batch_gt_boxes): moved_res.append( (each_gt_boxes.to(cpu_device), batch_pred_box[idx].to(cpu_device), batch_pred_box_topN[idx].to(cpu_device), batch_pred_box_det[idx].to(cpu_device), batch_pred_similarity[idx].to(cpu_device))) else: batch_gt_boxes, batch_pred_box, batch_pred_box_det, batch_pred_similarity = results for idx, each_gt_boxes in enumerate(batch_gt_boxes): moved_res.append( (each_gt_boxes.to(cpu_device), batch_pred_box[idx].to(cpu_device), batch_pred_box_det[idx].to(cpu_device), batch_pred_similarity[idx].to(cpu_device))) results_dict.update({ img_id + '_' + sent_id: result for img_id, sent_id, result in zip(image_ids, sent_ids, moved_res) }) synchronize() (predictions, image_ids) = _accumulate_predictions_from_multiple_gpus(results_dict) if output_folder: with open( os.path.join(output_folder, "predictions_{}.pkl".format(curr_iter)), 'wb') as f: pickle.dump(predictions, f) torch.save( predictions, os.path.join(output_folder, "predictions_{}.pth".format(curr_iter))) torch.cuda.empty_cache() if not is_main_process(): return logger.info('Total items num is {}'.format(len(predictions))) # with open(os.path.join(cfg.OUTPUT_DIR, 'prediction.pkl'), 'wb') as handle: # pickle.dump(predictions, handle, protocol=pickle.HIGHEST_PROTOCOL) iou_types = ("bbox", ) if cfg.MODEL.MASK_ON: iou_types = iou_types + ("segm", ) if cfg.MODEL.KEYPOINT_ON: iou_types = iou_types + ("keypoints", ) box_only = False if cfg.MODEL.RETINANET_ON else cfg.MODEL.RPN_ONLY expected_results = cfg.TEST.EXPECTED_RESULTS expected_results_sigma_tol = cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL extra_args = dict( box_only=False, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) acc, acc_topN, acc_det, acc_rel_softmax = evaluate( dataset=data_loader.dataset, predictions=predictions, image_ids=image_ids, curr_iter=curr_iter, output_folder=None, **extra_args) record = {val_tags[k]: v for (k, v) in meters.meters.items()} logger.log(TFBoardHandler_LEVEL, (record, curr_iter)) logger.info("current accuracy is: {}".format(acc)) logger.info("current topN accuracy is: {}".format(acc_topN)) logger.info("current accuracy with detection score is: {}".format(acc_det)) logger.info( "current rel constrain accuracy is: {}".format(acc_rel_softmax)) logger.log(TFBoardHandler_LEVEL, ({ val_tags['acc']: acc, val_tags['acc_topN']: acc_topN, val_tags['acc_det']: acc_det, val_tags['acc_rel_softmax']: acc_rel_softmax }, curr_iter)) logger.info("test done !")
output_name = 'BBAM_Mask_RCNN_logs_mcg85' output_folder = '%s/inference/voc_2012_val_cocostyle/' % output_name extra_args = dict( box_only=False, iou_types=["bbox", "segm"], expected_results=[], expected_results_sigma_tol=4, ) dataset_names = cfg.DATASETS.TEST data_loaders = make_data_loader(cfg, is_train=False, is_distributed=False)[0] dataset = data_loaders.dataset print(len(dataset)) predictions = torch.load(os.path.join(output_folder, "predictions.pth")) upscale = 0.6 output_folder = output_folder + 'after_crf' if not os.path.exists(output_folder): os.makedirs(output_folder) evaluate( dataset=dataset, predictions=predictions, output_folder=output_folder, do_CRF=True, upscale=upscale, **extra_args, ) # search "# job lib multiprocess here" to implement multi-processing print(output_folder) # print("17500")
def inference( model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, ): # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) total_timer = Timer() inference_timer = Timer() total_timer.tic() predictions, losses = compute_on_dataset(model, data_loader, device, inference_timer) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset), num_devices, )) (predictions, image_ids) = _accumulate_predictions_from_multiple_gpus(predictions) # torch.cuda.empty_cache() if not is_main_process(): return logger.info('Total items num is {}'.format(len(predictions))) if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) acc, acc_topN, acc_det, acc_rel_softmax = evaluate( dataset=data_loader.dataset, predictions=predictions, image_ids=image_ids, curr_iter='final', output_folder=None, **extra_args) logger.info("current accuracy is: {}".format(acc)) logger.info("current topN accuracy is: {}".format(acc_topN)) logger.info("current accuracy with detection score is: {}".format(acc_det)) logger.info("test done !")
def inference( model, cfg, data_loader, dataset_name, iou_types=("bbox",), box_only=False, bbox_aug=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, eval_attributes=False, save_predictions=False, skip_performance_eval=False, labelmap_file='', ): # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format(dataset_name, len(dataset))) total_timer = Timer() inference_timer = Timer() total_timer.tic() predictions = compute_on_dataset(model, data_loader, device, bbox_aug, inference_timer) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices ) ) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)".format( total_infer_time, inference_timer.total_time * num_devices / len(dataset), num_devices, ) ) predictions = _accumulate_predictions_from_multiple_gpus(predictions, cfg.TEST.GATHER_ON_CPU) if not is_main_process(): return if output_folder and save_predictions: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) if output_folder and cfg.TEST.SAVE_RESULTS_TO_TSV: logger.info("Convert prediction results to tsv format and save.") output_tsv_name = 'predictions_forcebox.tsv' if eval_attributes else 'predictions.tsv' convert_predictions_to_tsv( predictions, dataset, output_folder, data_subset=cfg.TEST.TSV_SAVE_SUBSET, labelmap_file=labelmap_file, output_tsv_name=output_tsv_name, relation_on=cfg.MODEL.RELATION_ON, ) if skip_performance_eval: logger.info("Skip performance evaluation and return.") return extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, save_predictions=save_predictions ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def main(): parser = argparse.ArgumentParser( description="PyTorch Object Detection Inference") parser.add_argument( "--config-file", default= "/private/home/fmassa/github/detectron.pytorch_v2/configs/e2e_faster_rcnn_R_50_C4_1x_caffe2.yaml", metavar="FILE", help="path to config file", ) parser.add_argument("--local_rank", type=int, default=0) parser.add_argument( "opts", help="Modify config options using the command-line", default=None, nargs=argparse.REMAINDER, ) args = parser.parse_args() num_gpus = int( os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1 distributed = num_gpus > 1 if distributed: torch.cuda.set_device(args.local_rank) torch.distributed.init_process_group(backend="nccl", init_method="env://") cfg.merge_from_file(args.config_file) cfg.merge_from_list(args.opts) cfg.freeze() save_dir = "" logger = setup_logger("maskrcnn_benchmark", save_dir, get_rank()) logger.info("Using {} GPUs".format(num_gpus)) logger.info(cfg) logger.info("Collecting env info (might take some time)") logger.info("\n" + collect_env_info()) model = build_detection_model(cfg) model.to(cfg.MODEL.DEVICE) output_dir = cfg.OUTPUT_DIR checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir) _ = checkpointer.load(cfg.MODEL.WEIGHT) iou_types = ("bbox", ) if cfg.MODEL.MASK_ON: iou_types = iou_types + ("segm", ) output_folders = [None] * len(cfg.DATASETS.TEST) dataset_names = cfg.DATASETS.TEST if cfg.OUTPUT_DIR: for idx, dataset_name in enumerate(dataset_names): output_folder = os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name) mkdir(output_folder) output_folders[idx] = output_folder if cfg.TEST.MULTI_SCALE: data_loaders_val = [] for min_size_test, max_size_test in cfg.TEST.MULTI_SIZES: cfg.defrost() cfg.INPUT.MIN_SIZE_TEST = min_size_test cfg.INPUT.MAX_SIZE_TEST = max_size_test cfg.freeze() data_loaders_val.extend( make_data_loader(cfg, is_train=False, is_distributed=distributed)) output_folders = output_folders * len(cfg.TEST.MULTI_SIZES) dataset_names = dataset_names * len(cfg.TEST.MULTI_SIZES) else: data_loaders_val = make_data_loader(cfg, is_train=False, is_distributed=distributed) predictions = [] for output_folder, dataset_name, data_loader_val in zip( output_folders, dataset_names, data_loaders_val): prediction = inference( model, data_loader_val, dataset_name=dataset_name, iou_types=iou_types, box_only=cfg.MODEL.RPN_ONLY, device=cfg.MODEL.DEVICE, expected_results=cfg.TEST.EXPECTED_RESULTS, expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL, output_folder=output_folder, ) synchronize() predictions.append(prediction) if cfg.TEST.MULTI_SCALE: logger.info("Processing multi-scale bbox voting....") voted_predictions = voting( predictions, args.local_rank) # box_voting(predictions, args.local_rank) torch.save(voted_predictions, os.path.join(output_folders[0], 'predictions.pth')) extra_args = dict( box_only=cfg.MODEL.RPN_ONLY, iou_types=iou_types, expected_results=cfg.TEST.EXPECTED_RESULTS, expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL, ) evaluate(dataset=data_loaders_val[0].dataset, predictions=voted_predictions, output_folder=output_folders[0], **extra_args) else: for prediction, output_folder, dataset_name, data_loader_val in zip( predictions, output_folders, dataset_names, data_loaders_val): extra_args = dict( box_only=cfg.MODEL.RPN_ONLY, iou_types=iou_types, expected_results=cfg.TEST.EXPECTED_RESULTS, expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL, ) evaluate(dataset=data_loader_val.dataset, predictions=prediction, output_folder=output_folder, **extra_args) return 0
def inference( model, iteration, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, ): # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) total_timer = Timer() inference_timer = Timer() total_timer.tic() predictions = compute_on_dataset(model, data_loader, device, synchronize_gather=False, timer=inference_timer) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset), num_devices, )) predictions = _accumulate_predictions_from_multiple_gpus( predictions, synchronize_gather=False) if not is_main_process(): return if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, iteration=iteration, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def inference( model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, bbox_aug=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, c2d_json_path=None, rngs=None, cfg=None, test_only=None, ): # import resource # rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) # resource.setrlimit(resource.RLIMIT_NOFILE, (2048, rlimit[1])) # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) total_timer = Timer() inference_timer = Timer() total_timer.tic() # calibrate bn statistics if 'search' in cfg.MODEL.BACKBONE.CONV_BODY: print('recalibrate bn') model = bn_statistic(model, rngs, cfg) print('recalibrate finished!') model.eval() predictions, seg_result = compute_on_dataset(model, data_loader, device, bbox_aug=bbox_aug, c2d_json_path=c2d_json_path, rngs=rngs, test_only=test_only) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset), num_devices, )) predictions = _accumulate_predictions_from_multiple_gpus( predictions, seg_result) if not is_main_process(): return if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, # test_only=test_only ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def inference( model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, ): # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) # compute predictions and do evaluation in the same time to save memory # TODO support multiple gpus evaluation if cfg.MODEL.YOLACT_ON and not cfg.MODEL.YOLACT.CONVERT_MASK_TO_POLY and \ isinstance(dataset, datasets.COCODataset): return do_coco_compute_and_evalute(model, data_loader, device, output_folder) total_timer = Timer() inference_timer = Timer() total_timer.tic() predictions = compute_on_dataset(model, data_loader, device, inference_timer) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset), num_devices, )) predictions = _accumulate_predictions_from_multiple_gpus(predictions) if not is_main_process(): return if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
from maskrcnn_benchmark.data.datasets.evaluation import evaluate from maskrcnn_benchmark.data.datasets.cityscapes import CityscapesDataset import torch from maskrcnn_benchmark.config import cfg from maskrcnn_benchmark.data import make_data_loader output_folder = '/home/brianyao/Documents/maskrcnn-benchmark/inference/cityscapes_fine_instanceonly_seg_val_cocostyle' anno_file = '/media/DATA/Cityscapes/annotations/instancesonly_filtered_gtFine_val.json' root = '/media/DATA/Cityscapes/leftImg8bit/val' dataset = CityscapesDataset(anno_file, root, True) cfg.merge_from_file('../configs/cityscapes/mask_rcnn_coco_eval.yaml') cfg.merge_from_list([]) cfg.freeze() data_loaders_val = make_data_loader(cfg, is_train=False, is_distributed=False) data_loader = data_loaders_val[0] extra_args = dict( box_only=False, iou_types=("bbox", "segm"), expected_results=[], expected_results_sigma_tol=4, ) predictions = torch.load( '../inference/cityscapes_fine_instanceonly_seg_val_cocostyle/predictions.pth' ) evaluate(data_loader.dataset, predictions, output_folder, **extra_args)
def inference( model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, ignore_uncertain=False, # add by hui use_iod_for_ignore=False, # add by hui eval_standard='coco', # add by hui use_last_prediction=False, # add by hui for debug evaluate_method='', # add by hui voc_iou_ths=(0.5, ), # add by hui gt_file=None, # add by hui use_ignore_attr=True): # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) start_time = time.time() if not use_last_prediction: # add by hui predictions = compute_on_dataset(model, data_loader, device) # wait for all processes to complete before measuring the time synchronize() total_time = time.time() - start_time total_time_str = str(datetime.timedelta(seconds=total_time)) logger.info( "Total inference time: {} ({} s / img per device, on {} devices)". format(total_time_str, total_time * num_devices / len(dataset), num_devices)) predictions = _accumulate_predictions_from_multiple_gpus(predictions) else: # add by hui predictions = torch.load(os.path.join(output_folder, 'predictions.pth')) # add by hui if not is_main_process(): return if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) # add by hui ##################################################3 from maskrcnn_benchmark.data import datasets if isinstance(dataset, datasets.COCODataset): extra_args.update( dict(ignore_uncertain=ignore_uncertain, use_iod_for_ignore=use_iod_for_ignore, eval_standard=eval_standard, gt_file=gt_file, use_ignore_attr=use_ignore_attr)) extra_args.update( dict(evaluate_method=evaluate_method, voc_iou_ths=voc_iou_ths)) # ###################################################################################################3 return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def inference( cfg, model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, logger=None, ): load_prediction_from_cache = cfg.TEST.ALLOW_LOAD_FROM_CACHE and output_folder is not None and os.path.exists( os.path.join(output_folder, "eval_results.pytorch")) # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() if logger is None: logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) total_timer = Timer() inference_timer = Timer() total_timer.tic() if load_prediction_from_cache: predictions = torch.load( os.path.join(output_folder, "eval_results.pytorch"), map_location=torch.device("cpu"))['predictions'] else: predictions = compute_on_dataset( model, data_loader, device, synchronize_gather=cfg.TEST.RELATION.SYNC_GATHER, timer=inference_timer) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset), num_devices, )) if not load_prediction_from_cache: predictions = _accumulate_predictions_from_multiple_gpus( predictions, synchronize_gather=cfg.TEST.RELATION.SYNC_GATHER) if not is_main_process(): return -1.0 #if output_folder is not None and not load_prediction_from_cache: # torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) if cfg.TEST.CUSTUM_EVAL: torch.save(predictions, os.path.join(cfg.OUTPUT_DIR, 'custom_prediction.pytorch')) print( str(os.path.join(cfg.OUTPUT_DIR, 'custom_prediction.pytorch')) + ' SAVED !') return -1.0 return evaluate(cfg=cfg, dataset=dataset, predictions=predictions, output_folder=output_folder, logger=logger, **extra_args)
def inference( model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, ): # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) total_timer = Timer() inference_timer = Timer() total_timer.tic() if 'search' in cfg.MODEL.BACKBONE.CONV_BODY: num_states = model.module.num_states if isinstance( model, torch.nn.parallel.DistributedDataParallel) else model.num_states rngs = [3] * num_states print('Evaluation on the single path of supernet: {}'.format(rngs)) model = bn_statistic(model, rngs) else: rngs = None predictions = compute_on_dataset(model, data_loader, device, rngs, inference_timer) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset), num_devices, )) predictions = _accumulate_predictions_from_multiple_gpus(predictions) if not is_main_process(): return if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
import torch from maskrcnn_benchmark.config import cfg from maskrcnn_benchmark.data import make_data_loader from maskrcnn_benchmark.data.datasets.evaluation import evaluate import logging cfg.merge_from_file('configs/vg_attribute.yaml') data_loaders_val = make_data_loader(cfg, is_train=False, is_distributed=False) output_folder = "/var/maskrcnn-benchmark/models/detection_with_attribute_bs8/inference/vg_val/" iou_types = "bbox" dataset = data_loaders_val[0].dataset eval_attributes = False box_only = True predictions = torch.load( "/var/maskrcnn-benchmark/models/detection_with_attribute_bs8/inference/vg_val/predictions.pth" ) evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, box_only=box_only, eval_attributes=eval_attributes, iou_types=iou_types)
def inference( model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, bbox_aug=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, ): # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset # zero-shot models should have class embeddings for inference to map predicted embeddings to classes module = model.module if isinstance( model, torch.nn.parallel.DistributedDataParallel) else model if hasattr(module, 'roi_heads') and 'box' in module.roi_heads: if module.roi_heads['box'].predictor.embedding_based: module.roi_heads['box'].predictor.set_class_embeddings( data_loader.dataset.class_emb_mtx) logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) total_timer = Timer() inference_timer = Timer() total_timer.tic() predictions = compute_on_dataset(model, data_loader, device, bbox_aug, inference_timer) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset), num_devices, )) predictions = _accumulate_predictions_from_multiple_gpus(predictions) if not is_main_process(): return if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def inference(model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, kitti_output=False): # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) total_timer = Timer() inference_timer = Timer() total_timer.tic() predictions = compute_on_dataset(model, data_loader, device, inference_timer) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset), num_devices, )) predictions_multi = _accumulate_predictions_from_multiple_gpus(predictions) if not is_main_process(): return if output_folder: torch.save(predictions_multi, os.path.join(output_folder, "predictions.pth")) if kitti_output: CATEGORIES = [ "__background", "Car", "Pedestrian", "Cyclist", ] # for image_id, prediction in enumerate(predictions): for image_id in range(len(predictions.keys())): original_id = dataset.id_to_img_map[image_id] img_info = dataset.get_img_info(image_id) image_width = img_info["width"] image_height = img_info["height"] prediction = predictions[image_id] prediction = prediction.resize((image_width, image_height)) file_name = os.path.join(output_folder, 'det_results', '{0:06d}.txt'.format(original_id)) with open(file_name, 'w') as output_file: scores = predictions[image_id].get_field("scores").tolist() labels = predictions[image_id].get_field("labels").tolist() labels = [CATEGORIES[i] for i in labels] boxes = prediction.bbox for box, score, label in zip(boxes, scores, labels): left, top, right, bottom = box if score > 0.0001: output_file.write( '{:s} {:.2f} {:d} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}\n' .format( label, -1, -1, -10, # type, truncated, occluded, alpha left, top, right, bottom, # bbox: left, top, right, bottom -1, -1, -1, # dimensions: height, width, length -1, -1, -1, # location: x,y,z -10, np.log(score) # rotation_y, score )) # print('{}: {} {} {} {} Score: {}'.format(label, left, top, right, bottom, score)) print('Saved results to ', file_name) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions_multi, output_folder=output_folder, **extra_args)
def inference( model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, ): # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) total_timer = Timer() inference_timer = Timer() total_timer.tic() predictions = compute_on_dataset(model, data_loader, device, inference_timer) # pdb.set_trace() # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset), num_devices, )) predictions = _accumulate_predictions_from_multiple_gpus(predictions) # pdb.set_trace() # (Pdb) type(predictions) # <class 'list'> # (Pdb) len(predictions) # 1 # (Pdb) predictions[0] # BoxList(num_boxes=1143, image_width=2133, image_height=1200, mode=xyxy) # (Pdb) predictions[0].get_field('labels').shape # torch.Size([1143]) # (Pdb) predictions[0].get_field('scores').shape # torch.Size([1143]) if not is_main_process(): return if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def inference(model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, skip_eval=False, dllogger=None): # convert to a torch.device for efficiency device = torch.device(device) num_devices = (torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1) dataset = data_loader.dataset dllogger.log(step="PARAMETER", data={ "eval_dataset_name": dataset_name, "eval_num_samples": len(dataset) }) start_time = time.time() predictions = compute_on_dataset(model, data_loader, device) # wait for all processes to complete before measuring the time synchronize() total_time = time.time() - start_time total_time_str = str(datetime.timedelta(seconds=total_time)) dllogger.log(step=tuple(), data={ "e2e_infer_time": total_time, "inference_perf_fps": len(dataset) / total_time }) logger = logging.getLogger("maskrcnn_benchmark.inference") logger.info( "Total inference time: {} ({} s / img per device, on {} devices)". format(total_time_str, total_time * num_devices / len(dataset), num_devices)) predictions = _accumulate_predictions_from_multiple_gpus(predictions) if not is_main_process(): return if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) if skip_eval: dllogger.log(step="PARAMETER", data={ "skip_eval": True, "predictions_saved_path": os.path.join(output_folder, "predictions.pth") }) return extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def inference( reid_model, model, data_loaders, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, ): # convert to a torch.device for efficiency modes = ['test', 'query'] # modes = ['query', 'test'] dataset_all = [] predictions_all = [] for mode, data_loader in zip(modes, data_loaders): # for i, batch in enumerate(tqdm(data_loader)): # images, targets, image_ids = batch print(mode, len(data_loader.dataset)) device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(data_loader.dataset))) total_timer = Timer() inference_timer = Timer() total_timer.tic() dataset = data_loader.dataset predictions = compute_on_dataset(reid_model, model, data_loader, device, mode) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(data_loader.dataset), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(data_loader.dataset), num_devices, )) # predictions = _accumulate_predictions_from_multiple_gpus(predictions) # we use 1 gpus if not is_main_process(): return if output_folder: if mode == 'test': torch.save(predictions, os.path.join(output_folder, "predictions.pth")) elif mode == 'query': torch.save(predictions, os.path.join(output_folder, "evaluations.pth")) else: raise KeyError(mode) dataset_all.append(dataset) predictions_all.append(predictions) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset_all, predictions=predictions_all, output_folder=output_folder, **extra_args)
def inference( model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, phase=1, shot=10, split=1, ): # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) total_timer = Timer() inference_timer = Timer() total_timer.tic() print('loading mean class attentions') input_dir = "saved_attentions/" meta_attentions = pickle.load( open( os.path.join( input_dir, 'meta_type_{}'.format(split), str(phase) + '_shots_' + str(shot) + '_mean_class_attentions.pkl'), 'rb')) predictions = compute_on_dataset(model, data_loader, device, inference_timer, meta_attentions) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset), num_devices, )) predictions = _accumulate_predictions_from_multiple_gpus(predictions) if not is_main_process(): return if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def inference( cfg, model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, logger=None, ): load_prediction_from_cache = cfg.TEST.ALLOW_LOAD_FROM_CACHE and output_folder is not None and os.path.exists( os.path.join(output_folder, "eval_results.pytorch")) # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() if logger is None: logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) total_timer = Timer() inference_timer = Timer() total_timer.tic() if load_prediction_from_cache: predictions = torch.load( os.path.join(output_folder, "eval_results.pytorch"), map_location=torch.device("cpu"))['predictions'] else: predictions = compute_on_dataset( model, data_loader, device, synchronize_gather=cfg.TEST.RELATION.SYNC_GATHER, timer=inference_timer) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset), num_devices, )) if not load_prediction_from_cache: predictions = _accumulate_predictions_from_multiple_gpus( predictions, synchronize_gather=cfg.TEST.RELATION.SYNC_GATHER) if not is_main_process(): return -1.0 #if output_folder is not None and not load_prediction_from_cache: # torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) # vocab_file = json.load(open('./datasets/vg/VG-SGG-dicts-with-attri.json')) # idx2label = vocab_file['idx_to_label'] # label is object,size 150 # idx2pred = vocab_file['idx_to_predicate'] # predicate is relation,size 50 # idx2attr = vocab_file['idx_to_attribute'] # attribute, size 200 # def get_info(prediction): # ## get obj # pred_obj = [idx2label[str(i)] for i in prediction.get_field('pred_labels').tolist()] # obj index to label # ## get relation # pred_rel_label = prediction.get_field('pred_rel_scores') # relation score, shape of (nbr_predition, 51) # pred_rel_label[:,0] = 0 # (nbr_predition, 0) # pred_rel_score, pred_rel_label = pred_rel_label.max(-1) # pred_rel_score -> score,pred_rel_label -> max index position # ## concate obj - rela - obj # pred_rel_pair = prediction.get_field('rel_pair_idxs').tolist() # obj1 * obj2 matrix # pred_rels = [(pred_obj[i[0]], idx2pred[str(j)], pred_obj[i[1]]) for i, j in # zip(pred_rel_pair, pred_rel_label.tolist())] # concate to obj1-rela-obj2 triplets # ## get attribute # pred_attr = prediction.get_field('pred_attributes').to(torch.float32) # gey attributes score, shape of (nbr_predition, 201) # pred_attr[:,0] = 0 # (nbr_predition, 0) # pred_attr_score, pred_attr_index = pred_attr.max(-1) # # pred_attr_label = [idx2attr[str(i)] for i in pred_attr_index.tolist()] # # ## concate obj - attr # pred_obj_attr = [(obj, attr) for obj, attr in zip(pred_obj, pred_attr_label)] # obj-attr # return pred_rels, pred_obj, pred_obj_attr, pred_rel_score, pred_rel_label # print('predicting attributes!') # rels, obj, attr, score, label = get_info(predictions) if cfg.TEST.CUSTUM_EVAL: detected_sgg = custom_sgg_post_precessing(predictions) with open(os.path.join(cfg.DETECTED_SGG_DIR, 'custom_prediction.json'), 'w') as outfile: json.dump(detected_sgg, outfile) print( '=====> ' + str(os.path.join(cfg.DETECTED_SGG_DIR, 'custom_prediction.json')) + ' SAVED !') return -1.0 return evaluate(cfg=cfg, dataset=dataset, predictions=predictions, output_folder=output_folder, logger=logger, **extra_args)
def ms_inference( model, data_loader_val_mstest, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, ): # convert to a torch.device for efficiency device = torch.device(device) num_devices = get_world_size() logger = logging.getLogger("maskrcnn_benchmark.inference") # pdb.set_trace() data_loader_small = data_loader_val_mstest[0][0] data_loader_medium = data_loader_val_mstest[1][0] data_loader_large = data_loader_val_mstest[2][0] #################################### small #################################### dataset_small = data_loader_small.dataset logger.info("Start evaluation on {} dataset_small({} images).".format( dataset_name, len(dataset_small))) total_timer = Timer() inference_timer = Timer() total_timer.tic() predictions_small = compute_on_dataset(model, data_loader_small, device, inference_timer) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset_small), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset_small), num_devices, )) predictions_small = _accumulate_predictions_from_multiple_gpus( predictions_small) #################################### medium #################################### dataset_medium = data_loader_medium.dataset logger.info("Start evaluation on {} dataset_medium({} images).".format( dataset_name, len(dataset_medium))) total_timer = Timer() inference_timer = Timer() total_timer.tic() predictions_medium = compute_on_dataset(model, data_loader_medium, device, inference_timer) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset_medium), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset_medium), num_devices, )) predictions_medium = _accumulate_predictions_from_multiple_gpus( predictions_medium) #################################### large #################################### dataset_large = data_loader_large.dataset logger.info("Start evaluation on {} dataset_large({} images).".format( dataset_name, len(dataset_large))) total_timer = Timer() inference_timer = Timer() total_timer.tic() predictions_large = compute_on_dataset(model, data_loader_large, device, inference_timer) # wait for all processes to complete before measuring the time synchronize() total_time = total_timer.toc() total_time_str = get_time_str(total_time) logger.info( "Total run time: {} ({} s / img per device, on {} devices)".format( total_time_str, total_time * num_devices / len(dataset_large), num_devices)) total_infer_time = get_time_str(inference_timer.total_time) logger.info( "Model inference time: {} ({} s / img per device, on {} devices)". format( total_infer_time, inference_timer.total_time * num_devices / len(dataset_large), num_devices, )) predictions_large = _accumulate_predictions_from_multiple_gpus( predictions_large) if not is_main_process(): return dataset = [dataset_small, dataset_medium, dataset_large] predictions = [predictions_small, predictions_medium, predictions_large] # dataset = dataset_large # predictions = predictions_large if output_folder: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict( box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, ) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)
def inference(model, data_loader, dataset_name, iou_types=("bbox", ), box_only=False, device="cuda", expected_results=(), expected_results_sigma_tol=4, output_folder=None, predictions=None, working_directory="/tmp", chunk_predictions=False, compute_pre_results=True, panoptic_confidence_thresh=0.6, panoptic_overlap_thresh=0.5, panoptic_stuff_min_area=(64 * 64)): # convert to a torch.device for efficiency device = torch.device(device) num_devices = (torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1) logger = logging.getLogger("maskrcnn_benchmark.inference") dataset = data_loader.dataset # for now, short circuit for "order". if "order" in iou_types: predictions = compute_order_on_dataset(model, data_loader, device) return given_predictions = not (predictions is None) if not given_predictions: # make sure these are divisible. if chunk_predictions: print("chunking predictions") logger.info("Start evaluation on {} dataset({} images).".format( dataset_name, len(dataset))) start_time = time.time() predictions = compute_on_dataset(model, data_loader, device) # wait for all processes to complete before measuring the time synchronize() total_time = time.time() - start_time total_time_str = str(datetime.timedelta(seconds=total_time)) logger.info( "Total inference time: {} ({} s / img per device, on {} devices)". format(total_time_str, total_time * num_devices / len(dataset), num_devices)) predictions = _accumulate_predictions_from_multiple_gpus( predictions, working_directory=working_directory, skip_gather=chunk_predictions) # if we decided to keep the chunks, we only get filenames back. if not is_main_process(): return # OK, should all be done within a single process now. if output_folder and not given_predictions: if chunk_predictions: parent_path = os.path.dirname(predictions[0]) for i, chunk_path in enumerate(predictions): chunk_save_path = os.path.join(output_folder, os.path.basename(chunk_path)) shutil.move(chunk_path, chunk_save_path) predictions[i] = chunk_save_path # remove the parent. os.rmdir(parent_path) print("chunks:") print(predictions) else: torch.save(predictions, os.path.join(output_folder, "predictions.pth")) extra_args = dict(box_only=box_only, iou_types=iou_types, expected_results=expected_results, expected_results_sigma_tol=expected_results_sigma_tol, working_directory=working_directory, save_panoptic_results=True, save_pre_results=compute_pre_results, panoptic_confidence_thresh=panoptic_confidence_thresh, panoptic_overlap_thresh=panoptic_overlap_thresh, panoptic_stuff_min_area=panoptic_stuff_min_area) return evaluate(dataset=dataset, predictions=predictions, output_folder=output_folder, **extra_args)