def build_dataloader(dataset_cfg, class_names, batch_size, dist, root_path=None, workers=4, logger=None, training=True): dataset = __all__[dataset_cfg.DATASET]( dataset_cfg=dataset_cfg, class_names=class_names, root_path=root_path, training=training, logger=logger, ) if dist: if training: sampler = torch.utils.data.distributed.DistributedSampler(dataset) else: rank, world_size = common_utils.get_dist_info() sampler = DistributedSampler(dataset, world_size, rank, shuffle=False) else: sampler = None dataloader = DataLoader( dataset, batch_size=batch_size, pin_memory=True, num_workers=workers, shuffle=(sampler is None) and training, collate_fn=dataset.collate_batch, drop_last=False, sampler=sampler, timeout=0 ) return dataset, dataloader, sampler
def build_dataloader(dataset_cfg, class_names, batch_size, dist, root_path=None, workers=4, logger=None, training=True, merge_all_iters_to_one_epoch=False, total_epochs=0): dataset = __all__[dataset_cfg.DATASET]( dataset_cfg=dataset_cfg, class_names=class_names, root_path=root_path, training=training, logger=logger, ) if merge_all_iters_to_one_epoch: assert hasattr(dataset, 'merge_all_iters_to_one_epoch') dataset.merge_all_iters_to_one_epoch(merge=True, epochs=total_epochs) if dist: if training: sampler = torch.utils.data.distributed.DistributedSampler(dataset) else: rank, world_size = common_utils.get_dist_info() sampler = DistributedSampler(dataset, world_size, rank, shuffle=False) else: sampler = None dataloader = DataLoader( dataset, batch_size=batch_size, pin_memory=True, num_workers=workers, shuffle=(sampler is None) and training, collate_fn=getattr(dataset, 'collate_batch', None), drop_last=False, sampler=sampler, timeout=0 ) return dataset, dataloader, sampler
def eval_one_epoch(cfg, model, dataloader, epoch_id, logger, dist_test=False, save_to_file=False, result_dir=None): result_dir.mkdir(parents=True, exist_ok=True) final_output_dir = result_dir / 'final_result' / 'data' if save_to_file: final_output_dir.mkdir(parents=True, exist_ok=True) metric = { 'gt_num': 0, } for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST: metric['recall_roi_%s' % str(cur_thresh)] = 0 metric['recall_rcnn_%s' % str(cur_thresh)] = 0 dataset = dataloader.dataset class_names = dataset.class_names det_annos = [] logger.info('*************** EPOCH %s EVALUATION *****************' % epoch_id) if dist_test: num_gpus = torch.cuda.device_count() local_rank = cfg.LOCAL_RANK % num_gpus model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[local_rank], broadcast_buffers=False) model.eval() if cfg.LOCAL_RANK == 0: progress_bar = tqdm.tqdm(total=len(dataloader), leave=True, desc='eval', dynamic_ncols=True) start_time = time.time() for i, batch_dict in enumerate(dataloader): load_data_to_gpu(batch_dict) with torch.no_grad(): pred_dicts, ret_dict = model(batch_dict) disp_dict = {} statistics_info(cfg, ret_dict, metric, disp_dict) annos = dataset.generate_prediction_dicts( batch_dict, pred_dicts, class_names, output_path=final_output_dir if save_to_file else None) det_annos += annos if cfg.LOCAL_RANK == 0: progress_bar.set_postfix(disp_dict) progress_bar.update() if cfg.LOCAL_RANK == 0: progress_bar.close() if dist_test: rank, world_size = common_utils.get_dist_info() det_annos = common_utils.merge_results_dist(det_annos, len(dataset), tmpdir=result_dir / 'tmpdir') metric = common_utils.merge_results_dist([metric], world_size, tmpdir=result_dir / 'tmpdir') logger.info('*************** Performance of EPOCH %s *****************' % epoch_id) sec_per_example = (time.time() - start_time) / len(dataloader.dataset) logger.info('Generate label finished(sec_per_example: %.4f second).' % sec_per_example) if cfg.LOCAL_RANK != 0: return {} ret_dict = {} if dist_test: for key, val in metric[0].items(): for k in range(1, world_size): metric[0][key] += metric[k][key] metric = metric[0] gt_num_cnt = metric['gt_num'] for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST: cur_roi_recall = metric['recall_roi_%s' % str(cur_thresh)] / max( gt_num_cnt, 1) cur_rcnn_recall = metric['recall_rcnn_%s' % str(cur_thresh)] / max( gt_num_cnt, 1) logger.info('recall_roi_%s: %f' % (cur_thresh, cur_roi_recall)) logger.info('recall_rcnn_%s: %f' % (cur_thresh, cur_rcnn_recall)) ret_dict['recall/roi_%s' % str(cur_thresh)] = cur_roi_recall ret_dict['recall/rcnn_%s' % str(cur_thresh)] = cur_rcnn_recall total_pred_objects = 0 for anno in det_annos: total_pred_objects += anno['name'].__len__() logger.info('Average predicted number of objects(%d samples): %.3f' % (len(det_annos), total_pred_objects / max(1, len(det_annos)))) with open(result_dir / 'result.pkl', 'wb') as f: pickle.dump(det_annos, f) result_str, result_dict = dataset.evaluation( det_annos, class_names, eval_metric=cfg.MODEL.POST_PROCESSING.EVAL_METRIC, output_path=final_output_dir) logger.info(result_str) ret_dict.update(result_dict) with open(result_dir / 'ap_dict.pkl', 'wb') as f: pickle.dump(ret_dict, f) logger.info('Result is save to %s' % result_dir) logger.info('****************Evaluation done.*****************') return ret_dict
def eval_one_epoch(cfg, model, dataloader, epoch_id, logger, dist_test=False, save_to_file=False, result_dir=None): result_dir.mkdir(parents=True, exist_ok=True) final_output_dir = result_dir / 'final_result' / 'data' if save_to_file: final_output_dir.mkdir(parents=True, exist_ok=True) metric = { 'gt_num': 0, } for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST: metric['recall_roi_%s' % str(cur_thresh)] = 0 metric['recall_rcnn_%s' % str(cur_thresh)] = 0 dataset = dataloader.dataset class_names = dataset.class_names det_annos = [] logger.info('*************** EPOCH %s EVALUATION *****************' % epoch_id) if dist_test: num_gpus = torch.cuda.device_count() local_rank = cfg.LOCAL_RANK % num_gpus model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[local_rank], broadcast_buffers=False) model.eval() if cfg.LOCAL_RANK == 0: progress_bar = tqdm.tqdm(total=len(dataloader), leave=True, desc='eval', dynamic_ncols=True) start_time = time.time() run_time = 0 query_batch = {} target_batch = {} # pred_dicts= {} # ret_dict = {} # print("FIRST FOWARD PASS") for i, batch_dict in enumerate(dataloader): # print("BATCH DICT",int(batch_dict['frame_id'])) run_start_time = time.time() load_data_to_gpu(batch_dict) if int(batch_dict['frame_id']) == 451: break # print("BATCH FRAME ID", batch_dict['frame_id']) if int(batch_dict['frame_id']) == 27: # print("TARGET LINE PASSED") target_batch = batch_dict.copy() # print("SIZE",len(target_batch)) if int(batch_dict['frame_id']) == 450: # print("QUERY LINE PASSED") query_batch = batch_dict.copy() # print("SIZE",len(query_batch)) # print("PRE MODEL RUN") if len(query_batch) != 0 and len(target_batch) != 0: # print("query_batch", query_batch.keys()) print("query_batch gt boxes", query_batch['gt_boxes']) print("query_batch voxels", query_batch['voxels']) print("query_batch voxel coords", query_batch['voxel_coords']) with torch.no_grad(): # print("query",len(query_batch)) # print("target",len(target_batch)) pred_dicts, ret_dict = model([query_batch, target_batch]) # pred_dicts, ret_dict = model(batch_dict) print(pred_dicts) # print("FORWARD PASS COMPLETE") disp_dict = {} run_end_time = time.time() run_duration = run_end_time - run_start_time run_time += run_duration statistics_info(cfg, ret_dict, metric, disp_dict) annos = dataset.generate_prediction_dicts( batch_dict, pred_dicts, class_names, output_path=final_output_dir if save_to_file else None) det_annos += annos if cfg.LOCAL_RANK == 0: progress_bar.set_postfix(disp_dict) progress_bar.update() if cfg.LOCAL_RANK == 0: progress_bar.close() if dist_test: rank, world_size = common_utils.get_dist_info() det_annos = common_utils.merge_results_dist(det_annos, len(dataset), tmpdir=result_dir / 'tmpdir') metric = common_utils.merge_results_dist([metric], world_size, tmpdir=result_dir / 'tmpdir') else: world_size = 1 logger.info('*************** Performance of EPOCH %s *****************' % epoch_id) logger.info('Run time per sample: %.4f second.' % (run_time / (len(dataloader.dataset) / world_size))) sec_per_example = (time.time() - start_time) / (len(dataloader.dataset) / world_size) logger.info('Generate label finished(sec_per_example: %.4f second).' % sec_per_example) if cfg.LOCAL_RANK != 0: return {} ret_dict = {} if dist_test: for key, val in metric[0].items(): for k in range(1, world_size): metric[0][key] += metric[k][key] metric = metric[0] gt_num_cnt = metric['gt_num'] for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST: cur_roi_recall = metric['recall_roi_%s' % str(cur_thresh)] / max( gt_num_cnt, 1) cur_rcnn_recall = metric['recall_rcnn_%s' % str(cur_thresh)] / max( gt_num_cnt, 1) logger.info('recall_roi_%s: %f' % (cur_thresh, cur_roi_recall)) logger.info('recall_rcnn_%s: %f' % (cur_thresh, cur_rcnn_recall)) ret_dict['recall/roi_%s' % str(cur_thresh)] = cur_roi_recall ret_dict['recall/rcnn_%s' % str(cur_thresh)] = cur_rcnn_recall total_pred_objects = 0 for anno in det_annos: total_pred_objects += anno['name'].__len__() logger.info('Average predicted number of objects(%d samples): %.3f' % (len(det_annos), total_pred_objects / max(1, len(det_annos)))) with open(result_dir / 'result.pkl', 'wb') as f: pickle.dump(det_annos, f) # result_str, result_dict = dataset.evaluation( # det_annos, class_names, # eval_metric=cfg.MODEL.POST_PROCESSING.EVAL_METRIC, # output_path=final_output_dir # ) # logger.info(result_str) # ret_dict.update(result_dict) # logger.info('Result is save to %s' % result_dir) logger.info('****************Evaluation done.*****************') return ret_dict
def eval_one_epoch(cfg, model, dataloader, epoch_id, logger, epsilon, ord, iterations, rec_type, pgd=True, momentum=False, dist_test=False, save_to_file=False, result_dir=None): result_dir.mkdir(parents=True, exist_ok=True) final_output_dir = result_dir / 'final_result' / 'data' if save_to_file: final_output_dir.mkdir(parents=True, exist_ok=True) metric = { 'gt_num': 0, } for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST: metric['recall_roi_%s' % str(cur_thresh)] = 0 metric['recall_rcnn_%s' % str(cur_thresh)] = 0 dataset = dataloader.dataset class_names = dataset.class_names det_annos = [] logger.info('*************** EPOCH %s EVALUATION *****************' % epoch_id) if dist_test: num_gpus = torch.cuda.device_count() local_rank = cfg.LOCAL_RANK % num_gpus model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[local_rank], broadcast_buffers=False) model.eval() if cfg.LOCAL_RANK == 0: progress_bar = tqdm.tqdm(total=len(dataloader), leave=True, desc='eval', dynamic_ncols=True) start_time = time.time() for i, batch_dict in enumerate(dataloader): load_data_to_gpu(batch_dict) original_voxels = batch_dict['voxels'].clone() fgsm(epsilon, batch_dict, model, ord, iterations, pgd=True, momentum=False) # rectification on both if rec_type == "both": rectification(batch_dict, original_voxels) # rectification just on pointcloud elif rec_type == "points": rectification_pointcloud(batch_dict, original_voxels) # rectification just on intencity elif rec_type == "intencity": rectification_angle(batch_dict, original_voxels) else: raise ValueError("3 type of rectification is accepted") with torch.no_grad(): pred_dicts, ret_dict = model(batch_dict) d = {'inf': 'inf', '1': 'L1', '2': 'L2', "2.5": 'L25'} # batch_dict['voxels'].view(-1,4).unique(dim=0).cpu().detach().numpy().astype('float32').tofile(f'/mrtstorage/users/jinwei/semanticvoxelsplus/data/kitti_{d[ord]}_{str(epsilon)[-1]}/training/velodyne/'+(str(batch_dict['frame_id'][0])+'.bin')) # print(f'/mrtstorage/users/jinwei/semanticvoxelsplus/data/kitti_{d[ord]}_{str(epsilon)[-1]}/training/velodyne/'+(str(batch_dict['frame_id'][0])+'.bin')) disp_dict = {} statistics_info(cfg, ret_dict, metric, disp_dict) annos = dataset.generate_prediction_dicts( batch_dict, pred_dicts, class_names, output_path=final_output_dir if save_to_file else None) det_annos += annos if cfg.LOCAL_RANK == 0: progress_bar.set_postfix(disp_dict) progress_bar.update() if cfg.LOCAL_RANK == 0: progress_bar.close() if dist_test: rank, world_size = common_utils.get_dist_info() det_annos = common_utils.merge_results_dist(det_annos, len(dataset), tmpdir=result_dir / 'tmpdir') metric = common_utils.merge_results_dist([metric], world_size, tmpdir=result_dir / 'tmpdir') logger.info('*************** Performance of EPOCH %s *****************' % epoch_id) sec_per_example = (time.time() - start_time) / len(dataloader.dataset) logger.info('Generate label finished(sec_per_example: %.4f second).' % sec_per_example) if cfg.LOCAL_RANK != 0: return {} ret_dict = {} if dist_test: for key, val in metric[0].items(): for k in range(1, world_size): metric[0][key] += metric[k][key] metric = metric[0] gt_num_cnt = metric['gt_num'] for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST: cur_roi_recall = metric['recall_roi_%s' % str(cur_thresh)] / max( gt_num_cnt, 1) cur_rcnn_recall = metric['recall_rcnn_%s' % str(cur_thresh)] / max( gt_num_cnt, 1) logger.info('recall_roi_%s: %f' % (cur_thresh, cur_roi_recall)) logger.info('recall_rcnn_%s: %f' % (cur_thresh, cur_rcnn_recall)) ret_dict['recall/roi_%s' % str(cur_thresh)] = cur_roi_recall ret_dict['recall/rcnn_%s' % str(cur_thresh)] = cur_rcnn_recall total_pred_objects = 0 for anno in det_annos: total_pred_objects += anno['name'].__len__() logger.info('Average predicted number of objects(%d samples): %.3f' % (len(det_annos), total_pred_objects / max(1, len(det_annos)))) with open(result_dir / 'result.pkl', 'wb') as f: pickle.dump(det_annos, f) result_str, result_dict = dataset.evaluation( det_annos, class_names, eval_metric=cfg.MODEL.POST_PROCESSING.EVAL_METRIC, output_path=final_output_dir) logger.info(result_str) print(result_str) ret_dict.update(result_dict) logger.info('Result is save to %s' % result_dir) logger.info('****************Evaluation done.*****************') return ret_dict
def eval_one_epoch(cfg, model, dataloader, epoch_id, logger, dist_test=False, save_to_file=False, result_dir=None): result_dir.mkdir(parents=True, exist_ok=True) final_output_dir = result_dir / 'final_result' / 'data' if save_to_file: final_output_dir.mkdir(parents=True, exist_ok=True) metric = { 'gt_num': 0, } for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST: metric['recall_roi_%s' % str(cur_thresh)] = 0 metric['recall_rcnn_%s' % str(cur_thresh)] = 0 dataset = dataloader.dataset class_names = dataset.class_names det_annos = [] logger.info('*************** EPOCH %s EVALUATION *****************' % epoch_id) if dist_test: num_gpus = torch.cuda.device_count() local_rank = cfg.LOCAL_RANK % num_gpus model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[local_rank], broadcast_buffers=False) model.eval() FPS = [] for i in range(0, 1): if cfg.LOCAL_RANK == 0: progress_bar = tqdm.tqdm(total=len(dataloader), leave=True, desc='eval', dynamic_ncols=True) start_time = time.time() #prefetcher = DataPrefetcher(dataloader) #batch_dict = prefetcher.next() #while batch_dict is not None: for i, batch_dict in enumerate(dataloader): #t0 = time.time() load_data_to_gpu(batch_dict) #print('load data to gpu :',time.time()-t0) with torch.no_grad(): t0 = time.time() pred_dicts, ret_dict = model(batch_dict) t1 = time.time() ''' path = '/media/ddd/data2/3d_MOTS_Ex./Code/OpenPCDet-RandlaNet/tools/time/' if os.path.exists(path+'total_pre.npy'): t = np.load(path+'total_pre.npy') t =list(t) t.append(t1-t0) np.save(path+'total_pre.npy',t) else: np.save(path+'total_pre.npy',[t1-t0]) ''' disp_dict = {} #statistics_info_t = time.time() statistics_info(cfg, ret_dict, metric, disp_dict) t0 = time.time() #print('statistics_info time:',t0-statistics_info_t) annos = dataset.generate_prediction_dicts( batch_dict, pred_dicts, class_names, output_path=final_output_dir if save_to_file else None) t1 = time.time() ''' path = '/media/ddd/data2/3d_MOTS_Ex./Code/OpenPCDet-RandlaNet/tools/time/' if os.path.exists(path+'generate_pre.npy'): t = np.load(path+'generate_pre.npy') t =list(t) t.append(t1-t0) np.save(path+'generate_pre.npy',t) else: np.save(path+'generate_pre.npy',[t1-t0]) ''' #print('generate_prediction_dicts: ', t) det_annos += annos #batch_dict = prefetcher.next() if cfg.LOCAL_RANK == 0: progress_bar.set_postfix(disp_dict) progress_bar.update() end_time = time.time() print('The speed of test is', len(dataloader.dataset) / (end_time - start_time), 'FPS') FPS.append(len(dataloader.dataset) / (end_time - start_time)) if cfg.LOCAL_RANK == 0: progress_bar.close() if dist_test: rank, world_size = common_utils.get_dist_info() det_annos = common_utils.merge_results_dist(det_annos, len(dataset), tmpdir=result_dir / 'tmpdir') metric = common_utils.merge_results_dist([metric], world_size, tmpdir=result_dir / 'tmpdir') logger.info('*************** Performance of EPOCH %s *****************' % epoch_id) sec_per_example = (time.time() - start_time) / len(dataloader.dataset) logger.info('Generate label finished(sec_per_example: %.4f second).' % sec_per_example) print('The speed of test is', np.max(FPS), ' FPS') if cfg.LOCAL_RANK != 0: return {} ret_dict = {} if dist_test: for key, val in metric[0].items(): for k in range(1, world_size): metric[0][key] += metric[k][key] metric = metric[0] gt_num_cnt = metric['gt_num'] for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST: cur_roi_recall = metric['recall_roi_%s' % str(cur_thresh)] / max( gt_num_cnt, 1) cur_rcnn_recall = metric['recall_rcnn_%s' % str(cur_thresh)] / max( gt_num_cnt, 1) logger.info('recall_roi_%s: %f' % (cur_thresh, cur_roi_recall)) logger.info('recall_rcnn_%s: %f' % (cur_thresh, cur_rcnn_recall)) ret_dict['recall/roi_%s' % str(cur_thresh)] = cur_roi_recall ret_dict['recall/rcnn_%s' % str(cur_thresh)] = cur_rcnn_recall total_pred_objects = 0 for anno in det_annos: total_pred_objects += anno['name'].__len__() logger.info('Average predicted number of objects(%d samples): %.3f' % (len(det_annos), total_pred_objects / max(1, len(det_annos)))) with open(result_dir / 'result.pkl', 'wb') as f: pickle.dump(det_annos, f) result_str, result_dict = dataset.evaluation( det_annos, class_names, eval_metric=cfg.MODEL.POST_PROCESSING.EVAL_METRIC, output_path=final_output_dir) logger.info(result_str) ret_dict.update(result_dict) logger.info('Result is save to %s' % result_dir) logger.info('****************Evaluation done.*****************') return ret_dict