def build_dataloader(dataset_cfg, class_names, batch_size, dist, root_path=None, workers=4,
logger=None, training=True):
dataset = __all__[dataset_cfg.DATASET](
dataset_cfg=dataset_cfg,
class_names=class_names,
root_path=root_path,
training=training,
logger=logger,
)
if dist:
if training:
sampler = torch.utils.data.distributed.DistributedSampler(dataset)
else:
rank, world_size = common_utils.get_dist_info()
sampler = DistributedSampler(dataset, world_size, rank, shuffle=False)
else:
sampler = None
dataloader = DataLoader(
dataset, batch_size=batch_size, pin_memory=True, num_workers=workers,
shuffle=(sampler is None) and training, collate_fn=dataset.collate_batch,
drop_last=False, sampler=sampler, timeout=0
)
return dataset, dataloader, sampler
def build_dataloader(dataset_cfg, class_names, batch_size, dist, root_path=None, workers=4,
logger=None, training=True, merge_all_iters_to_one_epoch=False, total_epochs=0):
dataset = __all__[dataset_cfg.DATASET](
dataset_cfg=dataset_cfg,
class_names=class_names,
root_path=root_path,
training=training,
logger=logger,
)
if merge_all_iters_to_one_epoch:
assert hasattr(dataset, 'merge_all_iters_to_one_epoch')
dataset.merge_all_iters_to_one_epoch(merge=True, epochs=total_epochs)
if dist:
if training:
sampler = torch.utils.data.distributed.DistributedSampler(dataset)
else:
rank, world_size = common_utils.get_dist_info()
sampler = DistributedSampler(dataset, world_size, rank, shuffle=False)
else:
sampler = None
dataloader = DataLoader(
dataset, batch_size=batch_size, pin_memory=True, num_workers=workers,
shuffle=(sampler is None) and training, collate_fn=getattr(dataset, 'collate_batch', None),
drop_last=False, sampler=sampler, timeout=0
)
return dataset, dataloader, sampler
def eval_one_epoch(cfg,
model,
dataloader,
epoch_id,
logger,
dist_test=False,
save_to_file=False,
result_dir=None):
result_dir.mkdir(parents=True, exist_ok=True)
final_output_dir = result_dir / 'final_result' / 'data'
if save_to_file:
final_output_dir.mkdir(parents=True, exist_ok=True)
metric = {
'gt_num': 0,
}
for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST:
metric['recall_roi_%s' % str(cur_thresh)] = 0
metric['recall_rcnn_%s' % str(cur_thresh)] = 0
dataset = dataloader.dataset
class_names = dataset.class_names
det_annos = []
logger.info('*************** EPOCH %s EVALUATION *****************' %
epoch_id)
if dist_test:
num_gpus = torch.cuda.device_count()
local_rank = cfg.LOCAL_RANK % num_gpus
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[local_rank], broadcast_buffers=False)
model.eval()
if cfg.LOCAL_RANK == 0:
progress_bar = tqdm.tqdm(total=len(dataloader),
leave=True,
desc='eval',
dynamic_ncols=True)
start_time = time.time()
for i, batch_dict in enumerate(dataloader):
load_data_to_gpu(batch_dict)
with torch.no_grad():
pred_dicts, ret_dict = model(batch_dict)
disp_dict = {}
statistics_info(cfg, ret_dict, metric, disp_dict)
annos = dataset.generate_prediction_dicts(
batch_dict,
pred_dicts,
class_names,
output_path=final_output_dir if save_to_file else None)
det_annos += annos
if cfg.LOCAL_RANK == 0:
progress_bar.set_postfix(disp_dict)
progress_bar.update()
if cfg.LOCAL_RANK == 0:
progress_bar.close()
if dist_test:
rank, world_size = common_utils.get_dist_info()
det_annos = common_utils.merge_results_dist(det_annos,
len(dataset),
tmpdir=result_dir /
'tmpdir')
metric = common_utils.merge_results_dist([metric],
world_size,
tmpdir=result_dir / 'tmpdir')
logger.info('*************** Performance of EPOCH %s *****************' %
epoch_id)
sec_per_example = (time.time() - start_time) / len(dataloader.dataset)
logger.info('Generate label finished(sec_per_example: %.4f second).' %
sec_per_example)
if cfg.LOCAL_RANK != 0:
return {}
ret_dict = {}
if dist_test:
for key, val in metric[0].items():
for k in range(1, world_size):
metric[0][key] += metric[k][key]
metric = metric[0]
gt_num_cnt = metric['gt_num']
for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST:
cur_roi_recall = metric['recall_roi_%s' % str(cur_thresh)] / max(
gt_num_cnt, 1)
cur_rcnn_recall = metric['recall_rcnn_%s' % str(cur_thresh)] / max(
gt_num_cnt, 1)
logger.info('recall_roi_%s: %f' % (cur_thresh, cur_roi_recall))
logger.info('recall_rcnn_%s: %f' % (cur_thresh, cur_rcnn_recall))
ret_dict['recall/roi_%s' % str(cur_thresh)] = cur_roi_recall
ret_dict['recall/rcnn_%s' % str(cur_thresh)] = cur_rcnn_recall
total_pred_objects = 0
for anno in det_annos:
total_pred_objects += anno['name'].__len__()
logger.info('Average predicted number of objects(%d samples): %.3f' %
(len(det_annos), total_pred_objects / max(1, len(det_annos))))
with open(result_dir / 'result.pkl', 'wb') as f:
pickle.dump(det_annos, f)
result_str, result_dict = dataset.evaluation(
det_annos,
class_names,
eval_metric=cfg.MODEL.POST_PROCESSING.EVAL_METRIC,
output_path=final_output_dir)
logger.info(result_str)
ret_dict.update(result_dict)
with open(result_dir / 'ap_dict.pkl', 'wb') as f:
pickle.dump(ret_dict, f)
logger.info('Result is save to %s' % result_dir)
logger.info('****************Evaluation done.*****************')
return ret_dict
def eval_one_epoch(cfg,
model,
dataloader,
epoch_id,
logger,
dist_test=False,
save_to_file=False,
result_dir=None):
result_dir.mkdir(parents=True, exist_ok=True)
final_output_dir = result_dir / 'final_result' / 'data'
if save_to_file:
final_output_dir.mkdir(parents=True, exist_ok=True)
metric = {
'gt_num': 0,
}
for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST:
metric['recall_roi_%s' % str(cur_thresh)] = 0
metric['recall_rcnn_%s' % str(cur_thresh)] = 0
dataset = dataloader.dataset
class_names = dataset.class_names
det_annos = []
logger.info('*************** EPOCH %s EVALUATION *****************' %
epoch_id)
if dist_test:
num_gpus = torch.cuda.device_count()
local_rank = cfg.LOCAL_RANK % num_gpus
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[local_rank], broadcast_buffers=False)
model.eval()
if cfg.LOCAL_RANK == 0:
progress_bar = tqdm.tqdm(total=len(dataloader),
leave=True,
desc='eval',
dynamic_ncols=True)
start_time = time.time()
run_time = 0
query_batch = {}
target_batch = {}
# pred_dicts= {}
# ret_dict = {}
# print("FIRST FOWARD PASS")
for i, batch_dict in enumerate(dataloader):
# print("BATCH DICT",int(batch_dict['frame_id']))
run_start_time = time.time()
load_data_to_gpu(batch_dict)
if int(batch_dict['frame_id']) == 451:
break
# print("BATCH FRAME ID", batch_dict['frame_id'])
if int(batch_dict['frame_id']) == 27:
# print("TARGET LINE PASSED")
target_batch = batch_dict.copy()
# print("SIZE",len(target_batch))
if int(batch_dict['frame_id']) == 450:
# print("QUERY LINE PASSED")
query_batch = batch_dict.copy()
# print("SIZE",len(query_batch))
# print("PRE MODEL RUN")
if len(query_batch) != 0 and len(target_batch) != 0:
# print("query_batch", query_batch.keys())
print("query_batch gt boxes", query_batch['gt_boxes'])
print("query_batch voxels", query_batch['voxels'])
print("query_batch voxel coords", query_batch['voxel_coords'])
with torch.no_grad():
# print("query",len(query_batch))
# print("target",len(target_batch))
pred_dicts, ret_dict = model([query_batch, target_batch])
# pred_dicts, ret_dict = model(batch_dict)
print(pred_dicts)
# print("FORWARD PASS COMPLETE")
disp_dict = {}
run_end_time = time.time()
run_duration = run_end_time - run_start_time
run_time += run_duration
statistics_info(cfg, ret_dict, metric, disp_dict)
annos = dataset.generate_prediction_dicts(
batch_dict,
pred_dicts,
class_names,
output_path=final_output_dir if save_to_file else None)
det_annos += annos
if cfg.LOCAL_RANK == 0:
progress_bar.set_postfix(disp_dict)
progress_bar.update()
if cfg.LOCAL_RANK == 0:
progress_bar.close()
if dist_test:
rank, world_size = common_utils.get_dist_info()
det_annos = common_utils.merge_results_dist(det_annos,
len(dataset),
tmpdir=result_dir /
'tmpdir')
metric = common_utils.merge_results_dist([metric],
world_size,
tmpdir=result_dir / 'tmpdir')
else:
world_size = 1
logger.info('*************** Performance of EPOCH %s *****************' %
epoch_id)
logger.info('Run time per sample: %.4f second.' %
(run_time / (len(dataloader.dataset) / world_size)))
sec_per_example = (time.time() - start_time) / (len(dataloader.dataset) /
world_size)
logger.info('Generate label finished(sec_per_example: %.4f second).' %
sec_per_example)
if cfg.LOCAL_RANK != 0:
return {}
ret_dict = {}
if dist_test:
for key, val in metric[0].items():
for k in range(1, world_size):
metric[0][key] += metric[k][key]
metric = metric[0]
gt_num_cnt = metric['gt_num']
for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST:
cur_roi_recall = metric['recall_roi_%s' % str(cur_thresh)] / max(
gt_num_cnt, 1)
cur_rcnn_recall = metric['recall_rcnn_%s' % str(cur_thresh)] / max(
gt_num_cnt, 1)
logger.info('recall_roi_%s: %f' % (cur_thresh, cur_roi_recall))
logger.info('recall_rcnn_%s: %f' % (cur_thresh, cur_rcnn_recall))
ret_dict['recall/roi_%s' % str(cur_thresh)] = cur_roi_recall
ret_dict['recall/rcnn_%s' % str(cur_thresh)] = cur_rcnn_recall
total_pred_objects = 0
for anno in det_annos:
total_pred_objects += anno['name'].__len__()
logger.info('Average predicted number of objects(%d samples): %.3f' %
(len(det_annos), total_pred_objects / max(1, len(det_annos))))
with open(result_dir / 'result.pkl', 'wb') as f:
pickle.dump(det_annos, f)
# result_str, result_dict = dataset.evaluation(
# det_annos, class_names,
# eval_metric=cfg.MODEL.POST_PROCESSING.EVAL_METRIC,
# output_path=final_output_dir
# )
# logger.info(result_str)
# ret_dict.update(result_dict)
# logger.info('Result is save to %s' % result_dir)
logger.info('****************Evaluation done.*****************')
return ret_dict
def eval_one_epoch(cfg,
model,
dataloader,
epoch_id,
logger,
epsilon,
ord,
iterations,
rec_type,
pgd=True,
momentum=False,
dist_test=False,
save_to_file=False,
result_dir=None):
result_dir.mkdir(parents=True, exist_ok=True)
final_output_dir = result_dir / 'final_result' / 'data'
if save_to_file:
final_output_dir.mkdir(parents=True, exist_ok=True)
metric = {
'gt_num': 0,
}
for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST:
metric['recall_roi_%s' % str(cur_thresh)] = 0
metric['recall_rcnn_%s' % str(cur_thresh)] = 0
dataset = dataloader.dataset
class_names = dataset.class_names
det_annos = []
logger.info('*************** EPOCH %s EVALUATION *****************' %
epoch_id)
if dist_test:
num_gpus = torch.cuda.device_count()
local_rank = cfg.LOCAL_RANK % num_gpus
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[local_rank], broadcast_buffers=False)
model.eval()
if cfg.LOCAL_RANK == 0:
progress_bar = tqdm.tqdm(total=len(dataloader),
leave=True,
desc='eval',
dynamic_ncols=True)
start_time = time.time()
for i, batch_dict in enumerate(dataloader):
load_data_to_gpu(batch_dict)
original_voxels = batch_dict['voxels'].clone()
fgsm(epsilon,
batch_dict,
model,
ord,
iterations,
pgd=True,
momentum=False)
# rectification on both
if rec_type == "both":
rectification(batch_dict, original_voxels)
# rectification just on pointcloud
elif rec_type == "points":
rectification_pointcloud(batch_dict, original_voxels)
# rectification just on intencity
elif rec_type == "intencity":
rectification_angle(batch_dict, original_voxels)
else:
raise ValueError("3 type of rectification is accepted")
with torch.no_grad():
pred_dicts, ret_dict = model(batch_dict)
d = {'inf': 'inf', '1': 'L1', '2': 'L2', "2.5": 'L25'}
# batch_dict['voxels'].view(-1,4).unique(dim=0).cpu().detach().numpy().astype('float32').tofile(f'/mrtstorage/users/jinwei/semanticvoxelsplus/data/kitti_{d[ord]}_{str(epsilon)[-1]}/training/velodyne/'+(str(batch_dict['frame_id'][0])+'.bin'))
# print(f'/mrtstorage/users/jinwei/semanticvoxelsplus/data/kitti_{d[ord]}_{str(epsilon)[-1]}/training/velodyne/'+(str(batch_dict['frame_id'][0])+'.bin'))
disp_dict = {}
statistics_info(cfg, ret_dict, metric, disp_dict)
annos = dataset.generate_prediction_dicts(
batch_dict,
pred_dicts,
class_names,
output_path=final_output_dir if save_to_file else None)
det_annos += annos
if cfg.LOCAL_RANK == 0:
progress_bar.set_postfix(disp_dict)
progress_bar.update()
if cfg.LOCAL_RANK == 0:
progress_bar.close()
if dist_test:
rank, world_size = common_utils.get_dist_info()
det_annos = common_utils.merge_results_dist(det_annos,
len(dataset),
tmpdir=result_dir /
'tmpdir')
metric = common_utils.merge_results_dist([metric],
world_size,
tmpdir=result_dir / 'tmpdir')
logger.info('*************** Performance of EPOCH %s *****************' %
epoch_id)
sec_per_example = (time.time() - start_time) / len(dataloader.dataset)
logger.info('Generate label finished(sec_per_example: %.4f second).' %
sec_per_example)
if cfg.LOCAL_RANK != 0:
return {}
ret_dict = {}
if dist_test:
for key, val in metric[0].items():
for k in range(1, world_size):
metric[0][key] += metric[k][key]
metric = metric[0]
gt_num_cnt = metric['gt_num']
for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST:
cur_roi_recall = metric['recall_roi_%s' % str(cur_thresh)] / max(
gt_num_cnt, 1)
cur_rcnn_recall = metric['recall_rcnn_%s' % str(cur_thresh)] / max(
gt_num_cnt, 1)
logger.info('recall_roi_%s: %f' % (cur_thresh, cur_roi_recall))
logger.info('recall_rcnn_%s: %f' % (cur_thresh, cur_rcnn_recall))
ret_dict['recall/roi_%s' % str(cur_thresh)] = cur_roi_recall
ret_dict['recall/rcnn_%s' % str(cur_thresh)] = cur_rcnn_recall
total_pred_objects = 0
for anno in det_annos:
total_pred_objects += anno['name'].__len__()
logger.info('Average predicted number of objects(%d samples): %.3f' %
(len(det_annos), total_pred_objects / max(1, len(det_annos))))
with open(result_dir / 'result.pkl', 'wb') as f:
pickle.dump(det_annos, f)
result_str, result_dict = dataset.evaluation(
det_annos,
class_names,
eval_metric=cfg.MODEL.POST_PROCESSING.EVAL_METRIC,
output_path=final_output_dir)
logger.info(result_str)
print(result_str)
ret_dict.update(result_dict)
logger.info('Result is save to %s' % result_dir)
logger.info('****************Evaluation done.*****************')
return ret_dict
def eval_one_epoch(cfg,
model,
dataloader,
epoch_id,
logger,
dist_test=False,
save_to_file=False,
result_dir=None):
result_dir.mkdir(parents=True, exist_ok=True)
final_output_dir = result_dir / 'final_result' / 'data'
if save_to_file:
final_output_dir.mkdir(parents=True, exist_ok=True)
metric = {
'gt_num': 0,
}
for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST:
metric['recall_roi_%s' % str(cur_thresh)] = 0
metric['recall_rcnn_%s' % str(cur_thresh)] = 0
dataset = dataloader.dataset
class_names = dataset.class_names
det_annos = []
logger.info('*************** EPOCH %s EVALUATION *****************' %
epoch_id)
if dist_test:
num_gpus = torch.cuda.device_count()
local_rank = cfg.LOCAL_RANK % num_gpus
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[local_rank], broadcast_buffers=False)
model.eval()
FPS = []
for i in range(0, 1):
if cfg.LOCAL_RANK == 0:
progress_bar = tqdm.tqdm(total=len(dataloader),
leave=True,
desc='eval',
dynamic_ncols=True)
start_time = time.time()
#prefetcher = DataPrefetcher(dataloader)
#batch_dict = prefetcher.next()
#while batch_dict is not None:
for i, batch_dict in enumerate(dataloader):
#t0 = time.time()
load_data_to_gpu(batch_dict)
#print('load data to gpu :',time.time()-t0)
with torch.no_grad():
t0 = time.time()
pred_dicts, ret_dict = model(batch_dict)
t1 = time.time()
'''
path = '/media/ddd/data2/3d_MOTS_Ex./Code/OpenPCDet-RandlaNet/tools/time/'
if os.path.exists(path+'total_pre.npy'):
t = np.load(path+'total_pre.npy')
t =list(t)
t.append(t1-t0)
np.save(path+'total_pre.npy',t)
else:
np.save(path+'total_pre.npy',[t1-t0])
'''
disp_dict = {}
#statistics_info_t = time.time()
statistics_info(cfg, ret_dict, metric, disp_dict)
t0 = time.time()
#print('statistics_info time:',t0-statistics_info_t)
annos = dataset.generate_prediction_dicts(
batch_dict,
pred_dicts,
class_names,
output_path=final_output_dir if save_to_file else None)
t1 = time.time()
'''
path = '/media/ddd/data2/3d_MOTS_Ex./Code/OpenPCDet-RandlaNet/tools/time/'
if os.path.exists(path+'generate_pre.npy'):
t = np.load(path+'generate_pre.npy')
t =list(t)
t.append(t1-t0)
np.save(path+'generate_pre.npy',t)
else:
np.save(path+'generate_pre.npy',[t1-t0])
'''
#print('generate_prediction_dicts: ', t)
det_annos += annos
#batch_dict = prefetcher.next()
if cfg.LOCAL_RANK == 0:
progress_bar.set_postfix(disp_dict)
progress_bar.update()
end_time = time.time()
print('The speed of test is',
len(dataloader.dataset) / (end_time - start_time), 'FPS')
FPS.append(len(dataloader.dataset) / (end_time - start_time))
if cfg.LOCAL_RANK == 0:
progress_bar.close()
if dist_test:
rank, world_size = common_utils.get_dist_info()
det_annos = common_utils.merge_results_dist(det_annos,
len(dataset),
tmpdir=result_dir /
'tmpdir')
metric = common_utils.merge_results_dist([metric],
world_size,
tmpdir=result_dir / 'tmpdir')
logger.info('*************** Performance of EPOCH %s *****************' %
epoch_id)
sec_per_example = (time.time() - start_time) / len(dataloader.dataset)
logger.info('Generate label finished(sec_per_example: %.4f second).' %
sec_per_example)
print('The speed of test is', np.max(FPS), ' FPS')
if cfg.LOCAL_RANK != 0:
return {}
ret_dict = {}
if dist_test:
for key, val in metric[0].items():
for k in range(1, world_size):
metric[0][key] += metric[k][key]
metric = metric[0]
gt_num_cnt = metric['gt_num']
for cur_thresh in cfg.MODEL.POST_PROCESSING.RECALL_THRESH_LIST:
cur_roi_recall = metric['recall_roi_%s' % str(cur_thresh)] / max(
gt_num_cnt, 1)
cur_rcnn_recall = metric['recall_rcnn_%s' % str(cur_thresh)] / max(
gt_num_cnt, 1)
logger.info('recall_roi_%s: %f' % (cur_thresh, cur_roi_recall))
logger.info('recall_rcnn_%s: %f' % (cur_thresh, cur_rcnn_recall))
ret_dict['recall/roi_%s' % str(cur_thresh)] = cur_roi_recall
ret_dict['recall/rcnn_%s' % str(cur_thresh)] = cur_rcnn_recall
total_pred_objects = 0
for anno in det_annos:
total_pred_objects += anno['name'].__len__()
logger.info('Average predicted number of objects(%d samples): %.3f' %
(len(det_annos), total_pred_objects / max(1, len(det_annos))))
with open(result_dir / 'result.pkl', 'wb') as f:
pickle.dump(det_annos, f)
result_str, result_dict = dataset.evaluation(
det_annos,
class_names,
eval_metric=cfg.MODEL.POST_PROCESSING.EVAL_METRIC,
output_path=final_output_dir)
logger.info(result_str)
ret_dict.update(result_dict)
logger.info('Result is save to %s' % result_dir)
logger.info('****************Evaluation done.*****************')
return ret_dict
