def kitti_eval(): if float(sys.version[:3]) < 3.6: print("KITTI mAP evaluation can only run with python3.6+") sys.exit(1) args = parse_args() label_dir = os.path.join(args.data_dir, 'KITTI/object/training', 'label_2') split_file = os.path.join(args.data_dir, 'KITTI/ImageSets', '{}.txt'.format(args.split)) final_output_dir = os.path.join(args.result_dir, 'final_result', 'data') name_to_class = {'Car': 0, 'Pedestrian': 1, 'Cyclist': 2} from tools.kitti_object_eval_python.evaluate import evaluate as kitti_evaluate ap_result_str, ap_dict = kitti_evaluate( label_dir, final_output_dir, label_split_file=split_file, current_class=name_to_class[args.class_name]) print("KITTI evaluate: ", ap_result_str, ap_dict)
def eval_one_epoch_joint(model, dataloader, epoch_id, result_dir, logger): np.random.seed(666) MEAN_SIZE = torch.from_numpy(cfg.CLS_MEAN_SIZE[0]).cuda() mode = 'TEST' if args.test else 'EVAL' final_output_dir = os.path.join(result_dir, 'final_result', 'data') os.makedirs(final_output_dir, exist_ok=True) if args.save_result: roi_output_dir = os.path.join(result_dir, 'roi_result', 'data') refine_output_dir = os.path.join(result_dir, 'refine_result', 'data') rpn_output_dir = os.path.join(result_dir, 'rpn_result', 'data') os.makedirs(rpn_output_dir, exist_ok=True) os.makedirs(roi_output_dir, exist_ok=True) os.makedirs(refine_output_dir, exist_ok=True) logger.info('---- EPOCH %s JOINT EVALUATION ----' % epoch_id) logger.info('==> Output file: %s' % result_dir) model.eval() thresh_list = [0.1, 0.3, 0.5, 0.7, 0.9] total_recalled_bbox_list, total_gt_bbox = [0] * 5, 0 total_roi_recalled_bbox_list = [0] * 5 dataset = dataloader.dataset cnt = final_total = total_cls_acc = total_cls_acc_refined = total_rpn_iou = 0 progress_bar = tqdm.tqdm(total=len(dataloader), leave=True, desc='eval') for data in dataloader: cnt += 1 sample_id, pts_rect, pts_features, pts_input = \ data['sample_id'], data['pts_rect'], data['pts_features'], data['pts_input'] batch_size = len(sample_id) inputs = torch.from_numpy(pts_input).cuda(non_blocking=True).float() input_data = {'pts_input': inputs} # model inference ret_dict = model(input_data) roi_scores_raw = ret_dict['roi_scores_raw'] # (B, M) roi_boxes3d = ret_dict['rois'] # (B, M, 7) seg_result = ret_dict['seg_result'].long() # (B, N) rcnn_cls = ret_dict['rcnn_cls'].view(batch_size, -1, ret_dict['rcnn_cls'].shape[1]) rcnn_reg = ret_dict['rcnn_reg'].view( batch_size, -1, ret_dict['rcnn_reg'].shape[1]) # (B, M, C) # bounding box regression anchor_size = MEAN_SIZE if cfg.RCNN.SIZE_RES_ON_ROI: assert False pred_boxes3d = decode_bbox_target( roi_boxes3d.view(-1, 7), rcnn_reg.view(-1, rcnn_reg.shape[-1]), anchor_size=anchor_size, loc_scope=cfg.RCNN.LOC_SCOPE, loc_bin_size=cfg.RCNN.LOC_BIN_SIZE, num_head_bin=cfg.RCNN.NUM_HEAD_BIN, get_xz_fine=True, get_y_by_bin=cfg.RCNN.LOC_Y_BY_BIN, loc_y_scope=cfg.RCNN.LOC_Y_SCOPE, loc_y_bin_size=cfg.RCNN.LOC_Y_BIN_SIZE, get_ry_fine=True).view(batch_size, -1, 7) # scoring if rcnn_cls.shape[2] == 1: raw_scores = rcnn_cls # (B, M, 1) norm_scores = torch.sigmoid(raw_scores) pred_classes = (norm_scores > cfg.RCNN.SCORE_THRESH).long() else: pred_classes = torch.argmax(rcnn_cls, dim=1).view(-1) cls_norm_scores = F.softmax(rcnn_cls, dim=1) raw_scores = rcnn_cls[:, pred_classes] norm_scores = cls_norm_scores[:, pred_classes] # evaluation recalled_num = gt_num = rpn_iou = 0 if not args.test: if not cfg.RPN.FIXED: rpn_cls_label, rpn_reg_label = data['rpn_cls_label'], data[ 'rpn_reg_label'] rpn_cls_label = torch.from_numpy(rpn_cls_label).cuda( non_blocking=True).long() gt_boxes3d = data['gt_boxes3d'] for k in range(batch_size): # calculate recall cur_gt_boxes3d = gt_boxes3d[k] tmp_idx = cur_gt_boxes3d.__len__() - 1 while tmp_idx >= 0 and cur_gt_boxes3d[tmp_idx].sum() == 0: tmp_idx -= 1 if tmp_idx >= 0: cur_gt_boxes3d = cur_gt_boxes3d[:tmp_idx + 1] cur_gt_boxes3d = torch.from_numpy(cur_gt_boxes3d).cuda( non_blocking=True).float() iou3d = iou3d_utils.boxes_iou3d_gpu( pred_boxes3d[k], cur_gt_boxes3d) gt_max_iou, _ = iou3d.max(dim=0) refined_iou, _ = iou3d.max(dim=1) for idx, thresh in enumerate(thresh_list): total_recalled_bbox_list[idx] += (gt_max_iou > thresh).sum().item() recalled_num += (gt_max_iou > 0.7).sum().item() gt_num += cur_gt_boxes3d.shape[0] total_gt_bbox += cur_gt_boxes3d.shape[0] # original recall iou3d_in = iou3d_utils.boxes_iou3d_gpu( roi_boxes3d[k], cur_gt_boxes3d) gt_max_iou_in, _ = iou3d_in.max(dim=0) for idx, thresh in enumerate(thresh_list): total_roi_recalled_bbox_list[idx] += ( gt_max_iou_in > thresh).sum().item() if not cfg.RPN.FIXED: fg_mask = rpn_cls_label > 0 correct = ((seg_result == rpn_cls_label) & fg_mask).sum().float() union = fg_mask.sum().float() + (seg_result > 0).sum().float() - correct rpn_iou = correct / torch.clamp(union, min=1.0) total_rpn_iou += rpn_iou.item() disp_dict = { 'mode': mode, 'recall': '%d/%d' % (total_recalled_bbox_list[3], total_gt_bbox) } progress_bar.set_postfix(disp_dict) progress_bar.update() if args.save_result: # save roi and refine results roi_boxes3d_np = roi_boxes3d.cpu().numpy() pred_boxes3d_np = pred_boxes3d.cpu().numpy() roi_scores_raw_np = roi_scores_raw.cpu().numpy() raw_scores_np = raw_scores.cpu().numpy() rpn_cls_np = ret_dict['rpn_cls'].cpu().numpy() rpn_xyz_np = ret_dict['backbone_xyz'].cpu().numpy() seg_result_np = seg_result.cpu().numpy() output_data = np.concatenate( (rpn_xyz_np, rpn_cls_np.reshape(batch_size, -1, 1), seg_result_np.reshape(batch_size, -1, 1)), axis=2) for k in range(batch_size): cur_sample_id = sample_id[k] calib = dataset.get_calib(cur_sample_id) image_shape = dataset.get_image_shape(cur_sample_id) save_kitti_format(cur_sample_id, calib, roi_boxes3d_np[k], roi_output_dir, roi_scores_raw_np[k], image_shape) save_kitti_format(cur_sample_id, calib, pred_boxes3d_np[k], refine_output_dir, raw_scores_np[k], image_shape) output_file = os.path.join(rpn_output_dir, '%06d.npy' % cur_sample_id) np.save(output_file, output_data.astype(np.float32)) # scores thresh inds = norm_scores > cfg.RCNN.SCORE_THRESH for k in range(batch_size): cur_inds = inds[k].view(-1) if cur_inds.sum() == 0: continue pred_boxes3d_selected = pred_boxes3d[k, cur_inds] raw_scores_selected = raw_scores[k, cur_inds] norm_scores_selected = norm_scores[k, cur_inds] # NMS thresh # rotated nms boxes_bev_selected = kitti_utils.boxes3d_to_bev_torch( pred_boxes3d_selected) keep_idx = iou3d_utils.nms_gpu(boxes_bev_selected, raw_scores_selected, cfg.RCNN.NMS_THRESH).view(-1) pred_boxes3d_selected = pred_boxes3d_selected[keep_idx] scores_selected = raw_scores_selected[keep_idx] pred_boxes3d_selected, scores_selected = pred_boxes3d_selected.cpu( ).numpy(), scores_selected.cpu().numpy() cur_sample_id = sample_id[k] calib = dataset.get_calib(cur_sample_id) final_total += pred_boxes3d_selected.shape[0] image_shape = dataset.get_image_shape(cur_sample_id) save_kitti_format(cur_sample_id, calib, pred_boxes3d_selected, final_output_dir, scores_selected, image_shape) progress_bar.close() # dump empty files split_file = os.path.join(dataset.imageset_dir, '..', '..', 'ImageSets', dataset.split + '.txt') split_file = os.path.abspath(split_file) image_idx_list = [x.strip() for x in open(split_file).readlines()] empty_cnt = 0 for k in range(image_idx_list.__len__()): cur_file = os.path.join(final_output_dir, '%s.txt' % image_idx_list[k]) if not os.path.exists(cur_file): with open(cur_file, 'w') as temp_f: pass empty_cnt += 1 logger.info('empty_cnt=%d: dump empty file %s' % (empty_cnt, cur_file)) ret_dict = {'empty_cnt': empty_cnt} logger.info( '-------------------performance of epoch %s---------------------' % epoch_id) logger.info(str(datetime.now())) avg_rpn_iou = (total_rpn_iou / max(cnt, 1.0)) avg_cls_acc = (total_cls_acc / max(cnt, 1.0)) avg_cls_acc_refined = (total_cls_acc_refined / max(cnt, 1.0)) avg_det_num = (final_total / max(len(dataset), 1.0)) logger.info('final average detections: %.3f' % avg_det_num) logger.info('final average rpn_iou refined: %.3f' % avg_rpn_iou) logger.info('final average cls acc: %.3f' % avg_cls_acc) logger.info('final average cls acc refined: %.3f' % avg_cls_acc_refined) ret_dict['rpn_iou'] = avg_rpn_iou ret_dict['rcnn_cls_acc'] = avg_cls_acc ret_dict['rcnn_cls_acc_refined'] = avg_cls_acc_refined ret_dict['rcnn_avg_num'] = avg_det_num for idx, thresh in enumerate(thresh_list): cur_roi_recall = total_roi_recalled_bbox_list[idx] / max( total_gt_bbox, 1.0) logger.info('total roi bbox recall(thresh=%.3f): %d / %d = %f' % (thresh, total_roi_recalled_bbox_list[idx], total_gt_bbox, cur_roi_recall)) ret_dict['rpn_recall(thresh=%.2f)' % thresh] = cur_roi_recall for idx, thresh in enumerate(thresh_list): cur_recall = total_recalled_bbox_list[idx] / max(total_gt_bbox, 1.0) logger.info( 'total bbox recall(thresh=%.3f): %d / %d = %f' % (thresh, total_recalled_bbox_list[idx], total_gt_bbox, cur_recall)) ret_dict['rcnn_recall(thresh=%.2f)' % thresh] = cur_recall if cfg.TEST.SPLIT != 'test': logger.info('Averate Precision:') name_to_class = {'Car': 0, 'Pedestrian': 1, 'Cyclist': 2} ap_result_str, ap_dict = kitti_evaluate( dataset.label_dir, final_output_dir, label_split_file=split_file, current_class=name_to_class[cfg.CLASSES]) logger.info(ap_result_str) ret_dict.update(ap_dict) logger.info('result is saved to: %s' % result_dir) return ret_dict
def eval_one_epoch_rcnn(model, dataloader, epoch_id, result_dir, logger): np.random.seed(1024) MEAN_SIZE = torch.from_numpy(cfg.CLS_MEAN_SIZE[0]).cuda() mode = 'TEST' if args.test else 'EVAL' final_output_dir = os.path.join(result_dir, 'final_result', 'data') os.makedirs(final_output_dir, exist_ok=True) if args.save_result: roi_output_dir = os.path.join(result_dir, 'roi_result', 'data') refine_output_dir = os.path.join(result_dir, 'refine_result', 'data') os.makedirs(roi_output_dir, exist_ok=True) os.makedirs(refine_output_dir, exist_ok=True) logger.info('---- EPOCH %s RCNN EVALUATION ----' % epoch_id) model.eval() thresh_list = [0.1, 0.3, 0.5, 0.7, 0.9] total_recalled_bbox_list, total_gt_bbox = [0] * 5, 0 total_roi_recalled_bbox_list = [0] * 5 dataset = dataloader.dataset cnt = final_total = total_cls_acc = total_cls_acc_refined = 0 progress_bar = tqdm.tqdm(total=len(dataloader), leave=True, desc='eval') for data in dataloader: sample_id = data['sample_id'] cnt += 1 assert args.batch_size == 1, 'Only support bs=1 here' input_data = {} for key, val in data.items(): if key != 'sample_id': input_data[key] = torch.from_numpy(val).contiguous().cuda( non_blocking=True).float() roi_boxes3d = input_data['roi_boxes3d'] roi_scores = input_data['roi_scores'] if cfg.RCNN.ROI_SAMPLE_JIT: for key, val in input_data.items(): if key in ['gt_iou', 'gt_boxes3d']: continue input_data[key] = input_data[key].unsqueeze(dim=0) else: pts_input = torch.cat( (input_data['pts_input'], input_data['pts_features']), dim=-1) input_data['pts_input'] = pts_input ret_dict = model(input_data) rcnn_cls = ret_dict['rcnn_cls'] rcnn_reg = ret_dict['rcnn_reg'] # bounding box regression anchor_size = MEAN_SIZE if cfg.RCNN.SIZE_RES_ON_ROI: roi_size = input_data['roi_size'] anchor_size = roi_size pred_boxes3d = decode_bbox_target( roi_boxes3d, rcnn_reg, anchor_size=anchor_size, loc_scope=cfg.RCNN.LOC_SCOPE, loc_bin_size=cfg.RCNN.LOC_BIN_SIZE, num_head_bin=cfg.RCNN.NUM_HEAD_BIN, get_xz_fine=True, get_y_by_bin=cfg.RCNN.LOC_Y_BY_BIN, loc_y_scope=cfg.RCNN.LOC_Y_SCOPE, loc_y_bin_size=cfg.RCNN.LOC_Y_BIN_SIZE, get_ry_fine=True) # scoring if rcnn_cls.shape[1] == 1: raw_scores = rcnn_cls.view(-1) norm_scores = torch.sigmoid(raw_scores) pred_classes = (norm_scores > cfg.RCNN.SCORE_THRESH).long() else: pred_classes = torch.argmax(rcnn_cls, dim=1).view(-1) cls_norm_scores = F.softmax(rcnn_cls, dim=1) raw_scores = rcnn_cls[:, pred_classes] norm_scores = cls_norm_scores[:, pred_classes] # evaluation disp_dict = {'mode': mode} if not args.test: gt_boxes3d = input_data['gt_boxes3d'] gt_iou = input_data['gt_iou'] # calculate recall gt_num = gt_boxes3d.shape[0] if gt_num > 0: iou3d = iou3d_utils.boxes_iou3d_gpu(pred_boxes3d, gt_boxes3d) gt_max_iou, _ = iou3d.max(dim=0) refined_iou, _ = iou3d.max(dim=1) for idx, thresh in enumerate(thresh_list): total_recalled_bbox_list[idx] += (gt_max_iou > thresh).sum().item() recalled_num = (gt_max_iou > 0.7).sum().item() total_gt_bbox += gt_num iou3d_in = iou3d_utils.boxes_iou3d_gpu(roi_boxes3d, gt_boxes3d) gt_max_iou_in, _ = iou3d_in.max(dim=0) for idx, thresh in enumerate(thresh_list): total_roi_recalled_bbox_list[idx] += (gt_max_iou_in > thresh).sum().item() # classification accuracy cls_label = (gt_iou > cfg.RCNN.CLS_FG_THRESH).float() cls_valid_mask = ((gt_iou >= cfg.RCNN.CLS_FG_THRESH) | (gt_iou <= cfg.RCNN.CLS_BG_THRESH)).float() cls_acc = ((pred_classes == cls_label.long()).float() * cls_valid_mask).sum() / max(cls_valid_mask.sum(), 1.0) iou_thresh = 0.7 if cfg.CLASSES == 'Car' else 0.5 cls_label_refined = (gt_iou >= iou_thresh).float() cls_acc_refined = ( pred_classes == cls_label_refined.long()).float().sum() / max( cls_label_refined.shape[0], 1.0) total_cls_acc += cls_acc.item() total_cls_acc_refined += cls_acc_refined.item() disp_dict['recall'] = '%d/%d' % (total_recalled_bbox_list[3], total_gt_bbox) disp_dict['cls_acc_refined'] = '%.2f' % cls_acc_refined.item() progress_bar.set_postfix(disp_dict) progress_bar.update() image_shape = dataset.get_image_shape(sample_id) if args.save_result: # save roi and refine results roi_boxes3d_np = roi_boxes3d.cpu().numpy() pred_boxes3d_np = pred_boxes3d.cpu().numpy() calib = dataset.get_calib(sample_id) save_kitti_format(sample_id, calib, roi_boxes3d_np, roi_output_dir, roi_scores, image_shape) save_kitti_format(sample_id, calib, pred_boxes3d_np, refine_output_dir, raw_scores.cpu().numpy(), image_shape) # NMS and scoring # scores thresh inds = norm_scores > cfg.RCNN.SCORE_THRESH if inds.sum() == 0: continue pred_boxes3d_selected = pred_boxes3d[inds] raw_scores_selected = raw_scores[inds] # NMS thresh boxes_bev_selected = kitti_utils.boxes3d_to_bev_torch( pred_boxes3d_selected) keep_idx = iou3d_utils.nms_gpu(boxes_bev_selected, raw_scores_selected, cfg.RCNN.NMS_THRESH) pred_boxes3d_selected = pred_boxes3d_selected[keep_idx] scores_selected = raw_scores_selected[keep_idx] pred_boxes3d_selected, scores_selected = pred_boxes3d_selected.cpu( ).numpy(), scores_selected.cpu().numpy() calib = dataset.get_calib(sample_id) final_total += pred_boxes3d_selected.shape[0] save_kitti_format(sample_id, calib, pred_boxes3d_selected, final_output_dir, scores_selected, image_shape) progress_bar.close() # dump empty files split_file = os.path.join(dataset.imageset_dir, '..', '..', 'ImageSets', dataset.split + '.txt') split_file = os.path.abspath(split_file) image_idx_list = [x.strip() for x in open(split_file).readlines()] empty_cnt = 0 for k in range(image_idx_list.__len__()): cur_file = os.path.join(final_output_dir, '%s.txt' % image_idx_list[k]) if not os.path.exists(cur_file): with open(cur_file, 'w') as temp_f: pass empty_cnt += 1 logger.info('empty_cnt=%d: dump empty file %s' % (empty_cnt, cur_file)) ret_dict = {'empty_cnt': empty_cnt} logger.info( '-------------------performance of epoch %s---------------------' % epoch_id) logger.info(str(datetime.now())) avg_cls_acc = (total_cls_acc / max(cnt, 1.0)) avg_cls_acc_refined = (total_cls_acc_refined / max(cnt, 1.0)) avg_det_num = (final_total / max(cnt, 1.0)) logger.info('final average detections: %.3f' % avg_det_num) logger.info('final average cls acc: %.3f' % avg_cls_acc) logger.info('final average cls acc refined: %.3f' % avg_cls_acc_refined) ret_dict['rcnn_cls_acc'] = avg_cls_acc ret_dict['rcnn_cls_acc_refined'] = avg_cls_acc_refined ret_dict['rcnn_avg_num'] = avg_det_num for idx, thresh in enumerate(thresh_list): cur_roi_recall = total_roi_recalled_bbox_list[idx] / max( total_gt_bbox, 1.0) logger.info('total roi bbox recall(thresh=%.3f): %d / %d = %f' % (thresh, total_roi_recalled_bbox_list[idx], total_gt_bbox, cur_roi_recall)) ret_dict['rpn_recall(thresh=%.2f)' % thresh] = cur_roi_recall for idx, thresh in enumerate(thresh_list): cur_recall = total_recalled_bbox_list[idx] / max(total_gt_bbox, 1.0) logger.info( 'total bbox recall(thresh=%.3f): %d / %d = %f' % (thresh, total_recalled_bbox_list[idx], total_gt_bbox, cur_recall)) ret_dict['rcnn_recall(thresh=%.2f)' % thresh] = cur_recall if cfg.TEST.SPLIT != 'test': logger.info('Averate Precision:') name_to_class = {'Car': 0, 'Pedestrian': 1, 'Cyclist': 2} ap_result_str, ap_dict = kitti_evaluate( dataset.label_dir, final_output_dir, label_split_file=split_file, current_class=name_to_class[cfg.CLASSES]) logger.info(ap_result_str) ret_dict.update(ap_dict) logger.info('result is saved to: %s' % result_dir) return ret_dict
ret_dict['rcnn_cls_acc_refined'] = avg_cls_acc_refined ret_dict['rcnn_avg_num'] = avg_det_num for idx, thresh in enumerate(thresh_list): cur_roi_recall = total_roi_recalled_bbox_list[idx] / max( total_gt_bbox, 1.0) logger.info('total roi bbox recall(thresh=%.3f): %d / %d = %f' % (thresh, total_roi_recalled_bbox_list[idx], total_gt_bbox, cur_roi_recall)) ret_dict['rpn_recall(thresh=%.2f)' % thresh] = cur_roi_recall for idx, thresh in enumerate(thresh_list): cur_recall = total_recalled_bbox_list[idx] / max(total_gt_bbox, 1.0) logger.info( 'total bbox recall(thresh=%.3f): %d / %d = %f' % (thresh, total_recalled_bbox_list[idx], total_gt_bbox, cur_recall)) ret_dict['rcnn_recall(thresh=%.2f)' % thresh] = cur_recall if cfg.TEST.SPLIT != 'test': logger.info('Averate Precision:') name_to_class = {'Car': 0, 'Pedestrian': 1, 'Cyclist': 2} ap_result_str, ap_dict = kitti_evaluate( dataset.label_dir, final_output_dir, label_split_file=split_file, current_class=name_to_class[cfg.CLASSES]) logger.info(ap_result_str) ret_dict.update(ap_dict) logger.info('result is saved to: %s' % result_dir)
def eval(): args = parse_args() print_arguments(args) # check whether the installed paddle is compiled with GPU # PointRCNN model can only run on GPU check_gpu(True) load_config(args.cfg) if args.set_cfgs is not None: set_config_from_list(args.set_cfgs) if not os.path.isdir(args.output_dir): os.makedirs(args.output_dir) if args.eval_mode == 'rpn': cfg.RPN.ENABLED = True cfg.RCNN.ENABLED = False elif args.eval_mode == 'rcnn': cfg.RCNN.ENABLED = True cfg.RPN.ENABLED = cfg.RPN.FIXED = True assert args.batch_size, "batch size must be 1 in rcnn evaluation" elif args.eval_mode == 'rcnn_offline': cfg.RCNN.ENABLED = True cfg.RPN.ENABLED = False assert args.batch_size, "batch size must be 1 in rcnn_offline evaluation" else: raise NotImplementedError("unkown eval mode: {}".format( args.eval_mode)) place = fluid.CUDAPlace(0) exe = fluid.Executor(place) # build model startup = fluid.Program() eval_prog = fluid.Program() with fluid.program_guard(eval_prog, startup): with fluid.unique_name.guard(): eval_model = PointRCNN(cfg, args.batch_size, True, 'TEST') eval_model.build() eval_loader = eval_model.get_loader() eval_feeds = eval_model.get_feeds() eval_outputs = eval_model.get_outputs() eval_prog = eval_prog.clone(True) extra_keys = [] if args.eval_mode == 'rpn': extra_keys.extend(['sample_id', 'rpn_cls_label', 'gt_boxes3d']) if args.save_rpn_feature: extra_keys.extend([ 'pts_rect', 'pts_features', 'pts_input', ]) eval_keys, eval_values = parse_outputs(eval_outputs, prog=eval_prog, extra_keys=extra_keys) eval_compile_prog = fluid.compiler.CompiledProgram( eval_prog).with_data_parallel() exe.run(startup) # load weights if not os.path.isdir(args.weights): assert os.path.exists("{}.pdparams".format(args.weights)), \ "Given resume weight {}.pdparams not exist.".format(args.weights) fluid.load(eval_prog, args.weights, exe) kitti_feature_dir = os.path.join(args.output_dir, 'features') kitti_output_dir = os.path.join(args.output_dir, 'detections', 'data') seg_output_dir = os.path.join(args.output_dir, 'seg_result') if args.save_rpn_feature: if os.path.exists(kitti_feature_dir): shutil.rmtree(kitti_feature_dir) os.makedirs(kitti_feature_dir) if os.path.exists(kitti_output_dir): shutil.rmtree(kitti_output_dir) os.makedirs(kitti_output_dir) if os.path.exists(seg_output_dir): shutil.rmtree(seg_output_dir) os.makedirs(seg_output_dir) # must make sure these dirs existing roi_output_dir = os.path.join('./result_dir', 'roi_result', 'data') refine_output_dir = os.path.join('./result_dir', 'refine_result', 'data') final_output_dir = os.path.join("./result_dir", 'final_result', 'data') if not os.path.exists(final_output_dir): os.makedirs(final_output_dir) if args.save_result: if not os.path.exists(roi_output_dir): os.makedirs(roi_output_dir) if not os.path.exists(refine_output_dir): os.makedirs(refine_output_dir) # get reader kitti_rcnn_reader = KittiRCNNReader( data_dir=args.data_dir, npoints=cfg.RPN.NUM_POINTS, split=cfg.TEST.SPLIT, mode='EVAL', classes=cfg.CLASSES, rcnn_eval_roi_dir=args.rcnn_eval_roi_dir, rcnn_eval_feature_dir=args.rcnn_eval_feature_dir) eval_reader = kitti_rcnn_reader.get_multiprocess_reader( args.batch_size, eval_feeds) eval_loader.set_sample_list_generator(eval_reader, place) thresh_list = [0.1, 0.3, 0.5, 0.7, 0.9] queue = multiprocessing.Queue(128) mgr = multiprocessing.Manager() lock = multiprocessing.Lock() mdict = mgr.dict() if cfg.RPN.ENABLED: mdict['exit_proc'] = 0 mdict['total_gt_bbox'] = 0 mdict['total_cnt'] = 0 mdict['total_rpn_iou'] = 0 for i in range(len(thresh_list)): mdict['total_recalled_bbox_list_{}'.format(i)] = 0 p_list = [] for i in range(METRIC_PROC_NUM): p_list.append( multiprocessing.Process(target=rpn_metric, args=(queue, mdict, lock, thresh_list, args.save_rpn_feature, kitti_feature_dir, seg_output_dir, kitti_output_dir, kitti_rcnn_reader, cfg.CLASSES))) p_list[-1].start() if cfg.RCNN.ENABLED: for i in range(len(thresh_list)): mdict['total_recalled_bbox_list_{}'.format(i)] = 0 mdict['total_roi_recalled_bbox_list_{}'.format(i)] = 0 mdict['exit_proc'] = 0 mdict['total_cls_acc'] = 0 mdict['total_cls_acc_refined'] = 0 mdict['total_det_num'] = 0 mdict['total_gt_bbox'] = 0 p_list = [] for i in range(METRIC_PROC_NUM): p_list.append( multiprocessing.Process( target=rcnn_metric, args=(queue, mdict, lock, thresh_list, kitti_rcnn_reader, roi_output_dir, refine_output_dir, final_output_dir, args.save_result))) p_list[-1].start() try: eval_loader.start() eval_iter = 0 start_time = time.time() cur_time = time.time() while True: eval_outs = exe.run(eval_compile_prog, fetch_list=eval_values, return_numpy=False) rets_dict = { k: (np.array(v), v.recursive_sequence_lengths()) for k, v in zip(eval_keys, eval_outs) } run_time = time.time() - cur_time cur_time = time.time() queue.put(rets_dict) eval_iter += 1 logger.info("[EVAL] iter {}, time: {:.2f}".format( eval_iter, run_time)) except fluid.core.EOFException: # terminate metric process for i in range(METRIC_PROC_NUM): queue.put(None) while mdict['exit_proc'] < METRIC_PROC_NUM: time.sleep(1) for p in p_list: if p.is_alive(): p.join() end_time = time.time() logger.info( "[EVAL] total {} iter finished, average time: {:.2f}".format( eval_iter, (end_time - start_time) / float(eval_iter))) if cfg.RPN.ENABLED: avg_rpn_iou = mdict['total_rpn_iou'] / max(len(kitti_rcnn_reader), 1.) logger.info("average rpn iou: {:.3f}".format(avg_rpn_iou)) total_gt_bbox = float(max(mdict['total_gt_bbox'], 1.0)) for idx, thresh in enumerate(thresh_list): recall = mdict['total_recalled_bbox_list_{}'.format( idx)] / total_gt_bbox logger.info( "total bbox recall(thresh={:.3f}): {} / {} = {:.3f}". format(thresh, mdict['total_recalled_bbox_list_{}'.format(idx)], mdict['total_gt_bbox'], recall)) if cfg.RCNN.ENABLED: cnt = float(max(eval_iter, 1.0)) avg_cls_acc = mdict['total_cls_acc'] / cnt avg_cls_acc_refined = mdict['total_cls_acc_refined'] / cnt avg_det_num = mdict['total_det_num'] / cnt logger.info("avg_cls_acc: {}".format(avg_cls_acc)) logger.info("avg_cls_acc_refined: {}".format(avg_cls_acc_refined)) logger.info("avg_det_num: {}".format(avg_det_num)) total_gt_bbox = float(max(mdict['total_gt_bbox'], 1.0)) for idx, thresh in enumerate(thresh_list): cur_roi_recall = mdict['total_roi_recalled_bbox_list_{}'. format(idx)] / total_gt_bbox logger.info( 'total roi bbox recall(thresh=%.3f): %d / %d = %f' % (thresh, mdict['total_roi_recalled_bbox_list_{}'.format(idx)], total_gt_bbox, cur_roi_recall)) for idx, thresh in enumerate(thresh_list): cur_recall = mdict['total_recalled_bbox_list_{}'.format( idx)] / total_gt_bbox logger.info( 'total bbox recall(thresh=%.2f) %d / %.2f = %.4f' % (thresh, mdict['total_recalled_bbox_list_{}'.format(idx)], total_gt_bbox, cur_recall)) split_file = os.path.join('./data/KITTI', 'ImageSets', 'val.txt') image_idx_list = [x.strip() for x in open(split_file).readlines()] for k in range(image_idx_list.__len__()): cur_file = os.path.join(final_output_dir, '%s.txt' % image_idx_list[k]) if not os.path.exists(cur_file): with open(cur_file, 'w') as temp_f: pass if float(sys.version[:3]) >= 3.6: label_dir = os.path.join('./data/KITTI/object/training', 'label_2') split_file = os.path.join('./data/KITTI', 'ImageSets', 'val.txt') final_output_dir = os.path.join("./result_dir", 'final_result', 'data') name_to_class = {'Car': 0, 'Pedestrian': 1, 'Cyclist': 2} from tools.kitti_object_eval_python.evaluate import evaluate as kitti_evaluate ap_result_str, ap_dict = kitti_evaluate( label_dir, final_output_dir, label_split_file=split_file, current_class=name_to_class["Car"]) logger.info("KITTI evaluate: {}, {}".format( ap_result_str, ap_dict)) else: logger.info( "KITTI mAP only support python version >= 3.6, users can " "run 'python3 tools/kitti_eval.py' to evaluate KITTI mAP.") finally: eval_loader.reset()
def eval_one_epoch_joint(model, dataloader, epoch_id, result_dir, logger): np.random.seed(666) MEAN_SIZE = torch.from_numpy(cfg.CLS_MEAN_SIZE[0]).cuda() mode = 'TEST' if args.test else 'EVAL' final_output_dir = os.path.join(result_dir, 'final_result', 'data') if os.path.exists(final_output_dir): shutil.rmtree(final_output_dir) os.makedirs(final_output_dir, exist_ok=True) logger.info('---- EPOCH %s JOINT EVALUATION ----' % epoch_id) logger.info('==> Output file: %s' % result_dir) model.eval() thresh_list = [0.1, 0.3, 0.5, 0.7, 0.9] total_recalled_bbox_list, total_gt_bbox = [0] * 5, 0 total_roi_recalled_bbox_list = [0] * 5 dataset = dataloader.dataset cnt = final_total = total_cls_acc = total_cls_acc_refined = total_rpn_iou = 0 obj_num = 0 progress_bar = tqdm.tqdm(total=len(dataloader), leave=True, desc='eval') iou_list = [] iou_p_score_list = [] rcnn_p_score_list = [] prop_count = 0 for data in dataloader: # Loading sample sample_id_list, pts_input = data['sample_id'], data['pts_input'] sample_id = sample_id_list[0] cnt += len(sample_id_list) #if cnt < 118: continue #load label if not args.test: gt_boxes3d = data['gt_boxes3d'] obj_num += gt_boxes3d.shape[1] # print(obj_num) if gt_boxes3d.shape[1] == 0: # (B, M, 7) pass else: gt_boxes3d = gt_boxes3d # rpn model inference inputs = torch.from_numpy(pts_input).cuda(non_blocking=True).float() #inputs = inputs[:,torch.argsort(-inputs[0,:,2])] input_data = {'pts_input': inputs} ret_dict = model.rpn_forward(input_data) rpn_cls, rpn_reg = ret_dict['rpn_cls'], ret_dict['rpn_reg'] rpn_backbone_xyz, rpn_backbone_features = ret_dict[ 'backbone_xyz'], ret_dict['backbone_features'] # stage score parsing rpn_scores_raw = rpn_cls[:, :, 0] rpn_scores_norm = torch.sigmoid(rpn_cls[:, :, 0]) rcnn_input_scores = rpn_scores_norm.view(-1).clone() inputs = inputs.view(-1, inputs.shape[-1]) rpn_backbone_features = rpn_backbone_features.view( -1, rpn_backbone_features.shape[-2]) rpn_backbone_xyz = rpn_backbone_xyz.view(-1, rpn_backbone_xyz.shape[-1]) # if VISUAL: # order = torch.argsort(-rpn_scores_norm).view(-1) # inputs = inputs.view(-1,inputs.shape[-1])[order] # rpn_scores_norm = rpn_scores_norm.view(-1)[order] # rpn_backbone_features = rpn_backbone_features.view(-1,rpn_backbone_features.shape[-1])[order] # # norm_feature = F.normalize(rpn_backbone_features) # similarity = norm_feature.mm(norm_feature.t()) # # inputs_plt = inputs.detach().cpu().numpy() # scores_plt = rpn_scores_norm.detach().cpu().numpy() # similarity_plt = similarity.detach().cpu().numpy() # # # fig = plt.figure(figsize=(10, 10)) # plt.axes(facecolor='silver') # plt.axis([-30,30,0,70]) # plt.title('point_regressed_center %06d'%sample_id) # plt.scatter(inputs_plt[:, 0], inputs_plt[:, 2], s=15, c=scores_plt[:], edgecolor='none', # cmap=plt.get_cmap('rainbow'), alpha=1, marker='.', vmin=0, vmax=1) # if args.test==False: # gt_boxes3d = gt_boxes3d.reshape(-1,7) # plt.scatter(gt_boxes3d[:, 0], gt_boxes3d[:, 2], s=200, c='blue', # alpha=0.5, marker='+', vmin=-1, vmax=1) # plt.show() # # for i in range(similarity_plt.shape[0]): # fig = plt.figure(figsize=(10, 10)) # plt.axes(facecolor='silver') # plt.axis([-30, 30, 0, 70]) # sm_plt = similarity_plt[i] # plt.scatter(inputs_plt[i, 0].reshape(-1), inputs_plt[i, 2].reshape(-1), s=400, c='blue', # alpha=0.5, marker='+', vmin=-1, vmax=1) # plt.scatter(inputs_plt[:, 0], inputs_plt[:, 2], s=15, c=(sm_plt[:]+scores_plt[:])/2, edgecolor='none', # cmap=plt.get_cmap('rainbow'), alpha=1, marker='.', vmin=0, vmax=1) # plt.show() # thresh select and jump out # rpn_mask = rpn_scores_norm.view(-1) > cfg.RPN.SCORE_THRESH # if rpn_mask.float().sum() == 0: continue # rpn_scores_raw = rpn_scores_raw.view(-1)[rpn_mask] # rpn_scores_norm = rpn_scores_norm.view(-1)[rpn_mask] # rpn_reg = rpn_reg.view(-1, rpn_reg.shape[-1])[rpn_mask] # rpn_backbone_xyz = rpn_backbone_xyz.view(-1, rpn_backbone_xyz.shape[-1])[rpn_mask] # generate rois rpn_rois = decode_center_target( rpn_backbone_xyz, rpn_reg.view(-1, rpn_reg.shape[-1]), loc_scope=cfg.RPN.LOC_SCOPE, loc_bin_size=cfg.RPN.LOC_BIN_SIZE, ).view(-1, 3) rpn_reg_dist = (rpn_rois - rpn_backbone_xyz).clone() #similarity = torch.cosine_similarity(rpn_backbone_xyz[:, [0, 2]], rpn_reg_dist[:, [0, 2]], dim=1) # # thresh select and jump out rpn_mask = (rpn_scores_norm.view(-1) > cfg.RPN.SCORE_THRESH) & ( rpn_reg_dist[:, [0, 2]].pow(2).sum(-1).sqrt() > 0.2) #\ #& (similarity > -0.7) if rpn_mask.float().sum() == 0: continue rpn_scores_raw = rpn_scores_raw.view(-1)[rpn_mask] rpn_scores_norm = rpn_scores_norm.view(-1)[rpn_mask] rpn_rois = rpn_rois[rpn_mask] rpn_backbone_xyz = rpn_backbone_xyz.view( -1, rpn_backbone_xyz.shape[-1])[rpn_mask] # radius NMS # sort by center score sort_points = torch.argsort(-rpn_scores_raw) rpn_rois = rpn_rois[sort_points] rpn_scores_norm = rpn_scores_norm[sort_points] rpn_scores_raw = rpn_scores_raw[sort_points] if rpn_rois.shape[0] > 1: keep_id = [0] prop_prop_distance = distance_2(rpn_rois[:, [0, 2]], rpn_rois[:, [0, 2]]) for i in range(1, rpn_rois.shape[0]): #if torch.min(prop_prop_distance[:i, i], dim=-1)[0] > 0.3: if torch.min(prop_prop_distance[keep_id, i], dim=-1)[0] > 0.3: keep_id.append(i) rpn_center = rpn_rois[keep_id][:, [0, 2]] rpn_scores_norm = rpn_scores_norm[keep_id] rpn_scores_raw = rpn_scores_raw[keep_id] else: rpn_center = rpn_rois[:, [0, 2]] rpn_scores_norm = rpn_scores_norm rpn_scores_raw = rpn_scores_raw # #rcnn input select: point_center_distance = distance_2(rpn_center, inputs[:, [0, 2]]) cur_proposal_points_index = (torch.min(point_center_distance, dim=-1)[0] < 4.0) point_center_distance = point_center_distance[ cur_proposal_points_index] inputs = inputs[cur_proposal_points_index] rcnn_input_scores = rcnn_input_scores.view( -1)[cur_proposal_points_index] if VISUAL: inputs_plt = inputs.detach().cpu().numpy() scores_plt = rcnn_input_scores.detach().cpu().numpy() # point_center= rpn_center[rpn_scores_norm > 0.5] # point_center_score = rpn_scores_norm[rpn_scores_norm > 0.5] point_center = rpn_center point_center_score = rpn_scores_norm fig = plt.figure(figsize=(10, 10)) plt.axes(facecolor='silver') plt.axis([-30, 30, 0, 70]) point_center_plt = point_center.cpu().numpy() plt.title('point_regressed_center %06d' % sample_id) plt.scatter(inputs_plt[:, 0], inputs_plt[:, 2], s=15, c=scores_plt[:], edgecolor='none', cmap=plt.get_cmap('rainbow'), alpha=1, marker='.', vmin=0, vmax=1) if point_center.shape[0] > 0: plt.scatter(point_center_plt[:, 0], point_center_plt[:, 1], s=200, c='white', alpha=0.5, marker='x', vmin=-1, vmax=1) if args.test == False: gt_boxes3d = gt_boxes3d.reshape(-1, 7) plt.scatter(gt_boxes3d[:, 0], gt_boxes3d[:, 2], s=200, c='blue', alpha=0.5, marker='+', vmin=-1, vmax=1) plt.savefig('../visual/rpn.jpg') # RCNN stage box_list = [] raw_score_list = [] iou_score_list = [] inputs[:, 1] -= 1.65 point_center_distance = distance_2(rpn_center[:, :], inputs[:, [0, 2]]) #for c in range(min(rpn_center.shape[0],100)): prop_count += rpn_center.shape[0] print('num %d' % (prop_count / float(cnt))) for c in range(rpn_center.shape[0]): # rcnn input generate cur_input = inputs.clone() cur_input_score = rcnn_input_scores.clone() # if COSINE_DISTANCE: # cur_center_points_index = ((point_center_distance[:, c] < 4.0) & \ # (point_prop_cos_matrix[:, c] > COS_THRESH) | \ # (point_center_distance[:, c].view(-1) < 0.7)).view(-1) # else: cur_center_points_index = (point_center_distance[:, c] < 4.0).view(-1) if cur_center_points_index.long().sum() == 0: continue cur_center_points_xyz = cur_input[cur_center_points_index, :3] cur_center_points_xyz[:, 0] -= rpn_center[c, 0] cur_center_points_xyz[:, 2] -= rpn_center[c, 1] cur_center_points_r = cur_input[cur_center_points_index, 3].view(-1, 1) cur_center_points_mask = (cur_input_score[cur_center_points_index] > 0.5).view(-1, 1).float() # # easy sample sampling # if pts_input.shape[0]>512: # cur_input = torch.cat((cur_center_points_xyz, cur_center_points_r, # (cur_input_score[cur_center_points_index] > 0.5).view(-1, 1).float()), dim=-1) # pts_input = cur_input # pts_input = pts_input[:min(pts_input.shape[0], 2000), :] # pts_input = pts_input[:, :] # sample_index = fps(pts_input[:, 0:3].contiguous(), ratio=min(512 / pts_input.shape[0], 0.99), # random_start=False) # perm = sample_index # while sample_index.shape[0] < 512: # sample_index = torch.cat( # (sample_index, perm[:min(perm.shape[0], 512 - sample_index.shape[0])]), dim=0) # # cur_center_points_xyz = pts_input[sample_index, 0:3] # cur_center_points_r = pts_input[sample_index, 3].reshape(-1, 1) # cur_center_points_mask = pts_input[sample_index, 4].reshape(-1, 1) cur_center_points_xyz = cur_center_points_xyz.unsqueeze(0).float() cur_center_points_r = cur_center_points_r.unsqueeze(0).float() cur_center_points_mask = cur_center_points_mask.unsqueeze( 0).float() - 0.5 input_data = { 'cur_box_point': cur_center_points_xyz, 'cur_box_reflect': cur_center_points_r, 'train_mask': cur_center_points_mask, } # # globaly random sampling # pts_input = pts_input[:min(pts_input.shape[0], self.npoints), :] # sample_index = np.arange(0, pts_input.shape[0], 1).astype(np.int) # perm = np.copy(sample_index) # while sample_index.shape[0] < self.npoints: # sample_index = np.concatenate( # (sample_index, perm[:min(perm.shape[0], self.npoints - sample_index.shape[0])])) # # cur_box_point = pts_input[sample_index, 0:3] # cur_box_reflect = pts_input[sample_index, 3].reshape(-1, 1) # cur_prob_mask = pts_input[sample_index, 4].reshape(-1, 1) # gt_mask = pts_input[sample_index, 5].reshape(-1, 1) # rcnn model inference ret_dict = model.rcnn_forward(input_data) rcnn_cls = ret_dict['rcnn_cls'] ioun_cls = ret_dict['ioun_cls'] rcnn_reg = ret_dict['rcnn_reg'] rcnn_iou = ret_dict['rcnn_iou'] rcnn_ref = ret_dict['rcnn_ref'].view(1, 1, -1) rcnn_box3d = ret_dict['pred_boxes3d'] refined_box = ret_dict['refined_box'] rcnn_box3d = refined_box rcnn_box3d[:, :, 6] = rcnn_box3d[:, :, 6] % (np.pi * 2) if rcnn_box3d[:, :, 6] > np.pi: rcnn_box3d[:, :, 6] -= np.pi * 2 rcnn_box3d[:, :, 0] += rpn_center[c][0] rcnn_box3d[:, :, 2] += rpn_center[c][1] rcnn_box3d[:, :, 1] += 1.65 box_list.append(rcnn_box3d) raw_score_list.append(rcnn_cls.view(1, 1)) #raw_score_list.append(ioun_cls.view(1,1)) iou_score_list.append(rcnn_iou.view(1, 1)) rcnn_box3d = torch.cat((box_list), dim=1) raw_rcnn_score = torch.cat((raw_score_list), dim=0).unsqueeze(0).float() norm_ioun_score = torch.cat((iou_score_list), dim=0).unsqueeze(0).float() # scoring pred_boxes3d = rcnn_box3d norm_ioun_score = norm_ioun_score raw_rcnn_score = raw_rcnn_score norm_rcnn_score = torch.sigmoid(raw_rcnn_score) # scores thresh pred_h = pred_boxes3d[:, :, 3].view(-1) pred_w = pred_boxes3d[:, :, 4].view(-1) pred_l = pred_boxes3d[:, :, 5].view(-1) inds = (norm_rcnn_score > cfg.RCNN.SCORE_THRESH) & ( norm_ioun_score > cfg.IOUN.SCORE_THRESH) inds = inds.view(-1) #size filiter # inds = inds & \ # (pred_h > 1.2) & (pred_h < 2.2) & \ # (pred_w > 1.3) & (pred_w < 2.0) & \ # (pred_l > 2.2) & (pred_l < 5.0) inds = inds & \ (pred_h > 1.1) & (pred_h < 2.3) & \ (pred_w > 1.2) & (pred_w < 2.1) & \ (pred_l > 2.1) & (pred_l < 5.1) pred_boxes3d = pred_boxes3d[:, inds] norm_rcnn_score = norm_rcnn_score[:, inds] norm_ioun_score = norm_ioun_score[:, inds] raw_rcnn_score = raw_rcnn_score[:, inds] if pred_boxes3d.shape[1] == 0: continue # evaluation recalled_num = gt_num = 0 if not args.test: gt_boxes3d = data['gt_boxes3d'] for k in range(1): # calculate recall cur_gt_boxes3d = gt_boxes3d[k] tmp_idx = cur_gt_boxes3d.__len__() - 1 while tmp_idx >= 0 and cur_gt_boxes3d[tmp_idx].sum() == 0: tmp_idx -= 1 if tmp_idx >= 0: cur_gt_boxes3d = cur_gt_boxes3d[:tmp_idx + 1] cur_gt_boxes3d = torch.from_numpy(cur_gt_boxes3d).cuda( non_blocking=True).float() _, iou3d = iou3d_utils.boxes_iou3d_gpu( pred_boxes3d[k], cur_gt_boxes3d) gt_max_iou, _ = iou3d.max(dim=0) refined_iou, _ = iou3d.max(dim=1) iou_list.append(refined_iou.view(-1, 1)) iou_p_score_list.append(norm_ioun_score.view(-1, 1)) rcnn_p_score_list.append(norm_rcnn_score.view(-1, 1)) for idx, thresh in enumerate(thresh_list): total_recalled_bbox_list[idx] += (gt_max_iou > thresh).sum().item() recalled_num += (gt_max_iou > 0.7).sum().item() gt_num += cur_gt_boxes3d.shape[0] total_gt_bbox += cur_gt_boxes3d.shape[0] if cnt == 1000: iou_clloe = torch.cat(iou_list, dim=0).detach().cpu().numpy() iou_score_clloe = torch.cat(iou_p_score_list, dim=0).detach().cpu().numpy() plt.axis([-.1, 1.1, -.1, 1.1]) plt.scatter(iou_clloe, iou_score_clloe, s=20, c='blue', edgecolor='none', cmap=plt.get_cmap('YlOrRd'), alpha=1, marker='.') plt.savefig(os.path.join(result_dir, 'distributercnn.png')) disp_dict = { 'mode': mode, 'recall': '%d/%d' % (total_recalled_bbox_list[3], total_gt_bbox) } progress_bar.set_postfix(disp_dict) progress_bar.update() if VISUAL: fig, ax = plt.subplots(figsize=(10, 10)) inputs_plt = inputs.detach().cpu().numpy() #plt.axes(facecolor='silver') plt.axis([-35, 35, 0, 70]) plt.scatter(inputs_plt[:, 0], inputs_plt[:, 2], s=15, c=inputs_plt[:, 1], edgecolor='none', cmap=plt.get_cmap('Blues'), alpha=1, marker='.', vmin=-1, vmax=2) pred_boxes3d_numpy = pred_boxes3d[0].detach().cpu().numpy() pred_boxes3d_corner = kitti_utils.boxes3d_to_corners3d( pred_boxes3d_numpy, rotate=True) for o in range(pred_boxes3d_corner.shape[0]): print_box_corner = pred_boxes3d_corner[o] x1, x2, x3, x4 = print_box_corner[0:4, 0] z1, z2, z3, z4 = print_box_corner[0:4, 2] polygon = np.zeros([5, 2], dtype=np.float32) polygon[0, 0] = x1 polygon[1, 0] = x2 polygon[2, 0] = x3 polygon[3, 0] = x4 polygon[4, 0] = x1 polygon[0, 1] = z1 polygon[1, 1] = z2 polygon[2, 1] = z3 polygon[3, 1] = z4 polygon[4, 1] = z1 line1 = [(x1, z1), (x2, z2)] line2 = [(x2, z2), (x3, z3)] line3 = [(x3, z3), (x4, z4)] line4 = [(x4, z4), (x1, z1)] (line1_xs, line1_ys) = zip(*line1) (line2_xs, line2_ys) = zip(*line2) (line3_xs, line3_ys) = zip(*line3) (line4_xs, line4_ys) = zip(*line4) ax.add_line( Line2D(line1_xs, line1_ys, linewidth=1, color='green')) ax.add_line( Line2D(line2_xs, line2_ys, linewidth=1, color='red')) ax.add_line( Line2D(line3_xs, line3_ys, linewidth=1, color='red')) ax.add_line( Line2D(line4_xs, line4_ys, linewidth=1, color='red')) # gt visualize if args.test == False and data['gt_boxes3d'].shape[1] > 0: gt_boxes3d_corner = kitti_utils.boxes3d_to_corners3d( data['gt_boxes3d'].reshape(-1, 7), rotate=True) for o in range(gt_boxes3d_corner.shape[0]): print_box_corner = gt_boxes3d_corner[o] x1, x2, x3, x4 = print_box_corner[0:4, 0] z1, z2, z3, z4 = print_box_corner[0:4, 2] polygon = np.zeros([5, 2], dtype=np.float32) polygon[0, 0] = x1 polygon[1, 0] = x2 polygon[2, 0] = x3 polygon[3, 0] = x4 polygon[4, 0] = x1 polygon[0, 1] = z1 polygon[1, 1] = z2 polygon[2, 1] = z3 polygon[3, 1] = z4 polygon[4, 1] = z1 line1 = [(x1, z1), (x2, z2)] line2 = [(x2, z2), (x3, z3)] line3 = [(x3, z3), (x4, z4)] line4 = [(x4, z4), (x1, z1)] (line1_xs, line1_ys) = zip(*line1) (line2_xs, line2_ys) = zip(*line2) (line3_xs, line3_ys) = zip(*line3) (line4_xs, line4_ys) = zip(*line4) ax.add_line( Line2D(line1_xs, line1_ys, linewidth=1, color='yellow')) ax.add_line( Line2D(line2_xs, line2_ys, linewidth=1, color='purple')) ax.add_line( Line2D(line3_xs, line3_ys, linewidth=1, color='purple')) ax.add_line( Line2D(line4_xs, line4_ys, linewidth=1, color='purple')) plt.savefig('../visual/rcnn.jpg') # scores thresh inds = (norm_rcnn_score > cfg.RCNN.SCORE_THRESH) & ( norm_ioun_score > cfg.IOUN.SCORE_THRESH) #inds = (norm_ioun_score > cfg.IOUN.SCORE_THRESH) for k in range(1): cur_inds = inds[k].view(-1) if cur_inds.sum() == 0: continue pred_boxes3d_selected = pred_boxes3d[k, cur_inds] norm_iou_scores_selected = norm_ioun_score[k, cur_inds] raw_rcnn_score_selected = raw_rcnn_score[k, cur_inds] #traditional nms # NMS thresh rotated nms # boxes_bev_selected = kitti_utils.boxes3d_to_bev_torch(pred_boxes3d_selected) # #score NMS # # boxes_bev_selected[:,-1] += np.pi/2 # keep_idx = iou3d_utils.nms_normal_gpu(boxes_bev_selected, norm_iou_scores_selected, cfg.RCNN.NMS_THRESH).view(-1) # pred_boxes3d_selected = pred_boxes3d_selected[keep_idx] # norm_iou_scores_selected = norm_iou_scores_selected[keep_idx] # raw_rcnn_score_selected = raw_rcnn_score_selected[keep_idx] #self NMS sort_boxes = torch.argsort(-norm_iou_scores_selected.view(-1)) pred_boxes3d_selected = pred_boxes3d_selected[sort_boxes] norm_iou_scores_selected = norm_iou_scores_selected[sort_boxes] if pred_boxes3d_selected.shape[0] > 1: keep_id = [0] iou2d, iou3d = iou3d_utils.boxes_iou3d_gpu( pred_boxes3d_selected, pred_boxes3d_selected) for i in range(1, pred_boxes3d_selected.shape[0]): # if torch.min(prop_prop_distance[:i, i], dim=-1)[0] > 0.3: if torch.max(iou2d[keep_id, i], dim=-1)[0] < 0.01: keep_id.append(i) pred_boxes3d_selected = pred_boxes3d_selected[keep_id] norm_iou_scores_selected = norm_iou_scores_selected[keep_id] else: pred_boxes3d_selected = pred_boxes3d_selected norm_iou_scores_selected = norm_iou_scores_selected pred_boxes3d_selected, norm_iou_scores_selected = pred_boxes3d_selected.cpu( ).numpy(), norm_iou_scores_selected.cpu().numpy() cur_sample_id = sample_id calib = dataset.get_calib(cur_sample_id) final_total += pred_boxes3d_selected.shape[0] image_shape = dataset.get_image_shape(cur_sample_id) save_kitti_format(cur_sample_id, calib, pred_boxes3d_selected, final_output_dir, norm_iou_scores_selected, image_shape) if VISUAL: fig, ax = plt.subplots(figsize=(10, 10)) inputs_plt = inputs.detach().cpu().numpy() # plt.axes(facecolor='silver') plt.axis([-35, 35, 0, 70]) plt.scatter(inputs_plt[:, 0], inputs_plt[:, 2], s=15, c=inputs_plt[:, 1], edgecolor='none', cmap=plt.get_cmap('Blues'), alpha=1, marker='.', vmin=-1, vmax=2) pred_boxes3d_numpy = pred_boxes3d_selected pred_boxes3d_corner = kitti_utils.boxes3d_to_corners3d( pred_boxes3d_numpy, rotate=True) for o in range(pred_boxes3d_corner.shape[0]): print_box_corner = pred_boxes3d_corner[o] x1, x2, x3, x4 = print_box_corner[0:4, 0] z1, z2, z3, z4 = print_box_corner[0:4, 2] polygon = np.zeros([5, 2], dtype=np.float32) polygon[0, 0] = x1 polygon[1, 0] = x2 polygon[2, 0] = x3 polygon[3, 0] = x4 polygon[4, 0] = x1 polygon[0, 1] = z1 polygon[1, 1] = z2 polygon[2, 1] = z3 polygon[3, 1] = z4 polygon[4, 1] = z1 line1 = [(x1, z1), (x2, z2)] line2 = [(x2, z2), (x3, z3)] line3 = [(x3, z3), (x4, z4)] line4 = [(x4, z4), (x1, z1)] (line1_xs, line1_ys) = zip(*line1) (line2_xs, line2_ys) = zip(*line2) (line3_xs, line3_ys) = zip(*line3) (line4_xs, line4_ys) = zip(*line4) ax.add_line( Line2D(line1_xs, line1_ys, linewidth=1, color='green')) ax.add_line( Line2D(line2_xs, line2_ys, linewidth=1, color='red')) ax.add_line( Line2D(line3_xs, line3_ys, linewidth=1, color='red')) ax.add_line( Line2D(line4_xs, line4_ys, linewidth=1, color='red')) # gt visualize if args.test == False and data['gt_boxes3d'].shape[1] > 0: gt_boxes3d_corner = kitti_utils.boxes3d_to_corners3d( data['gt_boxes3d'].reshape(-1, 7), rotate=True) for o in range(gt_boxes3d_corner.shape[0]): print_box_corner = gt_boxes3d_corner[o] x1, x2, x3, x4 = print_box_corner[0:4, 0] z1, z2, z3, z4 = print_box_corner[0:4, 2] polygon = np.zeros([5, 2], dtype=np.float32) polygon[0, 0] = x1 polygon[1, 0] = x2 polygon[2, 0] = x3 polygon[3, 0] = x4 polygon[4, 0] = x1 polygon[0, 1] = z1 polygon[1, 1] = z2 polygon[2, 1] = z3 polygon[3, 1] = z4 polygon[4, 1] = z1 line1 = [(x1, z1), (x2, z2)] line2 = [(x2, z2), (x3, z3)] line3 = [(x3, z3), (x4, z4)] line4 = [(x4, z4), (x1, z1)] (line1_xs, line1_ys) = zip(*line1) (line2_xs, line2_ys) = zip(*line2) (line3_xs, line3_ys) = zip(*line3) (line4_xs, line4_ys) = zip(*line4) ax.add_line( Line2D(line1_xs, line1_ys, linewidth=1, color='yellow')) ax.add_line( Line2D(line2_xs, line2_ys, linewidth=1, color='purple')) ax.add_line( Line2D(line3_xs, line3_ys, linewidth=1, color='purple')) ax.add_line( Line2D(line4_xs, line4_ys, linewidth=1, color='purple')) plt.savefig('../visual/ioun.jpg') progress_bar.close() # dump empty files split_file = os.path.join(dataset.imageset_dir, '..', 'ImageSets', dataset.split + '.txt') split_file = os.path.abspath(split_file) image_idx_list = [x.strip() for x in open(split_file).readlines()] empty_cnt = 0 for k in range(image_idx_list.__len__()): cur_file = os.path.join(final_output_dir, '%s.txt' % image_idx_list[k]) if not os.path.exists(cur_file): with open(cur_file, 'w') as temp_f: pass empty_cnt += 1 logger.info('empty_cnt=%d: dump empty file %s' % (empty_cnt, cur_file)) ret_dict = {'empty_cnt': empty_cnt} if not args.eval_all: logger.info( '-------------------performance of epoch %s---------------------' % epoch_id) logger.info(str(datetime.now())) avg_rpn_iou = (total_rpn_iou / max(cnt, 1.0)) avg_cls_acc = (total_cls_acc / max(cnt, 1.0)) avg_cls_acc_refined = (total_cls_acc_refined / max(cnt, 1.0)) avg_det_num = (final_total / max(len(dataset), 1.0)) logger.info('final average detections: %.3f' % avg_det_num) logger.info('final average rpn_iou refined: %.3f' % avg_rpn_iou) logger.info('final average cls acc: %.3f' % avg_cls_acc) logger.info('final average cls acc refined: %.3f' % avg_cls_acc_refined) ret_dict['rpn_iou'] = avg_rpn_iou ret_dict['rcnn_cls_acc'] = avg_cls_acc ret_dict['rcnn_cls_acc_refined'] = avg_cls_acc_refined ret_dict['rcnn_avg_num'] = avg_det_num for idx, thresh in enumerate(thresh_list): cur_recall = total_recalled_bbox_list[idx] / max( total_gt_bbox, 1.0) logger.info('total bbox recall(thresh=%.3f): %d / %d = %f' % (thresh, total_recalled_bbox_list[idx], total_gt_bbox, cur_recall)) ret_dict['rcnn_recall(thresh=%.2f)' % thresh] = cur_recall if thresh == 0.7: recall = cur_recall if cfg.TEST.SPLIT != 'test': logger.info('Averate Precision:') name_to_class = {'Car': 0, 'Pedestrian': 1, 'Cyclist': 2} ap_result_str, ap_dict = kitti_evaluate( dataset.label_dir, final_output_dir, label_split_file=split_file, current_class=name_to_class[cfg.CLASSES]) if not args.eval_all: logger.info(ap_result_str) ret_dict.update(ap_dict) logger.info('result is saved to: %s' % result_dir) precision = ap_dict['Car_3d_easy'] + ap_dict['Car_3d_moderate'] + ap_dict[ 'Car_3d_hard'] recall = total_recalled_bbox_list[3] / max(total_gt_bbox, 1.0) F2_score = 0 return precision, recall, F2_score