Example #1
0
def validate(m, opt, heatmap_to_coord, batch_size=64):
    det_dataset = builder.build_dataset(cfg.DATASET.TEST,
                                        preset_cfg=cfg.DATA_PRESET,
                                        train=False,
                                        opt=opt)
    det_loader = torch.utils.data.DataLoader(det_dataset,
                                             batch_size=batch_size,
                                             shuffle=False,
                                             num_workers=opt.nThreads,
                                             drop_last=False)
    kpt_json = []
    eval_joints = det_dataset.EVAL_JOINTS

    m.eval()

    norm_type = cfg.LOSS.get('NORM_TYPE', None)
    hm_size = cfg.DATA_PRESET.HEATMAP_SIZE

    for index, (inps, crop_bboxes, bboxes, img_ids, scores, imghts,
                imgwds) in tqdm(enumerate(det_loader), dynamic_ncols=True):
        if opt.device.type != "cpu":
            if isinstance(inps, list):
                inps = [inp.cuda() for inp in inps]
            else:
                inps = inps.cuda()
        full_output = m(inps)
        joints_map = full_output.joints_map

        pred = joints_map
        assert pred.dim() == 4
        pred = pred[:, eval_joints, :, :]

        for i in range(joints_map.shape[0]):
            bbox = crop_bboxes[i].tolist()
            pose_coords, pose_scores = heatmap_to_coord(
                pred[i][det_dataset.EVAL_JOINTS],
                bbox,
                hm_shape=hm_size,
                norm_type=norm_type)

            keypoints = np.concatenate((pose_coords, pose_scores), axis=1)
            keypoints = keypoints.reshape(-1).tolist()

            data = dict()
            data['bbox'] = bboxes[i, 0].tolist()
            data['image_id'] = int(img_ids[i])
            data['score'] = float(scores[i] + np.mean(pose_scores) +
                                  np.max(pose_scores))
            data['category_id'] = 1
            data['keypoints'] = keypoints

            kpt_json.append(data)

    with open(os.path.join(opt.work_dir, 'test_kpt.json'), 'w') as fid:
        json.dump(kpt_json, fid)
    res = evaluate_mAP(os.path.join(opt.work_dir, 'test_kpt.json'),
                       ann_type='keypoints',
                       ann_file=os.path.join(cfg.DATASET.VAL.ROOT,
                                             cfg.DATASET.VAL.ANN))
    return res
Example #2
0
def validate_gt(pose_model, cfg, heatmap_to_coord, batch_size=20):
    gt_val_dataset = builder.build_dataset(cfg.DATASET.VAL, preset_cfg=cfg.DATA_PRESET, train=False)
    eval_joints = gt_val_dataset.EVAL_JOINTS

    gt_val_loader = torch.utils.data.DataLoader(
        gt_val_dataset, batch_size=batch_size, shuffle=False, num_workers=20, drop_last=False)
    kpt_json = []
    # m.eval()

    norm_type = cfg.LOSS.get('NORM_TYPE', None)
    hm_size = cfg.DATA_PRESET.HEATMAP_SIZE

    for inps, labels, label_masks, img_ids, bboxes in tqdm(gt_val_loader, dynamic_ncols=True):
        if isinstance(inps, list):
            inps = [inp.cuda() for inp in inps]
        else:
            inps = inps.cuda()
        # 使用static shape的fast pose的engine进行推理
        output = pose_model.detect_context(inps.cpu().numpy())
        # 一定要转到cuda上,结果才不会出错
        output = torch.from_numpy(output).to(opt.device)
        if opt.flip_test:
            if isinstance(inps, list):
                inps_flip = [flip(inp).cuda() for inp in inps]
            else:
                inps_flip = flip(inps).cuda()
            inps_flip = pose_model.detect_context(inps_flip.cpu().numpy())
            inps_flip = torch.from_numpy(inps_flip).to(opt.device)
            # output_flip = flip_heatmap(m(inps_flip), gt_val_dataset.joint_pairs, shift=True)
            output_flip = flip_heatmap(inps_flip, gt_val_dataset.joint_pairs, shift=True)
            pred_flip = output_flip[:, eval_joints, :, :]
        else:
            output_flip = None

        pred = output
        assert pred.dim() == 4
        pred = pred[:, eval_joints, :, :]
        for i in range(bboxes.shape[0]):
            bbox = bboxes[i].tolist()
            pose_coords, pose_scores = heatmap_to_coord(
                pred[i], bbox, hms_flip=pred_flip[i], hm_shape=hm_size, norm_type=norm_type)

            keypoints = np.concatenate((pose_coords, pose_scores), axis=1)
            keypoints = keypoints.reshape(-1).tolist()

            data = dict()
            data['bbox'] = bboxes[i].tolist()
            data['image_id'] = int(img_ids[i])
            data['score'] = float(np.mean(pose_scores) + np.max(pose_scores))
            data['category_id'] = 1
            data['keypoints'] = keypoints

            kpt_json.append(data)

    with open('./exp/json/trt/validate_gt_kpt.json', 'w') as fid:
        json.dump(kpt_json, fid)
    res = evaluate_mAP('./exp/json/trt/validate_gt_kpt.json', ann_type='keypoints',
                       ann_file=os.path.join(cfg.DATASET.VAL.ROOT, cfg.DATASET.VAL.ANN))
    return res
Example #3
0
def validate(m, heatmap_to_coord, batch_size=20):
    det_dataset = builder.build_dataset(cfg.DATASET.TEST,
                                        preset_cfg=cfg.DATA_PRESET,
                                        train=False,
                                        opt=opt)
    eval_joints = det_dataset.EVAL_JOINTS

    det_loader = torch.utils.data.DataLoader(det_dataset,
                                             batch_size=batch_size,
                                             shuffle=False,
                                             num_workers=20,
                                             drop_last=False)
    kpt_json = []
    m.eval()

    for inps, crop_bboxes, bboxes, img_ids, scores, imghts, imgwds in tqdm(
            det_loader, dynamic_ncols=True):
        if isinstance(inps, list):
            inps = [inp.cuda() for inp in inps]
        else:
            inps = inps.cuda()
        output = m(inps)
        if opt.flip_test:
            if isinstance(inps, list):
                inps_flip = [flip(inp) for inp in inps]
            else:
                inps_flip = flip(inps)
            output_flip = flip_heatmap(m(inps_flip),
                                       det_dataset.joint_pairs,
                                       shift=True)
            output = (output + output_flip) / 2

        pred = output.cpu().data.numpy()
        assert pred.ndim == 4
        pred = pred[:, eval_joints, :, :]

        for i in range(output.shape[0]):
            bbox = crop_bboxes[i].tolist()
            pose_coords, pose_scores = heatmap_to_coord(
                pred[i][det_dataset.EVAL_JOINTS], bbox)

            keypoints = np.concatenate((pose_coords, pose_scores), axis=1)
            keypoints = keypoints.reshape(-1).tolist()

            data = dict()
            data['bbox'] = bboxes[i, 0].tolist()
            data['image_id'] = int(img_ids[i])
            data['score'] = float(scores[i] + np.mean(pose_scores) +
                                  np.max(pose_scores))
            data['category_id'] = 1
            data['keypoints'] = keypoints

            kpt_json.append(data)

    with open('./exp/json/validate_rcnn_kpt.json', 'w') as fid:
        json.dump(kpt_json, fid)
    res = evaluate_mAP('./exp/json/validate_rcnn_kpt.json',
                       ann_type='keypoints')
    return res['AP']
Example #4
0
def validate(m, heatmap_to_coord, batch_size=20):
    det_dataset = builder.build_dataset(cfg.DATASET.TEST, preset_cfg=cfg.DATA_PRESET, detector_cfg=cfg.DETECTOR, train=False, opt=opt)
    eval_joints = det_dataset.EVAL_JOINTS

    det_loader = torch.utils.data.DataLoader(
        det_dataset, batch_size=batch_size, shuffle=False, num_workers=20, drop_last=False)
    kpt_json = []
    m.eval()

    norm_type = cfg.LOSS.get('NORM_TYPE', None)
    hm_size = cfg.DATA_PRESET.HEATMAP_SIZE

    for inps, crop_bboxes, bboxes, img_ids, scores, imghts, imgwds in tqdm(det_loader, dynamic_ncols=True):
        if isinstance(inps, list):
            inps = [inp.cuda() for inp in inps]
        else:
            inps = inps.cuda()
        output = m(inps)
        if opt.flip_test:
            if isinstance(inps, list):
                inps_flip = [flip(inp).cuda() for inp in inps]
            else:
                inps_flip = flip(inps).cuda()
            output_flip = flip_heatmap(m(inps_flip), det_dataset.joint_pairs, shift=True)
            pred_flip = output_flip[:, eval_joints, :, :]
        else:
            output_flip = None

        pred = output
        assert pred.dim() == 4
        pred = pred[:, eval_joints, :, :]

        for i in range(output.shape[0]):
            bbox = crop_bboxes[i].tolist()
            pose_coords, pose_scores = heatmap_to_coord(
                pred[i], bbox, hms_flip=pred_flip[i], hm_shape=hm_size, norm_type=norm_type)

            keypoints = np.concatenate((pose_coords, pose_scores), axis=1)
            keypoints = keypoints.reshape(-1).tolist()

            data = dict()
            data['bbox'] = bboxes[i, 0].tolist()
            data['image_id'] = int(img_ids[i])
            data['area'] = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1])
            # data['score'] = float(scores[i] + np.mean(pose_scores) + np.max(pose_scores))
            data['score'] = float(scores[i])
            data['category_id'] = 1
            data['keypoints'] = keypoints

            kpt_json.append(data)

    kpt_json = oks_pose_nms(kpt_json)

    with open('./exp/json/validate_rcnn_kpt.json', 'w') as fid:
        json.dump(kpt_json, fid)
    res = evaluate_mAP('./exp/json/validate_rcnn_kpt.json', ann_type='keypoints')
    return res['AP']
Example #5
0
def validate_gt(m, opt, cfg, heatmap_to_coord, batch_size=20):
    gt_val_dataset = builder.build_dataset(cfg.DATASET.VAL,
                                           preset_cfg=cfg.DATA_PRESET,
                                           train=False)
    eval_joints = gt_val_dataset.EVAL_JOINTS

    gt_val_loader = torch.utils.data.DataLoader(gt_val_dataset,
                                                batch_size=batch_size,
                                                shuffle=False,
                                                num_workers=20,
                                                drop_last=False)
    kpt_json = []
    m.eval()

    norm_type = cfg.LOSS.get('NORM_TYPE', None)
    hm_size = cfg.DATA_PRESET.HEATMAP_SIZE

    for inps, labels, label_masks, img_ids, bboxes in tqdm(gt_val_loader,
                                                           dynamic_ncols=True):
        if isinstance(inps, list):
            inps = [inp.cuda() for inp in inps]
        else:
            inps = inps.cuda()
        output = m(inps)

        pred = output
        assert pred.dim() == 4
        pred = pred[:, eval_joints, :, :]

        for i in range(output.shape[0]):
            bbox = bboxes[i].tolist()
            pose_coords, pose_scores = heatmap_to_coord(
                pred[i][gt_val_dataset.EVAL_JOINTS],
                bbox,
                hm_shape=hm_size,
                norm_type=norm_type)

            keypoints = np.concatenate((pose_coords, pose_scores), axis=1)
            keypoints = keypoints.reshape(-1).tolist()

            data = dict()
            data['bbox'] = bboxes[i].tolist()
            data['image_id'] = int(img_ids[i])
            data['score'] = float(np.mean(pose_scores) + np.max(pose_scores))
            data['category_id'] = 1
            data['keypoints'] = keypoints

            kpt_json.append(data)

    with open(os.path.join(opt.work_dir, 'test_gt_kpt.json'), 'w') as fid:
        json.dump(kpt_json, fid)
    res = evaluate_mAP(os.path.join(opt.work_dir, 'test_gt_kpt.json'),
                       ann_type='keypoints',
                       ann_file=os.path.join(cfg.DATASET.VAL.ROOT,
                                             cfg.DATASET.VAL.ANN))
    return res
Example #6
0
def validate(m, opt, heatmap_to_coord, batch_size=20):
    det_dataset = builder.build_dataset(cfg.DATASET.TEST, preset_cfg=cfg.DATA_PRESET, train=False, opt=opt)
    det_loader = torch.utils.data.DataLoader(
        det_dataset, batch_size=batch_size, shuffle=False, num_workers=20, drop_last=False)
    kpt_json = []
    eval_joints = det_dataset.EVAL_JOINTS
    test_branch = cfg.OTHERS.get('TEST_BRANCH',True)
    m.eval()

    norm_type = cfg.LOSS.get('NORM_TYPE', None)
    hm_size = cfg.DATA_PRESET.HEATMAP_SIZE

    for inps, crop_bboxes, bboxes, img_ids, scores, imghts, imgwds in tqdm(det_loader, dynamic_ncols=True):
        if isinstance(inps, list):
            inps = [inp.cuda() for inp in inps]
        else:
            inps = inps.cuda()

        output,_ = m(inps,crop_bboxes[:,1,:],crop_bboxes[:,2,:],crop_bboxes[:,3,:])

        pred = output
        assert pred.dim() == 4
        pred = pred[:, eval_joints, :, :]

        for i in range(output.shape[0]):
            bbox = crop_bboxes[i][0].tolist()
            pose_coords, pose_scores = heatmap_to_coord(
                pred[i][det_dataset.EVAL_JOINTS], bbox, hm_shape=hm_size, norm_type=norm_type)

            keypoints = np.concatenate((pose_coords, pose_scores), axis=1)
            keypoints = keypoints.reshape(-1).tolist()

            data = dict()
            #data['bbox'] = bboxes[i, 0].tolist()
            data['bbox'] = bbox
            data['image_id'] = int(img_ids[i])
            data['score'] = float(scores[i] + np.mean(pose_scores) + np.max(pose_scores))
            data['category_id'] = 1
            data['keypoints'] = keypoints

            kpt_json.append(data)

    with open(os.path.join(opt.work_dir, 'test_kpt.json'), 'w') as fid:
                
        json.dump(kpt_json, fid)
    res = evaluate_mAP(os.path.join(opt.work_dir, 'test_kpt.json'), ann_type='keypoints', ann_file='/ssd3/Benchmark/coco/annotations/coco_wholebody_val_133.json')#ann_file=os.path.join(cfg.DATASET.VAL.ROOT, cfg.DATASET.VAL.ANN))
    return res
Example #7
0
def validate_gt(m, opt, cfg, heatmap_to_coord, batch_size=64):
    joint_radius_mse = DataLogger()
    gt_val_dataset = builder.build_dataset(cfg.DATASET.VAL,
                                           preset_cfg=cfg.DATA_PRESET,
                                           train=False)
    eval_joints = gt_val_dataset.EVAL_JOINTS

    gt_val_loader = torch.utils.data.DataLoader(gt_val_dataset,
                                                batch_size=batch_size,
                                                shuffle=False,
                                                num_workers=opt.nThreads,
                                                drop_last=False)
    kpt_json = []
    m.eval()

    norm_type = cfg.LOSS.get('NORM_TYPE', None)
    hm_size = cfg.DATA_PRESET.HEATMAP_SIZE

    mse_loss = nn.MSELoss()
    for index, (inps, labels, label_masks, joint_radius_gt, img_ids,
                bboxes) in tqdm(enumerate(gt_val_loader), dynamic_ncols=True):
        if opt.device.type != 'cpu':
            if isinstance(inps, list):
                inps = [inp.cuda() for inp in inps]
            else:
                inps = inps.cuda()
        full_output = m(inps)
        joints_map = full_output.joints_map
        joints_radius = full_output.joints_radius

        pred = joints_map
        assert pred.dim() == 4
        pred = pred[:, eval_joints, :, :]

        for i in range(joints_map.shape[0]):
            bbox = bboxes[i].tolist()
            pose_coords, pose_scores = heatmap_to_coord(
                pred[i][gt_val_dataset.EVAL_JOINTS],
                bbox,
                hm_shape=hm_size,
                norm_type=norm_type)

            keypoints = np.concatenate((pose_coords, pose_scores), axis=1)
            keypoints = keypoints.reshape(-1).tolist()

            data = dict()
            data['bbox'] = bboxes[i].tolist()
            data['image_id'] = int(img_ids[i])
            data['score'] = float(np.mean(pose_scores) + np.max(pose_scores))
            data['category_id'] = 1
            data['keypoints'] = keypoints

            kpt_json.append(data)

        radius_masks = (label_masks[:, :, 0, 0] != 0) & (joint_radius_gt != -1)
        joints_radius = joints_radius[:, eval_joints]
        joint_radius_gt = joint_radius_gt[:, eval_joints]
        joint_radius_gt = joint_radius_gt[radius_masks]
        joints_radius = joints_radius[radius_masks]
        joints_radius_error = mse_loss(joint_radius_gt, joints_radius.cpu())
        joint_radius_mse.update(joints_radius_error, joint_radius_gt.shape[0])

    with open(os.path.join(opt.work_dir, 'test_gt_kpt.json'), 'w') as fid:
        json.dump(kpt_json, fid)
    res = evaluate_mAP(os.path.join(opt.work_dir, 'test_gt_kpt.json'),
                       ann_type='keypoints',
                       ann_file=os.path.join(cfg.DATASET.VAL.ROOT,
                                             cfg.DATASET.VAL.ANN))
    return {
        "map": res,
        "radius_mse": joint_radius_mse.avg,
    }
Example #8
0
def validate(m, opt, heatmap_to_coord, batch_size=20):
    det_dataset = builder.build_dataset(cfg.DATASET.TEST, preset_cfg=cfg.DATA_PRESET, train=False, opt=opt)
    det_loader = torch.utils.data.DataLoader(
        det_dataset, batch_size=batch_size, shuffle=False, num_workers=20, drop_last=False)
    kpt_json = []
    eval_joints = det_dataset.EVAL_JOINTS

    m.eval()

    norm_type = cfg.LOSS.get('NORM_TYPE', None)
    hm_size = cfg.DATA_PRESET.HEATMAP_SIZE
    combined_loss = (cfg.LOSS.get('TYPE') == 'Combined')

    halpe = (cfg.DATA_PRESET.NUM_JOINTS == 133) or (cfg.DATA_PRESET.NUM_JOINTS == 136)

    for inps, crop_bboxes, bboxes, img_ids, scores, imghts, imgwds in tqdm(det_loader, dynamic_ncols=True):
        if isinstance(inps, list):
            inps = [inp.cuda() for inp in inps]
        else:
            inps = inps.cuda()
        output = m(inps)

        pred = output
        assert pred.dim() == 4
        pred = pred[:, eval_joints, :, :]

        if output.size()[1] == 68:
            face_hand_num = 42
        else:
            face_hand_num = 110

        for i in range(output.shape[0]):
            bbox = crop_bboxes[i].tolist()
            if combined_loss:
                pose_coords_body_foot, pose_scores_body_foot = heatmap_to_coord[0](
                    pred[i][det_dataset.EVAL_JOINTS[:-face_hand_num]], bbox, hm_shape=hm_size, norm_type=norm_type)
                pose_coords_face_hand, pose_scores_face_hand = heatmap_to_coord[1](
                    pred[i][det_dataset.EVAL_JOINTS[-face_hand_num:]], bbox, hm_shape=hm_size, norm_type=norm_type)
                pose_coords = np.concatenate((pose_coords_body_foot, pose_coords_face_hand), axis=0)
                pose_scores = np.concatenate((pose_scores_body_foot, pose_scores_face_hand), axis=0)
            else:
                pose_coords, pose_scores = heatmap_to_coord(
                    pred[i][det_dataset.EVAL_JOINTS], bbox, hm_shape=hm_size, norm_type=norm_type)

            keypoints = np.concatenate((pose_coords, pose_scores), axis=1)
            keypoints = keypoints.reshape(-1).tolist()

            data = dict()
            data['bbox'] = bboxes[i, 0].tolist()
            data['image_id'] = int(img_ids[i])
            data['score'] = float(scores[i] + np.mean(pose_scores) + 1.25 * np.max(pose_scores))
            data['category_id'] = 1
            data['keypoints'] = keypoints

            kpt_json.append(data)

    sysout = sys.stdout
    with open(os.path.join(opt.work_dir, 'test_kpt.json'), 'w') as fid:
        json.dump(kpt_json, fid)
    res = evaluate_mAP(os.path.join(opt.work_dir, 'test_kpt.json'), ann_type='keypoints', ann_file=os.path.join(cfg.DATASET.VAL.ROOT, cfg.DATASET.VAL.ANN), halpe=halpe)
    sys.stdout = sysout
    return res
Example #9
0
def validate_gt(cfg,
                batchsize=1,
                engine_file_path=None,
                m=None,
                heatmap_to_coord=None):

    gt_val_dataset = builder.build_dataset(cfg.DATASET.VAL,
                                           preset_cfg=cfg.DATA_PRESET,
                                           train=False)
    eval_joints = gt_val_dataset.EVAL_JOINTS

    gt_val_loader = torch.utils.data.DataLoader(gt_val_dataset,
                                                batch_size=batchsize,
                                                shuffle=False,
                                                num_workers=20,
                                                drop_last=True)
    kpt_json = []
    if m:
        m.eval()

    norm_type = cfg.LOSS.get('NORM_TYPE', None)
    hm_size = cfg.DATA_PRESET.HEATMAP_SIZE
    average_time = 0
    pytorch_all_infer_time = 0
    trt_all_infer_time = 0
    data_num = 0

    for inps, labels, label_masks, img_ids, bboxes in tqdm(gt_val_loader,
                                                           dynamic_ncols=True):
        data_num += 1
        if engine_file_path:

            # hm_data = []
            inps = inps.numpy()
            np.copyto(inputs[0].host, inps.ravel())

            trt_infer_before_time = count_time()
            trt_outputs = trt_common.do_inference(context,
                                                  bindings=bindings,
                                                  inputs=inputs,
                                                  outputs=outputs,
                                                  stream=stream)
            trt_infer_after_time = count_time()

            trt_all_infer_time += trt_infer_after_time - trt_infer_before_time

            pred = trt_outputs[0].reshape(-1, 17, 64, 48)
        else:
            if isinstance(inps, list):
                inps = [inp.cuda() for inp in inps]
            else:
                inps = inps.cuda()

            pytorch_infer_before_time = count_time()
            output = m(inps)
            pytorch_infer_after_time = count_time()
            pytorch_all_infer_time += pytorch_infer_after_time - pytorch_infer_before_time

            if opt.flip_test:
                if isinstance(inps, list):
                    inps_flip = [flip(inp).cuda() for inp in inps]
                else:
                    inps_flip = flip(inps).cuda()
                output_flip = flip_heatmap(m(inps_flip),
                                           gt_val_dataset.joint_pairs,
                                           shift=True)
                pred_flip = output_flip[:, eval_joints, :, :]
            else:
                output_flip = None

            pred = output
            assert pred.dim() == 4
            pred = pred[:, eval_joints, :, :]

        for i in range(pred.shape[0]):  #后处理过程
            bbox = bboxes[i].tolist()
            if engine_file_path:
                pose_coords, pose_scores = heatmap_to_coord_simple(
                    pred[i], bbox, hm_shape=hm_size, norm_type=norm_type)
            else:
                pose_coords, pose_scores = heatmap_to_coord(
                    pred[i], bbox, hm_shape=hm_size, norm_type=norm_type)

            keypoints = np.concatenate((pose_coords, pose_scores), axis=1)
            keypoints = keypoints.reshape(-1).tolist()

            data = dict()
            data['bbox'] = bboxes[i].tolist()
            data['image_id'] = int(img_ids[i])
            data['score'] = float(np.mean(pose_scores) + np.max(pose_scores))
            data['category_id'] = 1
            data['keypoints'] = keypoints

            kpt_json.append(data)
    if engine_file_path:
        average_time = float(trt_all_infer_time / data_num)
    else:
        average_time = (pytorch_all_infer_time / data_num)

    print("average_time:", average_time)
    res_file = r"data/coco/res.json"
    with open(res_file, 'w') as F:
        json.dump(kpt_json, F)
    res = evaluate_mAP(res_file,
                       ann_type='keypoints',
                       ann_file=os.path.join(cfg.DATASET.VAL.ROOT,
                                             cfg.DATASET.VAL.ANN))
    return res, average_time
Example #10
0
def validate_gt(m, opt, cfg, heatmap_to_coord, batch_size=20):
    gt_val_dataset = builder.build_dataset(cfg.DATASET.VAL, preset_cfg=cfg.DATA_PRESET, train=False)
    eval_joints = gt_val_dataset.EVAL_JOINTS
    test_branch = cfg.OTHERS.get('TEST_BRANCH',True)

    gt_val_loader = torch.utils.data.DataLoader(
        gt_val_dataset, batch_size=batch_size, shuffle=False, num_workers=20, drop_last=False)
    kpt_json = []
    kpt_json_branch = []
    m.eval()

    norm_type = cfg.LOSS.get('NORM_TYPE', None)
    hm_size = cfg.DATA_PRESET.HEATMAP_SIZE

    for inps, labels, label_masks, img_ids, bboxes in tqdm(gt_val_loader, dynamic_ncols=True):
        if isinstance(inps, list):
            inps = [inp.cuda() for inp in inps]
        else:
            inps = inps.cuda()
        output,feature = m(inps)

        pred = copy.deepcopy(output)
        assert pred.dim() == 4
        pred = pred[:, eval_joints, :, :]

        for i in range(output.shape[0]):
            bbox = bboxes[i][0].tolist()
            pose_coords, pose_scores = heatmap_to_coord(
                pred[i][gt_val_dataset.EVAL_JOINTS], bbox, hm_shape=hm_size, norm_type=norm_type)

            keypoints = np.concatenate((pose_coords, pose_scores), axis=1)
            keypoints = keypoints.reshape(-1).tolist()

            data = dict()
            #data['bbox'] = bboxes[i, 0].tolist()
            data['bbox'] = bbox
            data['image_id'] = int(img_ids[i])
            data['score'] = float(np.mean(pose_scores) + np.max(pose_scores))
            data['category_id'] = 1
            data['keypoints'] = keypoints

            kpt_json.append(data)

        if test_branch:
            hm_height, hm_width = hm_size
            # regression the joints of wholeboy in stage1
            pred_jts, pred_score = _integral_tensor(
            pred, 133, False, hm_width, hm_height, 1, integral_operation=integral_op, norm_type='sigmoid')
            pred_jts = pred_jts.reshape(pred_jts.shape[0], 133, 2)

            # get the coords with the size of heatmap
            coords_x = (pred_jts[:, :, 0] + 0.5) * hm_width
            coords_y = (pred_jts[:, :, 1] + 0.5) * hm_height

            # get the box of hands for roi align
            lefthand_boxes = get_box_for_align(coords_x[:,-42:-21],coords_y[:,-42:-21])
            righthand_boxes = get_box_for_align(coords_x[:,-21:],coords_y[:,-21:])
            # stage2 testing
            fine_out = m.forward_branch(output, feature, lefthand_boxes, righthand_boxes)
            # output contains the finer and amplified hands kpts, need to apply aff
            fine_pred_jts, fine_pred_score = _integral_tensor(
            fine_out[:,-42:,:,:], 42, False, hm_width, hm_height, 1, integral_operation=integral_op, norm_type='sigmoid')
            fine_pred_jts = fine_pred_jts.reshape(fine_pred_jts.shape[0], 42, 2)
            

            lefthand_jts = fine_pred_jts[:,:21,:]
            righthand_jts = fine_pred_jts[:,21:,:]
            lefthand_jts[:,:,0] = (lefthand_jts[:,:,0]+0.5)*hm_width
            lefthand_jts[:,:,1] = (lefthand_jts[:,:,1]+0.5)*hm_height
            righthand_jts[:,:,0] = (righthand_jts[:,:,0]+0.5)*hm_width
            righthand_jts[:,:,1] = (righthand_jts[:,:,1]+0.5)*hm_height

            center_hm = np.array([hm_width/2.0,hm_height/2.0])
            scale_hm = np.array([hm_size[1],hm_size[0]])

            lefthand_kpts = copy.deepcopy(lefthand_jts.cpu().numpy().astype(np.float32))
            righthand_kpts = copy.deepcopy(righthand_jts.cpu().numpy().astype(np.float32))
            # apply affine trans to lefthand and add offset
            for j in range(lefthand_jts.shape[0]):
                box = lefthand_boxes[j].tolist()
                width = np.array(box[2] - box[0])
                height = np.array(box[3] - box[1])
                output_size = [box[2]-box[0],box[3]-box[1]]
                offset = np.array([box[0],box[1]])
                trans = get_affine_transform(center_hm,scale_hm,0,output_size) 
                for k in range(21):
                    lefthand_kpts[j ,k, 0:2] = affine_transform(lefthand_kpts[j ,k, 0:2], trans)
        
                lefthand_kpts[j,:,0] = (lefthand_kpts[j,:,0]) + offset[0]
                lefthand_kpts[j,:,1] = (lefthand_kpts[j,:,1])+ offset[1]
            #--------------------------------------------------
            # apply affine trans to righthand and add offset
            for j in range(righthand_jts.shape[0]):
                box = righthand_boxes[j].tolist()
                width = np.array(box[2] - box[0])
                height = np.array(box[3] - box[1])
                output_size = [box[2]-box[0],box[3]-box[1]]
                offset = np.array([box[0],box[1]])
                trans = get_affine_transform(center_hm,scale_hm,0,output_size)
                for k in range(21):
                    righthand_kpts[j,k, 0:2] = affine_transform(righthand_kpts[j ,k, 0:2], trans)
                
                righthand_kpts[j,:,0] = (righthand_kpts[j,:,0]) + offset[0]
                righthand_kpts[j,:,1] = (righthand_kpts[j,:,1]) + offset[1]
            #--------------------------------------------------

            bodyface_kpts = copy.deepcopy(pred_jts[:,:-42,:].cpu().numpy().astype(np.float32))
            bodyface_kpts[:,:,0] = (bodyface_kpts[:,:,0]+0.5)*hm_width
            bodyface_kpts[:,:,1] = (bodyface_kpts[:,:,1]+0.5)*hm_height

            fine_kpts = np.concatenate((bodyface_kpts,lefthand_kpts,righthand_kpts), axis=1)
            fine_socre = np.concatenate((pred_score[:,:-42,:].cpu().numpy(),fine_pred_score.cpu().numpy()), axis=1)
            
            for n in range(output.shape[0]):
                bbox = bboxes[n][0].tolist()
                xmin, ymin, xmax, ymax = bbox
                w = xmax - xmin
                h = ymax - ymin
                center = np.array([xmin + w * 0.5, ymin + h * 0.5])
                scale = np.array([w, h])
                for l in range(fine_kpts.shape[1]):
                    fine_kpts[n, l, 0:2] = transform_preds(fine_kpts[n, l, 0:2], center, scale,
                                               [hm_size[1],hm_size[0]])

                keypoints = np.concatenate((fine_kpts[n], fine_socre[n]), axis=1)
                keypoints = keypoints.reshape(-1).tolist()

                data_branch = dict()
                #data['bbox'] = bboxes[i, 0].tolist()
                data_branch['bbox'] = bbox
                data_branch['image_id'] = int(img_ids[n])
                data_branch['score'] = float(np.mean(fine_socre) + np.max(fine_socre))
                data_branch['category_id'] = 1
                data_branch['keypoints'] = keypoints
                kpt_json_branch.append(data_branch)
    

    with open(os.path.join(opt.work_dir, 'test_gt_kpt.json'), 'w') as fid:
        json.dump(kpt_json, fid)

    res = evaluate_mAP(os.path.join(opt.work_dir, 'test_gt_kpt.json'), ann_type='keypoints', ann_file='/ssd3/Benchmark/coco/annotations/coco_wholebody_val_133.json')#ann_file=os.path.join(cfg.DATASET.VAL.ROOT, cfg.DATASET.VAL.ANN))
        
    if test_branch:
        with open(os.path.join(opt.work_dir, 'test_gt_kpt_2branch.json'), 'w') as fid2:
            json.dump(kpt_json_branch, fid2)
        res_branch = evaluate_mAP(os.path.join(opt.work_dir, 'test_gt_kpt_2branch.json'), ann_type='keypoints', ann_file='/ssd3/Benchmark/coco/annotations/coco_wholebody_val_133.json')
        
        return res,res_branch
    else:
        return res, 0
Example #11
0
def validate(m, heatmap_to_coord, batch_size=20):
    det_dataset = builder.build_dataset(cfg.DATASET.TEST,
                                        preset_cfg=cfg.DATA_PRESET,
                                        train=False,
                                        opt=opt)
    eval_joints = det_dataset.EVAL_JOINTS

    det_loader = torch.utils.data.DataLoader(det_dataset,
                                             batch_size=batch_size,
                                             shuffle=False,
                                             num_workers=20,
                                             drop_last=False)
    kpt_json = []
    m.eval()

    norm_type = cfg.LOSS.get('NORM_TYPE', None)
    hm_size = cfg.DATA_PRESET.HEATMAP_SIZE
    combined_loss = (cfg.LOSS.get('TYPE') == 'Combined')

    halpe = (cfg.DATA_PRESET.NUM_JOINTS == 133) or (cfg.DATA_PRESET.NUM_JOINTS
                                                    == 136)

    for inps, crop_bboxes, bboxes, img_ids, scores, imghts, imgwds in tqdm(
            det_loader, dynamic_ncols=True):
        if isinstance(inps, list):
            inps = [inp.cuda() for inp in inps]
        else:
            inps = inps.cuda()
        output = m(inps)
        if opt.flip_test:
            if isinstance(inps, list):
                inps_flip = [flip(inp).cuda() for inp in inps]
            else:
                inps_flip = flip(inps).cuda()
            output_flip = flip_heatmap(m(inps_flip),
                                       det_dataset.joint_pairs,
                                       shift=True)
            pred_flip = output_flip[:, eval_joints, :, :]
        else:
            output_flip = None
            pred_flip = None

        pred = output
        assert pred.dim() == 4
        pred = pred[:, eval_joints, :, :]

        if output.size()[1] == 68:
            face_hand_num = 42
        else:
            face_hand_num = 110

        for i in range(output.shape[0]):
            bbox = crop_bboxes[i].tolist()
            if combined_loss:
                pose_coords_body_foot, pose_scores_body_foot = heatmap_to_coord[
                    0](pred[i][det_dataset.EVAL_JOINTS[:-face_hand_num]],
                       bbox,
                       hm_shape=hm_size,
                       norm_type=norm_type,
                       hms_flip=pred_flip[i][
                           det_dataset.EVAL_JOINTS[:-face_hand_num]]
                       if pred_flip is not None else None)
                pose_coords_face_hand, pose_scores_face_hand = heatmap_to_coord[
                    1](pred[i][det_dataset.EVAL_JOINTS[-face_hand_num:]],
                       bbox,
                       hm_shape=hm_size,
                       norm_type=norm_type,
                       hms_flip=pred_flip[i][
                           det_dataset.EVAL_JOINTS[-face_hand_num:]]
                       if pred_flip is not None else None)
                pose_coords = np.concatenate(
                    (pose_coords_body_foot, pose_coords_face_hand), axis=0)
                pose_scores = np.concatenate(
                    (pose_scores_body_foot, pose_scores_face_hand), axis=0)
            else:
                pose_coords, pose_scores = heatmap_to_coord(
                    pred[i][det_dataset.EVAL_JOINTS],
                    bbox,
                    hm_shape=hm_size,
                    norm_type=norm_type,
                    hms_flip=pred_flip[i][det_dataset.EVAL_JOINTS]
                    if pred_flip is not None else None)

            keypoints = np.concatenate((pose_coords, pose_scores), axis=1)
            keypoints = keypoints.reshape(-1).tolist()

            data = dict()
            data['bbox'] = bboxes[i, 0].tolist()
            data['image_id'] = int(img_ids[i])
            data['area'] = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1])
            data['score'] = float(scores[i] + np.mean(pose_scores) +
                                  1.25 * np.max(pose_scores))
            # data['score'] = float(scores[i])
            data['category_id'] = 1
            data['keypoints'] = keypoints

            kpt_json.append(data)

    if opt.ppose_nms:
        from alphapose.utils.pPose_nms import ppose_nms_validate_preprocess, pose_nms, write_json
        final_result = []
        tmp_data = ppose_nms_validate_preprocess(kpt_json)
        for key in tmp_data:
            boxes, scores, ids, preds_img, preds_scores = tmp_data[key]
            boxes, scores, ids, preds_img, preds_scores, pick_ids = \
                        pose_nms(boxes, scores, ids, preds_img, preds_scores, 0, cfg.LOSS.get('TYPE') == 'MSELoss')

            _result = []
            for k in range(len(scores)):
                _result.append({
                    'keypoints':
                    preds_img[k],
                    'kp_score':
                    preds_scores[k],
                    'proposal_score':
                    torch.mean(preds_scores[k]) + scores[k] +
                    1.25 * max(preds_scores[k]),
                    'idx':
                    ids[k],
                    'box': [
                        boxes[k][0], boxes[k][1], boxes[k][2] - boxes[k][0],
                        boxes[k][3] - boxes[k][1]
                    ]
                })
            im_name = str(key).zfill(12) + '.jpg'
            result = {'imgname': im_name, 'result': _result}
            final_result.append(result)

        write_json(final_result,
                   './exp/json/',
                   form='coco',
                   for_eval=True,
                   outputfile='validate_rcnn_kpt.json')
    else:
        if opt.oks_nms:
            from alphapose.utils.pPose_nms import oks_pose_nms
            kpt_json = oks_pose_nms(kpt_json)

        with open('./exp/json/validate_rcnn_kpt.json', 'w') as fid:
            json.dump(kpt_json, fid)

    sysout = sys.stdout
    res = evaluate_mAP('./exp/json/validate_rcnn_kpt.json',
                       ann_type='keypoints',
                       ann_file=os.path.join(cfg.DATASET.TEST.ROOT,
                                             cfg.DATASET.TEST.ANN),
                       halpe=halpe)
    sys.stdout = sysout
    return res
Example #12
0
def validate_gt(m, cfg, heatmap_to_coord, batch_size=20):
    gt_val_dataset = builder.build_dataset(cfg.DATASET.VAL,
                                           preset_cfg=cfg.DATA_PRESET,
                                           train=False)
    eval_joints = gt_val_dataset.EVAL_JOINTS

    gt_val_loader = torch.utils.data.DataLoader(gt_val_dataset,
                                                batch_size=batch_size,
                                                shuffle=False,
                                                num_workers=20,
                                                drop_last=False)
    kpt_json = []
    m.eval()

    norm_type = cfg.LOSS.get('NORM_TYPE', None)
    hm_size = cfg.DATA_PRESET.HEATMAP_SIZE
    combined_loss = (cfg.LOSS.get('TYPE') == 'Combined')

    halpe = (cfg.DATA_PRESET.NUM_JOINTS == 133) or (cfg.DATA_PRESET.NUM_JOINTS
                                                    == 136)

    for inps, labels, label_masks, img_ids, bboxes in tqdm(gt_val_loader,
                                                           dynamic_ncols=True):
        if isinstance(inps, list):
            inps = [inp.cuda() for inp in inps]
        else:
            inps = inps.cuda()
        output = m(inps)
        if opt.flip_test:
            if isinstance(inps, list):
                inps_flip = [flip(inp).cuda() for inp in inps]
            else:
                inps_flip = flip(inps).cuda()
            output_flip = flip_heatmap(m(inps_flip),
                                       gt_val_dataset.joint_pairs,
                                       shift=True)
            pred_flip = output_flip[:, eval_joints, :, :]
        else:
            output_flip = None
            pred_flip = None

        pred = output
        assert pred.dim() == 4
        pred = pred[:, eval_joints, :, :]

        if output.size()[1] == 68:
            face_hand_num = 42
        else:
            face_hand_num = 110

        for i in range(output.shape[0]):
            bbox = bboxes[i].tolist()
            if combined_loss:
                pose_coords_body_foot, pose_scores_body_foot = heatmap_to_coord[
                    0](pred[i][gt_val_dataset.EVAL_JOINTS[:-face_hand_num]],
                       bbox,
                       hm_shape=hm_size,
                       norm_type=norm_type,
                       hms_flip=pred_flip[i][
                           gt_val_dataset.EVAL_JOINTS[:-face_hand_num]]
                       if pred_flip is not None else None)
                pose_coords_face_hand, pose_scores_face_hand = heatmap_to_coord[
                    1](pred[i][gt_val_dataset.EVAL_JOINTS[-face_hand_num:]],
                       bbox,
                       hm_shape=hm_size,
                       norm_type=norm_type,
                       hms_flip=pred_flip[i][
                           gt_val_dataset.EVAL_JOINTS[-face_hand_num:]]
                       if pred_flip is not None else None)
                pose_coords = np.concatenate(
                    (pose_coords_body_foot, pose_coords_face_hand), axis=0)
                pose_scores = np.concatenate(
                    (pose_scores_body_foot, pose_scores_face_hand), axis=0)
            else:
                pose_coords, pose_scores = heatmap_to_coord(
                    pred[i][gt_val_dataset.EVAL_JOINTS],
                    bbox,
                    hm_shape=hm_size,
                    norm_type=norm_type,
                    hms_flip=pred_flip[i][gt_val_dataset.EVAL_JOINTS]
                    if pred_flip is not None else None)

            keypoints = np.concatenate((pose_coords, pose_scores), axis=1)
            keypoints = keypoints.reshape(-1).tolist()

            data = dict()
            data['bbox'] = bboxes[i].tolist()
            data['image_id'] = int(img_ids[i])
            data['score'] = float(
                np.mean(pose_scores) + 1.25 * np.max(pose_scores))
            data['category_id'] = 1
            data['keypoints'] = keypoints

            kpt_json.append(data)

    sysout = sys.stdout
    with open('./exp/json/validate_gt_kpt.json', 'w') as fid:
        json.dump(kpt_json, fid)
    res = evaluate_mAP('./exp/json/validate_gt_kpt.json',
                       ann_type='keypoints',
                       ann_file=os.path.join(cfg.DATASET.VAL.ROOT,
                                             cfg.DATASET.VAL.ANN),
                       halpe=halpe)
    sys.stdout = sysout
    return res