def _report_metric(self, res_file, metrics, pck_thr=0.2, auc_nor=30):
"""Keypoint evaluation.
Args:
res_file (str): Json file stored prediction results.
metrics (str | list[str]): Metric to be performed.
Options: 'PCK', 'PCKh', 'AUC', 'EPE'.
pck_thr (float): PCK threshold, default as 0.2.
pckh_thr (float): PCKh threshold, default as 0.7.
auc_nor (float): AUC normalization factor, default as 30 pixel.
Returns:
List: Evaluation results for evaluation metric.
"""
info_str = []
with open(res_file, 'r') as fin:
preds = json.load(fin)
assert len(preds) == len(self.db)
outputs = []
gts = []
masks = []
threshold_bbox = []
for pred, item in zip(preds, self.db):
outputs.append(np.array(pred['keypoints'])[:, :-1])
gts.append(np.array(item['joints_3d'])[:, :-1])
masks.append((np.array(item['joints_3d_visible'])[:, 0]) > 0)
if 'PCK' in metrics:
bbox = np.array(item['bbox'])
bbox_thr = np.max(bbox[2:])
threshold_bbox.append(np.array([bbox_thr, bbox_thr]))
outputs = np.array(outputs)
gts = np.array(gts)
masks = np.array(masks)
threshold_bbox = np.array(threshold_bbox)
if 'PCK' in metrics:
_, pck, _ = keypoint_pck_accuracy(outputs, gts, masks, pck_thr,
threshold_bbox)
info_str.append(('PCK', pck))
if 'AUC' in metrics:
info_str.append(('AUC', keypoint_auc(outputs, gts, masks,
auc_nor)))
if 'EPE' in metrics:
info_str.append(('EPE', keypoint_epe(outputs, gts, masks)))
return info_str
def _report_metric(self, res_file, metrics):
"""Keypoint evaluation.
Report PCK, AUC or EPE.
"""
info_str = []
with open(res_file, 'r') as fin:
preds = json.load(fin)
assert len(preds) == len(self.db)
outputs = []
gts = []
for pred, item in zip(preds, self.db):
outputs.append(pred['keypoints'])
gts.append(item['joints_3d'])
outputs = np.array(outputs)[:, :, :-1]
gts = np.array(gts)[:, :, :-1]
if 'PCK' in metrics:
hit = 0
exist = 0
for pred, item in zip(preds, self.db):
bbox = np.array(item['bbox'])
threshold = np.max(bbox[2:]) * 0.2
h, _, e = keypoint_pck_accuracy(
np.array(pred['keypoints'])[None, :, :-1],
np.array(item['joints_3d'])[None, :, :-1], 1,
np.array([[threshold, threshold]]))
hit += len(h[h > 0])
exist += e
pck = hit / exist
info_str.append(('PCK', pck))
if 'AUC' in metrics:
info_str.append(('AUC', keypoint_auc(outputs, gts, 30)))
if 'EPE' in metrics:
info_str.append(('EPE', keypoint_epe(outputs, gts)))
return info_str
def _report_metric(self, res_file, metrics):
"""Keypoint evaluation.
Args:
res_file (str): Json file stored prediction results.
metrics (str | list[str]): Metric to be performed.
Options: 'MRRPE', 'MPJPE', 'Handedness_acc'.
Returns:
list: Evaluation results for evaluation metric.
"""
info_str = []
with open(res_file, 'r') as fin:
preds = json.load(fin)
assert len(preds) == len(self.db)
gts_rel_root = []
preds_rel_root = []
rel_root_masks = []
gts_joint_coord_cam = []
preds_joint_coord_cam = []
single_masks = []
interacting_masks = []
all_masks = []
gts_hand_type = []
preds_hand_type = []
hand_type_masks = []
for pred, item in zip(preds, self.db):
# mrrpe
if 'MRRPE' in metrics:
if item['hand_type'].all() and item['joints_3d_visible'][
20, 0] and item['joints_3d_visible'][41, 0]:
rel_root_masks.append(True)
pred_left_root_img = np.array(pred['keypoints'][41],
dtype=np.float32)[None, :]
pred_left_root_img[:, 2] += item['abs_depth'][0] + pred[
'rel_root_depth']
pred_left_root_cam = self._pixel2cam(
pred_left_root_img, item['focal'], item['princpt'])
pred_right_root_img = np.array(pred['keypoints'][20],
dtype=np.float32)[None, :]
pred_right_root_img[:, 2] += item['abs_depth'][0]
pred_right_root_cam = self._pixel2cam(
pred_right_root_img, item['focal'], item['princpt'])
preds_rel_root.append(pred_left_root_cam -
pred_right_root_cam)
gts_rel_root.append(
[item['joints_cam'][41] - item['joints_cam'][20]])
else:
rel_root_masks.append(False)
preds_rel_root.append([[0., 0., 0.]])
gts_rel_root.append([[0., 0., 0.]])
if 'MPJPE' in metrics:
pred_joint_coord_img = np.array(pred['keypoints'],
dtype=np.float32)
gt_joint_coord_cam = item['joints_cam'].copy()
pred_joint_coord_img[:21, 2] += item['abs_depth'][0]
pred_joint_coord_img[21:, 2] += item['abs_depth'][1]
pred_joint_coord_cam = self._pixel2cam(pred_joint_coord_img,
item['focal'],
item['princpt'])
pred_joint_coord_cam[:21] -= pred_joint_coord_cam[20]
pred_joint_coord_cam[21:] -= pred_joint_coord_cam[41]
gt_joint_coord_cam[:21] -= gt_joint_coord_cam[20]
gt_joint_coord_cam[21:] -= gt_joint_coord_cam[41]
preds_joint_coord_cam.append(pred_joint_coord_cam)
gts_joint_coord_cam.append(gt_joint_coord_cam)
mask = (np.array(item['joints_3d_visible'])[:, 0]) > 0
if item['hand_type'].all():
single_masks.append(
np.zeros(self.ann_info['num_joints'], dtype=bool))
interacting_masks.append(mask)
all_masks.append(mask)
else:
single_masks.append(mask)
interacting_masks.append(
np.zeros(self.ann_info['num_joints'], dtype=bool))
all_masks.append(mask)
if 'Handedness_acc' in metrics:
pred_hand_type = np.array(pred['hand_type'], dtype=int)
preds_hand_type.append(pred_hand_type)
gts_hand_type.append(item['hand_type'])
hand_type_masks.append(item['hand_type_valid'] > 0)
gts_rel_root = np.array(gts_rel_root, dtype=np.float32)
preds_rel_root = np.array(preds_rel_root, dtype=np.float32)
rel_root_masks = np.array(rel_root_masks, dtype=bool)[:, None]
gts_joint_coord_cam = np.array(gts_joint_coord_cam, dtype=np.float32)
preds_joint_coord_cam = np.array(preds_joint_coord_cam,
dtype=np.float32)
single_masks = np.array(single_masks, dtype=bool)
interacting_masks = np.array(interacting_masks, dtype=bool)
all_masks = np.array(all_masks, dtype=bool)
gts_hand_type = np.array(gts_hand_type, dtype=int)
preds_hand_type = np.array(preds_hand_type, dtype=int)
hand_type_masks = np.array(hand_type_masks, dtype=bool)
if 'MRRPE' in metrics:
info_str.append(('MRRPE',
keypoint_epe(preds_rel_root, gts_rel_root,
rel_root_masks)))
if 'MPJPE' in metrics:
info_str.append(('MPJPE_all',
keypoint_epe(preds_joint_coord_cam,
gts_joint_coord_cam, all_masks)))
info_str.append(('MPJPE_single',
keypoint_epe(preds_joint_coord_cam,
gts_joint_coord_cam, single_masks)))
info_str.append(
('MPJPE_interacting',
keypoint_epe(preds_joint_coord_cam, gts_joint_coord_cam,
interacting_masks)))
if 'Handedness_acc' in metrics:
info_str.append(('Handedness_acc',
self._get_accuracy(preds_hand_type, gts_hand_type,
hand_type_masks)))
return info_str
