def _report_metric(self, res_file, metrics, pck_thr=0.2, auc_nor=30): """Keypoint evaluation. Args: res_file (str): Json file stored prediction results. metrics (str | list[str]): Metric to be performed. Options: 'PCK', 'PCKh', 'AUC', 'EPE'. pck_thr (float): PCK threshold, default as 0.2. pckh_thr (float): PCKh threshold, default as 0.7. auc_nor (float): AUC normalization factor, default as 30 pixel. Returns: List: Evaluation results for evaluation metric. """ info_str = [] with open(res_file, 'r') as fin: preds = json.load(fin) assert len(preds) == len(self.db) outputs = [] gts = [] masks = [] threshold_bbox = [] for pred, item in zip(preds, self.db): outputs.append(np.array(pred['keypoints'])[:, :-1]) gts.append(np.array(item['joints_3d'])[:, :-1]) masks.append((np.array(item['joints_3d_visible'])[:, 0]) > 0) if 'PCK' in metrics: bbox = np.array(item['bbox']) bbox_thr = np.max(bbox[2:]) threshold_bbox.append(np.array([bbox_thr, bbox_thr])) outputs = np.array(outputs) gts = np.array(gts) masks = np.array(masks) threshold_bbox = np.array(threshold_bbox) if 'PCK' in metrics: _, pck, _ = keypoint_pck_accuracy(outputs, gts, masks, pck_thr, threshold_bbox) info_str.append(('PCK', pck)) if 'AUC' in metrics: info_str.append(('AUC', keypoint_auc(outputs, gts, masks, auc_nor))) if 'EPE' in metrics: info_str.append(('EPE', keypoint_epe(outputs, gts, masks))) return info_str
def _report_metric(self, res_file, metrics): """Keypoint evaluation. Report PCK, AUC or EPE. """ info_str = [] with open(res_file, 'r') as fin: preds = json.load(fin) assert len(preds) == len(self.db) outputs = [] gts = [] for pred, item in zip(preds, self.db): outputs.append(pred['keypoints']) gts.append(item['joints_3d']) outputs = np.array(outputs)[:, :, :-1] gts = np.array(gts)[:, :, :-1] if 'PCK' in metrics: hit = 0 exist = 0 for pred, item in zip(preds, self.db): bbox = np.array(item['bbox']) threshold = np.max(bbox[2:]) * 0.2 h, _, e = keypoint_pck_accuracy( np.array(pred['keypoints'])[None, :, :-1], np.array(item['joints_3d'])[None, :, :-1], 1, np.array([[threshold, threshold]])) hit += len(h[h > 0]) exist += e pck = hit / exist info_str.append(('PCK', pck)) if 'AUC' in metrics: info_str.append(('AUC', keypoint_auc(outputs, gts, 30))) if 'EPE' in metrics: info_str.append(('EPE', keypoint_epe(outputs, gts))) return info_str
def _report_metric(self, res_file, metrics): """Keypoint evaluation. Args: res_file (str): Json file stored prediction results. metrics (str | list[str]): Metric to be performed. Options: 'MRRPE', 'MPJPE', 'Handedness_acc'. Returns: list: Evaluation results for evaluation metric. """ info_str = [] with open(res_file, 'r') as fin: preds = json.load(fin) assert len(preds) == len(self.db) gts_rel_root = [] preds_rel_root = [] rel_root_masks = [] gts_joint_coord_cam = [] preds_joint_coord_cam = [] single_masks = [] interacting_masks = [] all_masks = [] gts_hand_type = [] preds_hand_type = [] hand_type_masks = [] for pred, item in zip(preds, self.db): # mrrpe if 'MRRPE' in metrics: if item['hand_type'].all() and item['joints_3d_visible'][ 20, 0] and item['joints_3d_visible'][41, 0]: rel_root_masks.append(True) pred_left_root_img = np.array(pred['keypoints'][41], dtype=np.float32)[None, :] pred_left_root_img[:, 2] += item['abs_depth'][0] + pred[ 'rel_root_depth'] pred_left_root_cam = self._pixel2cam( pred_left_root_img, item['focal'], item['princpt']) pred_right_root_img = np.array(pred['keypoints'][20], dtype=np.float32)[None, :] pred_right_root_img[:, 2] += item['abs_depth'][0] pred_right_root_cam = self._pixel2cam( pred_right_root_img, item['focal'], item['princpt']) preds_rel_root.append(pred_left_root_cam - pred_right_root_cam) gts_rel_root.append( [item['joints_cam'][41] - item['joints_cam'][20]]) else: rel_root_masks.append(False) preds_rel_root.append([[0., 0., 0.]]) gts_rel_root.append([[0., 0., 0.]]) if 'MPJPE' in metrics: pred_joint_coord_img = np.array(pred['keypoints'], dtype=np.float32) gt_joint_coord_cam = item['joints_cam'].copy() pred_joint_coord_img[:21, 2] += item['abs_depth'][0] pred_joint_coord_img[21:, 2] += item['abs_depth'][1] pred_joint_coord_cam = self._pixel2cam(pred_joint_coord_img, item['focal'], item['princpt']) pred_joint_coord_cam[:21] -= pred_joint_coord_cam[20] pred_joint_coord_cam[21:] -= pred_joint_coord_cam[41] gt_joint_coord_cam[:21] -= gt_joint_coord_cam[20] gt_joint_coord_cam[21:] -= gt_joint_coord_cam[41] preds_joint_coord_cam.append(pred_joint_coord_cam) gts_joint_coord_cam.append(gt_joint_coord_cam) mask = (np.array(item['joints_3d_visible'])[:, 0]) > 0 if item['hand_type'].all(): single_masks.append( np.zeros(self.ann_info['num_joints'], dtype=bool)) interacting_masks.append(mask) all_masks.append(mask) else: single_masks.append(mask) interacting_masks.append( np.zeros(self.ann_info['num_joints'], dtype=bool)) all_masks.append(mask) if 'Handedness_acc' in metrics: pred_hand_type = np.array(pred['hand_type'], dtype=int) preds_hand_type.append(pred_hand_type) gts_hand_type.append(item['hand_type']) hand_type_masks.append(item['hand_type_valid'] > 0) gts_rel_root = np.array(gts_rel_root, dtype=np.float32) preds_rel_root = np.array(preds_rel_root, dtype=np.float32) rel_root_masks = np.array(rel_root_masks, dtype=bool)[:, None] gts_joint_coord_cam = np.array(gts_joint_coord_cam, dtype=np.float32) preds_joint_coord_cam = np.array(preds_joint_coord_cam, dtype=np.float32) single_masks = np.array(single_masks, dtype=bool) interacting_masks = np.array(interacting_masks, dtype=bool) all_masks = np.array(all_masks, dtype=bool) gts_hand_type = np.array(gts_hand_type, dtype=int) preds_hand_type = np.array(preds_hand_type, dtype=int) hand_type_masks = np.array(hand_type_masks, dtype=bool) if 'MRRPE' in metrics: info_str.append(('MRRPE', keypoint_epe(preds_rel_root, gts_rel_root, rel_root_masks))) if 'MPJPE' in metrics: info_str.append(('MPJPE_all', keypoint_epe(preds_joint_coord_cam, gts_joint_coord_cam, all_masks))) info_str.append(('MPJPE_single', keypoint_epe(preds_joint_coord_cam, gts_joint_coord_cam, single_masks))) info_str.append( ('MPJPE_interacting', keypoint_epe(preds_joint_coord_cam, gts_joint_coord_cam, interacting_masks))) if 'Handedness_acc' in metrics: info_str.append(('Handedness_acc', self._get_accuracy(preds_hand_type, gts_hand_type, hand_type_masks))) return info_str