def evaluate(self, preds, result_dir):
print('Evaluation start...')
gts = self.data
assert len(gts) == len(preds)
sample_num = len(gts)
joint_num = self.joint_num
pred_save = []
for n in range(sample_num):
gt = gts[n]
image_id = gt['img_id']
f = gt['f']
c = gt['c']
bbox = gt['bbox']
gt_3d_root = gt['root_cam']
gt_3d_kpt = gt['joint_cam']
gt_vis = gt['joint_vis']
# restore coordinates to original space
pre_2d_kpt = preds[n].copy()
pre_2d_kpt[:,0], pre_2d_kpt[:,1], pre_2d_kpt[:,2] = warp_coord_to_original(pre_2d_kpt, bbox, gt_3d_root)
vis = False
if vis:
cvimg = cv2.imread(gt['img_path'], cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
filename = str(random.randrange(1,500))
tmpimg = cvimg.copy().astype(np.uint8)
tmpkps = np.zeros((3,joint_num))
tmpkps[0,:], tmpkps[1,:] = pre_2d_kpt[:,0], pre_2d_kpt[:,1]
tmpkps[2,:] = 1
tmpimg = vis_keypoints(tmpimg, tmpkps, self.skeleton)
cv2.imwrite(filename + '_output.jpg', tmpimg)
# back project to camera coordinate system
pred_3d_kpt = np.zeros((joint_num,3))
pred_3d_kpt[:,0], pred_3d_kpt[:,1], pred_3d_kpt[:,2] = pixel2cam(pre_2d_kpt, f, c)
# root joint alignment
pred_3d_kpt = pred_3d_kpt - pred_3d_kpt[self.root_idx]
gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx]
# rigid alignment for PA MPJPE (protocol #1)
pred_3d_kpt_align = rigid_align(pred_3d_kpt, gt_3d_kpt)
# exclude thorax
pred_3d_kpt = np.take(pred_3d_kpt, self.eval_joint, axis=0)
pred_3d_kpt_align = np.take(pred_3d_kpt_align, self.eval_joint, axis=0)
# prediction save
pred_save.append({'image_id': image_id, 'joint_cam': pred_3d_kpt.tolist(), 'joint_cam_aligned': pred_3d_kpt_align.tolist(), 'bbox': bbox.tolist(), 'root_cam': gt_3d_root.tolist()}) # joint_cams are root-relative
output_path = osp.join(result_dir, 'bbox_root_pose_human36m_output.json')
with open(output_path, 'w') as f:
json.dump(pred_save, f)
print("Test result is saved at " + output_path)
calculate_score(output_path, self.annot_dir, self.subject_list)
def evaluate(self, preds, result_dir):
print('Evaluation start...')
gts = self.data
assert len(gts) == len(preds)
sample_num = len(gts)
pred_save = []
error = np.zeros((sample_num, self.joint_num - 1)) # joint error
error_action = [[] for _ in range(len(self.action_name))
] # error for each sequence
for n in range(sample_num):
gt = gts[n]
image_id = gt['img_id']
f = gt['f']
c = gt['c']
bbox = gt['bbox']
gt_3d_root = gt['root_cam']
gt_3d_kpt = gt['joint_cam']
gt_vis = gt['joint_vis']
# restore coordinates to original space
pred_2d_kpt = preds[n].copy()
pred_2d_kpt[:, 0] = pred_2d_kpt[:, 0] / cfg.output_shape[1] * bbox[
2] + bbox[0]
pred_2d_kpt[:, 1] = pred_2d_kpt[:, 1] / cfg.output_shape[0] * bbox[
3] + bbox[1]
pred_2d_kpt[:,
2] = (pred_2d_kpt[:, 2] / cfg.depth_dim * 2 -
1) * (cfg.bbox_3d_shape[0] / 2) + gt_3d_root[2]
vis = False
if vis:
cvimg = cv2.imread(
gt['img_path'],
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
filename = str(random.randrange(1, 500))
tmpimg = cvimg.copy().astype(np.uint8)
tmpkps = np.zeros((3, self.joint_num))
tmpkps[0, :], tmpkps[1, :] = pred_2d_kpt[:, 0], pred_2d_kpt[:,
1]
tmpkps[2, :] = 1
tmpimg = vis_keypoints(tmpimg, tmpkps, self.skeleton)
cv2.imwrite(filename + '_output.jpg', tmpimg)
# back project to camera coordinate system
pred_3d_kpt = pixel2cam(pred_2d_kpt, f, c)
# root joint alignment
pred_3d_kpt = pred_3d_kpt - pred_3d_kpt[self.root_idx]
gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx]
if self.protocol == 1:
# rigid alignment for PA MPJPE (protocol #1)
pred_3d_kpt = rigid_align(pred_3d_kpt, gt_3d_kpt)
# exclude thorax
pred_3d_kpt = np.take(pred_3d_kpt, self.eval_joint, axis=0)
gt_3d_kpt = np.take(gt_3d_kpt, self.eval_joint, axis=0)
# error calculate
error[n] = np.sqrt(np.sum((pred_3d_kpt - gt_3d_kpt)**2, 1))
img_name = gt['img_path']
action_idx = int(img_name[img_name.find('act') +
4:img_name.find('act') + 6]) - 2
error_action[action_idx].append(error[n].copy())
# prediction save
pred_save.append({
'image_id': image_id,
'joint_cam': pred_3d_kpt.tolist(),
'bbox': bbox.tolist(),
'root_cam': gt_3d_root.tolist()
}) # joint_cam is root-relative coordinate
# total error
tot_err = np.mean(error)
metric = 'PA MPJPE' if self.protocol == 1 else 'MPJPE'
eval_summary = 'Protocol ' + str(
self.protocol) + ' error (' + metric + ') >> tot: %.2f\n' % (
tot_err)
# error for each action
for i in range(len(error_action)):
err = np.mean(np.array(error_action[i]))
eval_summary += (self.action_name[i] + ': %.2f ' % err)
print(eval_summary)
# prediction save
output_path = osp.join(result_dir,
'bbox_root_pose_human36m_output.json')
with open(output_path, 'w') as f:
json.dump(pred_save, f)
print("Test result is saved at " + output_path)
def evaluate(self, preds, result_dir):
print()
print('Evaluation start...')
gts = self.load_data()
assert len(gts) == len(preds)
sample_num = len(gts)
joint_num = self.joint_num
p1_error = np.zeros(
(sample_num, joint_num, 3)) # PA MPJPE (protocol #1 metric)
p2_error = np.zeros(
(sample_num, joint_num, 3)) # MPJPE (protocol #2 metroc)
p1_error_action = [[] for _ in range(len(self.action_idx))
] # PA MPJPE for each action
p2_error_action = [[] for _ in range(len(self.action_idx))
] # MPJPE error for each action
pred_to_save = []
for n in range(sample_num):
gt = gts[n]
f = gt['f']
c = gt['c']
bbox = gt['bbox']
gt_3d_center = gt['center_cam']
gt_3d_kpt = gt['joint_cam']
gt_vis = gt['joint_vis'].copy()
# restore coordinates to original space
pre_2d_kpt = preds[n].copy()
pre_2d_kpt[:,
0], pre_2d_kpt[:,
1], pre_2d_kpt[:,
2] = warp_coord_to_original(
pre_2d_kpt, bbox,
gt_3d_center)
vis = False
if vis:
cvimg = cv2.imread(
gt['img_path'],
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
filename = str(random.randrange(1, 500))
tmpimg = cvimg.copy().astype(np.uint8)
tmpkps = np.zeros((3, joint_num))
tmpkps[0, :], tmpkps[1, :] = pre_2d_kpt[:, 0], pre_2d_kpt[:, 1]
tmpkps[2, :] = 1
tmpimg = vis_keypoints(tmpimg, tmpkps, self.skeleton)
cv2.imwrite(osp.join(cfg.vis_dir, filename + '_output.jpg'),
tmpimg)
# back project to camera coordinate system
pre_3d_kpt = np.zeros((joint_num, 3))
pre_3d_kpt[:, 0], pre_3d_kpt[:, 1], pre_3d_kpt[:, 2] = pixel2cam(
pre_2d_kpt, f, c)
vis = False
if vis:
vis_3d_skeleton(pre_3d_kpt, gt_vis, self.skeleton, filename)
# root joint alignment
pre_3d_kpt = pre_3d_kpt - pre_3d_kpt[self.root_idx]
gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx]
# rigid alignment for PA MPJPE (protocol #1)
pre_3d_kpt_align = rigid_align(pre_3d_kpt, gt_3d_kpt)
# prediction save
pred_to_save.append({
'pred': pre_3d_kpt,
'align_pred': pre_3d_kpt_align,
'gt': gt_3d_kpt
})
# error save
p1_error[n] = np.power(pre_3d_kpt_align - gt_3d_kpt,
2) # PA MPJPE (protocol #1)
p2_error[n] = np.power(pre_3d_kpt - gt_3d_kpt,
2) # MPJPE (protocol #2)
img_name = gt['img_path']
action_idx = int(img_name[img_name.find('act') +
4:img_name.find('act') + 6]) - 2
p1_error_action[action_idx].append(p1_error[n].copy())
p2_error_action[action_idx].append(p2_error[n].copy())
# total error calculate
p1_error = np.take(p1_error, self.eval_joint, axis=1)
p2_error = np.take(p2_error, self.eval_joint, axis=1)
p1_error = np.mean(np.power(np.sum(p1_error, axis=2), 0.5))
p2_error = np.mean(np.power(np.sum(p2_error, axis=2), 0.5))
p1_eval_summary = 'Protocol #1 error (PA MPJPE) >> %.2f' % (p1_error)
p2_eval_summary = 'Protocol #2 error (MPJPE) >> %.2f' % (p2_error)
print()
print(p1_eval_summary)
print(p2_eval_summary)
# error for each action calculate
p1_action_eval_summary = 'Protocol #1 error (PA MPJPE) for each action: \n'
for i in range(len(p1_error_action)):
err = np.array(p1_error_action[i])
err = np.take(err, self.eval_joint, axis=1)
err = np.mean(np.power(np.sum(err, axis=2), 0.5))
action_name = self.action_name[i]
p1_action_eval_summary += (action_name + ': %.2f\n' % err)
p2_action_eval_summary = 'Protocol #2 error (MPJPE) for each action: \n'
for i in range(len(p2_error_action)):
err = np.array(p2_error_action[i])
err = np.take(err, self.eval_joint, axis=1)
err = np.mean(np.power(np.sum(err, axis=2), 0.5))
action_name = self.action_name[i]
p2_action_eval_summary += (action_name + ': %.2f\n' % err)
print()
print(p1_action_eval_summary)
print(p2_action_eval_summary)
# result save
f_pred_3d_kpt = open(osp.join(result_dir, 'pred_3d_kpt.txt'), 'w')
f_pred_3d_kpt_align = open(
osp.join(result_dir, 'pred_3d_kpt_align.txt'), 'w')
f_gt_3d_kpt = open(osp.join(result_dir, 'gt_3d_kpt.txt'), 'w')
for i in range(len(pred_to_save)):
for j in range(joint_num):
for k in range(3):
f_pred_3d_kpt.write('%.3f ' %
pred_to_save[i]['pred'][j][k])
f_pred_3d_kpt_align.write(
'%.3f ' % pred_to_save[i]['align_pred'][j][k])
f_gt_3d_kpt.write('%.3f ' % pred_to_save[i]['gt'][j][k])
f_pred_3d_kpt.write('\n')
f_pred_3d_kpt_align.write('\n')
f_gt_3d_kpt.write('\n')
f_pred_3d_kpt.close()
f_pred_3d_kpt_align.close()
f_gt_3d_kpt.close()
f_eval_result = open(osp.join(result_dir, 'eval_result.txt'), 'w')
f_eval_result.write(p1_eval_summary)
f_eval_result.write('\n')
f_eval_result.write(p2_eval_summary)
f_eval_result.write('\n')
f_eval_result.write(p1_action_eval_summary)
f_eval_result.write('\n')
f_eval_result.write(p2_action_eval_summary)
f_eval_result.write('\n')
f_eval_result.close()