def evaluate(self, outs):
print('Evaluation start...')
annots = self.datalist
assert len(annots) == len(outs)
sample_num = len(annots)
mesh_output_save = []
joint_output_save = []
for n in range(sample_num):
annot = annots[n]
out = outs[n]
mesh_coord_out = out['mesh_coord']
# joint_coord_out = coord_out_cam[self.vertex_num:,:]
joint_coord_out = np.dot(self.joint_regressor_mano, mesh_coord_out)
mesh_output_save.append(mesh_coord_out.tolist())
joint_output_save.append(joint_coord_out.tolist())
vis = cfg.TEST.vis
if vis and n % 500 == 0:
filename = str(n)
save_obj(mesh_coord_out, self.mesh_model.face, osp.join(cfg.vis_dir, filename + '.obj'))
output_save_path = osp.join(cfg.output_dir, 'pred.json')
with open(output_save_path, 'w') as f:
json.dump([joint_output_save, mesh_output_save], f)
def evaluate_joint(self, outs):
print('Evaluation start...')
annots = self.datalist
assert len(annots) == len(outs)
sample_num = len(annots)
mesh_output_save = []
joint_output_save = []
for n in range(sample_num):
annot = annots[n]
out = outs[n]
joint_coord_out = out['joint_coord']
mesh_coord_out = joint_coord_out # dummy
mesh_coord_out = mesh_coord_out - joint_coord_out[:1]
joint_coord_out = joint_coord_out - joint_coord_out[:1]
mesh_output_save.append(mesh_coord_out.tolist())
joint_output_save.append(joint_coord_out.tolist())
vis = False
if vis:
filename = str(n)
save_obj(mesh_coord_out, self.mano.face, osp.join(cfg.output_dir, filename + '.obj'))
output_save_path = osp.join(cfg.output_dir, 'pred.json')
with open(output_save_path, 'w') as f:
json.dump([joint_output_save, mesh_output_save], f)
def evaluate(self, outs):
annots = self.datalist
assert len(annots) == len(outs)
sample_num = len(annots)
eval_result = {'pose_error': [], 'mesh_error': []}
for n in range(sample_num):
out = outs[n]
annot = annots[n]
# root joint alignment
mesh_coord_out, mesh_coord_gt = out['mesh_coord'], out[
'mesh_coord_target']
joint_coord_out, joint_coord_gt = np.dot(
self.joint_regressor_smpl,
mesh_coord_out), np.dot(self.joint_regressor_smpl,
mesh_coord_gt)
mesh_coord_out = mesh_coord_out - joint_coord_out[:1]
mesh_coord_target = mesh_coord_gt - joint_coord_gt[:1]
joint_coord_out = joint_coord_out - joint_coord_out[:1]
joint_coord_target = joint_coord_gt - joint_coord_gt[:1]
# mesh error
eval_result['mesh_error'].append(
np.sqrt(np.sum((mesh_coord_out - mesh_coord_target)**2,
1))) # meter -> milimeter
# pose error
eval_result['pose_error'].append(
np.sqrt(np.sum((joint_coord_out - joint_coord_target)**2,
1))) # meter -> milimeter
vis = cfg.TEST.vis
if vis:
filename = annot['img_name'].split('/')[-1][:-4] #str(n)
save_obj(mesh_coord_out, self.mesh_model.face,
osp.join(cfg.vis_dir, filename + '_pred.obj'))
save_obj(mesh_coord_target, self.mesh_model.face,
osp.join(cfg.vis_dir, filename + '_gt.obj'))
print('Pose error (MPJPE): %.2f mm' %
np.mean(eval_result['pose_error']))
print('Mesh error (MPVPE): %.2f mm' %
np.mean(eval_result['mesh_error']))
def evaluate(self, outs):
print('Evaluation start...')
annots = self.datalist
assert len(annots) == len(outs)
sample_num = len(annots)
sample_num = len(outs)
mpjpe_h36m = np.zeros((sample_num, len(self.human36_eval_joint))) # pose error
pampjpe_h36m = np.zeros((sample_num, len(self.human36_eval_joint))) # pose error
mpjpe_smpl = np.zeros((sample_num, self.smpl_joint_num)) # pose error
mpvpe = np.zeros((sample_num, self.smpl_vertex_num)) # mesh error
pa_mpvpe = np.zeros((sample_num, self.smpl_vertex_num)) # mesh error
pred_j3d, gt_j3d = [], []
for n in range(sample_num):
out = outs[n]
annot = annots[n]
img_path = annot['img_path']
mesh_coord_out, mesh_coord_gt = out['mesh_coord'], out['mesh_coord_target']
joint_coord_out, joint_coord_gt = np.dot(self.joint_regressor_smpl, mesh_coord_out), np.dot(self.joint_regressor_smpl, mesh_coord_gt)
# root joint alignment
coord_out_cam = np.concatenate((mesh_coord_out, joint_coord_out))
coord_out_cam = coord_out_cam - coord_out_cam[self.smpl_vertex_num + self.smpl_root_joint_idx]
coord_gt_cam = np.concatenate((mesh_coord_gt, joint_coord_gt))
coord_gt_cam = coord_gt_cam - coord_gt_cam[self.smpl_vertex_num + self.smpl_root_joint_idx]
# pose error calculate
pose_coord_out = coord_out_cam[self.smpl_vertex_num:,:]
pose_coord_gt = coord_gt_cam[self.smpl_vertex_num:,:]
mpjpe_smpl[n] = np.sqrt(np.sum((pose_coord_out - pose_coord_gt)**2,1))
# mesh error calculate
mesh_coord_out = coord_out_cam[:self.smpl_vertex_num,:]
mesh_coord_gt = coord_gt_cam[:self.smpl_vertex_num,:]
mpvpe[n] = np.sqrt(np.sum((mesh_coord_out - mesh_coord_gt)**2,1))
mesh_coord_out_aligned, _, _, _ = rigid_align(mesh_coord_out, mesh_coord_gt)
pa_mpvpe[n] = np.sqrt(np.sum((mesh_coord_out_aligned - mesh_coord_gt)**2,1))
# pose error of h36m calculate
pose_coord_out_h36m = np.dot(self.mesh_model.joint_regressor_h36m, mesh_coord_out)
pose_coord_out_h36m = pose_coord_out_h36m - pose_coord_out_h36m[self.human36_root_joint_idx]
pose_coord_out_h36m = pose_coord_out_h36m[self.human36_eval_joint, :]
pose_coord_gt_h36m = np.dot(self.mesh_model.joint_regressor_h36m, mesh_coord_gt)
pose_coord_gt_h36m = pose_coord_gt_h36m - pose_coord_gt_h36m[self.human36_root_joint_idx]
pose_coord_gt_h36m = pose_coord_gt_h36m[self.human36_eval_joint, :]
pred_j3d.append(pose_coord_out_h36m); gt_j3d.append(pose_coord_gt_h36m)
mpjpe_h36m[n] = np.sqrt(np.sum((pose_coord_out_h36m - pose_coord_gt_h36m)**2,1))
pose_coord_out_h36m_aligned, _, _, _ = rigid_align(pose_coord_out_h36m, pose_coord_gt_h36m) # perform rigid alignment
pampjpe_h36m[n] = np.sqrt(np.sum((pose_coord_out_h36m_aligned - pose_coord_gt_h36m)**2,1))
vis = cfg.TEST.vis
if vis and (n % 10):
mesh_to_save = mesh_coord_out / 1000
obj_path = osp.join(cfg.vis_dir, f'3dpw_{img_path}.obj')
save_obj(mesh_to_save, self.mesh_model.face, obj_path)
"""
print("--------Smoothed output errors--------")
# print smoothed results
# compute accel
accel_error = []
mpjpe_list = []
pa_mpjpe_list = []
pred_j3d, gt_j3d = np.array(pred_j3d), np.array(gt_j3d)
for vid_idx in self.video_indices:
pred, gt = pred_j3d[vid_idx], gt_j3d[vid_idx]
pred = smooth_pose(pred, min_cutoff=0.004, beta=0.005)
vid_acc_err = compute_error_accel(gt, pred)
vid_acc_err = np.mean(vid_acc_err)
accel_error.append(vid_acc_err)
mpjpe = np.sqrt(np.sum((pred - gt)**2,2))
mpjpe_list.append(np.mean(mpjpe))
for idx in range(len(pred)):
pa_pred, _, _, _ = rigid_align(pred[idx], gt[idx])
pa_mpjpe = np.sqrt(np.sum((pa_pred - gt[idx])**2,1))
pa_mpjpe_list.append(pa_mpjpe)
accel_error = np.mean(accel_error)
eval_summary = 'H36M accel error (mm/s^2): tot: %.2f\n' % (accel_error)
print(eval_summary)
tot_err = np.mean(mpjpe_list)
eval_summary = 'H36M MPJPE (mm) >> tot: %.2f\n' % (tot_err)
print(eval_summary)
tot_err = np.mean(pa_mpjpe_list)
eval_summary = 'H36M PA-MPJPE (mm) >> tot: %.2f\n' % (tot_err)
print(eval_summary)
print("--------Original output errors--------")
"""
# total pose error (H36M joint set)
tot_err = np.mean(mpjpe_h36m)
eval_summary = 'H36M MPJPE (mm) >> tot: %.2f\n' % (tot_err)
print(eval_summary)
tot_err = np.mean(pampjpe_h36m)
eval_summary = 'H36M PA-MPJPE (mm) >> tot: %.2f\n' % (tot_err)
print(eval_summary)
# total pose error (SMPL joint set)
tot_err = np.mean(mpjpe_smpl)
eval_summary = 'SMPL MPJPE (mm) >> tot: %.2f\n' % (tot_err)
print(eval_summary)
# total mesh error
tot_err = np.mean(mpvpe)
eval_summary = 'MPVPE (mm) >> tot: %.2f\n' % (tot_err)
print(eval_summary)
# total mesh error
tot_err = np.mean(pa_mpvpe)
eval_summary = 'PA-MPVPE (mm) >> tot: %.2f\n' % (tot_err)
print(eval_summary)
def evaluate(self, outs):
print('Evaluation start...')
annots = self.datalist
# assert len(annots) == len(outs)
# sample_num = len(annots)
sample_num = len(outs)
mpjpe_h36m = np.zeros(
(sample_num, len(self.human36_eval_joint))) # pose error
pampjpe_h36m = np.zeros(
(sample_num, len(self.human36_eval_joint))) # pose error
mpjpe_smpl = np.zeros((sample_num, self.smpl_joint_num)) # pose error
mpvpe = np.zeros((sample_num, self.smpl_vertex_num)) # mesh error
for n in range(sample_num):
out = outs[n]
annot = annots[n]
mesh_coord_out, mesh_coord_gt = out['mesh_coord'], out[
'mesh_coord_target']
joint_coord_out, joint_coord_gt = np.dot(
self.joint_regressor_smpl,
mesh_coord_out), np.dot(self.joint_regressor_smpl,
mesh_coord_gt)
# root joint alignment
coord_out_cam = np.concatenate((mesh_coord_out, joint_coord_out))
coord_out_cam = coord_out_cam - coord_out_cam[
self.smpl_vertex_num + self.smpl_root_joint_idx]
coord_gt_cam = np.concatenate((mesh_coord_gt, joint_coord_gt))
coord_gt_cam = coord_gt_cam - coord_gt_cam[
self.smpl_vertex_num + self.smpl_root_joint_idx]
# pose error calculate
pose_coord_out = coord_out_cam[self.smpl_vertex_num:, :]
pose_coord_gt = coord_gt_cam[self.smpl_vertex_num:, :]
mpjpe_smpl[n] = np.sqrt(
np.sum((pose_coord_out - pose_coord_gt)**2, 1))
# mesh error calculate
mesh_coord_out = coord_out_cam[:self.smpl_vertex_num, :]
mesh_coord_gt = coord_gt_cam[:self.smpl_vertex_num, :]
mpvpe[n] = np.sqrt(np.sum((mesh_coord_out - mesh_coord_gt)**2, 1))
# pose error of h36m calculate
pose_coord_out_h36m = np.dot(self.mesh_model.joint_regressor_h36m,
mesh_coord_out)
pose_coord_out_h36m = pose_coord_out_h36m - pose_coord_out_h36m[
self.human36_root_joint_idx]
pose_coord_out_h36m = pose_coord_out_h36m[
self.human36_eval_joint, :]
pose_coord_gt_h36m = np.dot(self.mesh_model.joint_regressor_h36m,
mesh_coord_gt)
pose_coord_gt_h36m = pose_coord_gt_h36m - pose_coord_gt_h36m[
self.human36_root_joint_idx]
pose_coord_gt_h36m = pose_coord_gt_h36m[self.human36_eval_joint, :]
mpjpe_h36m[n] = np.sqrt(
np.sum((pose_coord_out_h36m - pose_coord_gt_h36m)**2, 1))
pose_coord_out_h36m = rigid_align(
pose_coord_out_h36m,
pose_coord_gt_h36m) # perform rigid alignment
pampjpe_h36m[n] = np.sqrt(
np.sum((pose_coord_out_h36m - pose_coord_gt_h36m)**2, 1))
vis = cfg.TEST.vis
if vis and (n % 500):
mesh_to_save = mesh_coord_out / 1000
obj_path = osp.join(cfg.vis_dir, f'3dpw_{n}.obj')
save_obj(mesh_to_save, self.mesh_model.face, obj_path)
# total pose error (H36M joint set)
tot_err = np.mean(mpjpe_h36m)
eval_summary = 'H36M MPJPE (mm) >> tot: %.2f\n' % (tot_err)
print(eval_summary)
tot_err = np.mean(pampjpe_h36m)
eval_summary = 'H36M PA-MPJPE (mm) >> tot: %.2f\n' % (tot_err)
print(eval_summary)
# total pose error (SMPL joint set)
tot_err = np.mean(mpjpe_smpl)
eval_summary = 'SMPL MPJPE (mm) >> tot: %.2f\n' % (tot_err)
print(eval_summary)
# total mesh error
tot_err = np.mean(mpvpe)
eval_summary = 'MPVPE (mm) >> tot: %.2f\n' % (tot_err)
print(eval_summary)
def evaluate(self, outs):
print('Evaluation start...')
annots = self.datalist
assert len(annots) == len(outs)
sample_num = len(outs)
# eval H36M joints
pose_error_h36m = np.zeros((sample_num, len(self.human36_eval_joint))) # pose error
pose_error_action_h36m = [[] for _ in range(len(self.action_name))] # pose error for each sequence
pose_pa_error_h36m = np.zeros((sample_num, len(self.human36_eval_joint))) # pose error
pose_pa_error_action_h36m = [[] for _ in range(len(self.action_name))] # pose error for each sequence
# eval SMPL joints and mesh vertices
pose_error = np.zeros((sample_num, self.smpl_joint_num)) # pose error
pose_error_action = [[] for _ in range(len(self.action_name))] # pose error for each sequence
mesh_error = np.zeros((sample_num, self.smpl_vertex_num)) # mesh error
mesh_error_action = [[] for _ in range(len(self.action_name))] # mesh error for each sequence
for n in range(sample_num):
annot = annots[n]
out = outs[n]
# render materials
img_path = annot['img_path']
obj_name = '_'.join(img_path.split('/')[-2:])[:-4]
# root joint alignment
mesh_coord_out, mesh_coord_gt = out['mesh_coord'], out['mesh_coord_target']
joint_coord_out, joint_coord_gt = np.dot(self.joint_regressor_smpl, mesh_coord_out), np.dot(self.joint_regressor_smpl, mesh_coord_gt)
mesh_coord_out = mesh_coord_out - joint_coord_out[self.smpl_root_joint_idx:self.smpl_root_joint_idx+1]
mesh_coord_gt = mesh_coord_gt - joint_coord_gt[self.smpl_root_joint_idx:self.smpl_root_joint_idx+1]
pose_coord_out = joint_coord_out - joint_coord_out[self.smpl_root_joint_idx:self.smpl_root_joint_idx+1]
pose_coord_gt = joint_coord_gt - joint_coord_gt[self.smpl_root_joint_idx:self.smpl_root_joint_idx+1]
# pose error calculate
pose_error[n] = np.sqrt(np.sum((pose_coord_out - pose_coord_gt) ** 2, 1))
img_name = annot['img_path']
action_idx = int(img_name[img_name.find('act') + 4:img_name.find('act') + 6]) - 2
pose_error_action[action_idx].append(pose_error[n].copy())
# mesh error calculate
mesh_error[n] = np.sqrt(np.sum((mesh_coord_out - mesh_coord_gt) ** 2, 1))
img_name = annot['img_path']
action_idx = int(img_name[img_name.find('act') + 4:img_name.find('act') + 6]) - 2
mesh_error_action[action_idx].append(mesh_error[n].copy())
# pose error of h36m calculate
pose_coord_out_h36m = np.dot(self.joint_regressor_human36, mesh_coord_out)
pose_coord_out_h36m = pose_coord_out_h36m - pose_coord_out_h36m[self.human36_root_joint_idx]
pose_coord_out_h36m = pose_coord_out_h36m[self.human36_eval_joint, :]
pose_coord_gt_h36m = annot['joint_cam']
pose_coord_gt_h36m = pose_coord_gt_h36m - pose_coord_gt_h36m[self.human36_root_joint_idx]
pose_coord_gt_h36m = pose_coord_gt_h36m[self.human36_eval_joint, :]
pose_error_h36m[n] = np.sqrt(np.sum((pose_coord_out_h36m - pose_coord_gt_h36m) ** 2, 1))
pose_coord_out_h36m = rigid_align(pose_coord_out_h36m, pose_coord_gt_h36m) # perform rigid alignment
pose_pa_error_h36m[n] = np.sqrt(np.sum((pose_coord_out_h36m - pose_coord_gt_h36m) ** 2, 1))
img_name = annot['img_path']
action_idx = int(img_name[img_name.find('act') + 4:img_name.find('act') + 6]) - 2
pose_error_action_h36m[action_idx].append(pose_error_h36m[n].copy())
pose_pa_error_action_h36m[action_idx].append(pose_pa_error_h36m[n].copy())
vis = cfg.TEST.vis
if vis and (n % 500 == 0):
mesh_to_save = mesh_coord_out / 1000
obj_path = osp.join(cfg.vis_dir, f'{obj_name}.obj')
save_obj(mesh_to_save, self.mesh_model.face, obj_path)
# total pose error (H36M joint set)
tot_err = np.mean(pose_error_h36m)
metric = 'MPJPE'
eval_summary = 'Protocol ' + str(self.protocol) + ' H36M pose error (' + metric + ') >> tot: %.2f\n' % (tot_err)
# pose error for each action
for i in range(len(pose_error_action_h36m)):
err = np.mean(np.array(pose_error_action_h36m[i]))
eval_summary += (self.action_name[i] + ': %.2f ' % err)
print(eval_summary)
tot_err = np.mean(pose_pa_error_h36m)
metric = 'PA-MPJPE'
eval_summary = 'Protocol ' + str(self.protocol) + ' H36M pose error (' + metric + ') >> tot: %.2f\n' % (tot_err)
# pose error for each action
for i in range(len(pose_pa_error_action_h36m)):
err = np.mean(np.array(pose_pa_error_action_h36m[i]))
eval_summary += (self.action_name[i] + ': %.2f ' % err)
print(eval_summary)
# total pose error (SMPL joint set)
tot_err = np.mean(pose_error)
metric = 'MPJPE'
eval_summary = 'Protocol ' + str(self.protocol) + ' SMPL pose error (' + metric + ') >> tot: %.2f\n' % (tot_err)
# pose error for each action
for i in range(len(pose_error_action)):
err = np.mean(np.array(pose_error_action[i]))
eval_summary += (self.action_name[i] + ': %.2f ' % err)
print(eval_summary)
# total mesh error
tot_err = np.mean(mesh_error)
metric = 'MPVPE'
eval_summary = 'Protocol ' + str(self.protocol) + ' SMPL mesh error (' + metric + ') >> tot: %.2f\n' % (tot_err)
# mesh error for each action
for i in range(len(mesh_error_action)):
err = np.mean(np.array(mesh_error_action[i]))
eval_summary += (self.action_name[i] + ': %.2f ' % err)
print(eval_summary)
# estimate mesh, pose
pred_mesh, _ = model(joint_img)
pred_mesh = pred_mesh[:, graph_perm_reverse[:mesh_model.face.max() + 1], :]
pred_3d_joint = torch.matmul(joint_regressor, pred_mesh)
out = {}
# assume batch=1
for j in range(0, 1500):
# projection
pred_2d_joint = project_net(pred_3d_joint.detach())
loss = criterion(pred_2d_joint, target_joint[:, :17, :])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if j == 500:
for param_group in optimizer.param_groups:
param_group['lr'] = 0.05
if j == 1000:
for param_group in optimizer.param_groups:
param_group['lr'] = 0.001
out['mesh'] = pred_mesh[0].detach().cpu().numpy()
out['cam_param'] = project_net.cam_param[0].detach().cpu().numpy()
out['trans'] = trans
out['pred2d_pose'] = proj_target_joint_img
render(out, output_path, FOCAL_LENGTH, IMG_RES, mesh_model.face, skeleton)
save_obj(out['mesh'], mesh_model.face,
osp.join(output_path, 'demo_output.obj'))
out = optimize_cam_param(project_net, joint_input, crop_size=virtual_crop_size)
# vis mesh
color = colorsys.hsv_to_rgb(np.random.rand(), 0.5, 1.0)
rendered_img = render(out, orig_height, orig_width, orig_img, mesh_model.face, color) # s[idx])
cv2.imwrite(output_path + f'demo_mesh.png', rendered_img)
# vis 2d pose
tmpkps = np.zeros((3, len(joint_input)))
tmpkps[0, :], tmpkps[1, :], tmpkps[2, :] = joint_input[:, 0], joint_input[:, 1], 1
tmpimg = orig_img.copy().astype(np.uint8)
pose_vis_img = vis_2d_keypoints(tmpimg, tmpkps, skeleton)
cv2.imwrite(output_path + f'demo_pose2d.png', pose_vis_img)
save_obj(out['mesh'], mesh_model.face, output_path + f'demo_mesh.obj')
else: # demo on CrowdPose dataset samples, dataset paper link: https://arxiv.org/abs/1812.00324
# only for vis
vis_joints_name = ('Nose', 'L_Eye', 'R_Eye', 'L_Ear', 'R_Ear', 'L_Shoulder', 'R_Shoulder', 'L_Elbow', 'R_Elbow', 'L_Wrist', 'R_Wrist', 'L_Hip', 'R_Hip', 'L_Knee', 'R_Knee', 'L_Ankle', 'R_Ankle', 'Thorax', 'Pelvis')
vis_skeleton = ((0, 1), (0, 2), (2, 4), (1, 3), (5, 7), (7, 9), (12, 14), (14, 16), (11, 13), (13, 15), (5, 17), (6, 17), (11, 18), (12, 18), (17, 18), (17, 0), (6, 8), (8, 10),)
# prepare input image and 2d pose
coco_joints_name = ('Nose', 'L_Eye', 'R_Eye', 'L_Ear', 'R_Ear', 'L_Shoulder', 'R_Shoulder', 'L_Elbow', 'R_Elbow', 'L_Wrist', 'R_Wrist', 'L_Hip', 'R_Hip', 'L_Knee', 'R_Knee', 'L_Ankle', 'R_Ankle', 'Pelvis', 'Neck')
pose2d_result_path = './demo/demo_input_2dpose.json' # coco 2d pose detection detected by HigherHRNet
with open(pose2d_result_path) as f:
pose2d_result = json.load(f)
img_dir = './demo/demo_input_img'
img_name = '106542.jpg' # '101570.jpg'
img_path = osp.join(img_dir, img_name)
orig_img = cv2.imread(img_path, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)