def evaluate(self, preds, result_dir):
print('Evaluation start...')
gts = self._labels
assert len(gts) == len(preds)
sample_num = len(gts)
seq_idx_dict = {k: [] for k in self.test_seqs}
act_idx_dict = {k: [] for k in range(len(self.activity_name))}
pred_save = []
error = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
error_pa = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
error_x = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
error_y = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
error_z = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
# error for each sequence
for n in range(sample_num):
gt = gts[n]
img_name = gt['img_name']
intrinsic_param = gt['intrinsic_param']
bbox = gt['bbox']
gt_3d_root = gt['root_cam']
gt_3d_kpt = gt['joint_cam']
# gt_vis = gt['joint_vis']
# restore coordinates to original space
pred_2d_kpt = preds[n].copy()
# pred_2d_kpt[:, 0] = pred_2d_kpt[:, 0] / self._output_size[1] * bbox[2] + bbox[0]
# pred_2d_kpt[:, 1] = pred_2d_kpt[:, 1] / self._output_size[0] * bbox[3] + bbox[1]
pred_2d_kpt[:, 2] = pred_2d_kpt[:, 2] * self.bbox_3d_shape[
0] + gt_3d_root[2]
# back project to camera coordinate system
pred_3d_kpt = pixel2cam_matrix(pred_2d_kpt, intrinsic_param)
# root joint alignment
pred_3d_kpt = pred_3d_kpt - pred_3d_kpt[self.root_idx]
gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx]
# if self.protocol == 1:
# # rigid alignment for PA MPJPE (protocol #1)
pred_3d_kpt_pa = reconstruction_error(pred_3d_kpt, gt_3d_kpt)
# exclude thorax
# pred_3d_kpt = np.take(pred_3d_kpt, self.EVAL_JOINTS, axis=0)
# pred_3d_kpt_pa = np.take(pred_3d_kpt_pa, self.EVAL_JOINTS, axis=0)
# gt_3d_kpt = np.take(gt_3d_kpt, self.EVAL_JOINTS, axis=0)
# error calculate
error[n] = np.sqrt(np.sum((pred_3d_kpt - gt_3d_kpt)**2, 1))
error_pa[n] = np.sqrt(np.sum((pred_3d_kpt_pa - gt_3d_kpt)**2, 1))
error_x[n] = np.abs(pred_3d_kpt[:, 0] - gt_3d_kpt[:, 0])
error_y[n] = np.abs(pred_3d_kpt[:, 1] - gt_3d_kpt[:, 1])
error_z[n] = np.abs(pred_3d_kpt[:, 2] - gt_3d_kpt[:, 2])
# record idx per seq or act
seq_id = int(img_name.split('/')[-3][2])
seq_idx_dict[seq_id].append(n)
act_idx_dict[int(gt['activity_id']) - 1].append(n)
img_name = gt['img_path']
# prediction save
pred_save.append({
'img_name': img_name,
'joint_cam': pred_3d_kpt.tolist(),
'bbox': [float(_) for _ in bbox],
'root_cam': gt_3d_root.tolist()
}) # joint_cam is root-relative coordinate
# total error
tot_err = np.mean(error)
tot_err_pa = np.mean(error_pa)
tot_err_x = np.mean(error_x)
tot_err_y = np.mean(error_y)
tot_err_z = np.mean(error_z)
eval_summary = f'PA MPJPE >> tot: {tot_err_pa:2f}; MPJPE >> tot: {tot_err:2f}, x: {tot_err_x:2f}, y: {tot_err_y:.2f}, z: {tot_err_z:2f}\n'
seq_mpjpes_list, seq_pck_array_list, seq_auc_array_list = self._calc_metric_per_class(
error, seq_idx_dict)
act_mpjpes_list, act_pck_array_list, act_auc_array_list = self._calc_metric_per_class(
error, act_idx_dict)
all_mpjpes_list, all_pck_array_list, all_auc_array_list = self._calc_metric_per_class(
error, {0: list(range(sample_num))})
# Summary mpjpe per sequence
eval_summary += '#' * 10 + 'MPJPE per sequence\n'
eval_summary += ''.join(
['MPJPE\t'] +
[self.joints_name[j] + ' '
for j in self.EVAL_JOINTS] + ['Average\n'])
total_mpjpe = 0
for i_idx, i in enumerate(self.test_seqs):
eval_summary += ''.join([f'TS{i}\t'] + [
'{:.2f}\t'.format(seq_mpjpes_list[i_idx][j])
for j in range(seq_mpjpes_list[i_idx].shape[0])
] + ['\n'])
total_mpjpe += seq_mpjpes_list[i_idx][-1]
total_mpjpe /= len(self.test_seqs)
eval_summary += f'Avg MPJPE >> tot: {total_mpjpe:2f}\n'
# Summary pck per sequence
eval_summary += '#' * 10 + 'PCK per sequence\n'
eval_summary += ''.join(['PCK\t'] +
[k + '\t' for k in self.joint_groups.keys()] +
['Total\n'])
total_pck = 0
for i_idx, i in enumerate(self.test_seqs):
eval_summary += ''.join([f'TS{i}\t'] + [
'{:.2f}\t'.format(seq_pck_array_list[i_idx][j])
for j in range(seq_pck_array_list[i_idx].shape[0])
] + ['\n'])
total_pck += seq_pck_array_list[i_idx][-1]
total_pck /= len(self.test_seqs)
eval_summary += f'Avg PCK >> tot: {total_pck:2f}\n'
# Summary auc per sequence
eval_summary += '#' * 10 + 'AUC per sequence\n'
eval_summary += ''.join(['AUC\t'] +
[k + '\t' for k in self.joint_groups.keys()] +
['Total\n'])
total_auc = 0
for i_idx, i in enumerate(self.test_seqs):
eval_summary += ''.join([f'TS{i}\t'] + [
'{:.2f}\t'.format(seq_auc_array_list[i_idx][j])
for j in range(seq_auc_array_list[i_idx].shape[0])
] + ['\n'])
total_auc += seq_auc_array_list[i_idx][-1]
total_auc /= len(self.test_seqs)
eval_summary += f'Avg AUC >> tot: {total_auc:2f}\n'
# Summary mpjpe per action
eval_summary += '#' * 10 + 'MPJPE per action\n'
eval_summary += ''.join(
['MPJPE\t'] +
[self.joints_name[j] + ' '
for j in self.EVAL_JOINTS] + ['Average\n'])
total_mpjpe = 0
for i_idx, i in enumerate(self.activity_name):
eval_summary += ''.join([f'{i}\t'] + [
'{:.2f}\t'.format(act_mpjpes_list[i_idx][j])
for j in range(act_mpjpes_list[i_idx].shape[0])
] + ['\n'])
total_mpjpe += act_mpjpes_list[i_idx][-1]
total_mpjpe /= len(self.activity_name)
eval_summary += ''.join(['All\t'] + [
'{:.2f}\t'.format(all_mpjpes_list[0][j])
for j in range(all_mpjpes_list[0].shape[0])
] + ['\n'])
eval_summary += f'Avg MPJPE >> tot: {total_mpjpe:2f}\n'
# Summary pck per action
eval_summary += '#' * 10 + 'PCK per action\n'
eval_summary += ''.join(['PCK\t'] +
[k + '\t' for k in self.joint_groups.keys()] +
['Total\n'])
total_pck = 0
for i_idx, i in enumerate(self.activity_name):
eval_summary += ''.join([f'{i}\t'] + [
'{:.2f}\t'.format(act_pck_array_list[i_idx][j])
for j in range(act_pck_array_list[i_idx].shape[0])
] + ['\n'])
total_pck += act_pck_array_list[i_idx][-1]
total_pck /= len(self.activity_name)
eval_summary += ''.join(['All\t'] + [
'{:.2f}\t'.format(all_pck_array_list[0][j])
for j in range(all_pck_array_list[0].shape[0])
] + ['\n'])
eval_summary += f'Avg PCK >> tot: {total_pck:2f}\n'
# Summary auc per action
eval_summary += '#' * 10 + 'AUC per action\n'
eval_summary += ''.join(['AUC\t'] +
[k + '\t' for k in self.joint_groups.keys()] +
['Total\n'])
total_auc = 0
for i_idx, i in enumerate(self.activity_name):
eval_summary += ''.join([f'{i}\t'] + [
'{:.2f}\t'.format(act_auc_array_list[i_idx][j])
for j in range(act_auc_array_list[i_idx].shape[0])
] + ['\n'])
total_auc += act_auc_array_list[i_idx][-1]
total_auc /= len(self.activity_name)
eval_summary += ''.join(['All\t'] + [
'{:.2f}\t'.format(all_auc_array_list[0][j])
for j in range(all_auc_array_list[0].shape[0])
] + ['\n'])
eval_summary += f'Avg AUC >> tot: {total_auc:2f}\n'
print(eval_summary)
# prediction save
with open(result_dir, 'w') as f:
json.dump(pred_save, f)
print("Test result is saved at " + result_dir)
return tot_err
def evaluate_xyz_17(self, preds, result_dir):
print('Evaluation start...')
gts = self._labels
assert len(gts) == len(preds)
sample_num = len(gts)
pred_save = []
error = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
error_pa = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
error_x = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
error_y = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
error_z = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
# error for each sequence
for n in range(sample_num):
gt = gts[n]
image_id = gt['img_id']
bbox = gt['bbox']
gt_3d_root = gt['root_cam']
gt_3d_kpt = gt['joint_relative_17']
# gt_vis = gt['joint_vis']
# restore coordinates to original space
pred_3d_kpt = preds[image_id]['xyz_17'].copy(
) * self.bbox_3d_shape[2]
# root joint alignment
pred_3d_kpt = pred_3d_kpt - pred_3d_kpt[self.root_idx_17]
gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx_17]
# select eval 14 joints
pred_3d_kpt = np.take(pred_3d_kpt, self.EVAL_JOINTS, axis=0)
gt_3d_kpt = np.take(gt_3d_kpt, self.EVAL_JOINTS, axis=0)
pred_3d_kpt_pa = reconstruction_error(pred_3d_kpt.copy(),
gt_3d_kpt.copy())
# error calculate
error[n] = np.sqrt(np.sum((pred_3d_kpt - gt_3d_kpt)**2, 1))
error_pa[n] = np.sqrt(np.sum((pred_3d_kpt_pa - gt_3d_kpt)**2, 1))
error_x[n] = np.abs(pred_3d_kpt[:, 0] - gt_3d_kpt[:, 0])
error_y[n] = np.abs(pred_3d_kpt[:, 1] - gt_3d_kpt[:, 1])
error_z[n] = np.abs(pred_3d_kpt[:, 2] - gt_3d_kpt[:, 2])
img_name = gt['img_path']
# prediction save
pred_save.append({
'img_name': img_name,
'joint_cam': pred_3d_kpt.tolist(),
'bbox': bbox,
'root_cam': gt_3d_root.tolist()
}) # joint_cam is root-relative coordinate
# total error
tot_err = np.mean(error) * 1000
tot_err_pa = np.mean(error_pa) * 1000
tot_err_x = np.mean(error_x) * 1000
tot_err_y = np.mean(error_y) * 1000
tot_err_z = np.mean(error_z) * 1000
eval_summary = f'PA MPJPE >> tot: {tot_err_pa:2f}; MPJPE >> tot: {tot_err:2f}, x: {tot_err_x:2f}, y: {tot_err_y:.2f}, z: {tot_err_z:2f}\n'
print(eval_summary)
# prediction save
with open(result_dir, 'w') as f:
json.dump(pred_save, f)
print("Test result is saved at " + result_dir)
return tot_err_pa
def evaluate_xyz_17(self, preds, result_dir):
print('Evaluation start...')
gts = self._labels
assert len(gts) == len(preds), (len(gts), len(preds))
sample_num = len(gts)
pred_save = []
error = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
error_align = np.zeros(
(sample_num, len(self.EVAL_JOINTS))) # joint error
error_x = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
error_y = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
error_z = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error
# error for each sequence
error_action = [[] for _ in range(len(self.action_name))]
for n in range(sample_num):
gt = gts[n]
image_id = gt['img_id']
bbox = gt['bbox']
gt_3d_root = gt['root_cam'].copy()
gt_3d_kpt = gt['joint_relative_17'].copy()
# gt_vis = gt['joint_vis']
# restore coordinates to original space
pred_3d_kpt = preds[image_id]['xyz_17'].copy(
) * self.bbox_3d_shape[2]
# root joint alignment
pred_3d_kpt = pred_3d_kpt - pred_3d_kpt[self.root_idx_17]
gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx_17]
# rigid alignment for PA MPJPE
# pred_3d_kpt_align = rigid_align(pred_3d_kpt.copy(), gt_3d_kpt.copy())
pred_3d_kpt_align = reconstruction_error(pred_3d_kpt.copy(),
gt_3d_kpt.copy())
# pred_3d_kpt_align = pred_3d_kpt_align - pred_3d_kpt_align[self.root_idx_17]
# select eval 14 joints
pred_3d_kpt = np.take(pred_3d_kpt, self.EVAL_JOINTS, axis=0)
gt_3d_kpt = np.take(gt_3d_kpt, self.EVAL_JOINTS, axis=0)
pred_3d_kpt_align = np.take(pred_3d_kpt_align,
self.EVAL_JOINTS,
axis=0)
# error calculate
error[n] = np.sqrt(np.sum((pred_3d_kpt - gt_3d_kpt)**2, 1))
error_align[n] = np.sqrt(
np.sum((pred_3d_kpt_align - gt_3d_kpt)**2, 1))
error_x[n] = np.abs(pred_3d_kpt[:, 0] - gt_3d_kpt[:, 0])
error_y[n] = np.abs(pred_3d_kpt[:, 1] - gt_3d_kpt[:, 1])
error_z[n] = np.abs(pred_3d_kpt[:, 2] - gt_3d_kpt[:, 2])
img_name = gt['img_path']
action_idx = int(img_name[img_name.find('act') +
4:img_name.find('act') + 6]) - 2
error_action[action_idx].append(error[n].copy())
# prediction save
pred_save.append({
'image_id': image_id,
'joint_cam': pred_3d_kpt.tolist(),
'bbox': bbox,
'root_cam': gt_3d_root.tolist()
}) # joint_cam is root-relative coordinate
# total error
tot_err = np.mean(error)
tot_err_align = np.mean(error_align)
tot_err_x = np.mean(error_x)
tot_err_y = np.mean(error_y)
tot_err_z = np.mean(error_z)
metric = 'PA MPJPE' if self.protocol == 1 else 'MPJPE'
eval_summary = f'XYZ_17 Protocol {self.protocol} error ({metric}) >> tot: {tot_err:2f}, tot_pa: {tot_err_align:2f}, x: {tot_err_x:2f}, y: {tot_err_y:.2f}, z: {tot_err_z:2f}\n'
# error for each action
for i in range(len(error_action)):
err = np.mean(np.array(error_action[i]))
eval_summary += (self.action_name[i] + ': %.2f ' % err)
print(eval_summary)
# prediction save
with open(result_dir, 'w') as f:
json.dump(pred_save, f)
print("Test result is saved at " + result_dir)
return tot_err_align
def evaluate_xyz_24(self, preds, result_dir):
print('Evaluation start...')
gts = self._labels
assert len(gts) == len(preds)
sample_num = len(gts)
pred_save = []
error = np.zeros((sample_num, 24)) # joint error
error_align = np.zeros((sample_num, 24)) # joint error
error_x = np.zeros((sample_num, 24)) # joint error
error_y = np.zeros((sample_num, 24)) # joint error
error_z = np.zeros((sample_num, 24)) # joint error
for n in range(sample_num):
gt = gts[n]
image_id = gt['img_id']
bbox = gt['bbox']
gt_3d_root = gt['root_cam'].copy()
gt_3d_kpt = gt['joint_cam_29'][:24, :].copy()
# gt_vis = gt['joint_vis']
# restore coordinates to original space
pred_3d_kpt = preds[image_id]['xyz_24'].copy(
) * self.bbox_3d_shape[2]
# root joint alignment
pred_3d_kpt = pred_3d_kpt - pred_3d_kpt[self.root_idx_smpl]
gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx_smpl]
# rigid alignment for PA MPJPE
pred_3d_kpt_align = reconstruction_error(pred_3d_kpt.copy(),
gt_3d_kpt.copy())
# error calculate
error[n] = np.sqrt(np.sum((pred_3d_kpt - gt_3d_kpt)**2, 1))
error_align[n] = np.sqrt(
np.sum((pred_3d_kpt_align - gt_3d_kpt)**2, 1))
error_x[n] = np.abs(pred_3d_kpt[:, 0] - gt_3d_kpt[:, 0])
error_y[n] = np.abs(pred_3d_kpt[:, 1] - gt_3d_kpt[:, 1])
error_z[n] = np.abs(pred_3d_kpt[:, 2] - gt_3d_kpt[:, 2])
img_name = gt['img_path']
# prediction save
pred_save.append({
'img_name': img_name,
'joint_cam': pred_3d_kpt.tolist(),
'bbox': bbox,
'root_cam': gt_3d_root.tolist()
}) # joint_cam is root-relative coordinate
# total error
tot_err = np.mean(error) * 1000
tot_err_align = np.mean(error_align) * 1000
tot_err_x = np.mean(error_x) * 1000
tot_err_y = np.mean(error_y) * 1000
tot_err_z = np.mean(error_z) * 1000
eval_summary = f'XYZ_24 >> tot: {tot_err:2f}, tot_pa: {tot_err_align:2f}, x: {tot_err_x:2f}, y: {tot_err_y:.2f}, z: {tot_err_z:2f}\n'
print(eval_summary)
# prediction save
with open(result_dir, 'w') as f:
json.dump(pred_save, f)
print("Test result is saved at " + result_dir)
return tot_err
def evaluate_uvd_24(self, preds, result_dir):
print('Evaluation start...')
gts = self._labels
assert len(gts) == len(preds)
sample_num = len(gts)
pred_save = []
error = np.zeros((sample_num, 24)) # joint error
error_x = np.zeros((sample_num, 24)) # joint error
error_y = np.zeros((sample_num, 24)) # joint error
error_z = np.zeros((sample_num, 24)) # joint error
# error for each sequence
error_action = [[] for _ in range(len(self.action_name))]
for n in range(sample_num):
gt = gts[n]
image_id = gt['img_id']
f = gt['f']
c = gt['c']
bbox = gt['bbox']
gt_3d_root = gt['root_cam'].copy()
gt_3d_kpt = gt['joint_cam_29'][:24].copy()
# restore coordinates to original space
pred_2d_kpt = preds[image_id]['uvd_29'][:24].copy()
# pred_2d_kpt[:, 0] = pred_2d_kpt[:, 0] / self._output_size[1] * bbox[2] + bbox[0]
# pred_2d_kpt[:, 1] = pred_2d_kpt[:, 1] / self._output_size[0] * bbox[3] + bbox[1]
pred_2d_kpt[:, 2] = pred_2d_kpt[:, 2] * self.bbox_3d_shape[
2] + gt_3d_root[2]
# back project to camera coordinate system
pred_3d_kpt = pixel2cam(pred_2d_kpt, f, c)
# root joint alignment
pred_3d_kpt = pred_3d_kpt - pred_3d_kpt[self.root_idx_smpl]
gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx_smpl]
if self.protocol == 1:
# rigid alignment for PA MPJPE (protocol #1)
pred_3d_kpt = reconstruction_error(pred_3d_kpt, gt_3d_kpt)
# error calculate
error[n] = np.sqrt(np.sum((pred_3d_kpt - gt_3d_kpt)**2, 1))
error_x[n] = np.abs(pred_3d_kpt[:, 0] - gt_3d_kpt[:, 0])
error_y[n] = np.abs(pred_3d_kpt[:, 1] - gt_3d_kpt[:, 1])
error_z[n] = np.abs(pred_3d_kpt[:, 2] - gt_3d_kpt[:, 2])
img_name = gt['img_path']
action_idx = int(img_name[img_name.find('act') +
4:img_name.find('act') + 6]) - 2
error_action[action_idx].append(error[n].copy())
# prediction save
pred_save.append({
'image_id': image_id,
'joint_cam': pred_3d_kpt.tolist(),
'bbox': bbox,
'root_cam': gt_3d_root.tolist()
}) # joint_cam is root-relative coordinate
# total error
tot_err = np.mean(error)
tot_err_x = np.mean(error_x)
tot_err_y = np.mean(error_y)
tot_err_z = np.mean(error_z)
metric = 'PA MPJPE' if self.protocol == 1 else 'MPJPE'
eval_summary = f'UVD_24 Protocol {self.protocol} error ({metric}) >> tot: {tot_err:2f}, x: {tot_err_x:2f}, y: {tot_err_y:.2f}, z: {tot_err_z:2f}\n'
# error for each action
for i in range(len(error_action)):
err = np.mean(np.array(error_action[i]))
eval_summary += (self.action_name[i] + ': %.2f ' % err)
print(eval_summary)
# prediction save
with open(result_dir, 'w') as f:
json.dump(pred_save, f)
print("Test result is saved at " + result_dir)
return tot_err
