def evaluate(self, preds, result_dir): print('Evaluation start...') gts = self._labels assert len(gts) == len(preds) sample_num = len(gts) seq_idx_dict = {k: [] for k in self.test_seqs} act_idx_dict = {k: [] for k in range(len(self.activity_name))} pred_save = [] error = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error error_pa = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error error_x = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error error_y = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error error_z = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error # error for each sequence for n in range(sample_num): gt = gts[n] img_name = gt['img_name'] intrinsic_param = gt['intrinsic_param'] bbox = gt['bbox'] gt_3d_root = gt['root_cam'] gt_3d_kpt = gt['joint_cam'] # gt_vis = gt['joint_vis'] # restore coordinates to original space pred_2d_kpt = preds[n].copy() # pred_2d_kpt[:, 0] = pred_2d_kpt[:, 0] / self._output_size[1] * bbox[2] + bbox[0] # pred_2d_kpt[:, 1] = pred_2d_kpt[:, 1] / self._output_size[0] * bbox[3] + bbox[1] pred_2d_kpt[:, 2] = pred_2d_kpt[:, 2] * self.bbox_3d_shape[ 0] + gt_3d_root[2] # back project to camera coordinate system pred_3d_kpt = pixel2cam_matrix(pred_2d_kpt, intrinsic_param) # root joint alignment pred_3d_kpt = pred_3d_kpt - pred_3d_kpt[self.root_idx] gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx] # if self.protocol == 1: # # rigid alignment for PA MPJPE (protocol #1) pred_3d_kpt_pa = reconstruction_error(pred_3d_kpt, gt_3d_kpt) # exclude thorax # pred_3d_kpt = np.take(pred_3d_kpt, self.EVAL_JOINTS, axis=0) # pred_3d_kpt_pa = np.take(pred_3d_kpt_pa, self.EVAL_JOINTS, axis=0) # gt_3d_kpt = np.take(gt_3d_kpt, self.EVAL_JOINTS, axis=0) # error calculate error[n] = np.sqrt(np.sum((pred_3d_kpt - gt_3d_kpt)**2, 1)) error_pa[n] = np.sqrt(np.sum((pred_3d_kpt_pa - gt_3d_kpt)**2, 1)) error_x[n] = np.abs(pred_3d_kpt[:, 0] - gt_3d_kpt[:, 0]) error_y[n] = np.abs(pred_3d_kpt[:, 1] - gt_3d_kpt[:, 1]) error_z[n] = np.abs(pred_3d_kpt[:, 2] - gt_3d_kpt[:, 2]) # record idx per seq or act seq_id = int(img_name.split('/')[-3][2]) seq_idx_dict[seq_id].append(n) act_idx_dict[int(gt['activity_id']) - 1].append(n) img_name = gt['img_path'] # prediction save pred_save.append({ 'img_name': img_name, 'joint_cam': pred_3d_kpt.tolist(), 'bbox': [float(_) for _ in bbox], 'root_cam': gt_3d_root.tolist() }) # joint_cam is root-relative coordinate # total error tot_err = np.mean(error) tot_err_pa = np.mean(error_pa) tot_err_x = np.mean(error_x) tot_err_y = np.mean(error_y) tot_err_z = np.mean(error_z) eval_summary = f'PA MPJPE >> tot: {tot_err_pa:2f}; MPJPE >> tot: {tot_err:2f}, x: {tot_err_x:2f}, y: {tot_err_y:.2f}, z: {tot_err_z:2f}\n' seq_mpjpes_list, seq_pck_array_list, seq_auc_array_list = self._calc_metric_per_class( error, seq_idx_dict) act_mpjpes_list, act_pck_array_list, act_auc_array_list = self._calc_metric_per_class( error, act_idx_dict) all_mpjpes_list, all_pck_array_list, all_auc_array_list = self._calc_metric_per_class( error, {0: list(range(sample_num))}) # Summary mpjpe per sequence eval_summary += '#' * 10 + 'MPJPE per sequence\n' eval_summary += ''.join( ['MPJPE\t'] + [self.joints_name[j] + ' ' for j in self.EVAL_JOINTS] + ['Average\n']) total_mpjpe = 0 for i_idx, i in enumerate(self.test_seqs): eval_summary += ''.join([f'TS{i}\t'] + [ '{:.2f}\t'.format(seq_mpjpes_list[i_idx][j]) for j in range(seq_mpjpes_list[i_idx].shape[0]) ] + ['\n']) total_mpjpe += seq_mpjpes_list[i_idx][-1] total_mpjpe /= len(self.test_seqs) eval_summary += f'Avg MPJPE >> tot: {total_mpjpe:2f}\n' # Summary pck per sequence eval_summary += '#' * 10 + 'PCK per sequence\n' eval_summary += ''.join(['PCK\t'] + [k + '\t' for k in self.joint_groups.keys()] + ['Total\n']) total_pck = 0 for i_idx, i in enumerate(self.test_seqs): eval_summary += ''.join([f'TS{i}\t'] + [ '{:.2f}\t'.format(seq_pck_array_list[i_idx][j]) for j in range(seq_pck_array_list[i_idx].shape[0]) ] + ['\n']) total_pck += seq_pck_array_list[i_idx][-1] total_pck /= len(self.test_seqs) eval_summary += f'Avg PCK >> tot: {total_pck:2f}\n' # Summary auc per sequence eval_summary += '#' * 10 + 'AUC per sequence\n' eval_summary += ''.join(['AUC\t'] + [k + '\t' for k in self.joint_groups.keys()] + ['Total\n']) total_auc = 0 for i_idx, i in enumerate(self.test_seqs): eval_summary += ''.join([f'TS{i}\t'] + [ '{:.2f}\t'.format(seq_auc_array_list[i_idx][j]) for j in range(seq_auc_array_list[i_idx].shape[0]) ] + ['\n']) total_auc += seq_auc_array_list[i_idx][-1] total_auc /= len(self.test_seqs) eval_summary += f'Avg AUC >> tot: {total_auc:2f}\n' # Summary mpjpe per action eval_summary += '#' * 10 + 'MPJPE per action\n' eval_summary += ''.join( ['MPJPE\t'] + [self.joints_name[j] + ' ' for j in self.EVAL_JOINTS] + ['Average\n']) total_mpjpe = 0 for i_idx, i in enumerate(self.activity_name): eval_summary += ''.join([f'{i}\t'] + [ '{:.2f}\t'.format(act_mpjpes_list[i_idx][j]) for j in range(act_mpjpes_list[i_idx].shape[0]) ] + ['\n']) total_mpjpe += act_mpjpes_list[i_idx][-1] total_mpjpe /= len(self.activity_name) eval_summary += ''.join(['All\t'] + [ '{:.2f}\t'.format(all_mpjpes_list[0][j]) for j in range(all_mpjpes_list[0].shape[0]) ] + ['\n']) eval_summary += f'Avg MPJPE >> tot: {total_mpjpe:2f}\n' # Summary pck per action eval_summary += '#' * 10 + 'PCK per action\n' eval_summary += ''.join(['PCK\t'] + [k + '\t' for k in self.joint_groups.keys()] + ['Total\n']) total_pck = 0 for i_idx, i in enumerate(self.activity_name): eval_summary += ''.join([f'{i}\t'] + [ '{:.2f}\t'.format(act_pck_array_list[i_idx][j]) for j in range(act_pck_array_list[i_idx].shape[0]) ] + ['\n']) total_pck += act_pck_array_list[i_idx][-1] total_pck /= len(self.activity_name) eval_summary += ''.join(['All\t'] + [ '{:.2f}\t'.format(all_pck_array_list[0][j]) for j in range(all_pck_array_list[0].shape[0]) ] + ['\n']) eval_summary += f'Avg PCK >> tot: {total_pck:2f}\n' # Summary auc per action eval_summary += '#' * 10 + 'AUC per action\n' eval_summary += ''.join(['AUC\t'] + [k + '\t' for k in self.joint_groups.keys()] + ['Total\n']) total_auc = 0 for i_idx, i in enumerate(self.activity_name): eval_summary += ''.join([f'{i}\t'] + [ '{:.2f}\t'.format(act_auc_array_list[i_idx][j]) for j in range(act_auc_array_list[i_idx].shape[0]) ] + ['\n']) total_auc += act_auc_array_list[i_idx][-1] total_auc /= len(self.activity_name) eval_summary += ''.join(['All\t'] + [ '{:.2f}\t'.format(all_auc_array_list[0][j]) for j in range(all_auc_array_list[0].shape[0]) ] + ['\n']) eval_summary += f'Avg AUC >> tot: {total_auc:2f}\n' print(eval_summary) # prediction save with open(result_dir, 'w') as f: json.dump(pred_save, f) print("Test result is saved at " + result_dir) return tot_err
def evaluate_xyz_17(self, preds, result_dir): print('Evaluation start...') gts = self._labels assert len(gts) == len(preds) sample_num = len(gts) pred_save = [] error = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error error_pa = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error error_x = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error error_y = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error error_z = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error # error for each sequence for n in range(sample_num): gt = gts[n] image_id = gt['img_id'] bbox = gt['bbox'] gt_3d_root = gt['root_cam'] gt_3d_kpt = gt['joint_relative_17'] # gt_vis = gt['joint_vis'] # restore coordinates to original space pred_3d_kpt = preds[image_id]['xyz_17'].copy( ) * self.bbox_3d_shape[2] # root joint alignment pred_3d_kpt = pred_3d_kpt - pred_3d_kpt[self.root_idx_17] gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx_17] # select eval 14 joints pred_3d_kpt = np.take(pred_3d_kpt, self.EVAL_JOINTS, axis=0) gt_3d_kpt = np.take(gt_3d_kpt, self.EVAL_JOINTS, axis=0) pred_3d_kpt_pa = reconstruction_error(pred_3d_kpt.copy(), gt_3d_kpt.copy()) # error calculate error[n] = np.sqrt(np.sum((pred_3d_kpt - gt_3d_kpt)**2, 1)) error_pa[n] = np.sqrt(np.sum((pred_3d_kpt_pa - gt_3d_kpt)**2, 1)) error_x[n] = np.abs(pred_3d_kpt[:, 0] - gt_3d_kpt[:, 0]) error_y[n] = np.abs(pred_3d_kpt[:, 1] - gt_3d_kpt[:, 1]) error_z[n] = np.abs(pred_3d_kpt[:, 2] - gt_3d_kpt[:, 2]) img_name = gt['img_path'] # prediction save pred_save.append({ 'img_name': img_name, 'joint_cam': pred_3d_kpt.tolist(), 'bbox': bbox, 'root_cam': gt_3d_root.tolist() }) # joint_cam is root-relative coordinate # total error tot_err = np.mean(error) * 1000 tot_err_pa = np.mean(error_pa) * 1000 tot_err_x = np.mean(error_x) * 1000 tot_err_y = np.mean(error_y) * 1000 tot_err_z = np.mean(error_z) * 1000 eval_summary = f'PA MPJPE >> tot: {tot_err_pa:2f}; MPJPE >> tot: {tot_err:2f}, x: {tot_err_x:2f}, y: {tot_err_y:.2f}, z: {tot_err_z:2f}\n' print(eval_summary) # prediction save with open(result_dir, 'w') as f: json.dump(pred_save, f) print("Test result is saved at " + result_dir) return tot_err_pa
def evaluate_xyz_17(self, preds, result_dir): print('Evaluation start...') gts = self._labels assert len(gts) == len(preds), (len(gts), len(preds)) sample_num = len(gts) pred_save = [] error = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error error_align = np.zeros( (sample_num, len(self.EVAL_JOINTS))) # joint error error_x = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error error_y = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error error_z = np.zeros((sample_num, len(self.EVAL_JOINTS))) # joint error # error for each sequence error_action = [[] for _ in range(len(self.action_name))] for n in range(sample_num): gt = gts[n] image_id = gt['img_id'] bbox = gt['bbox'] gt_3d_root = gt['root_cam'].copy() gt_3d_kpt = gt['joint_relative_17'].copy() # gt_vis = gt['joint_vis'] # restore coordinates to original space pred_3d_kpt = preds[image_id]['xyz_17'].copy( ) * self.bbox_3d_shape[2] # root joint alignment pred_3d_kpt = pred_3d_kpt - pred_3d_kpt[self.root_idx_17] gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx_17] # rigid alignment for PA MPJPE # pred_3d_kpt_align = rigid_align(pred_3d_kpt.copy(), gt_3d_kpt.copy()) pred_3d_kpt_align = reconstruction_error(pred_3d_kpt.copy(), gt_3d_kpt.copy()) # pred_3d_kpt_align = pred_3d_kpt_align - pred_3d_kpt_align[self.root_idx_17] # select eval 14 joints pred_3d_kpt = np.take(pred_3d_kpt, self.EVAL_JOINTS, axis=0) gt_3d_kpt = np.take(gt_3d_kpt, self.EVAL_JOINTS, axis=0) pred_3d_kpt_align = np.take(pred_3d_kpt_align, self.EVAL_JOINTS, axis=0) # error calculate error[n] = np.sqrt(np.sum((pred_3d_kpt - gt_3d_kpt)**2, 1)) error_align[n] = np.sqrt( np.sum((pred_3d_kpt_align - gt_3d_kpt)**2, 1)) error_x[n] = np.abs(pred_3d_kpt[:, 0] - gt_3d_kpt[:, 0]) error_y[n] = np.abs(pred_3d_kpt[:, 1] - gt_3d_kpt[:, 1]) error_z[n] = np.abs(pred_3d_kpt[:, 2] - gt_3d_kpt[:, 2]) img_name = gt['img_path'] action_idx = int(img_name[img_name.find('act') + 4:img_name.find('act') + 6]) - 2 error_action[action_idx].append(error[n].copy()) # prediction save pred_save.append({ 'image_id': image_id, 'joint_cam': pred_3d_kpt.tolist(), 'bbox': bbox, 'root_cam': gt_3d_root.tolist() }) # joint_cam is root-relative coordinate # total error tot_err = np.mean(error) tot_err_align = np.mean(error_align) tot_err_x = np.mean(error_x) tot_err_y = np.mean(error_y) tot_err_z = np.mean(error_z) metric = 'PA MPJPE' if self.protocol == 1 else 'MPJPE' eval_summary = f'XYZ_17 Protocol {self.protocol} error ({metric}) >> tot: {tot_err:2f}, tot_pa: {tot_err_align:2f}, x: {tot_err_x:2f}, y: {tot_err_y:.2f}, z: {tot_err_z:2f}\n' # error for each action for i in range(len(error_action)): err = np.mean(np.array(error_action[i])) eval_summary += (self.action_name[i] + ': %.2f ' % err) print(eval_summary) # prediction save with open(result_dir, 'w') as f: json.dump(pred_save, f) print("Test result is saved at " + result_dir) return tot_err_align
def evaluate_xyz_24(self, preds, result_dir): print('Evaluation start...') gts = self._labels assert len(gts) == len(preds) sample_num = len(gts) pred_save = [] error = np.zeros((sample_num, 24)) # joint error error_align = np.zeros((sample_num, 24)) # joint error error_x = np.zeros((sample_num, 24)) # joint error error_y = np.zeros((sample_num, 24)) # joint error error_z = np.zeros((sample_num, 24)) # joint error for n in range(sample_num): gt = gts[n] image_id = gt['img_id'] bbox = gt['bbox'] gt_3d_root = gt['root_cam'].copy() gt_3d_kpt = gt['joint_cam_29'][:24, :].copy() # gt_vis = gt['joint_vis'] # restore coordinates to original space pred_3d_kpt = preds[image_id]['xyz_24'].copy( ) * self.bbox_3d_shape[2] # root joint alignment pred_3d_kpt = pred_3d_kpt - pred_3d_kpt[self.root_idx_smpl] gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx_smpl] # rigid alignment for PA MPJPE pred_3d_kpt_align = reconstruction_error(pred_3d_kpt.copy(), gt_3d_kpt.copy()) # error calculate error[n] = np.sqrt(np.sum((pred_3d_kpt - gt_3d_kpt)**2, 1)) error_align[n] = np.sqrt( np.sum((pred_3d_kpt_align - gt_3d_kpt)**2, 1)) error_x[n] = np.abs(pred_3d_kpt[:, 0] - gt_3d_kpt[:, 0]) error_y[n] = np.abs(pred_3d_kpt[:, 1] - gt_3d_kpt[:, 1]) error_z[n] = np.abs(pred_3d_kpt[:, 2] - gt_3d_kpt[:, 2]) img_name = gt['img_path'] # prediction save pred_save.append({ 'img_name': img_name, 'joint_cam': pred_3d_kpt.tolist(), 'bbox': bbox, 'root_cam': gt_3d_root.tolist() }) # joint_cam is root-relative coordinate # total error tot_err = np.mean(error) * 1000 tot_err_align = np.mean(error_align) * 1000 tot_err_x = np.mean(error_x) * 1000 tot_err_y = np.mean(error_y) * 1000 tot_err_z = np.mean(error_z) * 1000 eval_summary = f'XYZ_24 >> tot: {tot_err:2f}, tot_pa: {tot_err_align:2f}, x: {tot_err_x:2f}, y: {tot_err_y:.2f}, z: {tot_err_z:2f}\n' print(eval_summary) # prediction save with open(result_dir, 'w') as f: json.dump(pred_save, f) print("Test result is saved at " + result_dir) return tot_err
def evaluate_uvd_24(self, preds, result_dir): print('Evaluation start...') gts = self._labels assert len(gts) == len(preds) sample_num = len(gts) pred_save = [] error = np.zeros((sample_num, 24)) # joint error error_x = np.zeros((sample_num, 24)) # joint error error_y = np.zeros((sample_num, 24)) # joint error error_z = np.zeros((sample_num, 24)) # joint error # error for each sequence error_action = [[] for _ in range(len(self.action_name))] for n in range(sample_num): gt = gts[n] image_id = gt['img_id'] f = gt['f'] c = gt['c'] bbox = gt['bbox'] gt_3d_root = gt['root_cam'].copy() gt_3d_kpt = gt['joint_cam_29'][:24].copy() # restore coordinates to original space pred_2d_kpt = preds[image_id]['uvd_29'][:24].copy() # pred_2d_kpt[:, 0] = pred_2d_kpt[:, 0] / self._output_size[1] * bbox[2] + bbox[0] # pred_2d_kpt[:, 1] = pred_2d_kpt[:, 1] / self._output_size[0] * bbox[3] + bbox[1] pred_2d_kpt[:, 2] = pred_2d_kpt[:, 2] * self.bbox_3d_shape[ 2] + gt_3d_root[2] # back project to camera coordinate system pred_3d_kpt = pixel2cam(pred_2d_kpt, f, c) # root joint alignment pred_3d_kpt = pred_3d_kpt - pred_3d_kpt[self.root_idx_smpl] gt_3d_kpt = gt_3d_kpt - gt_3d_kpt[self.root_idx_smpl] if self.protocol == 1: # rigid alignment for PA MPJPE (protocol #1) pred_3d_kpt = reconstruction_error(pred_3d_kpt, gt_3d_kpt) # error calculate error[n] = np.sqrt(np.sum((pred_3d_kpt - gt_3d_kpt)**2, 1)) error_x[n] = np.abs(pred_3d_kpt[:, 0] - gt_3d_kpt[:, 0]) error_y[n] = np.abs(pred_3d_kpt[:, 1] - gt_3d_kpt[:, 1]) error_z[n] = np.abs(pred_3d_kpt[:, 2] - gt_3d_kpt[:, 2]) img_name = gt['img_path'] action_idx = int(img_name[img_name.find('act') + 4:img_name.find('act') + 6]) - 2 error_action[action_idx].append(error[n].copy()) # prediction save pred_save.append({ 'image_id': image_id, 'joint_cam': pred_3d_kpt.tolist(), 'bbox': bbox, 'root_cam': gt_3d_root.tolist() }) # joint_cam is root-relative coordinate # total error tot_err = np.mean(error) tot_err_x = np.mean(error_x) tot_err_y = np.mean(error_y) tot_err_z = np.mean(error_z) metric = 'PA MPJPE' if self.protocol == 1 else 'MPJPE' eval_summary = f'UVD_24 Protocol {self.protocol} error ({metric}) >> tot: {tot_err:2f}, x: {tot_err_x:2f}, y: {tot_err_y:.2f}, z: {tot_err_z:2f}\n' # error for each action for i in range(len(error_action)): err = np.mean(np.array(error_action[i])) eval_summary += (self.action_name[i] + ': %.2f ' % err) print(eval_summary) # prediction save with open(result_dir, 'w') as f: json.dump(pred_save, f) print("Test result is saved at " + result_dir) return tot_err