def _top_down_pose_kernel(self, geo_affinity_mat, matched_list, pose_mat,
sub_imgid2cam):
multi_pose3d = list()
chosen_img = list()
for person in matched_list:
Graph = geo_affinity_mat[person][:, person].clone().numpy()
Graph *= (1 - np.eye(Graph.shape[0])) # make diagonal 0
if len(Graph) < 2:
continue
elif len(Graph) > 2:
if self.cfg.use_mincut:
cut0, cut1 = find_mincut(Graph.copy())
cut = cut0 if len(cut0) > len(cut1) else cut1
cut = cut.astype(int)
sub_imageid = person[cut]
else:
sub_imageid = get_min_reprojection_error(
person, self.dataset, pose_mat, sub_imgid2cam)
else:
sub_imageid = person
_, rank = torch.sort(
geo_affinity_mat[sub_imageid][:, sub_imageid].sum(dim=0))
sub_imageid = sub_imageid[rank[:2]]
cam_id_0, cam_id_1 = sub_imgid2cam[sub_imageid[0]], sub_imgid2cam[
sub_imageid[1]]
projmat_0, projmat_1 = self.dataset.P[cam_id_0], self.dataset.P[
cam_id_1]
pose2d_0, pose2d_1 = pose_mat[sub_imageid[0]].T, pose_mat[
sub_imageid[1]].T
pose3d_homo = cv2.triangulatePoints(projmat_0, projmat_1, pose2d_0,
pose2d_1)
if self.cfg.use_bundle:
pose3d_homo = bundle_adjustment(pose3d_homo,
person,
self.dataset,
pose_mat,
sub_imgid2cam,
logging=logger)
pose3d = pose3d_homo[:3] / (pose3d_homo[3] + 10e-6)
# pose3d -= ((pose3d[:, 11] + pose3d[:, 12]) / 2).reshape ( 3, -1 ) # No need to normalize to hip
if check_bone_length(pose3d):
multi_pose3d.append(pose3d)
else:
# logging.info ( f'A pose proposal deleted on {img_id}:{person}' )
sub_imageid = list()
pass
chosen_img.append(sub_imageid)
return multi_pose3d, chosen_img
def _hybrid_kernel(self, matched_list, pose_mat, sub_imgid2cam, img_id):
return pictorial.hybrid_kernel ( self, matched_list, pose_mat, sub_imgid2cam, img_id )
multi_pose3d = list ()
for person in matched_list:
# use bottom-up approach to get the 3D pose of person
if person.shape[0] <= 1:
continue
# step1: use the 2D joint of person to triangulate the 3D joints candidates
# person's 17 3D joints candidates
candidates = np.zeros ( (17, person.shape[0] * (person.shape[0] - 1) // 2, 3) )
# 17xC^2_nx3
cnt = 0
for i in range ( person.shape[0] ):
for j in range ( i + 1, person.shape[0] ):
cam_id_i, cam_id_j = sub_imgid2cam[person[i]], sub_imgid2cam[person[j]]
projmat_i, projmat_j = self.dataset.P[cam_id_i], self.dataset.P[cam_id_j]
pose2d_i, pose2d_j = pose_mat[person[i]].T, pose_mat[person[j]].T
pose3d_homo = cv2.triangulatePoints ( projmat_i, projmat_j, pose2d_i, pose2d_j )
pose3d_ij = pose3d_homo[:3] / pose3d_homo[3]
candidates[:, cnt] += pose3d_ij.T
cnt += 1
unary = self.dataset.get_unary ( person, sub_imgid2cam, candidates, img_id )
# step2: use the max-product algorithm to inference to get the 3d joint of the person
# change the coco order
coco_2_skel = [0, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
candidates = np.array ( candidates )[coco_2_skel]
unary = unary[coco_2_skel]
skel = pictorial.getskel ()
# construct pictorial model
edges = pictorial.getPictoStruct ( skel, self.dataset.distribution )
xp = pictorial.inferPict3D_MaxProd ( unary, edges, candidates )
human = np.array ( [candidates[i][j] for i, j in zip ( range ( xp.shape[0] ), xp )] )
human_coco = np.zeros ( (17, 3) )
human_coco[[0, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]] = human
human_coco[[1, 2, 3, 4]] = human_coco[0] # Just make visualize beauty not real ear and eye
human_coco = human_coco.T
if self.cfg.reprojection_refine and len ( person ) > 2:
for joint_idx in range ( human_coco.shape[1] ):
reprojected_error = np.zeros ( len ( person ) )
for idx, pid in enumerate ( person ):
human_coco_homo = np.ones ( 4 )
human_coco_homo[:3] = human_coco[:, joint_idx]
projected_pose_homo = self.dataset.P[sub_imgid2cam[pid]] @ human_coco_homo
projected_pose = projected_pose_homo[:2] / projected_pose_homo[2]
reprojected_error[idx] += np.linalg.norm ( projected_pose - pose_mat[pid, joint_idx] )
# import IPython; IPython.embed()
# pose_select = reprojected_error < self.cfg.refine_threshold
pose_select = (
reprojected_error - reprojected_error.mean ()) / reprojected_error.std () < self.cfg.refine_threshold
if pose_select.sum () >= 2:
Ps = list ()
Ys = list ()
for idx, is_selected in enumerate ( pose_select ):
if is_selected:
Ps.append ( self.dataset.P[sub_imgid2cam[person[idx]]] )
Ys.append ( pose_mat[person[idx], joint_idx].reshape ( -1, 1 ) )
Ps = torch.tensor ( Ps, dtype=torch.float32 )
Ys = torch.tensor ( Ys, dtype=torch.float32 )
Xs = multiTriIter ( Ps, Ys )
refined_pose = (Xs[:3] / Xs[3]).numpy ()
human_coco[:, joint_idx] = refined_pose.reshape ( -1 )
if True or check_bone_length ( human_coco ):
multi_pose3d.append ( human_coco )
return multi_pose3d