def get_joint_setting(mesh_model, joint_category='human36'):
joint_regressor, joint_num, skeleton, graph_L, graph_perm_reverse = None, None, None, None, None
if joint_category == 'human36':
joint_regressor = mesh_model.joint_regressor_h36m
joint_num = 17
skeleton = (
(0, 7), (7, 8), (8, 9), (9, 10), (8, 11), (11, 12), (12, 13), (8, 14), (14, 15), (15, 16), (0, 1), (1, 2),
(2, 3), (0, 4), (4, 5), (5, 6))
flip_pairs = ((1, 4), (2, 5), (3, 6), (14, 11), (15, 12), (16, 13))
graph_Adj, graph_L, graph_perm,graph_perm_reverse = \
build_coarse_graphs(mesh_model.face, joint_num, skeleton, flip_pairs, levels=9)
model_chk_path = './experiment/pose2mesh_human36J_train_human36/final.pth.tar'
elif joint_category == 'coco':
joint_regressor = mesh_model.joint_regressor_coco
joint_num = 19 # add pelvis and neck
skeleton = (
(1, 2), (0, 1), (0, 2), (2, 4), (1, 3), (6, 8), (8, 10), (5, 7), (7, 9), (12, 14), (14, 16), (11, 13),
(13, 15), # (5, 6), #(11, 12),
(17, 11), (17, 12), (17, 18), (18, 5), (18, 6), (18, 0))
flip_pairs = ((1, 2), (3, 4), (5, 6), (7, 8), (9, 10), (11, 12), (13, 14), (15, 16))
graph_Adj, graph_L, graph_perm, graph_perm_reverse = \
build_coarse_graphs(mesh_model.face, joint_num, skeleton, flip_pairs, levels=9)
model_chk_path = './experiment/pose2mesh_cocoJ_train_human36_coco_muco/final.pth.tar'
elif joint_category == 'smpl':
joint_regressor = mesh_model.layer['neutral'].th_J_regressor.numpy().astype(np.float32)
joint_num = 24
skeleton = (
(0, 1), (1, 4), (4, 7), (7, 10), (0, 2), (2, 5), (5, 8), (8, 11), (0, 3), (3, 6), (6, 9), (9, 14), (14, 17),
(17, 19), (19, 21), (21, 23), (9, 13), (13, 16), (16, 18), (18, 20), (20, 22), (9, 12), (12, 15))
flip_pairs = ((1, 2), (4, 5), (7, 8), (10, 11), (13, 14), (16, 17), (18, 19), (20, 21), (22, 23))
graph_Adj, graph_L, graph_perm, graph_perm_reverse = \
build_coarse_graphs(mesh_model.face, joint_num, skeleton, flip_pairs, levels=9)
model_chk_path = './experiment/pose2mesh_smplJ_train_surreal/final.pth.tar'
elif joint_category == 'mano':
joint_regressor = mesh_model.joint_regressor
joint_num = 21
skeleton = ( (0,1), (0,5), (0,9), (0,13), (0,17), (1,2), (2,3), (3,4), (5,6), (6,7), (7,8), (9,10), (10,11), (11,12), (13,14), (14,15), (15,16), (17,18), (18,19), (19,20) )
hori_conn = (
(1, 5), (5, 9), (9, 13), (13, 17), (2, 6), (6, 10), (10, 14), (14, 18), (3, 7), (7, 11), (11, 15), (15, 19),
(4, 8), (8, 12), (12, 16), (16, 20))
graph_Adj, graph_L, graph_perm, graph_perm_reverse = \
build_coarse_graphs(mesh_model.face, joint_num, skeleton, hori_conn, levels=6)
model_chk_path = './experiment/pose2mesh_manoJ_train_freihand/final.pth.tar'
else:
raise NotImplementedError(f"{joint_category}: unknown joint set category")
model = models.pose2mesh_net.get_model(joint_num, graph_L)
checkpoint = load_checkpoint(load_dir=model_chk_path)
model.load_state_dict(checkpoint['model_state_dict'])
return model, joint_regressor, joint_num, skeleton, graph_L, graph_perm_reverse
def __init__(self, img_res):
dataset_name = 'COCO'
self.data_split = 'val' # 'train' if data_split == 'train' else 'val'
self.img_path = osp.join(cfg.data_dir, dataset_name, 'images')
self.annot_path = osp.join(cfg.data_dir, dataset_name, 'annotations')
self.img_res = img_res
self.smpl = SMPL()
# COCO joint set
self.coco_joint_num = 19 # 17 + 2, manually added pelvis and neck
self.coco_joints_name = ('Nose', 'L_Eye', 'R_Eye', 'L_Ear', 'R_Ear',
'L_Shoulder', 'R_Shoulder', 'L_Elbow',
'R_Elbow', 'L_Wrist', 'R_Wrist', 'L_Hip',
'R_Hip', 'L_Knee', 'R_Knee', 'L_Ankle',
'R_Ankle', 'Pelvis', 'Neck')
self.coco_flip_pairs = ((1, 2), (3, 4), (5, 6), (7, 8), (9, 10),
(11, 12), (13, 14), (15, 16))
# self.coco_skeleton = (
# (1, 2), (0, 1), (0, 2), (2, 4), (1, 3), (6, 8), (8, 10), (5, 7), (7, 9), (12, 14), (14, 16), (11, 13), (13, 15),
# (5, 6), (11, 12))
self.coco_skeleton = (
(1, 2),
(0, 1),
(0, 2),
(2, 4),
(1, 3),
(6, 8),
(8, 10),
(5, 7),
(7, 9),
(11, 13),
(13, 15),
(17, 11),
(17, 12),
(17, 18),
(18, 5),
(18, 6),
(18, 0),
(12, 14),
(14, 16),
)
self.coco_root_joint_idx = self.coco_joints_name.index('Pelvis')
self.joint_regressor_coco = self.smpl.joint_regressor_coco
input_joint_name = 'coco'
self.joint_num, self.skeleton, self.flip_pairs = self.get_joint_setting(
input_joint_name)
self.datalist_cocoj_det = self.load_coco_joints_det(
osp.join(cfg.data_dir, dataset_name,
'hrnet_output_on_valset.json'))
self.datalist = self.load_data()[:6337]
self.graph_Adj, self.graph_L, self.graph_perm, self.graph_perm_reverse = \
build_coarse_graphs(self.smpl.face, self.joint_num, self.skeleton, self.flip_pairs, levels=9)
print("# of COCO annotation and detection data: ", len(self.datalist),
len(self.datalist_cocoj_det))
def __init__(self, data_split, args):
dataset_name = 'PW3D'
self.data_split = 'test'
self.data_path = osp.join(cfg.data_dir, dataset_name, 'data')
self.det_data_path = osp.join(self.data_path, 'darkpose_output_on_testset.json')
#'hrnet_output_on_testset.json')
self.img_path = osp.join(cfg.data_dir, dataset_name, 'imageFiles')
# SMPL joint set
self.mesh_model = SMPL()
self.smpl_root_joint_idx = self.mesh_model.root_joint_idx
self.face_kps_vertex = self.mesh_model.face_kps_vertex
self.smpl_vertex_num = 6890
self.smpl_joint_num = 24
self.smpl_flip_pairs = ((1, 2), (4, 5), (7, 8), (10, 11), (13, 14), (16, 17), (18, 19), (20, 21), (22, 23))
self.smpl_skeleton = (
(0, 1), (1, 4), (4, 7), (7, 10), (0, 2), (2, 5), (5, 8), (8, 11), (0, 3), (3, 6), (6, 9), (9, 14), (14, 17),
(17, 19), (19, 21), (21, 23), (9, 13), (13, 16), (16, 18), (18, 20), (20, 22), (9, 12), (12, 15))
self.joint_regressor_smpl = self.mesh_model.layer['neutral'].th_J_regressor
# H36M joint set
self.human36_root_joint_idx = 0
self.human36_eval_joint = (1, 2, 3, 4, 5, 6, 8, 10, 11, 12, 13, 14, 15, 16)
self.human36_skeleton = (
(0, 7), (7, 8), (8, 9), (9, 10), (8, 11), (11, 12), (12, 13), (8, 14), (14, 15), (15, 16), (0, 1), (1, 2),
(2, 3), (0, 4), (4, 5), (5, 6))
self.joint_regressor_human36 = torch.Tensor(self.mesh_model.joint_regressor_h36m)
# COCO joint set
self.coco_joint_num = 19 # 17 + 2, manually added pelvis and neck
self.coco_joints_name = (
'Nose', 'L_Eye', 'R_Eye', 'L_Ear', 'R_Ear', 'L_Shoulder', 'R_Shoulder', 'L_Elbow', 'R_Elbow', 'L_Wrist',
'R_Wrist', 'L_Hip', 'R_Hip', 'L_Knee', 'R_Knee', 'L_Ankle', 'R_Ankle', 'Pelvis', 'Neck')
self.coco_flip_pairs = ((1, 2), (3, 4), (5, 6), (7, 8), (9, 10), (11, 12), (13, 14), (15, 16))
self.coco_skeleton = (
(1, 2), (0, 1), (0, 2), (2, 4), (1, 3), (6, 8), (8, 10), (5, 7), (7, 9), (12, 14), (14, 16), (11, 13),
(13, 15), #(5, 6), (11, 12))
(17, 11), (17, 12), (17, 18), (18, 5), (18, 6), (18, 0))
self.joint_regressor_coco = torch.Tensor(self.mesh_model.joint_regressor_coco)
input_joint_name = 'coco'
self.joint_num, self.skeleton, self.flip_pairs = self.get_joint_setting(input_joint_name)
self.datalist, self.video_indices = self.load_data() # self.video_indexes: 37 video, and indices of each video
self.datadict_pose2d_det = self.load_pose2d_det(self.det_data_path)
self.graph_Adj, self.graph_L, self.graph_perm, self.graph_perm_reverse = \
build_coarse_graphs(self.mesh_model.face, self.joint_num, self.skeleton, self.flip_pairs, levels=9)
print(self.graph_perm_reverse)
print("Check lengths of annotation and detection output: ", len(self.datalist), len(self.datadict_pose2d_det))
def __init__(self, data_split, args):
dataset_name = 'SURREAL'
self.data_split = 'train' if data_split == 'train' else 'val'
self.debug = args.debug
self.data_path = osp.join(cfg.data_dir, dataset_name, 'data')
self.img_path = osp.join(cfg.data_dir, dataset_name, 'images',
self.data_split)
self.mesh_model = SMPL()
self.smpl_vertex_num = 6890
self.smpl_joint_num = 24
self.smpl_joints_name = self.mesh_model.joints_name
self.smpl_flip_pairs = ((1, 2), (4, 5), (7, 8), (10, 11), (13, 14),
(16, 17), (18, 19), (20, 21), (22, 23))
self.smpl_skeleton = ((0, 1), (1, 4), (4, 7), (7, 10), (0, 2), (2, 5),
(5, 8), (8, 11), (0, 3), (3, 6), (6, 9), (9, 14),
(14, 17), (17, 19), (19, 21), (21, 23), (9, 13),
(13, 16), (16, 18), (18, 20), (20, 22), (9, 12),
(12, 15))
self.smpl_root_joint_idx = self.mesh_model.root_joint_idx
self.joint_regressor_smpl = self.mesh_model.layer[
'neutral'].th_J_regressor
self.datalist = self.load_data()
det_file_path = osp.join(self.data_path,
f'hrnet_output_on_{self.data_split}set.json')
self.datalist_pose2d_det = self.load_pose2d_det(det_file_path)
print("Check lengths of annotation and detection output: ",
len(self.datalist), len(self.datalist_pose2d_det))
self.input_joint_name = cfg.DATASET.input_joint_set # 'coco'
self.joint_num, self.skeleton, self.flip_pairs = self.get_joint_setting(
self.input_joint_name)
self.graph_Adj, self.graph_L, self.graph_perm, self.graph_perm_reverse = \
build_coarse_graphs(self.mesh_model.face, self.joint_num, self.skeleton, self.flip_pairs, levels=9)
def __init__(self, mode, args):
dataset_name = 'FreiHAND'
self.data_split = mode
self.data_path = osp.join(cfg.data_dir, dataset_name, 'data')
# MANO joint set
self.mesh_model = MANO()
self.face = self.mesh_model.face
self.joint_regressor_mano = self.mesh_model.joint_regressor
self.vertex_num = self.mesh_model.vertex_num
self.joint_num = self.mesh_model.joint_num
self.joints_name = self.mesh_model.joints_name
self.skeleton = self.mesh_model.skeleton
self.root_joint_idx = self.mesh_model.root_joint_idx
self.joint_hori_conn = ((1,5), (5,9), (9,13), (13,17), (2,6),(6,10),(10,14), (14,18), (3,7), (7,11), (11,15), (15,19),(4,8),(8,12),(12,16),(16,20))
# ((1,5,9,13,17),(2,6,10,14,18),(3,7,11,15,19),(4,8,12,16,20))
self.datalist = self.load_data()
det_path = osp.join(self.data_path, f'hrnet_output_on_{mode}set.json')
self.datalist_pose2d_det = self.load_pose2d_det(det_path)
print("Check lengths of annotation and detection output: ", len(self.datalist), len(self.datalist_pose2d_det))
self.graph_Adj, self.graph_L, self.graph_perm, self.graph_perm_reverse = \
build_coarse_graphs(self.mesh_model.face, self.joint_num, self.skeleton, self.joint_hori_conn, levels=6)
def __init__(self, mode, args):
dataset_name = 'Human36M'
self.debug = args.debug
self.data_split = mode
self.img_dir = osp.join(cfg.data_dir, dataset_name, 'images')
self.annot_path = osp.join(cfg.data_dir, dataset_name, 'annotations')
self.subject_genders = {1: 'female', 5: 'female', 6: 'male', 7: 'female', 8: 'male', 9: 'male', 11: 'male'}
self.protocol = 2
self.action_name = ['Directions', 'Discussion', 'Eating', 'Greeting', 'Phoning', 'Posing', 'Purchases',
'Sitting', 'SittingDown', 'Smoking', 'Photo', 'Waiting', 'Walking', 'WalkDog',
'WalkTogether']
self.fitting_thr = 25 # milimeter
# SMPL joint set
self.mesh_model = SMPL()
self.smpl_root_joint_idx = self.mesh_model.root_joint_idx
self.smpl_face_kps_vertex = self.mesh_model.face_kps_vertex
self.smpl_vertex_num = 6890
self.smpl_joint_num = 24
self.smpl_flip_pairs = ((1, 2), (4, 5), (7, 8), (10, 11), (13, 14), (16, 17), (18, 19), (20, 21), (22, 23))
self.smpl_skeleton = (
(0, 1), (1, 4), (4, 7), (7, 10), (0, 2), (2, 5), (5, 8), (8, 11), (0, 3), (3, 6), (6, 9), (9, 14), (14, 17),
(17, 19), (19, 21), (21, 23), (9, 13), (13, 16), (16, 18), (18, 20), (20, 22), (9, 12), (12, 15))
self.joint_regressor_smpl = self.mesh_model.layer['neutral'].th_J_regressor
# H36M joint set
self.human36_joint_num = 17
self.human36_joints_name = (
'Pelvis', 'R_Hip', 'R_Knee', 'R_Ankle', 'L_Hip', 'L_Knee', 'L_Ankle', 'Torso', 'Neck', 'Nose', 'Head',
'L_Shoulder', 'L_Elbow', 'L_Wrist', 'R_Shoulder', 'R_Elbow', 'R_Wrist')
self.human36_flip_pairs = ((1, 4), (2, 5), (3, 6), (14, 11), (15, 12), (16, 13))
self.human36_skeleton = (
(0, 7), (7, 8), (8, 9), (9, 10), (8, 11), (11, 12), (12, 13), (8, 14), (14, 15), (15, 16), (0, 1), (1, 2),
(2, 3), (0, 4), (4, 5), (5, 6))
self.human36_root_joint_idx = self.human36_joints_name.index('Pelvis')
self.human36_error_distribution = self.get_stat()
self.human36_eval_joint = (1, 2, 3, 4, 5, 6, 8, 10, 11, 12, 13, 14, 15, 16)
self.joint_regressor_human36 = self.mesh_model.joint_regressor_h36m
# COCO joint set
self.coco_joint_num = 19 # 17 + 2, manually added pelvis and neck
self.coco_joints_name = (
'Nose', 'L_Eye', 'R_Eye', 'L_Ear', 'R_Ear', 'L_Shoulder', 'R_Shoulder', 'L_Elbow', 'R_Elbow', 'L_Wrist',
'R_Wrist', 'L_Hip', 'R_Hip', 'L_Knee', 'R_Knee', 'L_Ankle', 'R_Ankle', 'Pelvis', 'Neck')
self.coco_flip_pairs = ((1, 2), (3, 4), (5, 6), (7, 8), (9, 10), (11, 12), (13, 14), (15, 16))
self.coco_skeleton = (
(1, 2), (0, 1), (0, 2), (2, 4), (1, 3), (6, 8), (8, 10), (5, 7), (7, 9), (12, 14), (14, 16), (11, 13),
(13, 15), #(5, 6), #(11, 12),
(17, 11), (17, 12), (17, 18), (18, 5), (18, 6), (18, 0))
self.joint_regressor_coco = self.mesh_model.joint_regressor_coco
self.input_joint_name = cfg.DATASET.input_joint_set # 'coco'
self.joint_num, self.skeleton, self.flip_pairs = self.get_joint_setting(self.input_joint_name)
self.datalist, skip_idx, skip_img_path = self.load_data()
if self.data_split == 'test':
det_2d_data_path = osp.join(cfg.data_dir, dataset_name, 'absnet_output_on_testset.json')
self.datalist_pose2d_det = self.load_pose2d_det(det_2d_data_path, skip_img_path)
print("Check lengths of annotation and detection output: ", len(self.datalist), len(self.datalist_pose2d_det))
self.graph_Adj, self.graph_L, self.graph_perm, self.graph_perm_reverse = \
build_coarse_graphs(self.mesh_model.face, self.joint_num, self.skeleton, self.flip_pairs, levels=9)