def __getitem__(self, idx):
data = copy.deepcopy(self.datalist[idx])
img_id, img_name, cam_param, bbox, smpl_param, img_shape = data[
'img_id'], data['img_name'], data['cam_param'], data['bbox'], data[
'smpl_param'], data['img_shape']
flip, rot = augm_params(is_train=(self.data_split == 'train'))
# smpl coordinates
smpl_mesh_coord_cam, smpl_joint_coord_cam = self.get_smpl_coord(
smpl_param)
smpl_coord_cam = np.concatenate(
(smpl_mesh_coord_cam, smpl_joint_coord_cam))
smpl_coord_img = cam2pixel(smpl_coord_cam, cam_param['focal'],
cam_param['princpt'])
joint_coord_img = smpl_coord_img[self.smpl_vertex_num:][:, :2]
# vis_2d_pose(joint_coord_img, img_path, self.smpl_skeleton, prefix='gt')
# root relative cam coord
smpl_coord_cam = smpl_coord_cam - smpl_coord_cam[
self.smpl_vertex_num + self.smpl_root_joint_idx]
mesh_coord_cam = smpl_coord_cam[:self.smpl_vertex_num]
joint_coord_cam = smpl_coord_cam[self.smpl_vertex_num:]
if not cfg.DATASET.use_gt_input:
# train / test with 2d dection input
det_data = self.datalist_pose2d_det[idx]
assert img_id == det_data['img_id']
joint_coord_img = det_data['img_joint']
# vis_2d_pose(joint_coord_img, img_path, self.smpl_skeleton, prefix='det')
# vis_3d_pose(joint_coord_cam, self.smpl_skeleton, joint_set_name='smpl')
# make new bbox
bbox = get_bbox(joint_coord_img)
bbox = process_bbox(bbox.copy())
# aug
joint_coord_img, trans = j2d_processing(
joint_coord_img.copy(),
(cfg.MODEL.input_shape[1], cfg.MODEL.input_shape[0]), bbox, rot,
flip, self.flip_pairs)
joint_coord_cam = j3d_processing(joint_coord_cam, rot, flip,
self.flip_pairs)
# -> 0~1
joint_coord_img = joint_coord_img[:, :2]
joint_coord_img /= np.array(
[[cfg.MODEL.input_shape[1], cfg.MODEL.input_shape[0]]])
# normalize loc&scale
mean, std = np.mean(joint_coord_img, axis=0), np.std(joint_coord_img,
axis=0)
joint_coord_img = (joint_coord_img.copy() - mean) / std
if cfg.MODEL.name == 'pose2mesh_net':
# default valid
mesh_valid = np.ones((len(mesh_coord_cam), 1), dtype=np.float32)
reg_joint_valid = np.ones((len(joint_coord_cam), 1),
dtype=np.float32)
lift_joint_valid = np.ones((len(joint_coord_cam), 1),
dtype=np.float32)
inputs = {'pose2d': joint_coord_img}
targets = {
'mesh': mesh_coord_cam / 1000,
'lift_pose3d': joint_coord_cam,
'reg_pose3d': joint_coord_cam
}
meta = {
'mesh_valid': mesh_valid,
'lift_pose3d_valid': lift_joint_valid,
'reg_pose3d_valid': reg_joint_valid
}
elif cfg.MODEL.name == 'posenet':
# default valid
joint_valid = np.ones((len(joint_coord_cam), 1), dtype=np.float32)
return joint_coord_img, joint_coord_cam, joint_valid
return inputs, targets, meta
def __getitem__(self, idx):
data = copy.deepcopy(self.datalist[idx])
img_id, bbox, smpl_param, cam_param, img_shape = data['img_id'], data['bbox'].copy(), data['smpl_param'].copy(), data['cam_param'].copy(), data['img_hw']
flip, rot = augm_params(is_train=(self.data_split == 'train'))
# smpl coordinates
mesh_cam, joint_cam_smpl = self.get_smpl_coord(smpl_param, cam_param)
# regress coco joints
joint_cam_coco, joint_img_coco = self.get_coco_from_mesh(mesh_cam, cam_param)
# h36m joints from datasets
joint_cam_h36m, joint_img_h36m = data['joint_cam'], data['joint_img'][:, :2]
# root relative camera coordinate
mesh_cam = mesh_cam - joint_cam_h36m[:1]
# joint_cam_smpl = joint_cam_smpl - joint_cam_h36m[:1]
joint_cam_coco = joint_cam_coco - joint_cam_coco[-2:-1]
joint_cam_h36m = joint_cam_h36m - joint_cam_h36m[:1]
# joint_cam is PoseNet target
if self.input_joint_name == 'coco':
joint_img, joint_cam = joint_img_coco, joint_cam_coco
elif self.input_joint_name == 'human36':
joint_img, joint_cam = joint_img_h36m, joint_cam_h36m
# make new bbox
bbox = get_bbox(joint_img)
bbox = process_bbox(bbox.copy())
# aug
joint_img, trans = j2d_processing(joint_img.copy(), (cfg.MODEL.input_shape[1], cfg.MODEL.input_shape[0]),
bbox, rot, flip, self.flip_pairs)
joint_cam = j3d_processing(joint_cam, rot, flip, self.flip_pairs)
if not cfg.DATASET.use_gt_input:
joint_img = self.replace_joint_img(idx, img_id, joint_img, bbox, trans)
# vis
# img = cv2.imread(img_path, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
# new_img = cv2.warpAffine(img, trans, (cfg.MODEL.input_shape[1], cfg.MODEL.input_shape[0]), flags=cv2.INTER_LINEAR)
# vis_2d_pose(joint_img, new_img, self.human36_skeleton, prefix='detection')
# vis_3d_pose(joint_cam, self.human36_skeleton, joint_set_name='human36', gt=True)
# -> 0~1
joint_img = joint_img[:, :2]
joint_img /= np.array([[cfg.MODEL.input_shape[1], cfg.MODEL.input_shape[0]]])
# normalize loc&scale
mean, std = np.mean(joint_img, axis=0), np.std(joint_img, axis=0)
joint_img = (joint_img.copy() - mean) / std
if cfg.MODEL.name == 'pose2mesh_net':
# default valid
mesh_valid = np.ones((len(mesh_cam), 1), dtype=np.float32)
reg_joint_valid = np.ones((len(joint_cam_h36m), 1), dtype=np.float32)
lift_joint_valid = np.ones((len(joint_cam), 1), dtype=np.float32)
# if fitted mesh is too far from h36m gt, discard it
error = self.get_fitting_error(joint_cam_h36m, mesh_cam)
if error > self.fitting_thr:
mesh_valid[:] = 0
if self.input_joint_name == 'coco':
lift_joint_valid[:] = 0
inputs = {'pose2d': joint_img}
targets = {'mesh': mesh_cam / 1000, 'lift_pose3d': joint_cam, 'reg_pose3d': joint_cam_h36m}
meta = {'mesh_valid': mesh_valid, 'lift_pose3d_valid': lift_joint_valid, 'reg_pose3d_valid': reg_joint_valid}
return inputs, targets, meta
elif cfg.MODEL.name == 'posenet':
# default valid
joint_valid = np.ones((len(joint_cam), 1), dtype=np.float32)
# if fitted mesh is too far from h36m gt, discard it
error = self.get_fitting_error(joint_cam_h36m, mesh_cam)
if (error > self.fitting_thr) and (self.input_joint_name == 'coco'):
joint_valid[:] = 0
return joint_img, joint_cam, joint_valid
def __getitem__(self, idx):
data = copy.deepcopy(self.datalist[idx])
flip, rot = augm_params(is_train=(self.data_split == 'train'))
# get smpl mesh, joints
smpl_param, cam_param, img_shape = data['smpl_param'], data[
'cam_param'], data['img_shape']
mesh_cam, joint_cam_smpl = self.get_smpl_coord(smpl_param, cam_param)
# regress coco joints
joint_cam_h36m, joint_img_h36m = self.get_joints_from_mesh(
mesh_cam, 'human36', cam_param)
joint_cam_coco, joint_img_coco = self.get_joints_from_mesh(
mesh_cam, 'coco', cam_param)
# debug vis
# vis_3d_pose(joint_cam_coco, self.coco_skeleton, joint_set_name='coco', prefix=f'coco_joint_cam_{idx}')
# img = np.zeros((int(img_shape[0]), int(img_shape[1]), 3))
# vis_2d_pose(joint_img_coco, img, self.coco_skeleton, prefix='coco joint img')
# root relative camera coordinate
mesh_cam = mesh_cam - joint_cam_h36m[:1]
joint_cam_coco = joint_cam_coco - joint_cam_coco[-2:-1]
joint_cam_h36m = joint_cam_h36m - joint_cam_h36m[:1]
# joint_cam is PoseNet target
if self.input_joint_name == 'coco':
joint_img, joint_cam = joint_img_coco, joint_cam_coco
elif self.input_joint_name == 'human36':
joint_img, joint_cam = joint_img_h36m, joint_cam_h36m
# make new bbox
tight_bbox = get_bbox(joint_img)
bbox = process_bbox(tight_bbox.copy())
# aug
joint_img, trans = j2d_processing(
joint_img.copy(),
(cfg.MODEL.input_shape[1], cfg.MODEL.input_shape[0]), bbox, rot, 0,
None)
if not cfg.DATASET.use_gt_input:
joint_img = self.replace_joint_img(joint_img, tight_bbox, trans)
if flip:
joint_img = flip_2d_joint(joint_img, cfg.MODEL.input_shape[1],
self.flip_pairs)
joint_cam = j3d_processing(joint_cam, rot, flip, self.flip_pairs)
# -> 0~1
joint_img = joint_img[:, :2]
joint_img /= np.array(
[[cfg.MODEL.input_shape[1], cfg.MODEL.input_shape[0]]])
# normalize loc&scale
mean, std = np.mean(joint_img, axis=0), np.std(joint_img, axis=0)
joint_img = (joint_img.copy() - mean) / std
if cfg.MODEL.name == 'pose2mesh_net':
# default valid
mesh_valid = np.ones((len(mesh_cam), 1), dtype=np.float32)
reg_joint_valid = np.ones((len(joint_cam_h36m), 1),
dtype=np.float32)
lift_joint_valid = np.ones((len(joint_cam), 1), dtype=np.float32)
inputs = {'pose2d': joint_img}
targets = {
'mesh': mesh_cam / 1000,
'lift_pose3d': joint_cam,
'reg_pose3d': joint_cam_h36m
}
meta = {
'mesh_valid': mesh_valid,
'lift_pose3d_valid': lift_joint_valid,
'reg_pose3d_valid': reg_joint_valid
}
return inputs, targets, meta
elif cfg.MODEL.name == 'posenet':
# default valid
joint_valid = np.ones((len(joint_cam), 1), dtype=np.float32)
return joint_img, joint_cam, joint_valid
def __getitem__(self, idx):
data = copy.deepcopy(self.datalist[idx])
img_shape, bbox, dp_data, smplify_result = data['img_shape'], data[
'bbox'], data['dp_data'], data['smplify_result']
flip, rot = augm_params(is_train=(self.data_split == 'train'))
# img_name = img_path.split('/')[-1][:-4]
smpl_param, cam_param = smplify_result['smpl_param'], smplify_result[
'cam_param']
# regress h36m, coco joints
mesh_cam, joint_cam_smpl = self.get_smpl_coord(smpl_param)
joint_cam_h36m, joint_img_h36m = self.get_joints_from_mesh(
mesh_cam, 'human36', cam_param)
joint_cam_coco, joint_img_coco = self.get_joints_from_mesh(
mesh_cam, 'coco', cam_param)
# vis_2d_pose(joint_img_h36m, img_path, self.human36_skeleton, prefix='h36m joint')
# vis_2d_pose(joint_img_coco, img_path, self.coco_skeleton, prefix='coco joint')
# vis_3d_pose(joint_cam_h36m, self.human36_skeleton, 'human36', gt=True)
# root relative camera coordinate
mesh_cam = mesh_cam - joint_cam_h36m[:1]
joint_cam_coco = joint_cam_coco - joint_cam_coco[-2:-1]
joint_cam_h36m = joint_cam_h36m - joint_cam_h36m[:1]
if self.input_joint_name == 'coco':
joint_img, joint_cam = joint_img_coco, joint_cam_coco
elif self.input_joint_name == 'human36':
joint_img, joint_cam = joint_img_h36m, joint_cam_h36m
# make new bbox
tight_bbox = get_bbox(joint_img)
bbox = process_bbox(tight_bbox.copy())
# aug
joint_img, trans = j2d_processing(
joint_img.copy(),
(cfg.MODEL.input_shape[1], cfg.MODEL.input_shape[0]), bbox, rot, 0,
None)
if not cfg.DATASET.use_gt_input:
joint_img = self.replace_joint_img(joint_img, tight_bbox, trans)
if flip:
joint_img = flip_2d_joint(joint_img, cfg.MODEL.input_shape[1],
self.flip_pairs)
joint_cam = j3d_processing(joint_cam, rot, flip, self.flip_pairs)
# -> 0~1
joint_img = joint_img[:, :2]
joint_img /= np.array(
[[cfg.MODEL.input_shape[1], cfg.MODEL.input_shape[0]]])
# normalize loc&scale
mean, std = np.mean(joint_img, axis=0), np.std(joint_img, axis=0)
joint_img = (joint_img.copy() - mean) / std
if cfg.MODEL.name == 'pose2mesh_net':
# default valid
mesh_valid = np.ones((len(mesh_cam), 1), dtype=np.float32)
reg_joint_valid = np.ones((len(joint_cam_h36m), 1),
dtype=np.float32)
lift_joint_valid = np.ones((len(joint_cam), 1), dtype=np.float32)
error = self.get_fitting_error(tight_bbox, data['joint_img'],
joint_img_coco[:17],
data['joint_valid'])
if error > self.fitting_thr:
mesh_valid[:], reg_joint_valid[:], lift_joint_valid[:] = 0, 0, 0
inputs = {'pose2d': joint_img}
targets = {
'mesh': mesh_cam / 1000,
'lift_pose3d': joint_cam,
'reg_pose3d': joint_cam_h36m
}
meta = {
'mesh_valid': mesh_valid,
'lift_pose3d_valid': lift_joint_valid,
'reg_pose3d_valid': reg_joint_valid
}
return inputs, targets, meta
elif cfg.MODEL.name == 'posenet':
# default valid
joint_valid = np.ones((len(joint_cam), 1), dtype=np.float32)
# compute fitting error
error = self.get_fitting_error(tight_bbox, data['joint_img'],
joint_img_coco[:17],
data['joint_valid'])
if error > self.fitting_thr:
joint_valid[:, :] = 0
return joint_img, joint_cam, joint_valid