def get_bbox(joint_world, joint_valid, camrot, campos, focal, princpt):
joint_cam = []
for i in range(len(joint_world)):
joint_cam.append(world2cam(joint_world[i], camrot, campos))
joint_cam = np.array(joint_cam).reshape(-1, 3)
x_img, y_img, z_img = cam2pixel(joint_cam, focal, princpt)
x_img = x_img[joint_valid[:, 0] == 1]
y_img = y_img[joint_valid[:, 0] == 1]
xmin = min(x_img)
ymin = min(y_img)
xmax = max(x_img)
ymax = max(y_img)
x_center = (xmin + xmax) / 2.
width = xmax - xmin
xmin = x_center - 0.5 * width * 1.5
xmax = x_center + 0.5 * width * 1.5
y_center = (ymin + ymax) / 2.
height = ymax - ymin
ymin = y_center - 0.5 * height * 1.5
ymax = y_center + 0.5 * height * 1.5
bbox = np.array([xmin, ymin, xmax, ymax]).astype(np.float32)
return bbox
def __init__(self, transform, mode, annot_subset):
self.mode = mode # train, test, val
self.annot_subset = annot_subset # all, human_annot, machine_annot
self.img_path = '../data/InterHand2.6M/images'
self.annot_path = '../data/InterHand2.6M/annotations'
if self.annot_subset == 'machine_annot' and self.mode == 'val':
self.rootnet_output_path = '../data/InterHand2.6M/rootnet_output/rootnet_interhand2.6m_output_machine_annot_val.json'
else:
self.rootnet_output_path = '../data/InterHand2.6M/rootnet_output/rootnet_interhand2.6m_output_all_test.json'
self.transform = transform
self.joint_num = 21 # single hand
self.root_joint_idx = {'right': 20, 'left': 41}
self.joint_type = {'right': np.arange(0,self.joint_num), 'left': np.arange(self.joint_num,self.joint_num*2)}
self.skeleton = load_skeleton(osp.join(self.annot_path, 'skeleton.txt'), self.joint_num*2)
self.datalist = []
self.datalist_sh = []
self.datalist_ih = []
self.sequence_names = []
# load annotation
print("Load annotation from " + osp.join(self.annot_path, self.annot_subset))
db = COCO(osp.join(self.annot_path, self.annot_subset, 'InterHand2.6M_' + self.mode + '_data.json'))
with open(osp.join(self.annot_path, self.annot_subset, 'InterHand2.6M_' + self.mode + '_camera.json')) as f:
cameras = json.load(f)
with open(osp.join(self.annot_path, self.annot_subset, 'InterHand2.6M_' + self.mode + '_joint_3d.json')) as f:
joints = json.load(f)
if (self.mode == 'val' or self.mode == 'test') and cfg.trans_test == 'rootnet':
print("Get bbox and root depth from " + self.rootnet_output_path)
rootnet_result = {}
with open(self.rootnet_output_path) as f:
annot = json.load(f)
for i in range(len(annot)):
rootnet_result[str(annot[i]['annot_id'])] = annot[i]
else:
print("Get bbox and root depth from groundtruth annotation")
for aid in db.anns.keys():
ann = db.anns[aid]
image_id = ann['image_id']
img = db.loadImgs(image_id)[0]
capture_id = img['capture']
seq_name = img['seq_name']
cam = img['camera']
frame_idx = img['frame_idx']
img_path = osp.join(self.img_path, self.mode, img['file_name'])
campos, camrot = np.array(cameras[str(capture_id)]['campos'][str(cam)], dtype=np.float32), np.array(cameras[str(capture_id)]['camrot'][str(cam)], dtype=np.float32)
focal, princpt = np.array(cameras[str(capture_id)]['focal'][str(cam)], dtype=np.float32), np.array(cameras[str(capture_id)]['princpt'][str(cam)], dtype=np.float32)
joint_world = np.array(joints[str(capture_id)][str(frame_idx)], dtype=np.float32)
joint_cam = world2cam(joint_world.transpose(1,0), camrot, campos.reshape(3,1)).transpose(1,0)
joint_img = cam2pixel(joint_cam, focal, princpt)[:,:2]
joint_valid = np.array(ann['joint_valid'],dtype=np.float32).reshape(self.joint_num*2)
# if root is not valid -> root-relative 3D pose is also not valid. Therefore, mark all joints as invalid
joint_valid[self.joint_type['right']] *= joint_valid[self.root_joint_idx['right']]
joint_valid[self.joint_type['left']] *= joint_valid[self.root_joint_idx['left']]
hand_type = ann['hand_type']
hand_type_valid = np.array((ann['hand_type_valid']), dtype=np.float32)
if (self.mode == 'val' or self.mode == 'test') and cfg.trans_test == 'rootnet':
bbox = np.array(rootnet_result[str(aid)]['bbox'],dtype=np.float32)
abs_depth = {'right': rootnet_result[str(aid)]['abs_depth'][0], 'left': rootnet_result[str(aid)]['abs_depth'][1]}
else:
img_width, img_height = img['width'], img['height']
bbox = np.array(ann['bbox'],dtype=np.float32) # x,y,w,h
bbox = process_bbox(bbox, (img_height, img_width))
abs_depth = {'right': joint_cam[self.root_joint_idx['right'],2], 'left': joint_cam[self.root_joint_idx['left'],2]}
cam_param = {'focal': focal, 'princpt': princpt}
joint = {'cam_coord': joint_cam, 'img_coord': joint_img, 'valid': joint_valid}
data = {'img_path': img_path, 'seq_name': seq_name, 'cam_param': cam_param, 'bbox': bbox, 'joint': joint, 'hand_type': hand_type, 'hand_type_valid': hand_type_valid, 'abs_depth': abs_depth, 'file_name': img['file_name'], 'capture': capture_id, 'cam': cam, 'frame': frame_idx}
if hand_type == 'right' or hand_type == 'left':
self.datalist_sh.append(data)
else:
self.datalist_ih.append(data)
if seq_name not in self.sequence_names:
self.sequence_names.append(seq_name)
self.datalist = self.datalist_sh + self.datalist_ih
print('Number of annotations in single hand sequences: ' + str(len(self.datalist_sh)))
print('Number of annotations in interacting hand sequences: ' + str(len(self.datalist_ih)))
def load_data(self):
subject_list = self.get_subject()
sampling_ratio = self.get_subsampling_ratio()
# aggregate annotations from each subject
db = COCO()
cameras = {}
joints = {}
smpl_params = {}
for subject in subject_list:
# data load
with open(
osp.join(self.annot_path,
'Human36M_subject' + str(subject) + '_data.json'),
'r') as f:
annot = json.load(f)
if len(db.dataset) == 0:
for k, v in annot.items():
db.dataset[k] = v
else:
for k, v in annot.items():
db.dataset[k] += v
# camera load
with open(
osp.join(
self.annot_path,
'Human36M_subject' + str(subject) + '_camera.json'),
'r') as f:
cameras[str(subject)] = json.load(f)
# joint coordinate load
with open(
osp.join(
self.annot_path,
'Human36M_subject' + str(subject) + '_joint_3d.json'),
'r') as f:
joints[str(subject)] = json.load(f)
# smpl parameter load
with open(
osp.join(
self.annot_path, 'Human36M_subject' + str(subject) +
'_smpl_param.json'), 'r') as f:
smpl_params[str(subject)] = json.load(f)
db.createIndex()
if self.data_split == 'test' and not cfg.use_gt_info:
print("Get bounding box and root from " + self.human_bbox_root_dir)
bbox_root_result = {}
with open(self.human_bbox_root_dir) as f:
annot = json.load(f)
for i in range(len(annot)):
bbox_root_result[str(annot[i]['image_id'])] = {
'bbox': np.array(annot[i]['bbox']),
'root': np.array(annot[i]['root_cam'])
}
else:
print("Get bounding box and root from groundtruth")
datalist = []
for aid in db.anns.keys():
ann = db.anns[aid]
image_id = ann['image_id']
img = db.loadImgs(image_id)[0]
img_path = osp.join(self.img_dir, img['file_name'])
img_shape = (img['height'], img['width'])
# check subject and frame_idx
frame_idx = img['frame_idx']
if frame_idx % sampling_ratio != 0:
continue
# check smpl parameter exist
subject = img['subject']
action_idx = img['action_idx']
subaction_idx = img['subaction_idx']
frame_idx = img['frame_idx']
try:
smpl_param = smpl_params[str(subject)][str(action_idx)][str(
subaction_idx)][str(frame_idx)]
except KeyError:
smpl_param = None
# camera parameter
cam_idx = img['cam_idx']
cam_param = cameras[str(subject)][str(cam_idx)]
R, t, f, c = np.array(cam_param['R'], dtype=np.float32), np.array(
cam_param['t'], dtype=np.float32), np.array(
cam_param['f'],
dtype=np.float32), np.array(cam_param['c'],
dtype=np.float32)
cam_param = {'R': R, 't': t, 'focal': f, 'princpt': c}
# only use frontal camera following previous works (HMR and SPIN)
if self.data_split == 'test' and str(cam_idx) != '4':
continue
# project world coordinate to cam, image coordinate space
joint_world = np.array(joints[str(subject)][str(action_idx)][str(
subaction_idx)][str(frame_idx)],
dtype=np.float32)
joint_cam = world2cam(joint_world, R, t)
joint_img = cam2pixel(joint_cam, f, c)
joint_valid = np.ones((self.h36m_joint_num, 1))
if self.data_split == 'test' and not cfg.use_gt_info:
bbox = bbox_root_result[str(
image_id
)]['bbox'] # bbox should be aspect ratio preserved-extended. It is done in RootNet.
root_joint_depth = bbox_root_result[str(image_id)]['root'][2]
else:
bbox = process_bbox(np.array(ann['bbox']), img['width'],
img['height'])
if bbox is None: continue
root_joint_depth = joint_cam[self.h36m_root_joint_idx][2]
datalist.append({
'img_path': img_path,
'img_id': image_id,
'img_shape': img_shape,
'bbox': bbox,
'joint_img': joint_img,
'joint_cam': joint_cam,
'joint_valid': joint_valid,
'smpl_param': smpl_param,
'root_joint_depth': root_joint_depth,
'cam_param': cam_param
})
return datalist