def _check_load_keypoints(self, _mpii, entry):
"""Check and load ground-truth keypoints"""
ann_ids = _mpii.getAnnIds(imgIds=entry['id'], iscrowd=False)
objs = _mpii.loadAnns(ann_ids)
# check valid bboxes
valid_objs = []
width = entry['width']
height = entry['height']
for obj in objs:
if max(obj['keypoints']) == 0:
continue
# convert from (x, y, w, h) to (xmin, ymin, xmax, ymax) and clip bound
xmin, ymin, xmax, ymax = bbox_clip_xyxy(bbox_xywh_to_xyxy(obj['bbox']), width, height)
# require non-zero box area
if xmax <= xmin or ymax <= ymin:
continue
if obj['num_keypoints'] == 0:
continue
# joints 3d: (num_joints, 3, 2); 3 is for x, y, z; 2 is for position, visibility
joints_3d = np.zeros((self.num_joints, 3, 2), dtype=np.float32)
for i in range(self.num_joints):
joints_3d[i, 0, 0] = obj['keypoints'][i * 3 + 0]
joints_3d[i, 1, 0] = obj['keypoints'][i * 3 + 1]
# joints_3d[i, 2, 0] = 0
visible = min(1, obj['keypoints'][i * 3 + 2])
joints_3d[i, :2, 1] = visible
# joints_3d[i, 2, 1] = 0
if np.sum(joints_3d[:, 0, 1]) < 1:
# no visible keypoint
continue
if self._check_centers and self._train:
bbox_center, bbox_area = self._get_box_center_area((xmin, ymin, xmax, ymax))
kp_center, num_vis = self._get_keypoints_center_count(joints_3d)
ks = np.exp(-2 * np.sum(np.square(bbox_center - kp_center)) / bbox_area)
if (num_vis / 80.0 + 47 / 80.0) > ks:
continue
valid_objs.append({
'bbox': (xmin, ymin, xmax, ymax),
'width': width,
'height': height,
'joints_3d': joints_3d
})
if not valid_objs:
if not self._skip_empty:
# dummy invalid labels if no valid objects are found
valid_objs.append({
'bbox': np.array([-1, -1, 0, 0]),
'width': width,
'height': height,
'joints_3d': np.zeros((self.num_joints, 2, 2), dtype=np.float32)
})
return valid_objs
def _check_load_keypoints(self, coco, entry):
"""Check and load ground-truth keypoints"""
ann_ids = coco.getAnnIds(imgIds=entry['id'], iscrowd=False)
objs = coco.loadAnns(ann_ids)
# check valid bboxes
valid_objs = []
width = entry['width']
height = entry['height']
for obj in objs:
#obj['keypoints'].extend([0,0,0, 0,0,0, 0,0,0])
obj['keypoints'].extend(obj['foot_kpts'])
obj['keypoints'].extend(obj['face_kpts'])
obj['keypoints'].extend(obj['lefthand_kpts'])
obj['keypoints'].extend(obj['righthand_kpts'])
#'face_box', 'lefthand_box', 'righthand_box'
contiguous_cid = self.json_id_to_contiguous[obj['category_id']]
if contiguous_cid >= self.num_class:
# not class of interest
continue
if max(obj['keypoints']) == 0:
continue
# convert from (x, y, w, h) to (xmin, ymin, xmax, ymax) and clip bound
xmin, ymin, xmax, ymax = bbox_clip_xyxy(bbox_xywh_to_xyxy(obj['bbox']), width, height)
xmin_f, ymin_f, xmax_f, ymax_f = bbox_clip_xyxy(bbox_xywh_to_xyxy(obj['face_box']), width, height)
xmin_lh, ymin_lh, xmax_lh, ymax_lh = bbox_clip_xyxy(bbox_xywh_to_xyxy(obj['lefthand_box']), width, height)
xmin_rh, ymin_rh, xmax_rh, ymax_rh = bbox_clip_xyxy(bbox_xywh_to_xyxy(obj['righthand_box']), width, height)
# require non-zero box area
#if obj['area'] <= 0 or xmax <= xmin or ymax <= ymin:
if (xmax-xmin)*(ymax-ymin) <= 0 or xmax <= xmin or ymax <= ymin:
continue
# if (xmax_f-xmin_f)*(ymax_f-ymin_f) <= 0 or xmax_f <= xmin_f or ymax_f <= ymin_f:
# continue
# if (xmax_lh-xmin_lh)*(ymax_lh-ymin_lh) <= 0 or xmax_lh <= xmin_lh or ymax_lh <= ymin_lh:
# continue
# if (xmax_rh-xmin_rh)*(ymax_rh-ymin_rh) <= 0 or xmax_rh <= xmin_rh or ymax_rh <= ymin_rh:
# continue
if 'num_keypoints' in obj and obj['num_keypoints'] == 0:
continue
# joints 3d: (num_joints, 3, 2); 3 is for x, y, z; 2 is for position, visibility
joints_3d = np.zeros((self.num_joints, 3, 2), dtype=np.float32)
for i in range(self.num_joints):
joints_3d[i, 0, 0] = obj['keypoints'][i * 3 + 0]
joints_3d[i, 1, 0] = obj['keypoints'][i * 3 + 1]
# joints_3d[i, 2, 0] = 0
if obj['keypoints'][i * 3 + 2] >= 0.35:
visible = 1
else:
visible = 0
#visible = min(1, visible)
joints_3d[i, :2, 1] = visible
# joints_3d[i, 2, 1] = 0
if np.sum(joints_3d[:, 0, 1]) < 1:
# no visible keypoint
continue
if self._check_centers and self._train:
bbox_center, bbox_area = self._get_box_center_area((xmin, ymin, xmax, ymax))
kp_center, num_vis = self._get_keypoints_center_count(joints_3d)
ks = np.exp(-2 * np.sum(np.square(bbox_center - kp_center)) / bbox_area)
if (num_vis / 80.0 + 47 / 80.0) > ks:
continue
valid_objs.append({
'bbox': (xmin, ymin, xmax, ymax),
'face_bbox': (xmin_f, ymin_f, xmax_f, ymax_f),
'lefthand_bbox': (xmin_lh, ymin_lh, xmax_lh, ymax_lh),
'righthand_bbox': (xmin_rh, ymin_rh, xmax_rh, ymax_rh),
'width': width,
'height': height,
'joints_3d': joints_3d
})
if not valid_objs:
if not self._skip_empty:
# dummy invalid labels if no valid objects are found
valid_objs.append({
'bbox': np.array([-1, -1, 0, 0]),
'face_bbox': np.array([-1, -1, 0, 0]),
'lefthand_bbox': np.array([-1, -1, 0, 0]),
'righthand_bbox': np.array([-1, -1, 0, 0]),
'width': width,
'height': height,
'joints_3d': np.zeros((self.num_joints, 2, 2), dtype=np.float32)
})
return valid_objs
def _check_load_keypoints(self, coco, entry):
"""Check and load ground-truth keypoints"""
ann_ids = coco.getAnnIds(imgIds=entry['id'], iscrowd=False)
objs = coco.loadAnns(ann_ids)
# check valid bboxes
valid_objs = []
width = entry['width']
height = entry['height']
for obj in objs:
obj['keypoints'] = obj['keypoints'][-42 * 3:]
contiguous_cid = self.json_id_to_contiguous[obj['category_id']]
if contiguous_cid >= self.num_class:
# not class of interest
continue
if max(obj['keypoints']) == 0:
continue
# convert from (x, y, w, h) to (xmin, ymin, xmax, ymax) and clip bound
if 'bbox' not in obj:
obj['bbox'] = [1, 1, width - 1, height - 1]
xmin, ymin, xmax, ymax = bbox_clip_xyxy(
bbox_xywh_to_xyxy(obj['bbox']), width, height)
# require non-zero box area
if (xmax - xmin) * (ymax -
ymin) <= 0 or xmax <= xmin or ymax <= ymin:
continue
if 'num_keypoints' in obj and obj['num_keypoints'] == 0:
continue
# joints 3d: (num_joints, 3, 2); 3 is for x, y, z; 2 is for position, visibility
joints_3d = np.zeros((self.num_joints * 2, 3, 2), dtype=np.float32)
for i in range(self.num_joints * 2):
joints_3d[i, 0, 0] = obj['keypoints'][i * 3 + 0]
joints_3d[i, 1, 0] = obj['keypoints'][i * 3 + 1]
if obj['keypoints'][i * 3 + 2] >= 0.35 and obj['keypoints'][
i * 3 + 0] > 0 and obj['keypoints'][i * 3 + 1] > 0:
visible = 1
else:
visible = 0
joints_3d[i, :2, 1] = visible
if np.sum(joints_3d[:, 0, 1]) < 1:
# no visible keypoint
continue
# left hand
if np.sum(joints_3d[:21, 0, 1]) >= 10:
xmin = np.min(joints_3d[:21, 0, 0][joints_3d[:21, 0, 0] > 0])
ymin = np.min(joints_3d[:21, 1, 0][joints_3d[:21, 1, 0] > 0])
xmax = np.max(joints_3d[:21, 0, 0][joints_3d[:21, 0, 0] > 0])
ymax = np.max(joints_3d[:21, 1, 0][joints_3d[:21, 1, 0] > 0])
w = xmax - xmin
h = ymax - ymin
xmin = max(xmin - np.random.rand() * w / 2, 1)
xmax = min(xmax + np.random.rand() * w / 2, width)
ymin = max(ymin - np.random.rand() * h / 2, 1)
ymax = min(ymax + np.random.rand() * h / 2, height)
obj['bbox'] = [xmin, ymin, xmax - xmin, ymax - ymin]
if self._check_centers and self._train:
bbox_center, bbox_area = self._get_box_center_area(
(xmin, ymin, xmax, ymax))
kp_center, num_vis = self._get_keypoints_center_count(
joints_3d[:21, :, :])
ks = np.exp(-2 *
np.sum(np.square(bbox_center - kp_center)) /
bbox_area)
if (num_vis / 80.0 + 47 / 80.0) > ks:
continue
valid_objs.append({
'bbox': (xmin, ymin, xmax, ymax),
'width': width,
'height': height,
'joints_3d': joints_3d[:21, :, :]
})
# right hand
if np.sum(joints_3d[21:, 0, 1]) >= 10:
xmin = np.min(joints_3d[21:, 0, 0][joints_3d[21:, 0, 0] > 0])
ymin = np.min(joints_3d[21:, 1, 0][joints_3d[21:, 1, 0] > 0])
xmax = np.max(joints_3d[21:, 0, 0][joints_3d[21:, 0, 0] > 0])
ymax = np.max(joints_3d[21:, 1, 0][joints_3d[21:, 1, 0] > 0])
w = xmax - xmin
h = ymax - ymin
xmin = max(xmin - np.random.rand() * w / 2, 1)
xmax = min(xmax + np.random.rand() * w / 2, width)
ymin = max(ymin - np.random.rand() * h / 2, 1)
ymax = min(ymax + np.random.rand() * h / 2, height)
obj['bbox'] = [xmin, ymin, xmax - xmin, ymax - ymin]
if self._check_centers and self._train:
bbox_center, bbox_area = self._get_box_center_area(
(xmin, ymin, xmax, ymax))
kp_center, num_vis = self._get_keypoints_center_count(
joints_3d[21:, :, :])
ks = np.exp(-2 *
np.sum(np.square(bbox_center - kp_center)) /
bbox_area)
if (num_vis / 80.0 + 47 / 80.0) > ks:
continue
valid_objs.append({
'bbox': (xmin, ymin, xmax, ymax),
'width': width,
'height': height,
'joints_3d': joints_3d[21:, :, :]
})
if not valid_objs:
if not self._skip_empty:
# dummy invalid labels if no valid objects are found
valid_objs.append({
'bbox':
np.array([-1, -1, 0, 0]),
'width':
width,
'height':
height,
'joints_3d':
np.zeros((self.num_joints, 2, 2), dtype=np.float32)
})
return valid_objs
def _check_load_keypoints(self, coco, entry, img_path):
"""Check and load ground-truth keypoints"""
ann_ids = coco.getAnnIds(imgIds=entry['id'], iscrowd=False)
objs = coco.loadAnns(ann_ids)
# check valid bboxes
valid_objs = []
width = entry['width']
height = entry['height']
if width == 0 or height == 0:
image = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
height, width = image.shape[0], image.shape[1]
for obj in objs:
contiguous_cid = self.json_id_to_contiguous[obj['category_id']]
if contiguous_cid >= self.num_class:
# not class of interest
print("not class of interest: %d" % contiguous_cid)
continue
if max(obj['keypoints']) == 0:
print("no visible keypoints")
continue
# convert from (x, y, w, h) to (xmin, ymin, xmax, ymax) and clip bound
xmin, ymin, xmax, ymax = bbox_clip_xyxy(
bbox_xywh_to_xyxy(obj['bbox']), width, height)
# require non-zero box area
if obj['area'] <= 0 or xmax <= xmin or ymax <= ymin:
print(
"bbox area: Area - %d; Xmax - %d, Xmin - %d, Ymax - %d, Ymin - %d"
% (obj['area'], xmax, xmin, ymax, ymin))
continue
if obj['num_keypoints'] == 0:
print("no keypoints")
continue
# joints 3d: (num_joints, 3, 2); 3 is for x, y, z; 2 is for position, visibility
joints_3d = np.zeros((self.num_joints, 3, 2), dtype=np.float32)
for i in range(self.num_joints):
joints_3d[i, 0, 0] = obj['keypoints'][i * 3 + 0]
joints_3d[i, 1, 0] = obj['keypoints'][i * 3 + 1]
# joints_3d[i, 2, 0] = 0
visible = min(1, obj['keypoints'][i * 3 + 2])
joints_3d[i, :2, 1] = visible
# joints_3d[i, 2, 1] = 0
if np.sum(joints_3d[:, 0, 1]) < 1:
# no visible keypoint
print("no visible keypoints - summed validation")
continue
if self._check_centers and self._train:
bbox_center, bbox_area = self._get_box_center_area(
(xmin, ymin, xmax, ymax))
kp_center, num_vis = self._get_keypoints_center_count(
joints_3d)
ks = np.exp(-2 * np.sum(np.square(bbox_center - kp_center)) /
bbox_area)
if (num_vis / 80.0 + 47 / 80.0) > ks:
print("check center validation failed")
continue
valid_objs.append({
'file_name': entry['file_name'],
'bbox': (xmin, ymin, xmax, ymax),
'width': width,
'height': height,
'joints_3d': joints_3d
})
if not valid_objs:
if not self._skip_empty:
# dummy invalid labels if no valid objects are found
valid_objs.append({
'file_name':
entry['file_name'],
'bbox':
np.array([-1, -1, 0, 0]),
'width':
width,
'height':
height,
'joints_3d':
np.zeros((self.num_joints, 2, 2), dtype=np.float32)
})
return valid_objs
def _load_jsons(self):
"""Load all image paths and labels from JSON annotation files into buffer."""
cameras_path = self._root + '/cameras.h5'
cameras_dict = load_cameras(cameras_path)
camera_info = set([c[1] for c in cameras_dict.keys()])
if self._img_prefix == 'hm36train':
categories = ['s_01', 's_05', 's_06', 's_07', 's_08']
# categories = ['s_01']
else:
categories = ['s_09']
actions = []
for i in range(2, 16):
stri = str(i)
action = 'act_' + stri.zfill(2)
actions.append(action)
subact = ['subact_01', 'subact_02']
cameras = ['ca_01', 'ca_02', 'ca_03', 'ca_04']
items = []
labels = []
for cat in categories:
s = int(cat.split('_')[-1])
print('loading category ' + cat)
for act in actions:
for sub in subact:
for ca in cameras:
ci = int(ca[-1])
fname = cat + '_' + act + '_' + sub + '_' + ca
annopath = os.path.join(self._ann_file, fname,
'annotation.json')
if not os.path.exists(annopath):
continue
with open(annopath, "r") as f:
data = json.load(f)
anno = data['annotations']
R, T, f, c, k, p, name = cameras_dict[(s, ci)]
for item in anno:
file_name = os.path.join(self._root, 'images',
item['file_name'])
items.append(file_name)
bbox = item['bbox']
w_coord_xyz = np.array(item['joint_3d'])
width, height = item['width'], item['height']
xmin, ymin, xmax, ymax = bbox_clip_xyxy(
bbox_xywh_to_xyxy(bbox), width, height)
i_coord_xy, D, radial, tan, r2 = project_point_radial(
w_coord_xyz, R, T, f, c, k, p)
i_coord_xy = from_hm36_to_coco_single(
i_coord_xy, self.num_joints)
joints_3d = np.zeros((self.num_joints, 3, 2),
dtype=np.float32)
for i in range(self.num_joints):
joints_3d[i, 0, 0] = i_coord_xy[i][0]
joints_3d[i, 1, 0] = i_coord_xy[i][1]
joints_3d[i, :2, 1] = min(self.visibles[i], 1)
objdict = {
'bbox': (xmin, ymin, xmax, ymax),
'width': width,
'height': height,
'joints_3d': joints_3d
}
labels.append(objdict)
return items, labels
