def _check_load_keypoints(self, _mpii, entry): """Check and load ground-truth keypoints""" ann_ids = _mpii.getAnnIds(imgIds=entry['id'], iscrowd=False) objs = _mpii.loadAnns(ann_ids) # check valid bboxes valid_objs = [] width = entry['width'] height = entry['height'] for obj in objs: if max(obj['keypoints']) == 0: continue # convert from (x, y, w, h) to (xmin, ymin, xmax, ymax) and clip bound xmin, ymin, xmax, ymax = bbox_clip_xyxy(bbox_xywh_to_xyxy(obj['bbox']), width, height) # require non-zero box area if xmax <= xmin or ymax <= ymin: continue if obj['num_keypoints'] == 0: continue # joints 3d: (num_joints, 3, 2); 3 is for x, y, z; 2 is for position, visibility joints_3d = np.zeros((self.num_joints, 3, 2), dtype=np.float32) for i in range(self.num_joints): joints_3d[i, 0, 0] = obj['keypoints'][i * 3 + 0] joints_3d[i, 1, 0] = obj['keypoints'][i * 3 + 1] # joints_3d[i, 2, 0] = 0 visible = min(1, obj['keypoints'][i * 3 + 2]) joints_3d[i, :2, 1] = visible # joints_3d[i, 2, 1] = 0 if np.sum(joints_3d[:, 0, 1]) < 1: # no visible keypoint continue if self._check_centers and self._train: bbox_center, bbox_area = self._get_box_center_area((xmin, ymin, xmax, ymax)) kp_center, num_vis = self._get_keypoints_center_count(joints_3d) ks = np.exp(-2 * np.sum(np.square(bbox_center - kp_center)) / bbox_area) if (num_vis / 80.0 + 47 / 80.0) > ks: continue valid_objs.append({ 'bbox': (xmin, ymin, xmax, ymax), 'width': width, 'height': height, 'joints_3d': joints_3d }) if not valid_objs: if not self._skip_empty: # dummy invalid labels if no valid objects are found valid_objs.append({ 'bbox': np.array([-1, -1, 0, 0]), 'width': width, 'height': height, 'joints_3d': np.zeros((self.num_joints, 2, 2), dtype=np.float32) }) return valid_objs
def _check_load_keypoints(self, coco, entry): """Check and load ground-truth keypoints""" ann_ids = coco.getAnnIds(imgIds=entry['id'], iscrowd=False) objs = coco.loadAnns(ann_ids) # check valid bboxes valid_objs = [] width = entry['width'] height = entry['height'] for obj in objs: #obj['keypoints'].extend([0,0,0, 0,0,0, 0,0,0]) obj['keypoints'].extend(obj['foot_kpts']) obj['keypoints'].extend(obj['face_kpts']) obj['keypoints'].extend(obj['lefthand_kpts']) obj['keypoints'].extend(obj['righthand_kpts']) #'face_box', 'lefthand_box', 'righthand_box' contiguous_cid = self.json_id_to_contiguous[obj['category_id']] if contiguous_cid >= self.num_class: # not class of interest continue if max(obj['keypoints']) == 0: continue # convert from (x, y, w, h) to (xmin, ymin, xmax, ymax) and clip bound xmin, ymin, xmax, ymax = bbox_clip_xyxy(bbox_xywh_to_xyxy(obj['bbox']), width, height) xmin_f, ymin_f, xmax_f, ymax_f = bbox_clip_xyxy(bbox_xywh_to_xyxy(obj['face_box']), width, height) xmin_lh, ymin_lh, xmax_lh, ymax_lh = bbox_clip_xyxy(bbox_xywh_to_xyxy(obj['lefthand_box']), width, height) xmin_rh, ymin_rh, xmax_rh, ymax_rh = bbox_clip_xyxy(bbox_xywh_to_xyxy(obj['righthand_box']), width, height) # require non-zero box area #if obj['area'] <= 0 or xmax <= xmin or ymax <= ymin: if (xmax-xmin)*(ymax-ymin) <= 0 or xmax <= xmin or ymax <= ymin: continue # if (xmax_f-xmin_f)*(ymax_f-ymin_f) <= 0 or xmax_f <= xmin_f or ymax_f <= ymin_f: # continue # if (xmax_lh-xmin_lh)*(ymax_lh-ymin_lh) <= 0 or xmax_lh <= xmin_lh or ymax_lh <= ymin_lh: # continue # if (xmax_rh-xmin_rh)*(ymax_rh-ymin_rh) <= 0 or xmax_rh <= xmin_rh or ymax_rh <= ymin_rh: # continue if 'num_keypoints' in obj and obj['num_keypoints'] == 0: continue # joints 3d: (num_joints, 3, 2); 3 is for x, y, z; 2 is for position, visibility joints_3d = np.zeros((self.num_joints, 3, 2), dtype=np.float32) for i in range(self.num_joints): joints_3d[i, 0, 0] = obj['keypoints'][i * 3 + 0] joints_3d[i, 1, 0] = obj['keypoints'][i * 3 + 1] # joints_3d[i, 2, 0] = 0 if obj['keypoints'][i * 3 + 2] >= 0.35: visible = 1 else: visible = 0 #visible = min(1, visible) joints_3d[i, :2, 1] = visible # joints_3d[i, 2, 1] = 0 if np.sum(joints_3d[:, 0, 1]) < 1: # no visible keypoint continue if self._check_centers and self._train: bbox_center, bbox_area = self._get_box_center_area((xmin, ymin, xmax, ymax)) kp_center, num_vis = self._get_keypoints_center_count(joints_3d) ks = np.exp(-2 * np.sum(np.square(bbox_center - kp_center)) / bbox_area) if (num_vis / 80.0 + 47 / 80.0) > ks: continue valid_objs.append({ 'bbox': (xmin, ymin, xmax, ymax), 'face_bbox': (xmin_f, ymin_f, xmax_f, ymax_f), 'lefthand_bbox': (xmin_lh, ymin_lh, xmax_lh, ymax_lh), 'righthand_bbox': (xmin_rh, ymin_rh, xmax_rh, ymax_rh), 'width': width, 'height': height, 'joints_3d': joints_3d }) if not valid_objs: if not self._skip_empty: # dummy invalid labels if no valid objects are found valid_objs.append({ 'bbox': np.array([-1, -1, 0, 0]), 'face_bbox': np.array([-1, -1, 0, 0]), 'lefthand_bbox': np.array([-1, -1, 0, 0]), 'righthand_bbox': np.array([-1, -1, 0, 0]), 'width': width, 'height': height, 'joints_3d': np.zeros((self.num_joints, 2, 2), dtype=np.float32) }) return valid_objs
def _check_load_keypoints(self, coco, entry): """Check and load ground-truth keypoints""" ann_ids = coco.getAnnIds(imgIds=entry['id'], iscrowd=False) objs = coco.loadAnns(ann_ids) # check valid bboxes valid_objs = [] width = entry['width'] height = entry['height'] for obj in objs: obj['keypoints'] = obj['keypoints'][-42 * 3:] contiguous_cid = self.json_id_to_contiguous[obj['category_id']] if contiguous_cid >= self.num_class: # not class of interest continue if max(obj['keypoints']) == 0: continue # convert from (x, y, w, h) to (xmin, ymin, xmax, ymax) and clip bound if 'bbox' not in obj: obj['bbox'] = [1, 1, width - 1, height - 1] xmin, ymin, xmax, ymax = bbox_clip_xyxy( bbox_xywh_to_xyxy(obj['bbox']), width, height) # require non-zero box area if (xmax - xmin) * (ymax - ymin) <= 0 or xmax <= xmin or ymax <= ymin: continue if 'num_keypoints' in obj and obj['num_keypoints'] == 0: continue # joints 3d: (num_joints, 3, 2); 3 is for x, y, z; 2 is for position, visibility joints_3d = np.zeros((self.num_joints * 2, 3, 2), dtype=np.float32) for i in range(self.num_joints * 2): joints_3d[i, 0, 0] = obj['keypoints'][i * 3 + 0] joints_3d[i, 1, 0] = obj['keypoints'][i * 3 + 1] if obj['keypoints'][i * 3 + 2] >= 0.35 and obj['keypoints'][ i * 3 + 0] > 0 and obj['keypoints'][i * 3 + 1] > 0: visible = 1 else: visible = 0 joints_3d[i, :2, 1] = visible if np.sum(joints_3d[:, 0, 1]) < 1: # no visible keypoint continue # left hand if np.sum(joints_3d[:21, 0, 1]) >= 10: xmin = np.min(joints_3d[:21, 0, 0][joints_3d[:21, 0, 0] > 0]) ymin = np.min(joints_3d[:21, 1, 0][joints_3d[:21, 1, 0] > 0]) xmax = np.max(joints_3d[:21, 0, 0][joints_3d[:21, 0, 0] > 0]) ymax = np.max(joints_3d[:21, 1, 0][joints_3d[:21, 1, 0] > 0]) w = xmax - xmin h = ymax - ymin xmin = max(xmin - np.random.rand() * w / 2, 1) xmax = min(xmax + np.random.rand() * w / 2, width) ymin = max(ymin - np.random.rand() * h / 2, 1) ymax = min(ymax + np.random.rand() * h / 2, height) obj['bbox'] = [xmin, ymin, xmax - xmin, ymax - ymin] if self._check_centers and self._train: bbox_center, bbox_area = self._get_box_center_area( (xmin, ymin, xmax, ymax)) kp_center, num_vis = self._get_keypoints_center_count( joints_3d[:21, :, :]) ks = np.exp(-2 * np.sum(np.square(bbox_center - kp_center)) / bbox_area) if (num_vis / 80.0 + 47 / 80.0) > ks: continue valid_objs.append({ 'bbox': (xmin, ymin, xmax, ymax), 'width': width, 'height': height, 'joints_3d': joints_3d[:21, :, :] }) # right hand if np.sum(joints_3d[21:, 0, 1]) >= 10: xmin = np.min(joints_3d[21:, 0, 0][joints_3d[21:, 0, 0] > 0]) ymin = np.min(joints_3d[21:, 1, 0][joints_3d[21:, 1, 0] > 0]) xmax = np.max(joints_3d[21:, 0, 0][joints_3d[21:, 0, 0] > 0]) ymax = np.max(joints_3d[21:, 1, 0][joints_3d[21:, 1, 0] > 0]) w = xmax - xmin h = ymax - ymin xmin = max(xmin - np.random.rand() * w / 2, 1) xmax = min(xmax + np.random.rand() * w / 2, width) ymin = max(ymin - np.random.rand() * h / 2, 1) ymax = min(ymax + np.random.rand() * h / 2, height) obj['bbox'] = [xmin, ymin, xmax - xmin, ymax - ymin] if self._check_centers and self._train: bbox_center, bbox_area = self._get_box_center_area( (xmin, ymin, xmax, ymax)) kp_center, num_vis = self._get_keypoints_center_count( joints_3d[21:, :, :]) ks = np.exp(-2 * np.sum(np.square(bbox_center - kp_center)) / bbox_area) if (num_vis / 80.0 + 47 / 80.0) > ks: continue valid_objs.append({ 'bbox': (xmin, ymin, xmax, ymax), 'width': width, 'height': height, 'joints_3d': joints_3d[21:, :, :] }) if not valid_objs: if not self._skip_empty: # dummy invalid labels if no valid objects are found valid_objs.append({ 'bbox': np.array([-1, -1, 0, 0]), 'width': width, 'height': height, 'joints_3d': np.zeros((self.num_joints, 2, 2), dtype=np.float32) }) return valid_objs
def _check_load_keypoints(self, coco, entry, img_path): """Check and load ground-truth keypoints""" ann_ids = coco.getAnnIds(imgIds=entry['id'], iscrowd=False) objs = coco.loadAnns(ann_ids) # check valid bboxes valid_objs = [] width = entry['width'] height = entry['height'] if width == 0 or height == 0: image = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB) height, width = image.shape[0], image.shape[1] for obj in objs: contiguous_cid = self.json_id_to_contiguous[obj['category_id']] if contiguous_cid >= self.num_class: # not class of interest print("not class of interest: %d" % contiguous_cid) continue if max(obj['keypoints']) == 0: print("no visible keypoints") continue # convert from (x, y, w, h) to (xmin, ymin, xmax, ymax) and clip bound xmin, ymin, xmax, ymax = bbox_clip_xyxy( bbox_xywh_to_xyxy(obj['bbox']), width, height) # require non-zero box area if obj['area'] <= 0 or xmax <= xmin or ymax <= ymin: print( "bbox area: Area - %d; Xmax - %d, Xmin - %d, Ymax - %d, Ymin - %d" % (obj['area'], xmax, xmin, ymax, ymin)) continue if obj['num_keypoints'] == 0: print("no keypoints") continue # joints 3d: (num_joints, 3, 2); 3 is for x, y, z; 2 is for position, visibility joints_3d = np.zeros((self.num_joints, 3, 2), dtype=np.float32) for i in range(self.num_joints): joints_3d[i, 0, 0] = obj['keypoints'][i * 3 + 0] joints_3d[i, 1, 0] = obj['keypoints'][i * 3 + 1] # joints_3d[i, 2, 0] = 0 visible = min(1, obj['keypoints'][i * 3 + 2]) joints_3d[i, :2, 1] = visible # joints_3d[i, 2, 1] = 0 if np.sum(joints_3d[:, 0, 1]) < 1: # no visible keypoint print("no visible keypoints - summed validation") continue if self._check_centers and self._train: bbox_center, bbox_area = self._get_box_center_area( (xmin, ymin, xmax, ymax)) kp_center, num_vis = self._get_keypoints_center_count( joints_3d) ks = np.exp(-2 * np.sum(np.square(bbox_center - kp_center)) / bbox_area) if (num_vis / 80.0 + 47 / 80.0) > ks: print("check center validation failed") continue valid_objs.append({ 'file_name': entry['file_name'], 'bbox': (xmin, ymin, xmax, ymax), 'width': width, 'height': height, 'joints_3d': joints_3d }) if not valid_objs: if not self._skip_empty: # dummy invalid labels if no valid objects are found valid_objs.append({ 'file_name': entry['file_name'], 'bbox': np.array([-1, -1, 0, 0]), 'width': width, 'height': height, 'joints_3d': np.zeros((self.num_joints, 2, 2), dtype=np.float32) }) return valid_objs
def _load_jsons(self): """Load all image paths and labels from JSON annotation files into buffer.""" cameras_path = self._root + '/cameras.h5' cameras_dict = load_cameras(cameras_path) camera_info = set([c[1] for c in cameras_dict.keys()]) if self._img_prefix == 'hm36train': categories = ['s_01', 's_05', 's_06', 's_07', 's_08'] # categories = ['s_01'] else: categories = ['s_09'] actions = [] for i in range(2, 16): stri = str(i) action = 'act_' + stri.zfill(2) actions.append(action) subact = ['subact_01', 'subact_02'] cameras = ['ca_01', 'ca_02', 'ca_03', 'ca_04'] items = [] labels = [] for cat in categories: s = int(cat.split('_')[-1]) print('loading category ' + cat) for act in actions: for sub in subact: for ca in cameras: ci = int(ca[-1]) fname = cat + '_' + act + '_' + sub + '_' + ca annopath = os.path.join(self._ann_file, fname, 'annotation.json') if not os.path.exists(annopath): continue with open(annopath, "r") as f: data = json.load(f) anno = data['annotations'] R, T, f, c, k, p, name = cameras_dict[(s, ci)] for item in anno: file_name = os.path.join(self._root, 'images', item['file_name']) items.append(file_name) bbox = item['bbox'] w_coord_xyz = np.array(item['joint_3d']) width, height = item['width'], item['height'] xmin, ymin, xmax, ymax = bbox_clip_xyxy( bbox_xywh_to_xyxy(bbox), width, height) i_coord_xy, D, radial, tan, r2 = project_point_radial( w_coord_xyz, R, T, f, c, k, p) i_coord_xy = from_hm36_to_coco_single( i_coord_xy, self.num_joints) joints_3d = np.zeros((self.num_joints, 3, 2), dtype=np.float32) for i in range(self.num_joints): joints_3d[i, 0, 0] = i_coord_xy[i][0] joints_3d[i, 1, 0] = i_coord_xy[i][1] joints_3d[i, :2, 1] = min(self.visibles[i], 1) objdict = { 'bbox': (xmin, ymin, xmax, ymax), 'width': width, 'height': height, 'joints_3d': joints_3d } labels.append(objdict) return items, labels