def load_rpsm_testdata(testdata): with open(testdata, 'rb') as f: db = pickle.load(f) heatmaps = [] cameras = [] boxes = [] poses = [] for i in range(4): heatmap = db[i]['heatmap'] # [k, h, w] heatmaps.append(heatmap) camera = db[i]['cam_params'] cameras.append(camera) pose_camera = db[i]['joints_3d_cam'] pose_world = camera_to_world_frame(pose_camera, camera['R'], camera['T']) poses.append(pose_world) box = {} box['scale'] = np.array(db[i]['scale']) box['center'] = np.array(db[i]['center']) boxes.append(box) hms = np.array(heatmaps) grid_center = poses[0][0] body = HumanBody() limb_length = compute_limb_length(body, poses[0]) return cameras, hms, boxes, grid_center, limb_length, poses[0]
def __getitem__(self, idx): items = self.grouping[idx] heatmaps = [] boxes = [] poses = [] cameras = [] heatmap_start_idx = self.nviews * idx for itm_offset, itm in enumerate(items): datum = self.annot_db[itm] camera = datum['camera'] cameras.append(camera) joints_gt = cam_utils.camera_to_world_frame( datum['joints_3d'], datum['camera']['R'], datum['camera']['T']) datum['joints_gt'] = joints_gt # poses.append(datum['joints_gt']) poses.append(joints_gt) box = dict() box['scale'] = np.array(datum['scale']) box['center'] = np.array(datum['center']) boxes.append(box) heatmaps.append(self.heatmaps[heatmap_start_idx + itm_offset]) heatmaps = np.array(heatmaps) poses = poses[0] # 4 poses are identical # child - parent, parent is more close to root bone_vectors = dict() imu_edges_reverse = self.body.imu_edges_reverse for boneid in imu_edges_reverse: parent = imu_edges_reverse[boneid][0] child = imu_edges_reverse[boneid][1] bonevec = poses[child] - poses[parent] bone_vectors[boneid] = bonevec.astype(np.float32) # # use HumanBody.imubone as bone_vectors dict key # imubone_mapping = {'Head': 8, 'Pelvis': 2, 'L_UpArm': 11, 'R_UpArm': 13, 'L_LowArm': 12, 'R_LowArm': 14, # 'L_UpLeg': 5, 'R_UpLeg': 3, 'L_LowLeg': 6, 'R_LowLeg': 4} # todo read from body # bone_vectors = dict() # for bone_name in imubone_mapping: # bone_vectors[imubone_mapping[bone_name]] = bone_vec_tc[bone_name] limb_length = self.compute_limb_length(self.body, poses) # todo transform some vars into tensor # bone_vectors = torch.as_tensor(bone_vectors, dtype=torch.float32) return { 'heatmaps': heatmaps, 'datum': datum, 'boxes': boxes, 'poses': poses, 'cameras': cameras, 'limb_length': limb_length, 'bone_vectors': bone_vectors }
def main(): args = parse_args() logger, final_output_dir, tb_log_dir = create_logger( config, args.cfg, 'test3d') prediction_path = os.path.join(final_output_dir, config.TEST.HEATMAP_LOCATION_FILE) test_dataset = eval('dataset.' + config.DATASET.TEST_DATASET)( config, config.DATASET.TEST_SUBSET, False) all_heatmaps = h5py.File(prediction_path)['heatmaps'] pairwise_file = config.PICT_STRUCT.PAIRWISE_FILE with open(pairwise_file, 'rb') as f: pairwise = pickle.load(f)['pairwise_constrain'] cnt = 0 grouping = test_dataset.grouping mpjpes = [] for items in grouping: heatmaps = [] boxes = [] poses = [] cameras = [] for idx in items: datum = test_dataset.db[idx] camera = datum['camera'] cameras.append(camera) poses.append( camera_to_world_frame(datum['joints_3d'], camera['R'], camera['T'])) box = {} box['scale'] = np.array(datum['scale']) box['center'] = np.array(datum['center']) boxes.append(box) heatmaps.append(all_heatmaps[cnt]) cnt += 1 heatmaps = np.array(heatmaps) # This demo uses GT root locations and limb length; but can be replaced by statistics grid_center = poses[0][0] body = HumanBody() limb_length = compute_limb_length(body, poses[0]) prediction = rpsm(cameras, heatmaps, boxes, grid_center, limb_length, pairwise, config) mpjpe = np.mean(np.sqrt(np.sum((prediction - poses[0])**2, axis=1))) mpjpes.append(mpjpe) print(np.mean(mpjpes))
def load_rpsm_testdata_all(testdata): with open(testdata, 'rb') as f: db = pickle.load(f) assert len(db) % 4 == 0 all_heatmaps = [] all_cameras = [] all_boxes = [] all_grid_centers = [] all_limb_lengths = [] all_gts = [] body = HumanBody() for idx in range(0,len(db),4): group_hms = [] group_cameras = [] group_boxes = [] group_gts = [] for inner_idx in range(idx, idx+4): heatmap = db[inner_idx]['heatmap'] # [16, h, w], body order group_hms.append(heatmap) camera = db[inner_idx]['cam_params'] group_cameras.append(camera) pose_camera = db[inner_idx]['joints_3d_cam'] # [16,3] body order pose_world = camera_to_world_frame(pose_camera, camera['R'], camera['T']) group_gts.append(pose_world) box = {} box['scale'] = np.array(db[inner_idx]['scale']) box['center'] = np.array(db[inner_idx]['center']) group_boxes.append(box) group_hms = np.array(group_hms) all_heatmaps.append(group_hms) all_cameras.append(group_cameras) all_boxes.append(group_boxes) all_grid_centers.append(group_gts[0][body.root_idx]) all_limb_lengths.append(compute_limb_length(body, group_gts[0])) all_gts.append(group_gts[0]) return all_cameras, all_heatmaps, all_boxes, all_grid_centers, all_limb_lengths, all_gts
def evaluate_3d(self, preds3d, thresholds=None): if thresholds is None: thresholds = [ 5., 10., 15., 20., 25., 50., 75., 100., 125., 150., ] gt3d = [] for idx, items in enumerate(self.grouping): # note that h36m joints_3d is in camera frame db_rec = self.db[items[0]] j3d_global = cam_utils.camera_to_world_frame( db_rec['joints_3d'], db_rec['camera']['R'], db_rec['camera']['T']) gt3d.append(j3d_global) gt3d = np.array(gt3d) assert preds3d.shape == gt3d.shape, 'shape mismatch of preds and gt' distance = np.sum((preds3d - gt3d)**2, axis=2) num_groupings = len(gt3d) pcks = [] for thr in thresholds: detections = distance <= thr**2 detections_perjoint = np.sum(detections, axis=0) pck_perjoint = detections_perjoint / num_groupings # pck_avg = np.average(pck_perjoint, axis=0) pcks.append(pck_perjoint) return thresholds, pcks
def main(): device = torch.device('cuda:0') args = parse_args() logger, final_output_dir, tb_log_dir = create_logger( config, args.cfg, 'test3d') prediction_path = os.path.join(final_output_dir, config.TEST.HEATMAP_LOCATION_FILE) test_dataset = eval('dataset.' + config.DATASET.TEST_DATASET)( config, config.DATASET.TEST_SUBSET, False) all_heatmaps = h5py.File(prediction_path)['heatmaps'] pairwise_file = config.PICT_STRUCT.PAIRWISE_FILE with open(pairwise_file, 'rb') as f: pairwise = pickle.load(f)['pairwise_constrain'] for k, v in pairwise.items(): pairwise[k] = torch.as_tensor(v.todense().astype(np.float), device=device, dtype=torch.float) cnt = 0 grouping = test_dataset.grouping mpjpes = [] for items in grouping: heatmaps = [] boxes = [] poses = [] cameras = [] for idx in items: datum = test_dataset.db[idx] camera = datum['camera'] cameras.append(camera) poses.append( camera_to_world_frame(datum['joints_3d'], camera['R'], camera['T'])) box = {} box['scale'] = np.array(datum['scale']) box['center'] = np.array(datum['center']) boxes.append(box) heatmaps.append(all_heatmaps[cnt]) cnt += 1 heatmaps = np.array(heatmaps) grid_center = poses[0][0] body = HumanBody() limb_length = compute_limb_length(body, poses[0]) heatmaps = torch.as_tensor(heatmaps, device=device) kw = { 'body': body, 'boxes': boxes, 'center': grid_center, 'pairwise': pairwise, 'limb_length': limb_length } prediction = rpsm(cameras, heatmaps, config, kw) prediction = prediction.cpu().numpy() mpjpe = np.mean(np.sqrt(np.sum((prediction - poses[0])**2, axis=1))) mpjpes.append(mpjpe) print(mpjpe) print(np.mean(mpjpe))
def __getitem__(self, idx, source='h36m', **kwargs): db_rec = copy.deepcopy(self.db[idx]) image_dir = 'images.zip@' if self.data_format == 'zip' else '' image_file = osp.join(self.root, db_rec['source'], image_dir, 'images', db_rec['image']) if self.data_format == 'zip': from utils import zipreader data_numpy = zipreader.imread( image_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) else: data_numpy = cv2.imread( image_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) joints = db_rec['joints_2d'].copy() joints_vis = db_rec['joints_vis'].copy() center = np.array(db_rec['center']).copy() scale = np.array(db_rec['scale']).copy() rotation = 0 if self.is_train: sf = self.scale_factor rf = self.rotation_factor scale = scale * np.clip(np.random.randn() * sf + 1, 1 - sf, 1 + sf) rotation = np.clip(np.random.randn() * rf, -rf * 2, rf * 2) \ if random.random() <= 0.6 else 0 trans = get_affine_transform(center, scale, rotation, self.image_size) input = cv2.warpAffine( data_numpy, trans, (int(self.image_size[0]), int(self.image_size[1])), flags=cv2.INTER_LINEAR) if self.transform: input = self.transform(input) for i in range(self.num_joints): if joints_vis[i, 0] > 0.0: joints[i, 0:2] = affine_transform(joints[i, 0:2], trans) if (np.min(joints[i, :2]) < 0 or joints[i, 0] >= self.image_size[0] or joints[i, 1] >= self.image_size[1]): joints_vis[i, :] = 0 target, target_weight = self.generate_target(joints, joints_vis) target = torch.from_numpy(target) target_weight = torch.from_numpy(target_weight) meta = { 'scale': scale, 'center': center, 'rotation': rotation, 'joints_2d': db_rec['joints_2d'], 'joints_2d_transformed': joints, 'joints_vis': joints_vis, 'source': db_rec['source'], 'heatmap_size': self.heatmap_size } if source == 'totalcapture': imubone_mapping = kwargs['tc_imubone_map'] meta['joints_gt'] = db_rec['joints_gt'] meta['bone_vec'] = db_rec['bone_vec'] meta['camera'] = db_rec['camera'] bone_vec_tc = meta['bone_vec'] bone_vectors = dict() for bone_name in imubone_mapping: bone_vectors[ imubone_mapping[bone_name]] = bone_vec_tc[bone_name] meta['bone_vectors'] = bone_vectors # if self.totalcapture_template_meta is None: # self.totalcapture_template_meta = meta elif source == 'h36m': meta['camera'] = db_rec['camera'] meta['joints_gt'] = cam_utils.camera_to_world_frame( db_rec['joints_3d'], db_rec['camera']['R'], db_rec['camera']['T']) else: # since tc is mixed with mpii, they should have same keys in meta, # otherwise will lead to error when collate data in dataloader meta['joints_gt'] = self.totalcapture_template_meta['joints_gt'] # meta['joints_gt'] = np.zeros((16,3)) meta['bone_vec'] = self.totalcapture_template_meta['bone_vec'] meta['camera'] = self.totalcapture_template_meta['camera'] meta['bone_vectors'] = self.totalcapture_template_meta[ 'bone_vectors'] return input, target, target_weight, meta
def main(): args = parse_args() reset_config(config, args) no_distortion = True if args.no_distortion else False test_dataset = eval('dataset.' + config.DATASET.TEST_DATASET)( config, config.DATASET.TEST_SUBSET, False, pseudo_label_path='', no_distortion=no_distortion) grouping = test_dataset.grouping if args.heatmap == '': flag_test_gt = True else: flag_test_gt = False if not flag_test_gt: test_data_file = args.heatmap db = h5py.File(test_data_file, 'r') pred2d = np.array(db['locations'])[:, :, :2] # [8860, 16, 2] if flag_test_gt: pred2d = [] cameras = [] gt3d = [] for items in grouping: for item in items: cam = test_dataset.db[item]['camera'] cameras.append(cam) if flag_test_gt: pred2d.append(test_dataset.db[item]['joints_2d']) # [N, k_union, 2], already mapped gt = test_dataset.db[items[-1]]['joints_3d'] gt3d.append(camera_to_world_frame(gt, cameras[-1]['R'], cameras[-1]['T'])) if flag_test_gt: pred2d = np.array(pred2d) gt3d = np.array(gt3d) joints_vis = np.ones(pred2d.shape[:2]) joints_vis = ransac(camera_params=cameras, poses2d=pred2d, joints_vis=joints_vis, config=config) # [N, 16, 3] pred3d = triangulate_poses(cameras, pred2d, joints_vis=joints_vis, no_distortion=no_distortion) # [N, 16, 3] assert len(gt3d) == len(pred3d) u2a = test_dataset.u2a_mapping u2a = {k:v for k, v in u2a.items() if v != '*'} sorted_u2a = sorted(u2a.items(), key=lambda x: x[0]) u = np.array([mapping[0] for mapping in sorted_u2a]) a = np.array([mapping[1] for mapping in sorted_u2a]) if flag_test_gt: compatible_pred = pred3d[:, u, :] # [N, 16, 3] else: compatible_pred = pred3d # compatible_pred = pred3d[:, a, :] # for multivew mixed resutls compatible_gt = gt3d[:, a, :] # [N, 16, 3] norm = np.linalg.norm(compatible_pred - compatible_gt, axis=2) # [N, 16] print('Mean Error:', np.mean(norm)) print('Std Error:', np.std(norm)) print('Max Error:', np.amax(norm)) print('Larger than Mean+Std Error: {:.1%}'.format(np.sum(norm>np.mean(norm)+np.std(norm))/norm.size)) thre_list = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 500, 1000, 2000, 5000, 10000] print('| ' + ' | '.join(str(thre) for thre in thre_list) + ' |') print(''.join('| {:.1%} '.format(np.sum(norm<thre)/norm.size) for thre in thre_list) + '|')
def __getitem__(self, idx, source='h36m', **kwargs): db_rec = copy.deepcopy(self.db[idx]) image_dir = 'images.zip@' if self.data_format == 'zip' else '' image_file = osp.join(self.root, db_rec['source'], image_dir, 'images', db_rec['image']) if self.data_format == 'zip': from utils import zipreader data_numpy = zipreader.imread( image_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) else: data_numpy = cv2.imread( image_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) joints = db_rec['joints_2d'].copy() joints_vis = db_rec['joints_vis'].copy() center = np.array(db_rec['center']).copy() scale = np.array(db_rec['scale']).copy() rotation = 0 if self.is_train: sf = self.scale_factor rf = self.rotation_factor scale = scale * np.clip(np.random.randn() * sf + 1, 1 - sf, 1 + sf) rotation = np.clip(np.random.randn() * rf, -rf * 2, rf * 2) \ if random.random() <= 0.6 else 0 trans = get_affine_transform(center, scale, rotation, self.image_size) # ! Notice: this trans represents full image to cropped image, # not full image->heatmap input = cv2.warpAffine( data_numpy, trans, (int(self.image_size[0]), int(self.image_size[1])), flags=cv2.INTER_LINEAR) if self.transform: input = self.transform(input) for i in range(self.num_joints): if joints_vis[i, 0] > 0.0: joints[i, 0:2] = affine_transform(joints[i, 0:2], trans) if (np.min(joints[i, :2]) < 0 or joints[i, 0] >= self.image_size[0] or joints[i, 1] >= self.image_size[1]): joints_vis[i, :] = 0 target, target_weight = self.generate_target(joints, joints_vis) target = torch.from_numpy(target) target_weight = torch.from_numpy(target_weight) # 3x3 data augmentation affine trans (scale rotate) # !!! Notice: this transformation contains both heatmap->image scale affine # and data augmentation affine aug_trans = np.eye(3, 3) aug_trans[0:2] = trans # full img -> cropped img hm_scale = self.heatmap_size / self.image_size scale_trans = np.eye(3,3) # cropped img -> heatmap scale_trans[0,0] = hm_scale[1] scale_trans[1, 1] = hm_scale[0] aug_trans = scale_trans @ aug_trans meta = { 'scale': scale, 'center': center, 'rotation': rotation, 'joints_2d': db_rec['joints_2d'], 'joints_2d_transformed': joints, 'joints_vis': joints_vis, 'source': db_rec['source'], 'heatmap_size': self.heatmap_size, 'aug_trans': aug_trans, } if source == 'totalcapture': meta['joints_gt'] = db_rec['joints_gt'] meta['camera'] = db_rec['camera'] elif source in ['h36m']: meta['camera'] = db_rec['camera'] meta['joints_gt'] = cam_utils.camera_to_world_frame(db_rec['joints_3d'], db_rec['camera']['R'], db_rec['camera']['T']) elif source == 'panoptic': meta['camera'] = db_rec['camera'] meta['joints_gt'] = db_rec['joints_gt'] elif source in ['unrealcv']: meta['camera'] = db_rec['camera'] meta['joints_gt'] = db_rec['joints_gt'] else: assert 0==1, 'No such dataset definition in JointDataset' return input, target, target_weight, meta