def get_jointset(pose2d_type): if pose2d_type == 'openpose': return OpenPoseJoints() elif pose2d_type == 'hrnet': return CocoExJoints() else: raise Exception("Unknown 2D pose type: " + pose2d_type)
def unstack_mupots_poses(dataset, predictions): """ Converts output of the logger to dict of list of ndarrays. """ COCO_TO_MUPOTS = [] for i in range(MuPoTSJoints.NUM_JOINTS): try: COCO_TO_MUPOTS.append(CocoExJoints().index_of( MuPoTSJoints.NAMES[i])) except: COCO_TO_MUPOTS.append(-1) COCO_TO_MUPOTS = np.array(COCO_TO_MUPOTS) assert np.all(COCO_TO_MUPOTS[1:14] >= 0) pred_2d = {} pred_3d = {} for seq in range(1, 21): gt = mupots_3d.load_gt_annotations(seq) gt_len = len(gt["annot2"]) pred_2d[seq] = [] pred_3d[seq] = [] seq_inds = dataset.index.seq_num == seq for i in range(gt_len): frame_inds = dataset.index.frame == i valid = dataset.good_poses & seq_inds & frame_inds pred_2d[seq].append(dataset.poses2d[valid, :, :2][:, COCO_TO_MUPOTS]) pred_3d[seq].append(predictions[seq][frame_inds[dataset.good_poses & seq_inds]]) return pred_2d, pred_3d
def extend_hrnet_raw(raw): assert_shape(raw, (None, 17, 3)) js = CocoExJoints() result = np.zeros((len(raw), 19, 3), dtype='float32') result[:, :17, :] = raw _combine(result, js.index_of('hip'), js.index_of('left_hip'), js.index_of('right_hip')) _combine(result, js.index_of('neck'), js.index_of('left_shoulder'), js.index_of('right_shoulder')) return result
def extend_hrnet_raw(raw): """ Adds the hip and neck to a Coco skeleton by averaging left/right hips and shoulders. The score will be the harmonic mean of the two. """ assert_shape(raw, (None, 17, 3)) js = CocoExJoints() result = np.zeros((len(raw), 19, 3), dtype='float32') result[:, :17, :] = raw _combine(result, js.index_of('hip'), js.index_of('left_hip'), js.index_of('right_hip')) _combine(result, js.index_of('neck'), js.index_of('left_shoulder'), js.index_of('right_shoulder')) return result
def __init__(self, img_folder, metadata, poses_path, depth_folder): self.transform = None self.images = sorted(os.listdir(img_folder)) # Load camera parameters with open(metadata, 'r') as f: data = f.readlines() data = [x.split(',') for x in data] data = [[y.strip() for y in x] for x in data] camera_params = {x[0]: [float(y) for y in x[1:]] for x in data[1:]} # Prepare data poses2d = [] fx = [] fy = [] cx = [] cy = [] img_names = [] jointwise_depth = [] pred2d = load(poses_path) for image in self.images: poses = [np.array(x['keypoints']).reshape((17, 3)) for x in pred2d[image]] poses = np.stack(poses, axis=0) # (nPoses, 17, 3) poses = extend_hrnet_raw(poses) # (nPoses, 19, 3) img = cv2.imread(os.path.join(img_folder, image)) width, height = recommended_size(img.shape) depth = load(os.path.join(depth_folder, image + '.npy')) depth = depth_from_coords(depth, poses.reshape((1, -1, 3))[:, :, :2], width, height) # (nFrames(=1), nPoses*19) depth = depth.reshape((-1, 19)) # (nPoses, 19) jointwise_depth.append(depth) poses2d.append(poses) for i, field in enumerate([fx, fy, cx, cy]): field.extend([camera_params[image][i]] * len(poses)) img_names.extend([image] * len(poses)) self.poses2d = np.concatenate(poses2d).astype('float32') self.poses3d = np.ones_like(self.poses2d)[:, :17] self.fx = np.array(fx, dtype='float32') self.fy = np.array(fy, dtype='float32') self.cx = np.array(cx, dtype='float32') self.cy = np.array(cy, dtype='float32') self.img_names = np.array(img_names) self.pred_cdepths = np.concatenate(jointwise_depth).astype('float32') self.pose2d_jointset = CocoExJoints() self.pose3d_jointset = MuPoTSJoints()
def __init__(self, frame_folder, hrnet_keypoint_file, fx, fy, cx=None, cy=None): self.transform = None self.pose2d_jointset = CocoExJoints() self.pose3d_jointset = MuPoTSJoints() frame_list = sorted(os.listdir(frame_folder)) N = len(frame_list) hrnet_detections = load(hrnet_keypoint_file) self.poses2d, self.valid_2d_pred = stack_hrnet_raw( frame_list, hrnet_detections) assert len(self.poses2d) == N, "unexpected number of frames" index = [('vid', i) for i in range(N)] self.index = np.rec.array(index, dtype=[('seq', 'U4'), ('frame', 'int32')]) self.poses3d = np.ones( (N, self.pose3d_jointset.NUM_JOINTS, 3)) # dummy values # load first frame to get width/height frame = cv2.imread(os.path.join(frame_folder, frame_list[0])) self.width = frame.shape[1] self.fx = np.full(N, fx, dtype='float32') self.fy = np.full(N, fy, dtype='float32') self.cx = np.full(N, cx if cx is not None else frame.shape[1] / 2, dtype='float32') self.cy = np.full(N, cy if cy is not None else frame.shape[0] / 2, dtype='float32') assert self.poses2d.shape[1] == self.pose2d_jointset.NUM_JOINTS
def __init__(self, pose2d_type, pose3d_scaling, cap_at_25fps, stride=1): assert pose2d_type == 'hrnet', "Only hrnet 2d is implemented" assert pose3d_scaling in ['normal', 'univ'], \ "Unexpected pose3d scaling type: " + str(pose3d_scaling) self.transform = None pose3d_key = 'annot3' if pose3d_scaling == 'normal' else 'univ_annot3' poses2d = [] poses3d = [] valid_2d_pred = [] # True if HR-net found a pose fx = [] fy = [] cx = [] cy = [] index = [] sequences = [] calibs = mpii_3dhp.get_calibration_matrices() for sub in range(1, 9): # S1, ..., S8 for seq in range(1, 3): # 2 sequence per S gt = mpii_3dhp.train_ground_truth(sub, seq) for cam in range(11): # In S3/Seq2 cam2 there are some frame between 9400-9900 where the pose is # behind the camera/nearly in the camera plane. This breaks training. # For simplicity, ignore the whole set but ignoring frames 9400-9900 # would also work if seq == 2 and sub == 3 and cam == 2: continue # Find indices that are selected for the dataset inds = np.arange(len(gt[pose3d_key][cam])) if cap_at_25fps and mpii_3dhp.get_train_fps(sub, seq) == 50: inds = inds[::2] inds = inds[::stride] num_frames = len(inds) poses3d.append(gt[pose3d_key][cam][inds]) tmp = mpii_3dhp.train_poses_hrnet(sub, seq, cam) poses2d.append(tmp['poses'][inds]) valid_2d_pred.append(tmp['is_valid'][inds]) assert len(poses3d[-1]) == len( poses2d[-1] ), "Gt and predicted frames are not aligned, seq:" + str( seq) seq_name = 'S%d/Seq%d/%d' % (sub, seq, cam) sequences.append(seq_name) index.extend([(seq_name, sub, seq, cam, i) for i in inds]) calibration_mx = calibs[(sub, seq, cam)] fx.extend([calibration_mx[0, 0]] * num_frames) fy.extend([calibration_mx[1, 1]] * num_frames) cx.extend([calibration_mx[0, 2]] * num_frames) cy.extend([calibration_mx[1, 2]] * num_frames) self.pose2d_jointset = CocoExJoints() self.pose3d_jointset = MuPoTSJoints() self.poses2d = np.concatenate(poses2d) self.poses3d = np.concatenate(poses3d) self.valid_2d_pred = np.concatenate(valid_2d_pred) self.index = np.rec.array(index, dtype=[('seq', 'U12'), ('sub', 'int32'), ('subseq', 'int32'), ('cam', 'int32'), ('frame', 'int32')]) self.fx = np.array(fx, dtype='float32') self.fy = np.array(fy, dtype='float32') self.cx = np.array(cx, dtype='float32') self.cy = np.array(cy, dtype='float32') self.sequences = sorted(sequences) assert len(self.poses2d) == len(self.index), len(self.index) assert len(self.poses2d) == len(self.poses3d) assert len(self.poses2d) == len(self.index), len(self.index) assert len(self.poses2d) == len(self.valid_2d_pred), len( self.valid_2d_pred) assert len(self.poses2d) == len(self.fx), len(self.fx) assert len(self.poses2d) == len(self.fy), len(self.fy) assert len(self.poses2d) == len(self.cx), len(self.cx) assert len(self.poses2d) == len(self.cy), len(self.cy)
def __init__(self, pose2d_type, pose3d_scaling, eval_frames_only=False): assert pose2d_type == 'hrnet', "Only hrnet 2d is implemented" assert pose3d_scaling in ['normal', 'univ'], \ "Unexpected pose3d scaling type: " + str(pose3d_scaling) self.transform = None self.eval_frames_only = eval_frames_only pose3d_key = 'annot3' if pose3d_scaling == 'normal' else 'univ_annot3' poses2d = [] poses3d = [] valid_2d_pred = [] # True if HR-net found a pose valid_frame = [] # True if MPI-INF-3DHP marked the frame as valid fx = [] fy = [] cx = [] cy = [] width = [] index = [] for seq in range(1, 7): gt = h5py.File( os.path.join(mpii_3dhp.MPII_3DHP_PATH, 'mpi_inf_3dhp_test_set', 'TS%d' % seq, 'annot_data.mat'), 'r') poses3d.append(gt[pose3d_key][:, 0]) valid_frame.append(gt['valid_frame'][()] == 1) num_frames = len( poses3d[-1] ) # The annotations are shorter than the number of images tmp = mpii_3dhp.test_poses_hrnet(seq) poses2d.append(tmp['poses']) valid_2d_pred.append(tmp['is_valid']) assert len(poses3d[-1]) == len( poses2d[-1] ), "Gt and predicted frames are not aligned, seq:" + str(seq) index.extend([(seq, i) for i in range(num_frames)]) calibration_mx = mpii_3dhp.get_test_calib(seq) fx.extend([calibration_mx[0, 0]] * num_frames) fy.extend([calibration_mx[1, 1]] * num_frames) cx.extend([calibration_mx[0, 2]] * num_frames) cy.extend([calibration_mx[1, 2]] * num_frames) width.extend([2048 if seq < 5 else 1920] * num_frames) self.pose2d_jointset = CocoExJoints() self.pose3d_jointset = MuPoTSJoints() self.poses2d = np.concatenate(poses2d) self.poses3d = np.concatenate(poses3d) self.valid_2d_pred = np.concatenate(valid_2d_pred) valid_frame = np.concatenate(valid_frame) assert valid_frame.shape[1] == 1, valid_frame.shape valid_frame = valid_frame[:, 0] self.index = np.rec.array(index, dtype=[('seq', 'int32'), ('frame', 'int32')]) self.fx = np.array(fx, dtype='float32') self.fy = np.array(fy, dtype='float32') self.cx = np.array(cx, dtype='float32') self.cy = np.array(cy, dtype='float32') self.width = np.array(width, dtype='int32') assert len(self.poses2d) == len(self.index), len(self.index) # keep only those frame where a pose was detected good_poses = self.valid_2d_pred.copy() if eval_frames_only: good_poses = good_poses & valid_frame self.good_poses = good_poses assert len(self.poses2d) == len(self.poses3d) assert len(self.poses2d) == len(self.index), len(self.index) assert len(self.poses2d) == len(self.valid_2d_pred), len( self.valid_2d_pred) assert len(self.poses2d) == len(self.fx), len(self.fx) assert len(self.poses2d) == len(self.fy), len(self.fy) assert len(self.poses2d) == len(self.cx), len(self.cx) assert len(self.poses2d) == len(self.cy), len(self.cy) assert len(self.poses2d) == len(self.width), len(self.width) assert len(self.poses2d) == len(self.good_poses), len(self.good_poses)