def get_jointset(pose2d_type):
if pose2d_type == 'openpose':
return OpenPoseJoints()
elif pose2d_type == 'hrnet':
return CocoExJoints()
else:
raise Exception("Unknown 2D pose type: " + pose2d_type)
def unstack_mupots_poses(dataset, predictions):
""" Converts output of the logger to dict of list of ndarrays. """
COCO_TO_MUPOTS = []
for i in range(MuPoTSJoints.NUM_JOINTS):
try:
COCO_TO_MUPOTS.append(CocoExJoints().index_of(
MuPoTSJoints.NAMES[i]))
except:
COCO_TO_MUPOTS.append(-1)
COCO_TO_MUPOTS = np.array(COCO_TO_MUPOTS)
assert np.all(COCO_TO_MUPOTS[1:14] >= 0)
pred_2d = {}
pred_3d = {}
for seq in range(1, 21):
gt = mupots_3d.load_gt_annotations(seq)
gt_len = len(gt["annot2"])
pred_2d[seq] = []
pred_3d[seq] = []
seq_inds = dataset.index.seq_num == seq
for i in range(gt_len):
frame_inds = dataset.index.frame == i
valid = dataset.good_poses & seq_inds & frame_inds
pred_2d[seq].append(dataset.poses2d[valid, :, :2][:,
COCO_TO_MUPOTS])
pred_3d[seq].append(predictions[seq][frame_inds[dataset.good_poses
& seq_inds]])
return pred_2d, pred_3d
def extend_hrnet_raw(raw):
assert_shape(raw, (None, 17, 3))
js = CocoExJoints()
result = np.zeros((len(raw), 19, 3), dtype='float32')
result[:, :17, :] = raw
_combine(result, js.index_of('hip'), js.index_of('left_hip'), js.index_of('right_hip'))
_combine(result, js.index_of('neck'), js.index_of('left_shoulder'), js.index_of('right_shoulder'))
return result
def extend_hrnet_raw(raw):
"""
Adds the hip and neck to a Coco skeleton by averaging left/right hips and shoulders.
The score will be the harmonic mean of the two.
"""
assert_shape(raw, (None, 17, 3))
js = CocoExJoints()
result = np.zeros((len(raw), 19, 3), dtype='float32')
result[:, :17, :] = raw
_combine(result, js.index_of('hip'), js.index_of('left_hip'),
js.index_of('right_hip'))
_combine(result, js.index_of('neck'), js.index_of('left_shoulder'),
js.index_of('right_shoulder'))
return result
def __init__(self, img_folder, metadata, poses_path, depth_folder):
self.transform = None
self.images = sorted(os.listdir(img_folder))
# Load camera parameters
with open(metadata, 'r') as f:
data = f.readlines()
data = [x.split(',') for x in data]
data = [[y.strip() for y in x] for x in data]
camera_params = {x[0]: [float(y) for y in x[1:]] for x in data[1:]}
# Prepare data
poses2d = []
fx = []
fy = []
cx = []
cy = []
img_names = []
jointwise_depth = []
pred2d = load(poses_path)
for image in self.images:
poses = [np.array(x['keypoints']).reshape((17, 3)) for x in pred2d[image]]
poses = np.stack(poses, axis=0) # (nPoses, 17, 3)
poses = extend_hrnet_raw(poses) # (nPoses, 19, 3)
img = cv2.imread(os.path.join(img_folder, image))
width, height = recommended_size(img.shape)
depth = load(os.path.join(depth_folder, image + '.npy'))
depth = depth_from_coords(depth, poses.reshape((1, -1, 3))[:, :, :2], width, height) # (nFrames(=1), nPoses*19)
depth = depth.reshape((-1, 19)) # (nPoses, 19)
jointwise_depth.append(depth)
poses2d.append(poses)
for i, field in enumerate([fx, fy, cx, cy]):
field.extend([camera_params[image][i]] * len(poses))
img_names.extend([image] * len(poses))
self.poses2d = np.concatenate(poses2d).astype('float32')
self.poses3d = np.ones_like(self.poses2d)[:, :17]
self.fx = np.array(fx, dtype='float32')
self.fy = np.array(fy, dtype='float32')
self.cx = np.array(cx, dtype='float32')
self.cy = np.array(cy, dtype='float32')
self.img_names = np.array(img_names)
self.pred_cdepths = np.concatenate(jointwise_depth).astype('float32')
self.pose2d_jointset = CocoExJoints()
self.pose3d_jointset = MuPoTSJoints()
def __init__(self,
frame_folder,
hrnet_keypoint_file,
fx,
fy,
cx=None,
cy=None):
self.transform = None
self.pose2d_jointset = CocoExJoints()
self.pose3d_jointset = MuPoTSJoints()
frame_list = sorted(os.listdir(frame_folder))
N = len(frame_list)
hrnet_detections = load(hrnet_keypoint_file)
self.poses2d, self.valid_2d_pred = stack_hrnet_raw(
frame_list, hrnet_detections)
assert len(self.poses2d) == N, "unexpected number of frames"
index = [('vid', i) for i in range(N)]
self.index = np.rec.array(index,
dtype=[('seq', 'U4'), ('frame', 'int32')])
self.poses3d = np.ones(
(N, self.pose3d_jointset.NUM_JOINTS, 3)) # dummy values
# load first frame to get width/height
frame = cv2.imread(os.path.join(frame_folder, frame_list[0]))
self.width = frame.shape[1]
self.fx = np.full(N, fx, dtype='float32')
self.fy = np.full(N, fy, dtype='float32')
self.cx = np.full(N,
cx if cx is not None else frame.shape[1] / 2,
dtype='float32')
self.cy = np.full(N,
cy if cy is not None else frame.shape[0] / 2,
dtype='float32')
assert self.poses2d.shape[1] == self.pose2d_jointset.NUM_JOINTS
def __init__(self, pose2d_type, pose3d_scaling, cap_at_25fps, stride=1):
assert pose2d_type == 'hrnet', "Only hrnet 2d is implemented"
assert pose3d_scaling in ['normal', 'univ'], \
"Unexpected pose3d scaling type: " + str(pose3d_scaling)
self.transform = None
pose3d_key = 'annot3' if pose3d_scaling == 'normal' else 'univ_annot3'
poses2d = []
poses3d = []
valid_2d_pred = [] # True if HR-net found a pose
fx = []
fy = []
cx = []
cy = []
index = []
sequences = []
calibs = mpii_3dhp.get_calibration_matrices()
for sub in range(1, 9): # S1, ..., S8
for seq in range(1, 3): # 2 sequence per S
gt = mpii_3dhp.train_ground_truth(sub, seq)
for cam in range(11):
# In S3/Seq2 cam2 there are some frame between 9400-9900 where the pose is
# behind the camera/nearly in the camera plane. This breaks training.
# For simplicity, ignore the whole set but ignoring frames 9400-9900
# would also work
if seq == 2 and sub == 3 and cam == 2:
continue
# Find indices that are selected for the dataset
inds = np.arange(len(gt[pose3d_key][cam]))
if cap_at_25fps and mpii_3dhp.get_train_fps(sub,
seq) == 50:
inds = inds[::2]
inds = inds[::stride]
num_frames = len(inds)
poses3d.append(gt[pose3d_key][cam][inds])
tmp = mpii_3dhp.train_poses_hrnet(sub, seq, cam)
poses2d.append(tmp['poses'][inds])
valid_2d_pred.append(tmp['is_valid'][inds])
assert len(poses3d[-1]) == len(
poses2d[-1]
), "Gt and predicted frames are not aligned, seq:" + str(
seq)
seq_name = 'S%d/Seq%d/%d' % (sub, seq, cam)
sequences.append(seq_name)
index.extend([(seq_name, sub, seq, cam, i) for i in inds])
calibration_mx = calibs[(sub, seq, cam)]
fx.extend([calibration_mx[0, 0]] * num_frames)
fy.extend([calibration_mx[1, 1]] * num_frames)
cx.extend([calibration_mx[0, 2]] * num_frames)
cy.extend([calibration_mx[1, 2]] * num_frames)
self.pose2d_jointset = CocoExJoints()
self.pose3d_jointset = MuPoTSJoints()
self.poses2d = np.concatenate(poses2d)
self.poses3d = np.concatenate(poses3d)
self.valid_2d_pred = np.concatenate(valid_2d_pred)
self.index = np.rec.array(index,
dtype=[('seq', 'U12'), ('sub', 'int32'),
('subseq', 'int32'), ('cam', 'int32'),
('frame', 'int32')])
self.fx = np.array(fx, dtype='float32')
self.fy = np.array(fy, dtype='float32')
self.cx = np.array(cx, dtype='float32')
self.cy = np.array(cy, dtype='float32')
self.sequences = sorted(sequences)
assert len(self.poses2d) == len(self.index), len(self.index)
assert len(self.poses2d) == len(self.poses3d)
assert len(self.poses2d) == len(self.index), len(self.index)
assert len(self.poses2d) == len(self.valid_2d_pred), len(
self.valid_2d_pred)
assert len(self.poses2d) == len(self.fx), len(self.fx)
assert len(self.poses2d) == len(self.fy), len(self.fy)
assert len(self.poses2d) == len(self.cx), len(self.cx)
assert len(self.poses2d) == len(self.cy), len(self.cy)
def __init__(self, pose2d_type, pose3d_scaling, eval_frames_only=False):
assert pose2d_type == 'hrnet', "Only hrnet 2d is implemented"
assert pose3d_scaling in ['normal', 'univ'], \
"Unexpected pose3d scaling type: " + str(pose3d_scaling)
self.transform = None
self.eval_frames_only = eval_frames_only
pose3d_key = 'annot3' if pose3d_scaling == 'normal' else 'univ_annot3'
poses2d = []
poses3d = []
valid_2d_pred = [] # True if HR-net found a pose
valid_frame = [] # True if MPI-INF-3DHP marked the frame as valid
fx = []
fy = []
cx = []
cy = []
width = []
index = []
for seq in range(1, 7):
gt = h5py.File(
os.path.join(mpii_3dhp.MPII_3DHP_PATH, 'mpi_inf_3dhp_test_set',
'TS%d' % seq, 'annot_data.mat'), 'r')
poses3d.append(gt[pose3d_key][:, 0])
valid_frame.append(gt['valid_frame'][()] == 1)
num_frames = len(
poses3d[-1]
) # The annotations are shorter than the number of images
tmp = mpii_3dhp.test_poses_hrnet(seq)
poses2d.append(tmp['poses'])
valid_2d_pred.append(tmp['is_valid'])
assert len(poses3d[-1]) == len(
poses2d[-1]
), "Gt and predicted frames are not aligned, seq:" + str(seq)
index.extend([(seq, i) for i in range(num_frames)])
calibration_mx = mpii_3dhp.get_test_calib(seq)
fx.extend([calibration_mx[0, 0]] * num_frames)
fy.extend([calibration_mx[1, 1]] * num_frames)
cx.extend([calibration_mx[0, 2]] * num_frames)
cy.extend([calibration_mx[1, 2]] * num_frames)
width.extend([2048 if seq < 5 else 1920] * num_frames)
self.pose2d_jointset = CocoExJoints()
self.pose3d_jointset = MuPoTSJoints()
self.poses2d = np.concatenate(poses2d)
self.poses3d = np.concatenate(poses3d)
self.valid_2d_pred = np.concatenate(valid_2d_pred)
valid_frame = np.concatenate(valid_frame)
assert valid_frame.shape[1] == 1, valid_frame.shape
valid_frame = valid_frame[:, 0]
self.index = np.rec.array(index,
dtype=[('seq', 'int32'), ('frame', 'int32')])
self.fx = np.array(fx, dtype='float32')
self.fy = np.array(fy, dtype='float32')
self.cx = np.array(cx, dtype='float32')
self.cy = np.array(cy, dtype='float32')
self.width = np.array(width, dtype='int32')
assert len(self.poses2d) == len(self.index), len(self.index)
# keep only those frame where a pose was detected
good_poses = self.valid_2d_pred.copy()
if eval_frames_only:
good_poses = good_poses & valid_frame
self.good_poses = good_poses
assert len(self.poses2d) == len(self.poses3d)
assert len(self.poses2d) == len(self.index), len(self.index)
assert len(self.poses2d) == len(self.valid_2d_pred), len(
self.valid_2d_pred)
assert len(self.poses2d) == len(self.fx), len(self.fx)
assert len(self.poses2d) == len(self.fy), len(self.fy)
assert len(self.poses2d) == len(self.cx), len(self.cx)
assert len(self.poses2d) == len(self.cy), len(self.cy)
assert len(self.poses2d) == len(self.width), len(self.width)
assert len(self.poses2d) == len(self.good_poses), len(self.good_poses)