def unstack_mupots_poses(dataset, predictions):
""" Converts output of the logger to dict of list of ndarrays. """
COCO_TO_MUPOTS = []
for i in range(MuPoTSJoints.NUM_JOINTS):
try:
COCO_TO_MUPOTS.append(CocoExJoints().index_of(
MuPoTSJoints.NAMES[i]))
except:
COCO_TO_MUPOTS.append(-1)
COCO_TO_MUPOTS = np.array(COCO_TO_MUPOTS)
assert np.all(COCO_TO_MUPOTS[1:14] >= 0)
pred_2d = {}
pred_3d = {}
for seq in range(1, 21):
gt = mupots_3d.load_gt_annotations(seq)
gt_len = len(gt["annot2"])
pred_2d[seq] = []
pred_3d[seq] = []
seq_inds = dataset.index.seq_num == seq
for i in range(gt_len):
frame_inds = dataset.index.frame == i
valid = dataset.good_poses & seq_inds & frame_inds
pred_2d[seq].append(dataset.poses2d[valid, :, :2][:,
COCO_TO_MUPOTS])
pred_3d[seq].append(predictions[seq][frame_inds[dataset.good_poses
& seq_inds]])
return pred_2d, pred_3d
def get_mupo_poses(seq):
"""
Loads valid MuPo-TS poses for given sequence. A pose is valid if the hip is visible (has a score>0.5)
and was detected by OpenPose
"""
gt = mupots_3d.load_gt_annotations(seq)
op = mupots_3d.load_openpose_predictions(seq)
valid = gt['isValidFrame'].squeeze()
op_valid = op['valid_pose']
assert valid.dtype == 'bool'
assert op_valid.dtype == 'bool'
valid = np.logical_and(valid, op_valid)
pose2d = op['pose']
pose3d = gt['annot3']
assert pose2d.shape[:2] == valid.shape
assert pose3d.shape[:2] == valid.shape
assert pose2d.shape[2:] == (25, 3)
assert pose3d.shape[2:] == (17, 3)
assert valid.ndim == 2
pose2d = pose2d[valid]
pose3d = pose3d[valid]
good_poses = pose2d[:, 8, 2] > 0.5
pose2d = pose2d[good_poses]
pose3d = pose3d[good_poses]
assert len(pose2d) == len(pose3d)
return pose2d, pose3d
def __init__(self, pose2d_type, pose3d_scaling):
"""
Loads MuPoTS dataset but only those images where at least one person was detected. Each person on a frame
is loaded separately.
"""
assert pose3d_scaling in ['univ', 'normal']
self.pose2d_jointset = FilteredSinglePersonMuPoTsDataset.get_jointset(
pose2d_type)
self.pose3d_jointset = MuPoTSJoints()
poses2d = []
poses3d = []
pred_cdepths = []
index = []
for seq in range(1, 21):
depth_width = 512
depth_height = 512 if seq <= 5 else 288
gt = mupots_3d.load_gt_annotations(seq)
op = mupots_3d.load_2d_predictions(seq, pose2d_type)
pose2d = op['pose']
pose3d = gt['annot3' if pose3d_scaling ==
'normal' else 'univ_annot3']
depth = mupots_3d.load_jointwise_depth(seq)
good_poses = gt['isValidFrame'].squeeze()
good_poses = np.logical_and(good_poses, op['valid_pose'])
orig_frame = np.tile(
np.arange(len(good_poses)).reshape((-1, 1)),
(1, good_poses.shape[1]))
orig_pose = np.tile(
np.arange(good_poses.shape[1]).reshape((1, -1)),
(good_poses.shape[0], 1))
assert pose2d.shape[:2] == good_poses.shape # (nFrames, nPeople)
assert pose3d.shape[:2] == good_poses.shape
assert depth.shape[:2] == good_poses.shape
assert orig_frame.shape == good_poses.shape
assert orig_pose.shape == good_poses.shape
assert pose2d.shape[2:] == (self.pose2d_jointset.NUM_JOINTS, 3)
assert pose3d.shape[2:] == (17, 3)
assert good_poses.ndim == 2
# Keep only those poses where good_poses is True
pose2d = pose2d[good_poses]
pose3d = pose3d[good_poses]
orig_frame = orig_frame[good_poses]
orig_pose = orig_pose[good_poses]
depth = depth[good_poses]
index.extend([(seq, orig_frame[i], orig_pose[i], depth_width,
depth_height) for i in range(len(orig_frame))])
assert len(pose2d) == len(pose3d)
poses2d.append(pose2d)
poses3d.append(pose3d)
pred_cdepths.append(depth)
self.poses2d = np.concatenate(poses2d).astype('float32')
self.poses3d = np.concatenate(poses3d).astype('float32')
self.pred_cdepths = np.concatenate(pred_cdepths).astype('float32')
self.index = np.rec.array(index,
dtype=[('seq', 'int32'), ('frame', 'int32'),
('pose', 'int32'),
('depth_width', 'int32'),
('depth_height', 'int32')])
# Load calibration matrices
N = len(self.poses2d)
self.fx = np.zeros(N, dtype='float32')
self.fy = np.zeros(N, dtype='float32')
self.cx = np.zeros(N, dtype='float32')
self.cy = np.zeros(N, dtype='float32')
mupots_calibs = mupots_3d.get_calibration_matrices()
for seq in range(1, 21):
inds = (self.index.seq == seq)
self.fx[inds] = mupots_calibs[seq][0, 0]
self.fy[inds] = mupots_calibs[seq][1, 1]
self.cx[inds] = mupots_calibs[seq][0, 2]
self.cy[inds] = mupots_calibs[seq][1, 2]
assert np.all(self.fx > 0), "Some fields were not filled"
assert np.all(self.fy > 0), "Some fields were not filled"
assert np.all(np.abs(self.cx) > 0), "Some fields were not filled"
assert np.all(np.abs(self.cy) > 0), "Some fields were not filled"
self.transform = None
def __init__(self, seq, calibs, mean3d, std3d):
"""
seq: sequence number
calibs: calibration matrices for MuPo-TS, as returned by `mupots_3d.get_calibration_matrices()`
"""
assert 1 <= seq <= 20, "Invalid seq: " + str(seq)
gt = mupots_3d.load_gt_annotations(seq)
op = mupots_3d.load_openpose_predictions(seq)
pose2d = op['pose']
pose3d = gt['annot3']
good_poses = gt['isValidFrame'].squeeze()
good_poses = np.logical_and(good_poses, op['valid_pose'])
good_poses = np.logical_and(good_poses, pose2d[:, :, 8, 2] > 0.5)
self.valid = good_poses
self.orig_frame = np.tile(
np.arange(len(good_poses)).reshape((-1, 1)),
(1, good_poses.shape[1]))
self.orig_pose = np.tile(
np.arange(good_poses.shape[1]).reshape((1, -1)),
(good_poses.shape[0], 1))
assert pose2d.shape[:2] == good_poses.shape
assert pose3d.shape[:2] == good_poses.shape
assert self.orig_frame.shape == good_poses.shape
assert self.orig_pose.shape == good_poses.shape
assert pose2d.shape[2:] == (25, 3)
assert pose3d.shape[2:] == (17, 3)
assert good_poses.ndim == 2
# Keep only those frames where there is at least one pose
# Having empty frames causes issues with the model forward pass creating zero dimensions
good_frame = np.any(good_poses, axis=1)
pose2d = pose2d[good_frame]
pose3d = pose3d[good_frame]
self.orig_frame = self.orig_frame[good_frame]
self.orig_pose = self.orig_pose[good_frame]
good_poses = good_poses[good_frame]
self.orig_frame = self.orig_frame[good_poses]
self.orig_pose = self.orig_pose[good_poses]
width, height = mupots_3d.image_size(seq)
depth_width = 512
depth_height = 512 if seq <= 5 else 288
self.normed_poses = preprocess_2d(pose2d, calibs[seq][0, 0],
calibs[seq][0, 2], calibs[seq][1, 1],
calibs[seq][1, 2])
pose2d = scale_and_round_pose(pose2d, width, height, depth_width,
depth_height)
pose3d = preprocess_3d(pose3d, True)
pose3d = normalize_arr(pose3d, mean3d, std3d)
assert len(pose2d) == len(pose3d)
self.seq = seq
self.depth_width = depth_width
self.depth_height = depth_height
self.pose2d = pose2d
self.pose3d = pose3d[good_poses]
self.good_poses = good_poses.astype('uint8')
def __init__(self,
pose2d_type,
pose3d_scaling,
pose_validity='detected_only',
hip_threshold=-1):
"""
Loads MuPoTS dataset but only those images where at least one person was detected. Each person on a frame
is loaded separately.
:param pose_validity: one of 'all', 'detected_only', 'valid_only'; specifies which poses are marked valid
all - all of them; valid_only - those that are valid according to the GT annotations
detected_only - those that were successfuly detected by the 2D algon and also valid
:param hip_threshold: only those poses are loaded, where the score of the hip is larger than this value
:param filter_incorrect_match: MuPoTS's pose matching script has some erroneous matching. If filter_incorrect_match is True,
these are not loaded.
"""
assert pose_validity in ['all', 'detected_only', 'valid_only']
assert pose3d_scaling in ['univ', 'normal']
self.pose2d_jointset = PersonStackedMuPoTsDataset.get_jointset(
pose2d_type)
self.pose3d_jointset = MuPoTSJoints()
self.pose3d_scaling = pose3d_scaling
pred2d_root_ind = self.pose2d_jointset.index_of('hip')
poses2d = []
poses3d = []
joint3d_visible = []
all_good_poses = []
valid_annotations = []
width = []
index = []
for seq in range(1, 21):
img_width, img_height = mupots_3d.image_size(seq)
gt = mupots_3d.load_gt_annotations(seq)
pred2d = mupots_3d.load_2d_predictions(seq, pose2d_type)
pose2d = pred2d['pose']
pose3d = gt['annot3' if pose3d_scaling ==
'normal' else 'univ_annot3']
visibility = ~gt['occlusions']
if pose_validity == 'all':
good_poses = np.full(pose3d.shape[:2], True, dtype='bool')
elif pose_validity == 'valid_only':
good_poses = gt['isValidFrame'].squeeze()
elif pose_validity == 'detected_only':
good_poses = gt['isValidFrame'].squeeze()
good_poses = np.logical_and(good_poses, pred2d['valid_pose'])
good_poses = np.logical_and(
good_poses,
pose2d[:, :, pred2d_root_ind, 2] > hip_threshold)
else:
raise NotImplementedError("Unknown pose_validity value:" +
pose_validity)
orig_frame = np.tile(
np.arange(len(good_poses)).reshape(-1, 1),
(1, good_poses.shape[1]))
orig_pose = np.tile(
np.arange(good_poses.shape[1]).reshape(1, -1),
(good_poses.shape[0], 1))
assert pose2d.shape[:2] == good_poses.shape # (nFrames, nPeople)
assert pose3d.shape[:2] == good_poses.shape
assert orig_frame.shape == good_poses.shape
assert orig_pose.shape == good_poses.shape
assert pose2d.shape[2:] == (self.pose2d_jointset.NUM_JOINTS, 3)
assert pose3d.shape[2:] == (17, 3)
assert visibility.shape[2] == 17
assert good_poses.ndim == 2
orig_frame = _column_stack(orig_frame)
orig_pose = _column_stack(orig_pose)
index.extend([('%d/%d' % (seq, orig_pose[i]), seq, orig_frame[i],
orig_pose[i]) for i in range(len(orig_frame))])
poses2d.append(_column_stack(pose2d))
poses3d.append(_column_stack(pose3d))
joint3d_visible.append(_column_stack(visibility))
all_good_poses.append(_column_stack(good_poses))
valid_annotations.append(_column_stack(gt['isValidFrame']))
width.extend([img_width] * len(orig_frame))
self.poses2d = np.concatenate(poses2d).astype('float32')
self.poses3d = np.concatenate(poses3d).astype('float32')
self.joint3d_visible = np.concatenate(joint3d_visible)
self.good_poses = np.concatenate(all_good_poses)
self.valid_annotations = np.concatenate(valid_annotations)
self.width = np.array(width)
self.index = np.rec.array(index,
dtype=[('seq', 'U5'), ('seq_num', 'int32'),
('frame', 'int32'),
('pose', 'int32')])
assert self.valid_annotations.shape == self.good_poses.shape
assert len(self.valid_annotations) == len(self.poses2d)
# Load calibration matrices
N = len(self.poses2d)
self.fx = np.zeros(N, dtype='float32')
self.fy = np.zeros(N, dtype='float32')
self.cx = np.zeros(N, dtype='float32')
self.cy = np.zeros(N, dtype='float32')
mupots_calibs = mupots_3d.get_calibration_matrices()
for seq in range(1, 21):
inds = (self.index.seq_num == seq)
self.fx[inds] = mupots_calibs[seq][0, 0]
self.fy[inds] = mupots_calibs[seq][1, 1]
self.cx[inds] = mupots_calibs[seq][0, 2]
self.cy[inds] = mupots_calibs[seq][1, 2]
assert np.all(self.fx > 0), "Some fields were not filled"
assert np.all(self.fy > 0), "Some fields were not filled"
assert np.all(np.abs(self.cx) > 0), "Some fields were not filled"
assert np.all(np.abs(self.cy) > 0), "Some fields were not filled"
self.transform = None