def scale_and_round_pose(poses,
img_width,
img_height,
target_width,
target_height,
verbose=True):
"""
Converts the input OpenPose coordinates to integer pixel numbers on a resized image,
keeping only the Common14 joints.
Parameters:
poses: (nFrames, nPoses, 25, 3[x, y, score])
img_width, img_height: width and height of the original image
target_width, target_height: width and height of the resized image
"""
assert_shape(poses, (None, None, 25, 3))
poses = poses[:, :, mupots_3d.OPENPOSE25_TO_COMMON2D14, :2]
poses[:, :, :, 0] *= target_width / float(img_width)
poses[:, :, :, 1] *= target_height / float(img_height)
poses = np.around(poses)
if verbose:
if np.any(poses < -1):
print "too small value"
if np.any(poses[:, :, :, 0] > target_width):
print "too large width"
if np.any(poses[:, :, :, 1] > target_height):
print "too large height"
poses[:, :, :, 0] = np.clip(poses[:, :, :, 0], 0, target_width - 1)
poses[:, :, :, 1] = np.clip(poses[:, :, :, 1], 0, target_height - 1)
poses = poses.astype('int64')
return poses
def preprocess_3d(data, add_root, log_root_z, joint_set, root_name):
"""
3D preprocessing:
1. Removes the root joint
2. If add_root is True, append the root joint at the end of the pose. The
The logarithm of the z coordinate of the root is taken.
3. Flattens the data.
:param data: ndarray(nFrames, [nPoses], nJoints, 3[x, y, z]) 3D coordinates in MuPoTS order
:param add_root: True if the absolute coordinates of the hip should be included in the output
:param log_root_z:if true, the log of the z coordinate of the root is used
:param root_name: name of the root joint, must be a MuPoTS joint
:return: ndarray(nPoses, 3*nJoints|3*(nJoints-1)), 3*nJoints if add_root is true otherwise 3*(nJoints-1)
"""
assert_shape(data, ("*", joint_set.NUM_JOINTS, 3))
root_ind = joint_set.index_of(root_name)
root3d = data[..., root_ind, :].copy()
if log_root_z:
root3d[..., 2] = np.log(root3d[..., 2])
data = remove_root(data, root_ind) # (nFrames, [nPoses], nJoints-1, 3)
data = data.reshape(data.shape[:-2] + (-1,)) # (nFrames, [nPoses], (nJoints-1)*3)
if add_root:
data = np.concatenate([data, root3d], axis=-1) # (nFrames, [nPoses], nJoints*3)
return data.astype('float32')
def combine_pose_and_trans(data3d, std3d, mean3d, joint_set, root_name):
"""
3D result postprocess: unnormalizes data3d and reconstructs the absolute pose from relative + absolute split.
Parameters:
data3d: output of the PyTorch model, ndarray(nPoses, 3*nJoints), in the format created by preprocess3d
std3d: normalization standard deviations
mean3d: normalization means
root_name: name of the root joint
Returns:
ndarray(nPoses, nJoints, 3)
"""
assert_shape(data3d, (None, joint_set.NUM_JOINTS * 3))
data3d = data3d * std3d + mean3d
root = data3d[:, -3:]
rel_pose = data3d[:, :-3].reshape((len(data3d), joint_set.NUM_JOINTS - 1, 3))
root[:, 2] = np.exp(root[:, 2])
rel_pose += root[:, np.newaxis, :]
result = np.zeros((len(data3d), joint_set.NUM_JOINTS, 3), dtype='float32')
root_ind = joint_set.index_of(root_name)
result[:, :root_ind, :] = rel_pose[:, :root_ind, :]
result[:, root_ind, :] = root
result[:, root_ind + 1:, :] = rel_pose[:, root_ind:, :]
return result
def preprocess_2d(data, fx, cx, fy, cy, joint_set, root_name):
"""
2D data preprocessing, performing the following:
1. Keeps only COMMON14 joints
2. Normalizes coordinates by multiplying with the inverse of the calibration matrix
3. Converts numbers in a root-relative form
4. Invisible joints are replaced by a single value
5. Convert data into float
:param data: (nPoses, 25, 3[x, y, scores]) - OpenPose detected coordinates
:param fx: ndarray(nPoses) or float, horizontal focal length
:param cx: ndarray(nPoses) or float, horizontal principal point
:param fy: ndarray(nPoses) or float, vertical focal length
:param cy: ndarray(nPoses) or float, horizontal principal point
:param joint_set: the JointSet object describing the order of joints
:param root_name: name of the root joint, must be a COMMON14 joint
:return: ndarray(nPoses, 42), First 39 numbers are the non-root joints, last one is the root
"""
assert_shape(data, ("*", None, joint_set.NUM_JOINTS, 3))
assert not isinstance(fx, np.ndarray) or len(fx) == len(data)
assert not isinstance(fy, np.ndarray) or len(fy) == len(data)
if isinstance(fx, np.ndarray):
N = len(data)
shape = [1] * (data.ndim - 1)
shape[0] = N
fx = fx.reshape(shape)
fy = fy.reshape(shape)
cx = cx.reshape(shape)
cy = cy.reshape(shape)
data = data[..., joint_set.TO_COMMON14, :]
data[..., :, 0] -= cx
data[..., :, 1] -= cy
data[..., :, 0] /= fx
data[..., :, 1] /= fy
root_ind = np.where(Common14Joints.NAMES == root_name)[0][0]
root2d = data[..., root_ind, :].copy()
data = remove_root_keepscore(data, root_ind) # (nPoses, 13, 3), modifies data
bad_frames = data[..., 2] < 0.1
# replace joints having low scores with 1700/focus
# this is to prevent leaking cx/cy
if isinstance(fx, np.ndarray):
fx = np.tile(fx, (1,) + data.shape[1:-1])
fy = np.tile(fy, (1,) + data.shape[1:-1])
data[bad_frames, 0] = -1700 / fx[bad_frames]
data[bad_frames, 1] = -1700 / fy[bad_frames]
else:
data[bad_frames, 0] = -1700 / fx
data[bad_frames, 1] = -1700 / fy
# stack root next to the pose
data = data.reshape(data.shape[:-2] + (-1,)) # (nPoses, 13*3)
data = np.concatenate([data, root2d], axis=-1) # (nPoses, 14*3)
return data.astype('float32')
def abs_to_hiprel(poses, joint_set):
""" Converts an absolute pose into [hi]+relative_pose. """
assert_shape(poses, (None, joint_set.NUM_JOINTS, 3))
root = poses[:, [joint_set.index_of('hip')]].copy()
rel = remove_root(poses, joint_set.index_of('hip'))
return np.concatenate([root, rel], axis=-2)
def mrpe(pred, gt, joint_set):
""" Mean Roo Position Error. """
assert_shape(pred, ('*', None, 3))
assert pred.shape == gt.shape
hip_ind = joint_set.index_of('hip')
assert gt[..., hip_ind, :].shape[-1] == 3
return np.nanmean(
np.linalg.norm(gt[..., hip_ind, :] - pred[..., hip_ind, :], axis=-1))
def flip(self, data):
""" Flips a dataset """
assert_shape(data, ('*', self.NUM_JOINTS, None))
data = data.copy()
data[..., self.JOINTS_LEFT +
self.JOINTS_RIGHT, :] = data[..., self.JOINTS_RIGHT +
self.JOINTS_LEFT, :]
return data
def keep_hrnet_c14(data):
"""
Keeps only COMMON-14 joints from hrnet.
data - ndarray(..., 19), along the last dimension, each slice corresponds to a joint In CocoEx joint order.
"""
assert_shape(data, ('*', None, CocoExJoints.NUM_JOINTS))
data = data[..., CocoExJoints.TO_COMMON14]
return data
def add_back_hip(poses, joint_set):
""" Inverse of abs_to_hiprel """
assert_shape(poses, (None, joint_set.NUM_JOINTS, 3))
root = poses[:, [0]].copy()
hip_ind = joint_set.index_of('hip')
result = insert_zero_joint(poses[:, 1:], hip_ind)
result += root
return result
def extend_hrnet_raw(raw):
assert_shape(raw, (None, 17, 3))
js = CocoExJoints()
result = np.zeros((len(raw), 19, 3), dtype='float32')
result[:, :17, :] = raw
_combine(result, js.index_of('hip'), js.index_of('left_hip'), js.index_of('right_hip'))
_combine(result, js.index_of('neck'), js.index_of('left_shoulder'), js.index_of('right_shoulder'))
return result
def flip(self, data): # TODO Torchify
""" Flips a dataset """
assert_shape(data, ('*', self.NUM_JOINTS, None))
if type(data) != torch.Tensor:
data = data.copy()
data[..., self.JOINTS_LEFT +
self.JOINTS_RIGHT, :] = data[..., self.JOINTS_RIGHT +
self.JOINTS_LEFT, :]
return data
def n_mpjpe(pred, gt):
"""
Based on https://github.com/hrhodin/UnsupervisedGeometryAwareRepresentationLearning, losses/poses.py
"""
assert pred.shape == gt.shape
assert_shape(pred, ('*', None, 3))
s_opt = optimal_scaling(pred, gt)
return mpjpe(pred * s_opt[..., np.newaxis, np.newaxis], gt)
def abs_to_hiprel(poses, joint_set):
""" Converts an absolute pose into [hi]+relative_pose. """
assert_shape(poses, (None, joint_set.NUM_JOINTS, 3))
if isinstance(poses, torch.Tensor):
root = poses[:, [joint_set.index_of('hip')]]
rel = remove_root(poses, joint_set.index_of('hip'))
return torch.cat([root, rel], dim=-2)
else:
root = poses[:, [joint_set.index_of('hip')]].copy()
rel = remove_root(poses, joint_set.index_of('hip'))
return np.concatenate([root, rel], axis=-2)
def create_prediction_viz_frames(gt, pred, valid, out_path, draw_func=None):
"""
Creates frames showing the GT and predicted poses.
Parameters:
gt: ndarray(nFrames, nPoses, 17, 3) - ground-truth 3D coordinates using MuPoTS joints
pred: ndarray(nFrames, nPoses, 17, 3) - predicted 3D coordinates using MuPoTS joints
valid: ndarray(nFrames, nPoses) - true if the given pose was detected
out_path: path to the frame, must have a %d format specifier for the image index.
draw_func: optional function that draws the 3D plot It receives the gt, predicted joints, valid joint index and a 3D matplotlib axes.
if None, it simply draws gt skeletons in dark, predicted skeletons light.
"""
assert_shape(gt, (None, None, 17, 3))
assert_shape(pred, (None, None, 17, 3))
assert gt.shape[:2] == valid.shape
assert gt.shape[:2] == pred.shape[:2]
plt.ioff()
RADIUS = 1500
plt.clf()
ax = get_3d_axes()
xroot, yroot, zroot = np.mean(gt[0][:, 14, :], axis=0)
bottom = np.max(gt[0][:, :, 1])
def default_drawer(gts, preds, valid, ax):
show3Dpose(gts,
MuPoTSJoints(),
ax=ax,
invert_vertical=True,
show_numbers=False,
lcolor="#911f1f",
rcolor="#874924",
ccolor="#1b4882")
for p in preds[valid]:
add3Dpose(p, ax, MuPoTSJoints())
ax.set_xlim3d([-RADIUS - 400 + xroot, RADIUS + xroot + 600])
ax.set_ylim3d([-RADIUS + zroot - 200, RADIUS + zroot + 100])
ax.set_zlim3d([bottom + 10, bottom - 2500])
if draw_func is None:
draw_func = default_drawer
for i in range(len(gt)):
plt.cla()
draw_func(gt[i], pred[i], valid[i], ax)
plt.savefig(out_path % i)
plt.ion()
def rn_mpjpe(pred, gt, root_ind):
"""
N-MPJPE, when optimal scaling factor is calculated on relative pose.
This hsould be a good comparison to height based scaling
Based on https://github.com/hrhodin/UnsupervisedGeometryAwareRepresentationLearning, losses/poses.py
"""
assert pred.shape == gt.shape
assert_shape(pred, ('*', None, 3))
s_opt = optimal_scaling(remove_root(pred, root_ind),
remove_root(gt, root_ind))
return mpjpe(pred * s_opt[..., np.newaxis, np.newaxis], gt)
def combine_pose_and_trans(data3d,
std3d,
mean3d,
joint_set,
root_name,
log_root_z=True): # TODO Torchify
"""
3D result postprocess: unnormalizes data3d and reconstructs the absolute pose from relative + absolute split.
Parameters:
data3d: output of the PyTorch model, ndarray(nPoses, 3*nJoints), in the format created by preprocess3d
std3d: normalization standard deviations
mean3d: normalization means
root_name: name of the root joint
log_root_z: The z coordinate of the depth is in logarithms
Returns:
ndarray(nPoses, nJoints, 3)
"""
assert_shape(data3d, (None, joint_set.NUM_JOINTS * 3))
if type(data3d) == torch.Tensor:
data3d = data3d * torch.from_numpy(std3d).cuda() + torch.from_numpy(
mean3d).cuda()
else:
data3d = data3d * std3d + mean3d
root = data3d[:, -3:] # (201, 3)
rel_pose = data3d[:, :-3].reshape(
(len(data3d), joint_set.NUM_JOINTS - 1, 3)) # (201, 16, 3)
if log_root_z:
if type(data3d) == torch.Tensor:
root[:, 2] = torch.exp(root[:, 2])
else:
root[:, 2] = np.exp(root[:, 2])
rel_pose += root[:, np.newaxis, :]
if type(data3d) == torch.Tensor:
result = torch.zeros((len(data3d), joint_set.NUM_JOINTS, 3)).cuda()
else:
result = np.zeros((len(data3d), joint_set.NUM_JOINTS, 3),
dtype='float32')
root_ind = joint_set.index_of(root_name)
result[:, :root_ind, :] = rel_pose[:, :root_ind, :]
result[:, root_ind, :] = root
result[:, root_ind + 1:, :] = rel_pose[:, root_ind:, :]
return result
def extend_hrnet_raw(raw):
"""
Adds the hip and neck to a Coco skeleton by averaging left/right hips and shoulders.
The score will be the harmonic mean of the two.
"""
assert_shape(raw, (None, 17, 3))
js = CocoExJoints()
result = np.zeros((len(raw), 19, 3), dtype='float32')
result[:, :17, :] = raw
_combine(result, js.index_of('hip'), js.index_of('left_hip'),
js.index_of('right_hip'))
_combine(result, js.index_of('neck'), js.index_of('left_shoulder'),
js.index_of('right_shoulder'))
return result
def show_result(image_path, poses):
assert_shape(poses, (None, MuPoTSJoints.NUM_JOINTS, 3))
# import here so it's not needed for prediction
import matplotlib.pyplot as plt
from util import viz
img = cv2.imread(image_path)
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
plt.figure(figsize=(9, 4.5))
plt.subplot(1, 2, 1)
plt.imshow(img)
ax = viz.subplot(1, 2, 2)
viz.show3Dpose(poses, MuPoTSJoints(), ax, invert_vertical=True)
plt.show()
def _calc_limb_length(poses, joint_set, bones):
"""
calculates the length of a limb that contains multiple bones.
:param bones: list of (joint1, joint2) pairs, where joint1 and joint2 determines the bone.
:return: For each pose, the sum of the lengths of the bones in `bones`
"""
assert_shape(poses, ('*', joint_set.NUM_JOINTS, 3))
bone_inds = [[joint_set.index_of(j) for j in b] for b in bones]
height = np.zeros(poses.shape[:-2], dtype='float32')
for bone in bone_inds:
bones = poses[..., bone[0], :] - poses[...,
bone[1], :] # (shapePose, 3)
bones = np.linalg.norm(bones, axis=-1) # (shapePose)
height += bones
return height
def preprocess_3d(data, add_hip):
"""
:param data:
:param add_hip: True if the absolute coordinates of the hip should be included in the output
:return:
"""
assert_shape(data, ("*", 17, 3))
hip3d = data[..., 14, :].copy()
hip3d[..., 2] = np.log(hip3d[..., 2])
data = remove_root(data, 14) # (nFrames[*nPoses], 16, 3)
data = data.reshape(data.shape[:-2] + (-1, )) # (nFrames[*nPoses], 16*3)
if add_hip:
data = np.concatenate([data, hip3d],
axis=-1) # (nFrames[*nPoses], 17*3)
return data.astype('float32')
def _scale_to_gt(pred_poses, gt_poses):
""" Scales bone lengths in pred_poses to match gt_poses. Corresponds to ``mpii_map_to_gt_bone_lengths.m``."""
assert_shape(pred_poses, (None, 17, 3))
assert_shape(gt_poses, (None, 17, 3))
rescaled_pred_poses = pred_poses.copy()
for ind in _TRAVERSAL_ORDER:
parent = _JOINT_PARENTS[ind]
gt_bone_length = np.linalg.norm(gt_poses[:, ind] - gt_poses[:, parent],
axis=1) # (nPoses,)
pred_bone = pred_poses[:, ind] - pred_poses[:, parent] # (nPoses, 3)
pred_bone = pred_bone * gt_bone_length[:, np.newaxis] / \
(np.linalg.norm(pred_bone, axis=1, keepdims=True) + 1e-8)
rescaled_pred_poses[:,
ind] = rescaled_pred_poses[:, parent] + pred_bone
return rescaled_pred_poses
def preprocess_2d(data, fx, cx, fy, cy):
assert_shape(data, ("*", 25, 3))
assert not isinstance(fx, np.ndarray) or len(fx) == len(data)
assert not isinstance(fy, np.ndarray) or len(fy) == len(data)
if isinstance(fx, np.ndarray):
N = len(data)
shape = [1] * (data.ndim - 1)
shape[0] = N
fx = fx.reshape(shape)
fy = fy.reshape(shape)
cx = cx.reshape(shape)
cy = cy.reshape(shape)
root_ind = 0 # 0-common14, 8-openpose
data = data[..., mupots_3d.OPENPOSE25_TO_COMMON2D14, :]
data[..., :, 0] -= cx
data[..., :, 1] -= cy
data[..., :, 0] /= fx
data[..., :, 1] /= fy
hip2d = data[..., root_ind, :].copy()
data = remove_openpose_root(data, root_ind) # (nPoses, 13, 3)
bad_frames = data[..., 2] < 0.1
# replace joints having low scores with 1700/focus
# this is to prevent leaking cx/cy
if isinstance(fx, np.ndarray):
fx = np.tile(fx, (1, ) + data.shape[1:-1])
fy = np.tile(fy, (1, ) + data.shape[1:-1])
data[bad_frames, 0] = -1700 / fx[bad_frames]
data[bad_frames, 1] = -1700 / fy[bad_frames]
else:
data[bad_frames, 0] = -1700 / fx
data[bad_frames, 1] = -1700 / fy
# stack hip next to pose
data = data.reshape(data.shape[:-2] + (-1, )) # (nPoses, 13*3)
data = np.concatenate([data, hip2d], axis=-1) # (nPoses, 14*3)
return data.astype('float32')
def optimal_scaling(pred, gt):
"""
Calculates optimal scaling factor for a given set of points. Optimal scaling is the scalar s,
with which the pred points scaled become the closest to gt points, in L2 sense.
:param pred: array(nFrames, nPoints, 3)
:param gt: array(nFrames, nPoints, 3)
:return: array(nFrames,3)
"""
assert pred.shape == gt.shape
assert_shape(pred, ('*', None, 3))
# Optimal scale transform
dot_pose_pose = np.sum(
pred * pred,
axis=(-1,
-2)) # (nShape) torch.sum(torch.mul(pred,pred),1,keepdim=True)
dot_pose_gt = np.sum(pred * gt, axis=(-1, -2))
return dot_pose_gt / dot_pose_pose # (nShape), the optimal scaling factor s
def procrustes_depth(coords2d, coords3d, focal_length, verbose=False, approximate=False):
"""
Absolute depth prediction based on Mehta et al. (https://arxiv.org/pdf/1611.09813.pdf) .
Parameters:
pose3d: ndarray(nJoints, 3[x,y,z), the relative 3D coordinates
pose2d: ndarray(nJoints, 3[x,y]), the 2D coordinates, relative to the centerpoint of the camera
focal_length: scalar, focus distance
approximate: if True, uses the formula in https://arxiv.org/pdf/1611.09813.pdf, otherwise uses the solution without
any approximation. The latter gives better results.
Returns:
ndarray(3,), the optimal translation vector
"""
assert len(coords2d) == len(coords3d)
assert coords2d.ndim == 2
assert coords3d.ndim == 2
assert coords3d.shape[1] == 3
coords3d = coords3d[:, :2]
mean2d = np.mean(coords2d, axis=0, keepdims=True)
mean3d = np.mean(coords3d, axis=0, keepdims=True)
assert_shape(mean2d, (1, 2))
assert_shape(mean3d, (1, 2))
# orig method using an approximation (does not provide any visible speedup)
if approximate:
numer = np.sqrt(np.sum(np.square(coords3d - mean3d)))
denom = np.sqrt(np.sum(np.square(coords2d - mean2d)))
else:
# no cos approximation
numer = np.sum(np.square(coords3d - mean3d))
denom = np.trace(np.dot((coords2d - mean2d), (coords3d - mean3d).T))
if verbose:
print "proc: %f / %f" % (numer, denom)
return numer / denom * np.array([mean2d[0, 0], mean2d[0, 1], focal_length]) - np.array([mean3d[0, 0], mean3d[0, 1], 0])
def eval_results(pred3d,
gt3d,
joint_set,
verbose=True,
pck_threshold=150,
pctiles=[99]):
"""
Evaluates the results by printing various statistics. Also returns those results.
Poses can be represented either in hipless 16 joints or 17 joints with hip format.
Order is MuPo-TS order in all cases.
Parameters:
pred3d: dictionary of predictions in mm, seqname -> (nSample, [16|17], 3)
gt3d: dictionary of ground truth in mm, seqname -> (nSample, [16|17], 3)
joint_set; JointSet instance describing the order of joints
verbose: if True, a table of the results is printed
pctiles: list of percentiles of the errors to calculate
Returns:
sequence_mpjpes, sequence_pcks, sequence_pctiles, joint_means, joint_pctiles
"""
has_hip = list(
pred3d.values()
)[0].shape[1] == joint_set.NUM_JOINTS # whether it contains the hip or not
if has_hip:
common14_joints = joint_set.TO_COMMON14
else:
common14_joints = np.array(
joint_set.TO_COMMON14[1:]).copy() # a copy of original list
hip_ind = joint_set.index_of('hip')
common14_joints[common14_joints > hip_ind] -= 1
sequence_mpjpes = {}
sequence_pcks = {}
sequence_aucs = {}
sequence_common14_pcks = {
} # pck for the common 14 joints (used by Mehta et al.)
sequence_pctiles = {}
all_errs = []
for k in sorted(pred3d.keys()):
pred = pred3d[k]
gt = gt3d[k]
assert pred.shape == gt.shape, "Pred shape:%s, gt shape:%s" % (
pred.shape, gt.shape)
assert (not has_hip and pred.shape[1:] == (joint_set.NUM_JOINTS - 1, 3)) or \
(has_hip and pred.shape[1:] == (joint_set.NUM_JOINTS, 3)), \
"Unexpected shape:" + str(pred.shape)
errs = np.linalg.norm(pred - gt, axis=2, ord=2) # (nSample, nJoints)
sequence_pctiles[k] = np.nanpercentile(errs, pctiles)
sequence_pcks[k] = np.nanmean(
(errs < pck_threshold).astype(np.float64))
pck_curve = []
for t in range(0, 151, 5): # go from 0 to 150, 150 inclusive
pck_curve.append(np.mean((errs < t).astype(np.float64)))
sequence_aucs[k] = np.mean(pck_curve)
sequence_common14_pcks[k] = np.nanmean(
(errs[:, common14_joints] < pck_threshold).astype(np.float64))
sequence_mpjpes[k] = np.nanmean(errs)
# Adjusting results for missing hip
if not has_hip:
N = float(joint_set.NUM_JOINTS)
sequence_pcks[k] = sequence_pcks[k] * ((N - 1) / N) + 1. / N
sequence_aucs[k] = sequence_aucs[k] * ((N - 1) / N) + 1. / N
sequence_common14_pcks[k] = sequence_common14_pcks[k] * (
(N - 1) / N) + 1. / N
sequence_mpjpes[k] = sequence_mpjpes[k] * ((N - 1) / N)
all_errs.append(errs)
all_errs = np.concatenate(all_errs) # errors per joint, (nPoses, nJoints)
joint_mpjpes = np.nanmean(all_errs, axis=0)
joint_pctiles = np.nanpercentile(all_errs, pctiles, axis=0)
num_joints = joint_set.NUM_JOINTS if has_hip else joint_set.NUM_JOINTS - 1
assert_shape(all_errs, (None, num_joints))
assert_shape(joint_mpjpes, (num_joints, ))
assert_shape(joint_pctiles, (len(pctiles), num_joints))
if verbose:
joint_names = joint_set.NAMES.copy()
if not has_hip:
joint_names = np.delete(joint_names,
joint_set.index_of('hip')) # remove root
# Index of the percentile that will be printed. If 99 is calculated it is selected,
# otherwise the last one
pctile_ind = len(pctiles) - 1
if 99 in pctiles:
pctile_ind = pctiles.index(99)
print(
" ----- Per sequence and joint errors in millimeter on the validation set ----- "
)
print(" %s %6s %5s %6s \t %22s %6s %6s" %
('Sequence', 'Avg', 'PCK', str(pctiles[pctile_ind]) + '%', '',
'Avg', str(pctiles[pctile_ind]) + '%'))
for seq, joint_id in zip_longest(sorted(pred3d.keys()),
range(num_joints)):
if seq is not None:
seq_str = " %-8s: %6.2f mm %4.1f%% %6.2f mm\t " \
% (str(seq), sequence_mpjpes[seq], sequence_pcks[seq] * 100, sequence_pctiles[seq][pctile_ind])
else:
seq_str = " " * 49
if joint_id is not None:
print('%s%15s (#%2d): %6.2f mm %6.2f mm ' %
(seq_str, joint_names[joint_id], joint_id,
joint_mpjpes[joint_id], joint_pctiles[pctile_ind,
joint_id]))
else:
print(seq_str)
mean_sequence_err = np.mean(list(sequence_mpjpes.values()))
print("\nMean absolute error: %6.2f mm" % mean_sequence_err)
return sequence_mpjpes, sequence_pcks, sequence_pctiles, joint_mpjpes, joint_pctiles
def eval_results(pred3d,
gt3d,
joint_set,
verbose=True,
pck_threshold=150,
pctiles=[99]):
"""
Evaluates the results by printing various statistics. Also returns those results.
Poses can be represented either in hipless 16 joints or 17 joints with hip format.
Order is MuPo-TS order in all cases.
Parameters:
pred3d: dictionary of predictions in mm, seqname -> (nSample, [16|17], 3)
gt3d: dictionary of ground truth in mm, seqname -> (nSample, [16|17], 3)
joint_set; JointSet instance describing the order of joints
verbose: if True, a table of the results is printed
pctiles: list of percentiles of the errors to calculate
Returns:
sequence_mpjpes, sequence_pcks, sequence_pctiles, joint_means, joint_pctiles
"""
has_hip = (list(pred3d.values())[0].shape[1] == joint_set.NUM_JOINTS
) # whether it contains the hip or not
sequence_mpjpes = {}
sequence_pcks = {}
sequence_pctiles = {}
all_errs = []
for k in sorted(pred3d.keys()):
pred = pred3d[k]
gt = gt3d[k]
assert pred.shape == gt.shape, "Pred shape:%s, gt shape:%s" % (
pred.shape,
gt.shape,
)
assert (not has_hip and pred.shape[1:] == (joint_set.NUM_JOINTS - 1, 3)
) or (has_hip and pred.shape[1:] == (joint_set.NUM_JOINTS, 3)
), "Unexpected shape:" + str(pred.shape)
errs = np.linalg.norm(pred - gt, axis=2, ord=2) # (nSample, nJoints)
sequence_pctiles[k] = np.nanpercentile(errs, pctiles)
sequence_pcks[k] = np.nanmean(
(errs < pck_threshold).astype(np.float64))
sequence_mpjpes[k] = np.nanmean(errs)
# Adjusting results for missing hip
if not has_hip:
N = float(joint_set.NUM_JOINTS)
sequence_pcks[k] = sequence_pcks[k] * ((N - 1) / N) + 1.0 / N
sequence_mpjpes[k] = sequence_mpjpes[k] * ((N - 1) / N)
all_errs.append(errs)
all_errs = np.concatenate(all_errs) # errors per joint, (nPoses, nJoints)
joint_mpjpes = np.nanmean(all_errs, axis=0)
joint_pctiles = np.nanpercentile(all_errs, pctiles, axis=0)
num_joints = joint_set.NUM_JOINTS if has_hip else joint_set.NUM_JOINTS - 1
assert_shape(all_errs, (None, num_joints))
assert_shape(joint_mpjpes, (num_joints, ))
assert_shape(joint_pctiles, (len(pctiles), num_joints))
if verbose:
joint_names = joint_set.NAMES.copy()
if not has_hip:
joint_names = np.delete(joint_names,
joint_set.index_of("hip")) # remove root
# Index of the percentile that will be printed. If 99 is calculated it is selected,
# otherwise the last one
pctile_ind = len(pctiles) - 1
if 99 in pctiles:
pctile_ind = pctiles.index(99)
print(
"----- Per sequence and joint errors in millimeter on the validation set ----- "
)
print("%s %6s %5s %6s \t %22s %6s %6s" % (
"Sequence",
"Avg",
"PCK",
str(pctiles[pctile_ind]) + "%",
"",
"Avg",
str(pctiles[pctile_ind]) + "%",
))
for seq, joint_id in zip_longest(sorted(pred3d.keys()),
range(num_joints)):
if seq is not None:
seq_str = "%-8s: %6.2f mm %4.1f%% %6.2f mm\t " % (
str(seq),
sequence_mpjpes[seq],
sequence_pcks[seq] * 100,
sequence_pctiles[seq][pctile_ind],
)
else:
seq_str = " " * 49
if joint_id is not None:
print("%s%15s (#%2d): %6.2f mm %6.2f mm " % (
seq_str,
joint_names[joint_id],
joint_id,
joint_mpjpes[joint_id],
joint_pctiles[pctile_ind, joint_id],
))
else:
print(seq_str)
mean_sequence_err = np.mean(
np.asarray(list(sequence_mpjpes.values()), dtype=np.float32))
print("\nMean sequence error (Absolute MPJPE) is %6.2f mm" %
mean_sequence_err)
print(
"---------------------------------------------------------------- "
)
print(
"MRPE: %.1f" %
np.mean([mrpe(pred3d[k], gt3d[k], joint_set)
for k in gt3d.keys()]))
return sequence_mpjpes, sequence_pcks, sequence_pctiles, joint_mpjpes, joint_pctiles
def mpjpe(pred, gt):
assert_shape(pred, ('*', None, 3))
assert pred.shape == gt.shape
return np.mean(np.linalg.norm(gt - pred, axis=-1))
