def sampleRandomPoses(importer,
rng,
base_poses,
base_com,
base_cube,
num_poses,
aug_modes,
retall=False,
rot3D=False):
"""
Sample random poses such that we can estimate the subspace more robustly
:param importer: importer
:param rng: RandomState
:param base_poses: set of base 3D poses
:param base_com: corresponding 3D crop locations
:param base_cube: corresponding crop cubes
:param num_poses: number of poses to sample
:param aug_modes: augmentation modes (comb, com, rot, sc, none)
:param retall: return all random parameters
:param rot3D: augment rotation in 3D, which is only possible with poses not images
:return: random poses
"""
all_modes = [
'none', 'rot', 'sc', 'com', 'rot+com', 'com+rot', 'rot+com+sc',
'rot+sc+com', 'sc+rot+com', 'sc+com+rot', 'com+sc+rot',
'com+rot+sc'
]
assert all([aug_modes[i] in all_modes for i in xrange(len(aug_modes))])
new_poses = numpy.zeros(
(int(num_poses), base_poses.shape[1], base_poses.shape[2]),
dtype=base_poses.dtype)
new_com = numpy.zeros((int(num_poses), 3), dtype=base_poses.dtype)
new_cube = numpy.zeros((int(num_poses), 3), dtype=base_poses.dtype)
modes = rng.randint(0, len(aug_modes), int(num_poses))
ridxs = rng.randint(0, base_poses.shape[0], int(num_poses))
base_com2D = importer.joints3DToImg(base_com)
off = rng.randn(int(num_poses), 3) * 7.
sc = numpy.fabs(rng.randn(int(num_poses)) * 0.1 + 1.)
rot = rng.uniform(0, 360, size=(int(num_poses), 3))
if aug_modes == ['none']:
if retall is True:
return base_poses / (base_cube[:, 2] /
2.)[:, None, None], base_com, base_cube
else:
return base_poses / (base_cube[:, 2] / 2.)[:, None, None]
for i in xrange(int(num_poses)):
mode = modes[i]
ridx = ridxs[i]
cube = base_cube[ridx]
com3D = base_com[ridx]
com = base_com2D[ridx]
pose = base_poses[ridx]
if aug_modes[mode] == 'com':
# augment com
new_com[i] = importer.jointImgTo3D(com + off[i])
new_cube[i] = cube
new_poses[i] = (pose + com3D - new_com[i]) / (new_cube[i][2] /
2.)
elif aug_modes[mode] == 'rot':
# augment rotation
new_com[i] = com3D
new_cube[i] = cube
if rot3D is False:
joint_2D = importer.joints3DToImg(pose + new_com[i])
data_2D = rotatePoints2D(joint_2D, com[0:2], rot[i, 0])
new_poses[i] = (importer.jointsImgTo3D(data_2D) -
new_com[i]) / (new_cube[i][2] / 2.)
else:
new_poses[i] = (rotatePoints3D(
pose + new_com[i], new_com[i], rot[i, 0], rot[i, 1],
rot[i, 2]) - new_com[i]) / (new_cube[i][2] / 2.)
elif aug_modes[mode] == 'sc':
# augment cube
new_com[i] = com3D
new_cube[i] = cube * sc[i]
new_poses[i] = pose / (new_cube[i][2] / 2.)
elif aug_modes[mode] == 'none':
# no augmentation
new_com[i] = com3D
new_cube[i] = cube
new_poses[i] = pose / (new_cube[i][2] / 2.)
elif aug_modes[mode] == 'rot+com' or aug_modes[mode] == 'com+rot':
# augment com+rot
new_com[i] = importer.jointImgTo3D(com + off[i])
new_cube[i] = cube
pose = (pose + com3D - new_com[i])
if rot3D is False:
joint_2D = importer.joints3DToImg(pose + com3D)
data_2D = rotatePoints2D(
joint_2D,
importer.joint3DToImg(new_com[i])[0:2], rot[i, 0])
new_poses[i] = (importer.jointsImgTo3D(data_2D) -
com3D) / (new_cube[i][2] / 2.)
else:
new_poses[i] = (rotatePoints3D(
pose + new_com[i], new_com[i], rot[i, 0], rot[i, 1],
rot[i, 2]) - new_com[i]) / (new_cube[i][2] / 2.)
elif aug_modes[mode] == 'rot+com+sc' or aug_modes[
mode] == 'rot+sc+com' or aug_modes == 'sc+rot+com' or aug_modes == 'sc+com+rot' or aug_modes == 'com+sc+rot' or aug_modes == 'com+rot+sc':
# augment com+scale+rot
new_com[i] = importer.jointImgTo3D(com + off[i])
new_cube[i] = cube
pose = (pose + com3D - new_com[i])
pose = pose * sc[i]
if rot3D is False:
joint_2D = importer.joints3DToImg(pose + com3D)
data_2D = rotatePoints2D(
joint_2D,
importer.joint3DToImg(new_com[i])[0:2], rot[i, 0])
new_poses[i] = (importer.jointsImgTo3D(data_2D) -
com3D) / (new_cube[i][2] / 2.)
else:
new_poses[i] = (rotatePoints3D(
pose + new_com[i], new_com[i], rot[i, 0], rot[i, 1],
rot[i, 2]) - new_com[i]) / (new_cube[i][2] / 2.)
else:
raise NotImplementedError()
if retall is True:
return new_poses, new_com, new_cube, rot
else:
return new_poses
def show(self, frame, handpose):
"""
Show depth with overlaid joints
:param frame: depth frame
:param handpose: joint positions
:return: image
"""
upsample = 1.
if 'upsample' in self.sync['config']:
upsample = self.sync['config']['upsample']
# plot depth image with annotations
imgcopy = frame.copy()
# display hack to hide nd depth
msk = numpy.logical_and(32001 > imgcopy, imgcopy > 0)
msk2 = numpy.logical_or(imgcopy == 0, imgcopy == 32001)
min = imgcopy[msk].min()
max = imgcopy[msk].max()
imgcopy = (imgcopy - min) / (max - min) * 255.
imgcopy[msk2] = 255.
imgcopy = imgcopy.astype('uint8')
imgcopy = cv2.cvtColor(imgcopy, cv2.COLOR_GRAY2BGR)
if not numpy.allclose(upsample, 1):
imgcopy = cv2.resize(imgcopy,
dsize=None,
fx=upsample,
fy=upsample,
interpolation=cv2.INTER_LINEAR)
if handpose.shape[0] == 16:
hpe = ICVLHandposeEvaluation(numpy.zeros((3, 3)),
numpy.zeros((3, 3)))
elif handpose.shape[0] == 14:
hpe = NYUHandposeEvaluation(numpy.zeros((3, 3)), numpy.zeros(
(3, 3)))
elif handpose.shape[0] == 21:
hpe = MSRAHandposeEvaluation(numpy.zeros((3, 3)),
numpy.zeros((3, 3)))
else:
raise ValueError("Invalid number of joints {}".format(
handpose.shape[0]))
jtI = self.importer.joints3DToImg(handpose)
jtI[:,
0:2] -= numpy.asarray([frame.shape[0] // 2, frame.shape[1] // 2])
jtI[:, 0:2] *= upsample
jtI[:, 0:2] += numpy.asarray(
[imgcopy.shape[0] // 2, imgcopy.shape[1] // 2])
for i in range(handpose.shape[0]):
cv2.circle(imgcopy, (jtI[i, 0], jtI[i, 1]), 3, (255, 0, 0), -1)
for i in range(len(hpe.jointConnections)):
cv2.line(imgcopy, (jtI[hpe.jointConnections[i][0],
0], jtI[hpe.jointConnections[i][0], 1]),
(jtI[hpe.jointConnections[i][1],
0], jtI[hpe.jointConnections[i][1], 1]),
255. * hpe.jointConnectionColors[i], 2)
# comI = self.importer.joint3DToImg(com3D)
# comI[0:2] -= numpy.asarray([frame.shape[0]//2, frame.shape[1]//2])
# comI[0:2] *= upsample
# comI[0:2] += numpy.asarray([imgcopy.shape[0]//2, imgcopy.shape[1]//2])
# cv2.circle(imgcopy, (comI[0], comI[1]), 3, (0, 255, 0), 1)
poseimg = numpy.zeros_like(imgcopy)
# rotate 3D pose and project to 2D
jtP = self.importer.joints3DToImg(
rotatePoints3D(handpose, handpose[self.importer.crop_joint_idx],
0., 90., 0.))
jtP[:,
0:2] -= numpy.asarray([frame.shape[0] // 2, frame.shape[1] // 2])
jtP[:, 0:2] *= upsample
jtP[:, 0:2] += numpy.asarray(
[imgcopy.shape[0] // 2, imgcopy.shape[1] // 2])
for i in range(handpose.shape[0]):
cv2.circle(poseimg, (jtP[i, 0], jtP[i, 1]), 3, (255, 0, 0), -1)
for i in range(len(hpe.jointConnections)):
cv2.line(poseimg, (jtP[hpe.jointConnections[i][0],
0], jtP[hpe.jointConnections[i][0], 1]),
(jtP[hpe.jointConnections[i][1],
0], jtP[hpe.jointConnections[i][1], 1]),
255. * hpe.jointConnectionColors[i], 2)
# comP = self.importer.joint3DToImg(rotatePoint3D(com3D, handpose[self.importer.crop_joint_idx], 0., 90., 0.))
# comP[0:2] -= numpy.asarray([frame.shape[0]//2, frame.shape[1]//2])
# comP[0:2] *= upsample
# comP[0:2] += numpy.asarray([imgcopy.shape[0]//2, imgcopy.shape[1]//2])
# cv2.circle(poseimg, (comP[0], comP[1]), 3, (0, 255, 0), 1)
return imgcopy, poseimg