def doHandPoseEstimate(image_cv, sess, args):
image_tf = args['image_tf']
hand_side_tf = args['hand_side_tf']
evaluation = args['evaluation']
net = args['net']
hand_scoremap_tf = args['hand_scoremap_tf']
image_crop_tf = args['image_crop_tf']
scale_tf = args['scale_tf']
center_tf = args['center_tf']
keypoints_scoremap_tf = args['keypoints_scoremap_tf']
keypoint_coord3d_tf = args['keypoint_coord3d_tf']
image_raw = image_cv[:, :, ::-1]
image_raw = cv2.resize(image_raw, (320, 240))
image_v = np.expand_dims((image_raw.astype('float') / 255.0) - 0.5, 0)
hand_scoremap_v, image_crop_v, scale_v, center_v,\
keypoints_scoremap_v, keypoint_coord3d_v = sess.run([hand_scoremap_tf, image_crop_tf, scale_tf, center_tf,
keypoints_scoremap_tf, keypoint_coord3d_tf],
feed_dict={image_tf: image_v})
hand_scoremap_v = np.squeeze(hand_scoremap_v)
image_crop_v = np.squeeze(image_crop_v)
keypoints_scoremap_v = np.squeeze(keypoints_scoremap_v)
keypoint_coord3d_v = np.squeeze(keypoint_coord3d_v)
# post processing
image_crop_v = ((image_crop_v + 0.5) * 255).astype('uint8')
coord_hw_crop = detect_keypoints(np.squeeze(keypoints_scoremap_v))
coord_hw = trafo_coords(coord_hw_crop, center_v, scale_v, 256)
# visualize
fig = plt.figure(1)
plt.ion()
plt.clf()
ax1 = fig.add_subplot(221)
ax2 = fig.add_subplot(222)
ax3 = fig.add_subplot(223)
ax4 = fig.add_subplot(224, projection='3d')
ax1.imshow(image_raw)
plot_hand(coord_hw, ax1)
ax2.imshow(image_crop_v)
plot_hand(coord_hw_crop, ax2)
ax3.imshow(np.argmax(hand_scoremap_v, 2))
plot_hand_3d(keypoint_coord3d_v, ax4)
ax4.view_init(azim=-90.0,
elev=-90.0) # aligns the 3d coord with the camera view
ax4.set_xlim([-3, 3])
ax4.set_ylim([-3, 1])
ax4.set_zlim([-3, 3])
plt.show()
plt.pause(0.0001)
plt.show()
def visualize(heatmap_3d, image_crop):
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from utils.general import detect_keypoints_3d, plot_hand_3d
heatmap_3d = np.squeeze(heatmap_3d)
image_crop = np.squeeze(255 * (image_crop + 0.5)).astype(np.uint8)
fig = plt.figure()
ax = fig.add_subplot(121)
ax.imshow(image_crop[0])
ax = fig.add_subplot(122, projection='3d')
keypoints = detect_keypoints_3d(heatmap_3d[0]).astype(np.float32)
plot_hand_3d(keypoints, ax)
plt.show()
def get_coords_and_figure_from_name(img_name):
image_raw = scipy.misc.imread(img_name)
hand_scoremap_v, image_crop_v, scale_v, center_v, \
keypoints_scoremap_v, keypoint_coord3d_v = run_model_on_image(image_raw)
# from here on: saving stuff
basename = os.path.splitext(os.path.basename(img_name))[0]
# post processing
hand_scoremap_v = np.squeeze(hand_scoremap_v)
image_crop_v = np.squeeze(image_crop_v)
keypoints_scoremap_v = np.squeeze(keypoints_scoremap_v)
image_crop_v = ((image_crop_v + 0.5) * 255).astype('uint8')
coord_hw_crop = detect_keypoints(np.squeeze(keypoints_scoremap_v))
coord_hw = trafo_coords(coord_hw_crop, center_v, scale_v, 256)
# save keypoint coordinates
keypoint_save_filename = "{:s}/{:s}_coords.pkl".format(
output_coords_dir, basename)
with open(keypoint_save_filename, 'wb') as f:
pickle.dump((keypoint_coord3d_v, scale_v, center_v), f, protocol=-1)
print("Saved keypoint coordinates to {:s}".format(keypoint_save_filename))
# save image
image_save_filename = "{:s}/{:s}_figures.png".format(
output_figures_dir, basename)
fig = plt.figure(1)
ax1 = fig.add_subplot(221)
ax2 = fig.add_subplot(222)
ax3 = fig.add_subplot(223)
ax4 = fig.add_subplot(224, projection='3d')
ax1.imshow(image_raw)
plot_hand(coord_hw, ax1)
ax2.imshow(image_crop_v)
plot_hand(coord_hw_crop, ax2)
ax3.imshow(np.argmax(hand_scoremap_v, 2))
plot_hand_3d(keypoint_coord3d_v, ax4)
ax4.view_init(azim=-90.0,
elev=-90.0) # aligns the 3d coord with the camera view
ax4.set_xlim([-3, 3])
ax4.set_ylim([-3, 1])
ax4.set_zlim([-3, 3])
plt.savefig(image_save_filename)
plt.close()
print("Saved figure to {:s}".format(image_save_filename))
def visualize(scoremap, hand_side, rot_mat, coord3d_can, coord3d, coord2d):
import pdb
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from utils.general import plot_hand_3d, plot_hand
import numpy as np
l = scoremap.shape[0]
for i in range(l):
l_scoremap = scoremap[i, :, :, :]
l_coord3d = coord3d[i, :, :]
l_coord3d_can = coord3d_can[i, :, :] * 6.0
l_rot_mat = rot_mat[i, :, :]
l_coord2d = coord2d[i, :, :]
if hand_side[i, 1] == 1:
print('flip')
# l_coord3d_can[:, 2] = -l_coord3d_can[:, 2]
# l_coord3d_rotate = np.dot(l_coord3d_can, l_rot_mat)
l_coord3d -= l_coord3d[0, :]
# l_coord3d_rotate *= 2.0
s = l_scoremap.shape
keypoint_coords = np.zeros((s[2], 2))
for i in range(s[2]):
v, u = np.unravel_index(np.argmax(l_scoremap[:, :, i]),
(s[0], s[1]))
keypoint_coords[i, 0] = v
keypoint_coords[i, 1] = u
fig = plt.figure(1)
ax1 = fig.add_subplot(131)
ax1.imshow(np.amax(l_scoremap, axis=2))
ax2 = fig.add_subplot(132, projection='3d')
plot_hand_3d(l_coord3d, ax2, color_fixed=np.array([1.0, 0.0, 1.0]))
plot_hand_3d(l_coord3d_can, ax2, color_fixed=np.array([0.0, 1.0, 0.0]))
ax2.view_init(azim=-90.0,
elev=-90.0) # aligns the 3d coord with the camera view
ax3 = fig.add_subplot(133)
plot_hand(keypoint_coords, ax3, color_fixed=np.array([0.0, 1.0, 0.0]))
plt.gca().invert_yaxis()
plt.xlabel('x')
plt.ylabel('y')
plt.show()
feed_dict={image_tf: image_v})
hand_scoremap_v = np.squeeze(hand_scoremap_v)
image_crop_v = np.squeeze(image_crop_v)
keypoints_scoremap_v = np.squeeze(keypoints_scoremap_v)
keypoint_coord3d_v = np.squeeze(keypoint_coord3d_v)
# post processing
image_crop_v = ((image_crop_v + 0.5) * 255).astype('uint8')
coord_hw_crop = detect_keypoints(np.squeeze(keypoints_scoremap_v))
coord_hw = trafo_coords(coord_hw_crop, center_v, scale_v, 256)
# visualize
fig = plt.figure(1)
ax1 = fig.add_subplot(221)
ax2 = fig.add_subplot(222)
ax3 = fig.add_subplot(223)
ax4 = fig.add_subplot(224, projection='3d')
ax1.imshow(image_raw)
plot_hand(coord_hw, ax1)
ax2.imshow(image_crop_v)
plot_hand(coord_hw_crop, ax2)
ax3.imshow(np.argmax(hand_scoremap_v, 2))
plot_hand_3d(keypoint_coord3d_v, ax4)
ax4.view_init(azim=-90.0,
elev=-90.0) # aligns the 3d coord with the camera view
ax4.set_xlim([-3, 3])
ax4.set_ylim([-3, 1])
ax4.set_zlim([-3, 3])
plt.show()
scoremap_v[:, :, ik:ik + 4] = scoremap_v[:, :, ik + 3:ik - 1:-1]
coord2d_v = detect_keypoints(scoremap_v) * 8
# center gt
keypoint_xyz21 -= keypoint_xyz21[0, :]
if (i % 100) == 0:
print('%d / %d images done: %.3f percent' %
(i, dataset.num_samples, i * 100.0 / dataset.num_samples))
if args.visualize:
fig = plt.figure(1)
ax1 = fig.add_subplot(121, projection='3d')
plot_hand_3d(keypoint_xyz21,
ax1,
color_fixed=np.array([1.0, 0.0, 0.0]))
ax1.view_init(
azim=-90.0,
elev=-90.0) # aligns the 3d coord with the camera view
plt.xlabel('x')
plt.ylabel('y')
ax1.set_xlim(-0.1, 0.1)
ax1.set_ylim(-0.1, 0.1)
ax1.set_zlim(-0.1, 0.1)
ax2 = fig.add_subplot(122)
plt.imshow(image_crop_v)
plot_hand(coord2d_v, ax2)
plt.show()
scoremap_3d, keypoint_xyz21_normed, image_crop, keypoint_uv21, img_dir, scoremap, hand_side \
= sess.run([data['scoremap_3d'], data['keypoint_xyz21_normed'], data['image_crop'], data['keypoint_uv21'], data['img_dir'], data['scoremap'], data['hand_side']])
print(img_dir[0].decode())
print(hand_side)
scoremap_3d = np.squeeze(scoremap_3d)
keypoint_xyz21_normed = np.squeeze(keypoint_xyz21_normed)
image_crop = np.squeeze((image_crop + 0.5) * 255).astype(np.uint8)
keypoint_uv21 = np.squeeze(keypoint_uv21)
scoremap = np.squeeze(scoremap)
keypoints = detect_keypoints_3d(scoremap_3d)
keypoints2d = detect_keypoints(scoremap)
fig = plt.figure()
ax = fig.add_subplot(221, projection='3d')
plot_hand_3d(keypoints, ax)
ax.invert_yaxis()
ax.invert_zaxis()
ax = fig.add_subplot(222, projection='3d')
plot_hand_3d(keypoint_xyz21_normed, ax)
ax.invert_yaxis()
ax.invert_zaxis()
ax = fig.add_subplot(223)
ax.imshow(image_crop)
plot_hand(keypoint_uv21[:, ::-1], ax)
ax = fig.add_subplot(224)
ax.imshow(image_crop)
plot_hand(keypoints2d, ax)
plt.show()
coord3d_pred_v *= keypoint_scale
# center gt
keypoint_xyz21 -= keypoint_xyz21[0, :]
util.feed(keypoint_xyz21, keypoint_vis21, coord3d_pred_v)
if (i % 100) == 0:
print('%d / %d images done: %.3f percent' %
(i, dataset.num_samples, i * 100.0 / dataset.num_samples))
if args.visualize:
fig = plt.figure(1)
ax1 = fig.add_subplot(121, projection='3d')
plot_hand_3d(coord3d_pred_v,
ax1,
color_fixed=np.array([0.0, 0.0, 1.0]))
plot_hand_3d(keypoint_xyz21,
ax1,
color_fixed=np.array([1.0, 0.0, 0.0]))
ax1.view_init(
azim=-90.0,
elev=-90.0) # aligns the 3d coord with the camera view
plt.xlabel('x')
plt.ylabel('y')
ax2 = fig.add_subplot(122)
plt.imshow(image_scaled_v)
plt.show()
# pdb.set_trace()
scoremap_v = np.squeeze(scoremap_v)
key = key[0].decode()
coord3d_pred_v[0, :] = 2 * coord3d_pred_v[0, :] - coord3d_pred_v[12, :]
coord3d_pred_v -= coord3d_pred_v[0, :]
filename = 'handTest_' + key + '_rendered.png'
img = os.path.join(input_dir, filename)
print(img)
img = cv2.imread(img)
assert img is not None
fig = plt.figure(filename)
ax1 = fig.add_subplot(111, projection='3d')
plot_hand_3d(coord3d_pred_v * 15,
ax1,
color_fixed=np.array([1.0, 0.0, 0.0]))
plt.xlabel('x')
plt.ylabel('y')
ax1.view_init(azim=-90.0, elev=-70.0)
ax1.set_xlim(-3., 3.)
ax1.set_ylim(-3., 3.)
ax1.set_zlim(-3., 3.)
# ax2 = fig.add_subplot(122)
# keypoints2d = detect_keypoints(scoremap_v)
# plot_hand(keypoints2d, ax2, color_fixed=np.array([1.0, 0.0, 0.0]))
# ax2.set_xlim(0.0, 256.0)
# ax2.set_ylim(0.0, 256.0)
# ax2.invert_yaxis()
# plt.show()
scoremap_3d_v = np.squeeze(scoremap_3d_v)
coord3d_pred_v = detect_keypoints_3d(scoremap_3d_v)
coord3d_pred_v = coord3d_pred_v[:21, :]
coord3d_pred_v -= coord3d_pred_v[0, :]
coord3d_pred_v /= hand_size(coord3d_pred_v)
if (i % 100) == 0:
print('%d / %d images done: %.3f percent' %
(i, dataset.num_samples, i * 100.0 / dataset.num_samples))
if args.visualize:
fig = plt.figure(1)
ax1 = fig.add_subplot(121, projection='3d')
plot_hand_3d(coord3d_pred_v,
ax1,
color_fixed=np.array([0.0, 0.0, 1.0]))
ax1.view_init(
azim=-90.0,
elev=-90.0) # aligns the 3d coord with the camera view
plt.xlabel('x')
plt.ylabel('y')
ax1.set_xlim(-0.1, 0.1)
ax1.set_ylim(-0.1, 0.1)
ax1.set_zlim(-0.1, 0.1)
ax2 = fig.add_subplot(122)
plt.imshow(image_crop_v)
plot_hand(coord2d_v, ax2, color_fixed=np.array([0.0, 0.0, 1.0]))
plot_hand(keypoint_uv21[:, ::-1],
ax2,
def get_pic(image_list):
# images to be shown
# image_list = list()
# image_list.append('./data/img30.jpg')
#image_list.append('./data/img31.jpg')
#image_list.append('./data/img32.jpg')
#image_list.append('./data/img33.jpg')
# network input
image_tf = tf.placeholder(tf.float32, shape=(1, 320, 240, 3))
hand_side_tf = tf.constant([[1.0, 0.0]]) # left hand (true for all samples provided)
evaluation = tf.placeholder_with_default(True, shape=())
# build network
net = ColorHandPose3DNetwork()
hand_scoremap_tf, image_crop_tf, scale_tf, center_tf,\
keypoints_scoremap_tf, keypoint_coord3d_tf = net.inference(image_tf, hand_side_tf, evaluation)
# Start TF
# gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
sess = tf.Session()#config=tf.ConfigProto(gpu_options=gpu_options))
# initialize network
net.init(sess)
# Feed image list through network
final = list()
for i, image in tqdm(enumerate(image_list)):
image_raw = image
# image_raw = cv2.resize(image_raw, dsize=(240, 320))#, interpolation=cv2.INTER_CUBI)
image_raw = make_it_small.small(image_raw)
for row in image_raw:
for pixel in row:
temp = pixel[0]
pixel[0] = pixel[2]
pixel[2] = temp
image_v = np.expand_dims((image_raw.astype('float') / 255.0) - 0.5, 0)
hand_scoremap_v, image_crop_v, scale_v, center_v,\
keypoints_scoremap_v, keypoint_coord3d_v = sess.run([hand_scoremap_tf, image_crop_tf, scale_tf, center_tf,
keypoints_scoremap_tf, keypoint_coord3d_tf],
feed_dict={image_tf: image_v})
img_angle = description_of_hand_position(keypoint_coord3d_v)
hand_scoremap_v = np.squeeze(hand_scoremap_v)
image_crop_v = np.squeeze(image_crop_v)
keypoints_scoremap_v = np.squeeze(keypoints_scoremap_v)
keypoint_coord3d_v = np.squeeze(keypoint_coord3d_v)
# post processing
image_crop_v = ((image_crop_v + 0.5) * 255).astype('uint8')
coord_hw_crop = detect_keypoints(np.squeeze(keypoints_scoremap_v))
coord_hw = trafo_coords(coord_hw_crop, center_v, scale_v, 256)
# visualize
fig = plt.figure(1)
ax1 = fig.add_subplot(221)
ax2 = fig.add_subplot(222)
ax3 = fig.add_subplot(223)
ax4 = fig.add_subplot(224, projection='3d')
ax1.imshow(image_raw)
plot_hand(coord_hw, ax1)
ax2.imshow(image_crop_v)
plot_hand(coord_hw_crop, ax2)
ax3.imshow(np.argmax(hand_scoremap_v, 2))
plot_hand_3d(keypoint_coord3d_v, ax4)
ax4.view_init(azim=-90.0, elev=-90.0) # aligns the 3d coord with the camera view
ax4.set_xlim([-3, 3])
ax4.set_ylim([-3, 1])
ax4.set_zlim([-3, 3])
fig.suptitle(img_angle,fontsize = 10)
plt.savefig("imgs/{}.png".format(str(i)))
plt.close(fig)
img = imageio.imread("imgs/{}.png".format(str(i)))
final.append(img)
return final
