def post_process():
# global joints_2d, joints_3d
joints_2d = np.zeros(shape=(args.num_of_joints, 2), dtype=np.int32)
joints_3d = np.zeros(shape=(args.num_of_joints, 3), dtype=np.float32)
hm_size = args.input_size // args.pool_scale
hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
x_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
y_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
z_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
cam_img = np.zeros(shape=(args.input_size, args.input_size, 3),
dtype=np.uint8)
while True:
if not model_post_q.empty():
[hm_avg, x_hm_avg, y_hm_avg, z_hm_avg,
cam_img] = model_post_q.get(False)
# print('post get')
t1 = time.time()
# Get 2d joints
joints_2d = utils.extract_2d_joint_from_heatmap(
hm_avg, args.input_size, joints_2d)
# Get 3d joints
if args.plot_3d:
joints_3d = utils.extract_3d_joints_from_heatmap(
joints_2d, x_hm_avg, y_hm_avg, z_hm_avg, args.input_size,
joints_3d)
print('Post FPS', 1 / (time.time() - t1))
if not post_render_q.full():
post_render_q.put([joints_2d, joints_3d, cam_img])
def post_process():
# global joints_2d, joints_3d
joints_2d = np.zeros(shape=(args.num_of_joints, 2), dtype=np.int32)
joints_3d = np.zeros(shape=(args.num_of_joints, 3), dtype=np.float32)
hm_size = args.input_size // args.pool_scale
hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
x_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
y_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
z_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
cam_img = np.zeros(shape=(args.input_size, args.input_size, 3), dtype=np.uint8)
while True:
if not model_post_q.empty():
[hm_avg, x_hm_avg, y_hm_avg, z_hm_avg, cam_img] = model_post_q.get(False)
# print('post get')
t1 = time.time()
# Get 2d joints
joints_2d = utils.extract_2d_joint_from_heatmap(hm_avg, args.input_size, joints_2d)
# Get 3d joints
if args.plot_3d:
joints_3d = utils.extract_3d_joints_from_heatmap(joints_2d, x_hm_avg, y_hm_avg, z_hm_avg, args.input_size,
joints_3d)
print('Post FPS', 1/(time.time()-t1))
if not post_render_q.full():
post_render_q.put([joints_2d, joints_3d, cam_img])
def __call__(self, img_input):
t = time.time()
img_input = np.transpose(
img_input, axes=[1, 0, 2]).copy() if self.T else img_input
img_batch = self._gen_input_batch(img_input, self._box_size,
self.scales)
# inference
hm, xm, ym, zm = self.sess.run(
[self.heatmap, self.x_heatmap, self.y_heatmap, self.z_heatmap],
{self.input_crops: img_batch})
# average scale outputs
hm_size = self._box_size // self._hm_factor
hm_avg = np.zeros((hm_size, hm_size, self._joints_num))
xm_avg = np.zeros((hm_size, hm_size, self._joints_num))
ym_avg = np.zeros((hm_size, hm_size, self._joints_num))
zm_avg = np.zeros((hm_size, hm_size, self._joints_num))
for i in range(len(self.scales)):
rescale = 1.0 / self.scales[i]
scaled_hm = utils.img_scale(hm[i, :, :, :], rescale)
scaled_x_hm = utils.img_scale(xm[i, :, :, :], rescale)
scaled_y_hm = utils.img_scale(ym[i, :, :, :], rescale)
scaled_z_hm = utils.img_scale(zm[i, :, :, :], rescale)
mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2]
hm_avg += scaled_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] + hm_size // 2, :]
xm_avg += scaled_x_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] +
hm_size // 2, :]
ym_avg += scaled_y_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] +
hm_size // 2, :]
zm_avg += scaled_z_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] +
hm_size // 2, :]
hm_avg /= len(self.scales)
xm_avg /= len(self.scales)
ym_avg /= len(self.scales)
zm_avg /= len(self.scales)
joints_2d = utils.extract_2d_joints_from_heatmap(
hm_avg, self._box_size, self._hm_factor)
joints_3d = utils.extract_3d_joints_from_heatmap(
joints_2d, xm_avg, ym_avg, zm_avg, self._box_size, self._hm_factor)
joints_2d, joints_3d = self._joint_filter(joints_2d, joints_3d)
# if self.T:
# joints_2d = joints_2d[:, ::-1]
# joints_3d = joints_3d[:, [1, 0, 2]]
print('FPS: {:>2.2f}'.format(1 / (time.time() - t)))
if self.plot:
# 2d plotting
frame_square = utils.img_scale_squareify(img_input, self._box_size)
frame_square = utils.draw_limbs_2d(frame_square, joints_2d,
self._joint_parents)
cv2.imshow('2D Prediction', frame_square)
# 3d plotting
self.imshow_3d(self.ax_3d, joints_3d, self._joint_parents)
return joints_2d, joints_3d
def __call__(self, img_input):
t = time.time()
img_batch = self._gen_input_batch(img_input, self._box_size,
self.scales)
# inference
hm, xm, ym, zm = self.sess.run(
[self.heatmap, self.x_heatmap, self.y_heatmap, self.z_heatmap],
{self.input_crops: img_batch})
# average scale outputs
hm_size = self._box_size // self._hm_factor
hm_avg = np.zeros((hm_size, hm_size, self._joints_num))
xm_avg = np.zeros((hm_size, hm_size, self._joints_num))
ym_avg = np.zeros((hm_size, hm_size, self._joints_num))
zm_avg = np.zeros((hm_size, hm_size, self._joints_num))
for i in range(len(self.scales)):
rescale = 1.0 / self.scales[i]
scaled_hm = utils.img_scale(hm[i, :, :, :], rescale)
scaled_x_hm = utils.img_scale(xm[i, :, :, :], rescale)
scaled_y_hm = utils.img_scale(ym[i, :, :, :], rescale)
scaled_z_hm = utils.img_scale(zm[i, :, :, :], rescale)
mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2]
hm_avg += scaled_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] + hm_size // 2, :]
xm_avg += scaled_x_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] +
hm_size // 2, :]
ym_avg += scaled_y_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] +
hm_size // 2, :]
zm_avg += scaled_z_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] +
hm_size // 2, :]
hm_avg /= len(self.scales)
xm_avg /= len(self.scales)
ym_avg /= len(self.scales)
zm_avg /= len(self.scales)
joints_2d = utils.extract_2d_joints_from_heatmap(
hm_avg, self._box_size, self._hm_factor)
joints_3d = utils.extract_3d_joints_from_heatmap(
joints_2d, xm_avg, ym_avg, zm_avg, self._box_size, self._hm_factor)
joints_2d, joints_3d = self._joint_filter(joints_2d, joints_3d)
print('FPS: {:>2.2f}'.format(1 / (time.time() - t)))
return joints_2d, joints_3d
def _run_net(self):
# inference
hm, xm, ym, zm = self.sess.run(
[self.heatmap, self.x_heatmap, self.y_heatmap, self.z_heatmap],
{self.input_crops: self.input_batch})
# average scale outputs
hm_size = self.box_size // self.hm_factor
hm_avg = np.zeros((hm_size, hm_size, self.joints_num))
xm_avg = np.zeros((hm_size, hm_size, self.joints_num))
ym_avg = np.zeros((hm_size, hm_size, self.joints_num))
zm_avg = np.zeros((hm_size, hm_size, self.joints_num))
for i in range(len(self.scales)):
rescale = 1.0 / self.scales[i]
scaled_hm = utils.img_scale(hm[i, :, :, :], rescale)
scaled_x_hm = utils.img_scale(xm[i, :, :, :], rescale)
scaled_y_hm = utils.img_scale(ym[i, :, :, :], rescale)
scaled_z_hm = utils.img_scale(zm[i, :, :, :], rescale)
mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2]
hm_avg += scaled_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] + hm_size // 2, :]
xm_avg += scaled_x_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] +
hm_size // 2, :]
ym_avg += scaled_y_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] +
hm_size // 2, :]
zm_avg += scaled_z_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] +
hm_size // 2, :]
hm_avg /= len(self.scales)
xm_avg /= len(self.scales)
ym_avg /= len(self.scales)
zm_avg /= len(self.scales)
self.joints_2d = utils.extract_2d_joints_from_heatmap(
hm_avg, self.box_size, self.hm_factor)
self.joints_3d = utils.extract_3d_joints_from_heatmap(
self.joints_2d, xm_avg, ym_avg, zm_avg, self.box_size,
self.hm_factor)
def demo():
joints_2d = np.zeros(shape=(args.num_of_joints, 2), dtype=np.int32)
joints_3d = np.zeros(shape=(args.num_of_joints, 3), dtype=np.float32)
if args.plot_3d:
plt.ion()
fig = plt.figure()
ax = fig.add_subplot(121, projection='3d')
ax2 = fig.add_subplot(122)
plt.show()
if args.device == 'cpu':
caffe.set_mode_cpu()
elif args.device == 'gpu':
caffe.set_mode_gpu()
caffe.set_device(0)
else:
raise ValueError('No such device')
model_prototxt_path = os.path.join(args.model_dir, 'vnect_net.prototxt')
model_weight_path = os.path.join(args.model_dir, 'vnect_model.caffemodel')
# Load model
model = caffe.Net(model_prototxt_path, model_weight_path, caffe.TEST)
# Show network structure and shape
for layer_name in model.params.keys():
print(layer_name, model.params[layer_name][0].data.shape)
print('')
for i in model.blobs.keys():
print(i, model.blobs[i].data.shape)
cam = cv2.VideoCapture(0)
is_tracking = False
# for img_name in os.listdir('test_imgs'):
while True:
# if not is_tracking:
img_path = 'test_imgs/{}'.format('dance.jpg')
t1 = time.time()
input_batch = []
cam_img = utils.read_square_image('', cam, args.input_size, 'WEBCAM')
# cam_img = utils.read_square_image(img_path, '', args.input_size, 'IMAGE')
# cv2.imshow('', cam_img)
# cv2.waitKey(0)
orig_size_input = cam_img.astype(np.float32)
for scale in scales:
resized_img = utils.resize_pad_img(orig_size_input, scale,
args.input_size)
input_batch.append(resized_img)
input_batch = np.asarray(input_batch, dtype=np.float32)
input_batch = np.transpose(input_batch, (0, 3, 1, 2))
input_batch /= 255.0
input_batch -= 0.4
model.blobs['data'].data[...] = input_batch
# Forward
model.forward()
# Get output data
x_hm = model.blobs['x_heatmap'].data
y_hm = model.blobs['y_heatmap'].data
z_hm = model.blobs['z_heatmap'].data
hm = model.blobs['heatmap'].data
# Trans coordinates
x_hm = x_hm.transpose([0, 2, 3, 1])
y_hm = y_hm.transpose([0, 2, 3, 1])
z_hm = z_hm.transpose([0, 2, 3, 1])
hm = hm.transpose([0, 2, 3, 1])
# Average scale outputs
hm_size = args.input_size // args.pool_scale
hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
x_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
y_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
z_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
for i in range(len(scales)):
rescale = 1.0 / scales[i]
scaled_hm = cv2.resize(hm[i, :, :, :], (0, 0),
fx=rescale,
fy=rescale,
interpolation=cv2.INTER_LINEAR)
scaled_x_hm = cv2.resize(x_hm[i, :, :, :], (0, 0),
fx=rescale,
fy=rescale,
interpolation=cv2.INTER_LINEAR)
scaled_y_hm = cv2.resize(y_hm[i, :, :, :], (0, 0),
fx=rescale,
fy=rescale,
interpolation=cv2.INTER_LINEAR)
scaled_z_hm = cv2.resize(z_hm[i, :, :, :], (0, 0),
fx=rescale,
fy=rescale,
interpolation=cv2.INTER_LINEAR)
mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2]
hm_avg += scaled_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] + hm_size // 2, :]
x_hm_avg += scaled_x_hm[mid[0] - hm_size // 2:mid[0] +
hm_size // 2, mid[1] -
hm_size // 2:mid[1] + hm_size // 2, :]
y_hm_avg += scaled_y_hm[mid[0] - hm_size // 2:mid[0] +
hm_size // 2, mid[1] -
hm_size // 2:mid[1] + hm_size // 2, :]
z_hm_avg += scaled_z_hm[mid[0] - hm_size // 2:mid[0] +
hm_size // 2, mid[1] -
hm_size // 2:mid[1] + hm_size // 2, :]
hm_avg /= len(scales)
x_hm_avg /= len(scales)
y_hm_avg /= len(scales)
z_hm_avg /= len(scales)
t2 = time.time()
# Get 2d joints
joints_2d = utils.extract_2d_joint_from_heatmap(
hm_avg, args.input_size, joints_2d)
# Get 3d joints
joints_3d = utils.extract_3d_joints_from_heatmap(
joints_2d, x_hm_avg, y_hm_avg, z_hm_avg, args.input_size,
joints_3d)
print('Post FPS', 1 / (time.time() - t2))
# Plot 2d location heatmap
joint_map = np.zeros(shape=(args.input_size, args.input_size, 3))
for joint_num in range(joints_2d.shape[0]):
cv2.circle(joint_map,
center=(joints_2d[joint_num][1],
joints_2d[joint_num][0]),
radius=3,
color=(255, 0, 0),
thickness=-1)
# Plot 2d limbs
limb_img = utils.draw_limbs_2d(cam_img, joints_2d, limb_parents)
# Plot 3d limbs
if args.plot_3d:
ax.clear()
ax.view_init(azim=0, elev=90)
ax.set_xlim(-700, 700)
ax.set_ylim(-800, 800)
ax.set_zlim(-700, 700)
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_zlabel('z')
utils.draw_limbs_3d(joints_3d, limb_parents, ax)
# draw heatmap
# hm_img = utils.draw_predicted_heatmap(hm_avg*200, args.input_size)
# cv2.imshow('hm', hm_img.astype(np.uint8))
# cv2.waitKey(0)
concat_img = np.concatenate((limb_img, joint_map), axis=1)
# ax2.imshow(concat_img[..., ::-1].astype(np.uint8))
cv2.imshow('2d', concat_img.astype(np.uint8))
cv2.waitKey(1)
# ax2.imshow(concat_img.astype(np.uint8))
# plt.pause(0.0001)
# plt.show(block=False)
print('Forward FPS', 1 / (time.time() - t1))
def forward():
# global hm_avg, x_hm_avg, y_hm_avg, z_hm_avg
cam_img = np.zeros(shape=(args.input_size, args.input_size, 3),
dtype=np.uint8)
joints_2d = np.zeros(shape=(args.num_of_joints, 2), dtype=np.int32)
joints_3d = np.zeros(shape=(args.num_of_joints, 3), dtype=np.float32)
if args.device == 'cpu':
caffe.set_mode_cpu()
elif args.device == 'gpu':
caffe.set_mode_gpu()
caffe.set_device(1)
else:
raise ValueError('No such device')
model_prototxt_path = os.path.join(args.model_dir, 'vnect_net.prototxt')
model_weight_path = os.path.join(args.model_dir, 'vnect_model.caffemodel')
# Load model
model = caffe.Net(model_prototxt_path, model_weight_path, caffe.TEST)
# Show network structure and shape
print('##################################################')
print('################Network Structures################')
print('##################################################')
for layer_name in model.params.keys():
print(layer_name, model.params[layer_name][0].data.shape)
print('')
print('##################################################')
print('##################################################')
print('##################################################')
print('\n\n\n\n')
print('##################################################')
print('################Input Output Blobs################')
print('##################################################')
for i in model.blobs.keys():
print(i, model.blobs[i].data.shape)
print('##################################################')
print('##################################################')
print('##################################################')
# cam = cv2.VideoCapture(0)
is_tracking = False
# for img_name in os.listdir('test_imgs'):
while True:
# if not is_tracking:
img_path = 'test_imgs/{}'.format('dance.jpg')
t1 = time.time()
input_batch = []
if not cam_model_q.empty():
cam_img = cam_model_q.get()
# print('forward get')
# cam_img = utils.read_square_image('', cam, args.input_size, 'WEBCAM')
# cam_img = utils.read_square_image(img_path, '', args.input_size, 'IMAGE')
# cv2.imshow('', cam_img)
# cv2.waitKey(0)
orig_size_input = cam_img.astype(np.float32)
for scale in scales:
resized_img = utils.resize_pad_img(orig_size_input, scale,
args.input_size)
input_batch.append(resized_img)
input_batch = np.asarray(input_batch, dtype=np.float32)
input_batch = np.transpose(input_batch, (0, 3, 1, 2))
input_batch /= 255.0
input_batch -= 0.4
model.blobs['data'].data[...] = input_batch
# Forward
model.forward()
# Get output data
x_hm = model.blobs['x_heatmap'].data
y_hm = model.blobs['y_heatmap'].data
z_hm = model.blobs['z_heatmap'].data
hm = model.blobs['heatmap'].data
# Trans coordinates
x_hm = x_hm.transpose([0, 2, 3, 1])
y_hm = y_hm.transpose([0, 2, 3, 1])
z_hm = z_hm.transpose([0, 2, 3, 1])
hm = hm.transpose([0, 2, 3, 1])
# Average scale outputs
hm_size = args.input_size // args.pool_scale
hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
x_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
y_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
z_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
for i in range(len(scales)):
rescale = 1.0 / scales[i]
scaled_hm = cv2.resize(hm[i, :, :, :], (0, 0),
fx=rescale,
fy=rescale,
interpolation=cv2.INTER_LINEAR)
scaled_x_hm = cv2.resize(x_hm[i, :, :, :], (0, 0),
fx=rescale,
fy=rescale,
interpolation=cv2.INTER_LINEAR)
scaled_y_hm = cv2.resize(y_hm[i, :, :, :], (0, 0),
fx=rescale,
fy=rescale,
interpolation=cv2.INTER_LINEAR)
scaled_z_hm = cv2.resize(z_hm[i, :, :, :], (0, 0),
fx=rescale,
fy=rescale,
interpolation=cv2.INTER_LINEAR)
mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2]
hm_avg += scaled_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] + hm_size // 2, :]
x_hm_avg += scaled_x_hm[mid[0] - hm_size // 2:mid[0] +
hm_size // 2, mid[1] -
hm_size // 2:mid[1] + hm_size // 2, :]
y_hm_avg += scaled_y_hm[mid[0] - hm_size // 2:mid[0] +
hm_size // 2, mid[1] -
hm_size // 2:mid[1] + hm_size // 2, :]
z_hm_avg += scaled_z_hm[mid[0] - hm_size // 2:mid[0] +
hm_size // 2, mid[1] -
hm_size // 2:mid[1] + hm_size // 2, :]
hm_avg /= len(scales)
x_hm_avg /= len(scales)
y_hm_avg /= len(scales)
z_hm_avg /= len(scales)
t2 = time.time()
# Get 2d joints
joints_2d = utils.extract_2d_joint_from_heatmap(
hm_avg, args.input_size, joints_2d)
# Get 3d joints
joints_3d = utils.extract_3d_joints_from_heatmap(
joints_2d, x_hm_avg, y_hm_avg, z_hm_avg, args.input_size,
joints_3d)
print('Post FPS', 1 / (time.time() - t2))
if not model_post_q.full():
# model_post_q.put([hm_avg, x_hm_avg, y_hm_avg, z_hm_avg, cam_img])
model_post_q.put([joints_2d, joints_3d, cam_img])
# print('forward put')
print('Forward FPS', 1 / (time.time() - t1))
def forward():
# global hm_avg, x_hm_avg, y_hm_avg, z_hm_avg
cam_img = np.zeros(shape=(args.input_size, args.input_size, 3), dtype=np.uint8)
joints_2d = np.zeros(shape=(args.num_of_joints, 2), dtype=np.int32)
joints_3d = np.zeros(shape=(args.num_of_joints, 3), dtype=np.float32)
if args.device == 'cpu':
caffe.set_mode_cpu()
elif args.device == 'gpu':
caffe.set_mode_gpu()
caffe.set_device(1)
else:
raise ValueError('No such device')
model_prototxt_path = os.path.join(args.model_dir, 'vnect_net.prototxt')
model_weight_path = os.path.join(args.model_dir, 'vnect_model.caffemodel')
# Load model
model = caffe.Net(model_prototxt_path,
model_weight_path,
caffe.TEST)
# Show network structure and shape
print('##################################################')
print('################Network Structures################')
print('##################################################')
for layer_name in model.params.keys():
print(layer_name, model.params[layer_name][0].data.shape)
print('')
print('##################################################')
print('##################################################')
print('##################################################')
print('\n\n\n\n')
print('##################################################')
print('################Input Output Blobs################')
print('##################################################')
for i in model.blobs.keys():
print(i, model.blobs[i].data.shape)
print('##################################################')
print('##################################################')
print('##################################################')
# cam = cv2.VideoCapture(0)
is_tracking = False
# for img_name in os.listdir('test_imgs'):
while True:
# if not is_tracking:
img_path = 'test_imgs/{}'.format('dance.jpg')
t1 = time.time()
input_batch = []
if not cam_model_q.empty():
cam_img = cam_model_q.get()
# print('forward get')
# cam_img = utils.read_square_image('', cam, args.input_size, 'WEBCAM')
# cam_img = utils.read_square_image(img_path, '', args.input_size, 'IMAGE')
# cv2.imshow('', cam_img)
# cv2.waitKey(0)
orig_size_input = cam_img.astype(np.float32)
for scale in scales:
resized_img = utils.resize_pad_img(orig_size_input, scale, args.input_size)
input_batch.append(resized_img)
input_batch = np.asarray(input_batch, dtype=np.float32)
input_batch = np.transpose(input_batch, (0, 3, 1, 2))
input_batch /= 255.0
input_batch -= 0.4
model.blobs['data'].data[...] = input_batch
# Forward
model.forward()
# Get output data
x_hm = model.blobs['x_heatmap'].data
y_hm = model.blobs['y_heatmap'].data
z_hm = model.blobs['z_heatmap'].data
hm = model.blobs['heatmap'].data
# Trans coordinates
x_hm = x_hm.transpose([0, 2, 3, 1])
y_hm = y_hm.transpose([0, 2, 3, 1])
z_hm = z_hm.transpose([0, 2, 3, 1])
hm = hm.transpose([0, 2, 3, 1])
# Average scale outputs
hm_size = args.input_size // args.pool_scale
hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
x_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
y_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
z_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
for i in range(len(scales)):
rescale = 1.0 / scales[i]
scaled_hm = cv2.resize(hm[i, :, :, :], (0, 0), fx=rescale, fy=rescale, interpolation=cv2.INTER_LINEAR)
scaled_x_hm = cv2.resize(x_hm[i, :, :, :], (0, 0), fx=rescale, fy=rescale, interpolation=cv2.INTER_LINEAR)
scaled_y_hm = cv2.resize(y_hm[i, :, :, :], (0, 0), fx=rescale, fy=rescale, interpolation=cv2.INTER_LINEAR)
scaled_z_hm = cv2.resize(z_hm[i, :, :, :], (0, 0), fx=rescale, fy=rescale, interpolation=cv2.INTER_LINEAR)
mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2]
hm_avg += scaled_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
x_hm_avg += scaled_x_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
y_hm_avg += scaled_y_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
z_hm_avg += scaled_z_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
hm_avg /= len(scales)
x_hm_avg /= len(scales)
y_hm_avg /= len(scales)
z_hm_avg /= len(scales)
t2 = time.time()
# Get 2d joints
joints_2d = utils.extract_2d_joint_from_heatmap(hm_avg, args.input_size, joints_2d)
# Get 3d joints
joints_3d = utils.extract_3d_joints_from_heatmap(joints_2d, x_hm_avg, y_hm_avg, z_hm_avg, args.input_size,
joints_3d)
print('Post FPS', 1/(time.time()-t2))
if not model_post_q.full():
# model_post_q.put([hm_avg, x_hm_avg, y_hm_avg, z_hm_avg, cam_img])
model_post_q.put([joints_2d, joints_3d, cam_img])
# print('forward put')
print('Forward FPS', 1 / (time.time() - t1))
def demo():
joints_2d = np.zeros(shape=(args.num_of_joints, 2), dtype=np.int32)
joints_3d = np.zeros(shape=(args.num_of_joints, 3), dtype=np.float32)
if args.plot_3d:
plt.ion()
fig = plt.figure()
ax = fig.add_subplot(121, projection='3d')
ax2 = fig.add_subplot(122)
plt.show()
if args.device == 'cpu':
caffe.set_mode_cpu()
elif args.device == 'gpu':
caffe.set_mode_gpu()
caffe.set_device(0)
else:
raise ValueError('No such device')
model_prototxt_path = os.path.join(args.model_dir, 'vnect_net.prototxt')
model_weight_path = os.path.join(args.model_dir, 'vnect_model.caffemodel')
# Load model
model = caffe.Net(model_prototxt_path,
model_weight_path,
caffe.TEST)
# Show network structure and shape
for layer_name in model.params.keys():
print(layer_name, model.params[layer_name][0].data.shape)
print('')
for i in model.blobs.keys():
print(i, model.blobs[i].data.shape)
cam = cv2.VideoCapture(0)
is_tracking = False
# for img_name in os.listdir('test_imgs'):
while True:
# if not is_tracking:
img_path = 'test_imgs/{}'.format('dance.jpg')
t1 = time.time()
input_batch = []
cam_img = utils.read_square_image('', cam, args.input_size, 'WEBCAM')
# cam_img = utils.read_square_image(img_path, '', args.input_size, 'IMAGE')
# cv2.imshow('', cam_img)
# cv2.waitKey(0)
orig_size_input = cam_img.astype(np.float32)
for scale in scales:
resized_img = utils.resize_pad_img(orig_size_input, scale, args.input_size)
input_batch.append(resized_img)
input_batch = np.asarray(input_batch, dtype=np.float32)
input_batch = np.transpose(input_batch, (0, 3, 1, 2))
input_batch /= 255.0
input_batch -= 0.4
model.blobs['data'].data[...] = input_batch
# Forward
model.forward()
# Get output data
x_hm = model.blobs['x_heatmap'].data
y_hm = model.blobs['y_heatmap'].data
z_hm = model.blobs['z_heatmap'].data
hm = model.blobs['heatmap'].data
# Trans coordinates
x_hm = x_hm.transpose([0, 2, 3, 1])
y_hm = y_hm.transpose([0, 2, 3, 1])
z_hm = z_hm.transpose([0, 2, 3, 1])
hm = hm.transpose([0, 2, 3, 1])
# Average scale outputs
hm_size = args.input_size // args.pool_scale
hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
x_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
y_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
z_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
for i in range(len(scales)):
rescale = 1.0 / scales[i]
scaled_hm = cv2.resize(hm[i, :, :, :], (0, 0), fx=rescale, fy=rescale, interpolation=cv2.INTER_LINEAR)
scaled_x_hm = cv2.resize(x_hm[i, :, :, :], (0, 0), fx=rescale, fy=rescale, interpolation=cv2.INTER_LINEAR)
scaled_y_hm = cv2.resize(y_hm[i, :, :, :], (0, 0), fx=rescale, fy=rescale, interpolation=cv2.INTER_LINEAR)
scaled_z_hm = cv2.resize(z_hm[i, :, :, :], (0, 0), fx=rescale, fy=rescale, interpolation=cv2.INTER_LINEAR)
mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2]
hm_avg += scaled_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
x_hm_avg += scaled_x_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
y_hm_avg += scaled_y_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
z_hm_avg += scaled_z_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
hm_avg /= len(scales)
x_hm_avg /= len(scales)
y_hm_avg /= len(scales)
z_hm_avg /= len(scales)
t2 = time.time()
# Get 2d joints
joints_2d = utils.extract_2d_joint_from_heatmap(hm_avg, args.input_size, joints_2d)
# Get 3d joints
joints_3d = utils.extract_3d_joints_from_heatmap(joints_2d, x_hm_avg, y_hm_avg, z_hm_avg, args.input_size,
joints_3d)
print('Post FPS', 1/(time.time()-t2))
# Plot 2d location heatmap
joint_map = np.zeros(shape=(args.input_size, args.input_size, 3))
for joint_num in range(joints_2d.shape[0]):
cv2.circle(joint_map, center=(joints_2d[joint_num][1], joints_2d[joint_num][0]), radius=3,
color=(255, 0, 0), thickness=-1)
# Plot 2d limbs
limb_img = utils.draw_limbs_2d(cam_img, joints_2d, limb_parents)
# Plot 3d limbs
if args.plot_3d:
ax.clear()
ax.view_init(azim=0, elev=90)
ax.set_xlim(-700, 700)
ax.set_ylim(-800, 800)
ax.set_zlim(-700, 700)
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_zlabel('z')
utils.draw_limbs_3d(joints_3d, limb_parents, ax)
# draw heatmap
# hm_img = utils.draw_predicted_heatmap(hm_avg*200, args.input_size)
# cv2.imshow('hm', hm_img.astype(np.uint8))
# cv2.waitKey(0)
concat_img = np.concatenate((limb_img, joint_map), axis=1)
# ax2.imshow(concat_img[..., ::-1].astype(np.uint8))
cv2.imshow('2d', concat_img.astype(np.uint8))
cv2.waitKey(1)
# ax2.imshow(concat_img.astype(np.uint8))
# plt.pause(0.0001)
# plt.show(block=False)
print('Forward FPS', 1 / (time.time() - t1))