def __call__(self, img_input):
t = time.time()
img_input = np.transpose(
img_input, axes=[1, 0, 2]).copy() if self.T else img_input
img_batch = self._gen_input_batch(img_input, self._box_size,
self.scales)
# inference
hm, xm, ym, zm = self.sess.run(
[self.heatmap, self.x_heatmap, self.y_heatmap, self.z_heatmap],
{self.input_crops: img_batch})
# average scale outputs
hm_size = self._box_size // self._hm_factor
hm_avg = np.zeros((hm_size, hm_size, self._joints_num))
xm_avg = np.zeros((hm_size, hm_size, self._joints_num))
ym_avg = np.zeros((hm_size, hm_size, self._joints_num))
zm_avg = np.zeros((hm_size, hm_size, self._joints_num))
for i in range(len(self.scales)):
rescale = 1.0 / self.scales[i]
scaled_hm = utils.img_scale(hm[i, :, :, :], rescale)
scaled_x_hm = utils.img_scale(xm[i, :, :, :], rescale)
scaled_y_hm = utils.img_scale(ym[i, :, :, :], rescale)
scaled_z_hm = utils.img_scale(zm[i, :, :, :], rescale)
mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2]
hm_avg += scaled_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] + hm_size // 2, :]
xm_avg += scaled_x_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] +
hm_size // 2, :]
ym_avg += scaled_y_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] +
hm_size // 2, :]
zm_avg += scaled_z_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] +
hm_size // 2, :]
hm_avg /= len(self.scales)
xm_avg /= len(self.scales)
ym_avg /= len(self.scales)
zm_avg /= len(self.scales)
joints_2d = utils.extract_2d_joints_from_heatmap(
hm_avg, self._box_size, self._hm_factor)
joints_3d = utils.extract_3d_joints_from_heatmap(
joints_2d, xm_avg, ym_avg, zm_avg, self._box_size, self._hm_factor)
joints_2d, joints_3d = self._joint_filter(joints_2d, joints_3d)
# if self.T:
# joints_2d = joints_2d[:, ::-1]
# joints_3d = joints_3d[:, [1, 0, 2]]
print('FPS: {:>2.2f}'.format(1 / (time.time() - t)))
if self.plot:
# 2d plotting
frame_square = utils.img_scale_squareify(img_input, self._box_size)
frame_square = utils.draw_limbs_2d(frame_square, joints_2d,
self._joint_parents)
cv2.imshow('2D Prediction', frame_square)
# 3d plotting
self.imshow_3d(self.ax_3d, joints_3d, self._joint_parents)
return joints_2d, joints_3d
def run(self):
# initial BB by HOG detection
self.BB_init()
success, frame = self.cameraCapture.read()
frame = frame.T if self.T else frame
while success and cv2.waitKey(1) == -1:
found, w = self.hog.detectMultiScale(frame)
if len(found) > 0:
self._draw_BB_rect(frame, found[np.argmax(w)])
scale = 400 / self.H
frame = cv2.resize(frame, (0, 0), fx=scale, fy=scale)
cv2.imshow(self._box_init_window_name, frame)
if self._clicked:
self._clicked = False
cv2.destroyWindow(self._box_init_window_name)
break
success, frame = self.cameraCapture.read()
frame = frame.T if self.T else frame
x, y, w, h = self.rect
plt.show()
# main loop
success, frame = self.cameraCapture.read()
frame = frame.T if self.T else frame
while success and cv2.waitKey(1) == -1:
t = time.time()
# crop bounding box from the raw frame
frame_cropped = frame[y:y + h, x:x + w, :]
# crop --> one sqrare input img for CNN
self.frame_square = utils.img_scale_squareify(
frame_cropped, self.box_size)
# one sqrare input img --> a batch of sqrare input imgs
self._create_input_batch()
# sqrare input img batch --CNN net--> 2d and 3d skeleton joint coordinates
self._run_net()
# filter to smooth the joint coordinate results
self._joint_coord_filter()
## plot ##
# 2d plotting
self.frame_square = utils.draw_limbs_2d(self.frame_square,
self.joints_2d,
self.joint_parents)
cv2.imshow('2D Prediction', self.frame_square)
# 3d plotting
self._imshow_3d()
print('FPS: {:>2.2f}'.format(1 / (time.time() - t)))
success, frame = self.cameraCapture.read()
frame = frame.T if self.T else frame
self._exit()
def demo():
joints_2d = np.zeros(shape=(args.num_of_joints, 2), dtype=np.int32)
joints_3d = np.zeros(shape=(args.num_of_joints, 3), dtype=np.float32)
if args.plot_3d:
plt.ion()
fig = plt.figure()
ax = fig.add_subplot(121, projection='3d')
ax2 = fig.add_subplot(122)
plt.show()
if args.device == 'cpu':
caffe.set_mode_cpu()
elif args.device == 'gpu':
caffe.set_mode_gpu()
caffe.set_device(0)
else:
raise ValueError('No such device')
model_prototxt_path = os.path.join(args.model_dir, 'vnect_net.prototxt')
model_weight_path = os.path.join(args.model_dir, 'vnect_model.caffemodel')
# Load model
model = caffe.Net(model_prototxt_path, model_weight_path, caffe.TEST)
# Show network structure and shape
for layer_name in model.params.keys():
print(layer_name, model.params[layer_name][0].data.shape)
print('')
for i in model.blobs.keys():
print(i, model.blobs[i].data.shape)
cam = cv2.VideoCapture(0)
is_tracking = False
# for img_name in os.listdir('test_imgs'):
while True:
# if not is_tracking:
img_path = 'test_imgs/{}'.format('dance.jpg')
t1 = time.time()
input_batch = []
cam_img = utils.read_square_image('', cam, args.input_size, 'WEBCAM')
# cam_img = utils.read_square_image(img_path, '', args.input_size, 'IMAGE')
# cv2.imshow('', cam_img)
# cv2.waitKey(0)
orig_size_input = cam_img.astype(np.float32)
for scale in scales:
resized_img = utils.resize_pad_img(orig_size_input, scale,
args.input_size)
input_batch.append(resized_img)
input_batch = np.asarray(input_batch, dtype=np.float32)
input_batch = np.transpose(input_batch, (0, 3, 1, 2))
input_batch /= 255.0
input_batch -= 0.4
model.blobs['data'].data[...] = input_batch
# Forward
model.forward()
# Get output data
x_hm = model.blobs['x_heatmap'].data
y_hm = model.blobs['y_heatmap'].data
z_hm = model.blobs['z_heatmap'].data
hm = model.blobs['heatmap'].data
# Trans coordinates
x_hm = x_hm.transpose([0, 2, 3, 1])
y_hm = y_hm.transpose([0, 2, 3, 1])
z_hm = z_hm.transpose([0, 2, 3, 1])
hm = hm.transpose([0, 2, 3, 1])
# Average scale outputs
hm_size = args.input_size // args.pool_scale
hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
x_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
y_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
z_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
for i in range(len(scales)):
rescale = 1.0 / scales[i]
scaled_hm = cv2.resize(hm[i, :, :, :], (0, 0),
fx=rescale,
fy=rescale,
interpolation=cv2.INTER_LINEAR)
scaled_x_hm = cv2.resize(x_hm[i, :, :, :], (0, 0),
fx=rescale,
fy=rescale,
interpolation=cv2.INTER_LINEAR)
scaled_y_hm = cv2.resize(y_hm[i, :, :, :], (0, 0),
fx=rescale,
fy=rescale,
interpolation=cv2.INTER_LINEAR)
scaled_z_hm = cv2.resize(z_hm[i, :, :, :], (0, 0),
fx=rescale,
fy=rescale,
interpolation=cv2.INTER_LINEAR)
mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2]
hm_avg += scaled_hm[mid[0] - hm_size // 2:mid[0] + hm_size // 2,
mid[1] - hm_size // 2:mid[1] + hm_size // 2, :]
x_hm_avg += scaled_x_hm[mid[0] - hm_size // 2:mid[0] +
hm_size // 2, mid[1] -
hm_size // 2:mid[1] + hm_size // 2, :]
y_hm_avg += scaled_y_hm[mid[0] - hm_size // 2:mid[0] +
hm_size // 2, mid[1] -
hm_size // 2:mid[1] + hm_size // 2, :]
z_hm_avg += scaled_z_hm[mid[0] - hm_size // 2:mid[0] +
hm_size // 2, mid[1] -
hm_size // 2:mid[1] + hm_size // 2, :]
hm_avg /= len(scales)
x_hm_avg /= len(scales)
y_hm_avg /= len(scales)
z_hm_avg /= len(scales)
t2 = time.time()
# Get 2d joints
joints_2d = utils.extract_2d_joint_from_heatmap(
hm_avg, args.input_size, joints_2d)
# Get 3d joints
joints_3d = utils.extract_3d_joints_from_heatmap(
joints_2d, x_hm_avg, y_hm_avg, z_hm_avg, args.input_size,
joints_3d)
print('Post FPS', 1 / (time.time() - t2))
# Plot 2d location heatmap
joint_map = np.zeros(shape=(args.input_size, args.input_size, 3))
for joint_num in range(joints_2d.shape[0]):
cv2.circle(joint_map,
center=(joints_2d[joint_num][1],
joints_2d[joint_num][0]),
radius=3,
color=(255, 0, 0),
thickness=-1)
# Plot 2d limbs
limb_img = utils.draw_limbs_2d(cam_img, joints_2d, limb_parents)
# Plot 3d limbs
if args.plot_3d:
ax.clear()
ax.view_init(azim=0, elev=90)
ax.set_xlim(-700, 700)
ax.set_ylim(-800, 800)
ax.set_zlim(-700, 700)
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_zlabel('z')
utils.draw_limbs_3d(joints_3d, limb_parents, ax)
# draw heatmap
# hm_img = utils.draw_predicted_heatmap(hm_avg*200, args.input_size)
# cv2.imshow('hm', hm_img.astype(np.uint8))
# cv2.waitKey(0)
concat_img = np.concatenate((limb_img, joint_map), axis=1)
# ax2.imshow(concat_img[..., ::-1].astype(np.uint8))
cv2.imshow('2d', concat_img.astype(np.uint8))
cv2.waitKey(1)
# ax2.imshow(concat_img.astype(np.uint8))
# plt.pause(0.0001)
# plt.show(block=False)
print('Forward FPS', 1 / (time.time() - t1))
{input_batch: img_square / 255 - 0.4})
joints_2d = utils.extract_2d_joints_from_heatmap(hm[0, ...], box_size,
hm_factor)
for i in range(21):
if i == 0:
himg = hm[0, :, :, i]
ximg = xm[0, :, :, i]
yimg = ym[0, :, :, i]
zimg = zm[0, :, :, i]
else:
tmp = hm[0, :, :, i]
himg = np.hstack([himg, tmp])
tmp = xm[0, :, :, i]
ximg = np.hstack([ximg, tmp])
tmp = ym[0, :, :, i]
yimg = np.hstack([yimg, tmp])
tmp = zm[0, :, :, i]
zimg = np.hstack([zimg, tmp])
all_hm = np.vstack([himg, ximg, yimg, zimg])
cv2.imshow('all heatmaps', all_hm)
img_res2d = utils.draw_limbs_2d(img_square[0, ...], joints_2d,
limb_parents)
cv2.imshow('2D results', img_res2d)
cv2.waitKey()
cv2.destroyAllWindows()
def render_plt():
joints_2d = np.zeros(shape=(args.num_of_joints, 2), dtype=np.int32)
joints_3d = np.zeros(shape=(args.num_of_joints, 3), dtype=np.float32)
cam_img = np.zeros(shape=(args.input_size, args.input_size, 3),
dtype=np.uint8)
if args.plot_3d and args.plot_2d:
plt.ion()
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(121, projection='3d')
ax2 = fig.add_subplot(122)
plt.show()
elif args.plot_3d:
plt.ion()
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(111, projection='3d')
while True:
if model_post_q.qsize() != 0:
[joints_2d, joints_3d, cam_img] = model_post_q.get(False)
else:
print('render old')
t1 = time.time()
# Plot 2d location heatmap
if args.plot_2d:
joint_map = np.zeros(shape=(args.input_size, args.input_size, 3))
for joint_num in range(joints_2d.shape[0]):
cv2.circle(joint_map,
center=(joints_2d[joint_num][1],
joints_2d[joint_num][0]),
radius=3,
color=(255, 0, 0),
thickness=-1)
# Plot 2d limbs
limb_img = utils.draw_limbs_2d(cam_img, joints_2d, limb_parents)
# Plot 3d limbs
if args.plot_3d:
ax.clear()
ax.view_init(azim=0, elev=90)
ax.set_xlim(-700, 700)
ax.set_ylim(-800, 800)
ax.set_zlim(-700, 700)
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_zlabel('z')
utils.draw_limbs_3d(joints_3d, limb_parents, ax)
# draw heatmap
# hm_img = utils.draw_predicted_heatmap(hm_avg*200, args.input_size)
# cv2.imshow('hm', hm_img.astype(np.uint8))
# cv2.waitKey(0)
if args.plot_2d and args.plot_3d:
concat_img = np.concatenate((limb_img, joint_map), axis=1)
ax2.imshow(concat_img[..., ::-1].astype(np.uint8))
plt.pause(1e-10)
elif args.plot_3d:
plt.pause(1e-10)
else:
concat_img = np.concatenate((limb_img, joint_map), axis=1)
cv2.imshow('2d', concat_img.astype(np.uint8))
cv2.waitKey(1)
# ax2.imshow(concat_img.astype(np.uint8))
print('Render FPS', 1 / (time.time() - t1))
def render_plt():
joints_2d = np.zeros(shape=(args.num_of_joints, 2), dtype=np.int32)
joints_3d = np.zeros(shape=(args.num_of_joints, 3), dtype=np.float32)
cam_img = np.zeros(shape=(args.input_size, args.input_size, 3), dtype=np.uint8)
if args.plot_3d and args.plot_2d:
plt.ion()
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(121, projection='3d')
ax2 = fig.add_subplot(122)
plt.show()
elif args.plot_3d:
plt.ion()
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111, projection='3d')
while True:
if model_post_q.qsize() != 0:
[joints_2d, joints_3d, cam_img] = model_post_q.get(False)
else:
print('render old')
t1 = time.time()
# Plot 2d location heatmap
if args.plot_2d:
joint_map = np.zeros(shape=(args.input_size, args.input_size, 3))
for joint_num in range(joints_2d.shape[0]):
cv2.circle(joint_map, center=(joints_2d[joint_num][1], joints_2d[joint_num][0]), radius=3,
color=(255, 0, 0), thickness=-1)
# Plot 2d limbs
limb_img = utils.draw_limbs_2d(cam_img, joints_2d, limb_parents)
# Plot 3d limbs
if args.plot_3d:
ax.clear()
ax.view_init(azim=0, elev=90)
ax.set_xlim(-700, 700)
ax.set_ylim(-800, 800)
ax.set_zlim(-700, 700)
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_zlabel('z')
utils.draw_limbs_3d(joints_3d, limb_parents, ax)
# draw heatmap
# hm_img = utils.draw_predicted_heatmap(hm_avg*200, args.input_size)
# cv2.imshow('hm', hm_img.astype(np.uint8))
# cv2.waitKey(0)
if args.plot_2d and args.plot_3d:
concat_img = np.concatenate((limb_img, joint_map), axis=1)
ax2.imshow(concat_img[..., ::-1].astype(np.uint8))
plt.pause(1e-10)
elif args.plot_3d:
plt.pause(1e-10)
else:
concat_img = np.concatenate((limb_img, joint_map), axis=1)
cv2.imshow('2d', concat_img.astype(np.uint8))
cv2.waitKey(1)
# ax2.imshow(concat_img.astype(np.uint8))
print('Render FPS', 1 / (time.time() - t1))
def demo():
joints_2d = np.zeros(shape=(args.num_of_joints, 2), dtype=np.int32)
joints_3d = np.zeros(shape=(args.num_of_joints, 3), dtype=np.float32)
if args.plot_3d:
plt.ion()
fig = plt.figure()
ax = fig.add_subplot(121, projection='3d')
ax2 = fig.add_subplot(122)
plt.show()
if args.device == 'cpu':
caffe.set_mode_cpu()
elif args.device == 'gpu':
caffe.set_mode_gpu()
caffe.set_device(0)
else:
raise ValueError('No such device')
model_prototxt_path = os.path.join(args.model_dir, 'vnect_net.prototxt')
model_weight_path = os.path.join(args.model_dir, 'vnect_model.caffemodel')
# Load model
model = caffe.Net(model_prototxt_path,
model_weight_path,
caffe.TEST)
# Show network structure and shape
for layer_name in model.params.keys():
print(layer_name, model.params[layer_name][0].data.shape)
print('')
for i in model.blobs.keys():
print(i, model.blobs[i].data.shape)
cam = cv2.VideoCapture(0)
is_tracking = False
# for img_name in os.listdir('test_imgs'):
while True:
# if not is_tracking:
img_path = 'test_imgs/{}'.format('dance.jpg')
t1 = time.time()
input_batch = []
cam_img = utils.read_square_image('', cam, args.input_size, 'WEBCAM')
# cam_img = utils.read_square_image(img_path, '', args.input_size, 'IMAGE')
# cv2.imshow('', cam_img)
# cv2.waitKey(0)
orig_size_input = cam_img.astype(np.float32)
for scale in scales:
resized_img = utils.resize_pad_img(orig_size_input, scale, args.input_size)
input_batch.append(resized_img)
input_batch = np.asarray(input_batch, dtype=np.float32)
input_batch = np.transpose(input_batch, (0, 3, 1, 2))
input_batch /= 255.0
input_batch -= 0.4
model.blobs['data'].data[...] = input_batch
# Forward
model.forward()
# Get output data
x_hm = model.blobs['x_heatmap'].data
y_hm = model.blobs['y_heatmap'].data
z_hm = model.blobs['z_heatmap'].data
hm = model.blobs['heatmap'].data
# Trans coordinates
x_hm = x_hm.transpose([0, 2, 3, 1])
y_hm = y_hm.transpose([0, 2, 3, 1])
z_hm = z_hm.transpose([0, 2, 3, 1])
hm = hm.transpose([0, 2, 3, 1])
# Average scale outputs
hm_size = args.input_size // args.pool_scale
hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
x_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
y_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
z_hm_avg = np.zeros(shape=(hm_size, hm_size, args.num_of_joints))
for i in range(len(scales)):
rescale = 1.0 / scales[i]
scaled_hm = cv2.resize(hm[i, :, :, :], (0, 0), fx=rescale, fy=rescale, interpolation=cv2.INTER_LINEAR)
scaled_x_hm = cv2.resize(x_hm[i, :, :, :], (0, 0), fx=rescale, fy=rescale, interpolation=cv2.INTER_LINEAR)
scaled_y_hm = cv2.resize(y_hm[i, :, :, :], (0, 0), fx=rescale, fy=rescale, interpolation=cv2.INTER_LINEAR)
scaled_z_hm = cv2.resize(z_hm[i, :, :, :], (0, 0), fx=rescale, fy=rescale, interpolation=cv2.INTER_LINEAR)
mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2]
hm_avg += scaled_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
x_hm_avg += scaled_x_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
y_hm_avg += scaled_y_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
z_hm_avg += scaled_z_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
hm_avg /= len(scales)
x_hm_avg /= len(scales)
y_hm_avg /= len(scales)
z_hm_avg /= len(scales)
t2 = time.time()
# Get 2d joints
joints_2d = utils.extract_2d_joint_from_heatmap(hm_avg, args.input_size, joints_2d)
# Get 3d joints
joints_3d = utils.extract_3d_joints_from_heatmap(joints_2d, x_hm_avg, y_hm_avg, z_hm_avg, args.input_size,
joints_3d)
print('Post FPS', 1/(time.time()-t2))
# Plot 2d location heatmap
joint_map = np.zeros(shape=(args.input_size, args.input_size, 3))
for joint_num in range(joints_2d.shape[0]):
cv2.circle(joint_map, center=(joints_2d[joint_num][1], joints_2d[joint_num][0]), radius=3,
color=(255, 0, 0), thickness=-1)
# Plot 2d limbs
limb_img = utils.draw_limbs_2d(cam_img, joints_2d, limb_parents)
# Plot 3d limbs
if args.plot_3d:
ax.clear()
ax.view_init(azim=0, elev=90)
ax.set_xlim(-700, 700)
ax.set_ylim(-800, 800)
ax.set_zlim(-700, 700)
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_zlabel('z')
utils.draw_limbs_3d(joints_3d, limb_parents, ax)
# draw heatmap
# hm_img = utils.draw_predicted_heatmap(hm_avg*200, args.input_size)
# cv2.imshow('hm', hm_img.astype(np.uint8))
# cv2.waitKey(0)
concat_img = np.concatenate((limb_img, joint_map), axis=1)
# ax2.imshow(concat_img[..., ::-1].astype(np.uint8))
cv2.imshow('2d', concat_img.astype(np.uint8))
cv2.waitKey(1)
# ax2.imshow(concat_img.astype(np.uint8))
# plt.pause(0.0001)
# plt.show(block=False)
print('Forward FPS', 1 / (time.time() - t1))