def evaluateHmImageFormdata():
with tf.device(tf_device):
if 'file' not in request.files:
abort(409)
abort(Response('No field `file`'))
file = request.files['file']
test_img = cpm_utils.read_image(file.stream.read(), [],
FLAGS.input_size, 'BIN')
stage_heatmap_np = prepare_heatmap(test_img)
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np,
kalman_filter_array)
ret_val, image_binary = cv2.imencode('.png',
(demo_img).astype(np.uint8))
if ret_val:
response = make_response(image_binary.tobytes())
response.headers.set('Content-Type', 'image/png')
response.headers.set('Content-Disposition',
'attachment',
filename='%s.png' % 'img')
return response
else:
abort(500)
abort(Response('failed to encode image'))
def evaluateHm():
with tf.device(tf_device):
t1 = time.time()
test_img = cpm_utils.read_image(request.data, [], FLAGS.input_size,
'BIN')
stage_heatmap_np = prepare_heatmap(test_img)
return jsonify({
'stage_heatmap_np': stage_heatmap_np[0].tolist(),
'time': time.time() - t1
})
def evaluateHmImage():
with tf.device(tf_device):
test_img = cpm_utils.read_image(request.data, [], FLAGS.input_size,
'BIN')
stage_heatmap_np = prepare_heatmap(test_img)
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np,
kalman_filter_array)
ret_val, image_binary = cv2.imencode('.png',
(demo_img).astype(np.uint8))
if ret_val:
response = make_response(image_binary.tobytes())
response.headers.set('Content-Type', 'image/png')
response.headers.set('Content-Disposition',
'attachment',
filename='%s.png' % 'img')
return response
else:
abort(500)
def predict_batch(self, input_test_img):
with tf.device(self.tf_device):
test_img = cpm_utils.read_image(input_test_img, [],
self.input_size, 'IMAGE')
test_img_resize = cv2.resize(test_img,
(self.input_size, self.input_size))
if self.color_channel == 'GRAY':
test_img_resize = np.dot(test_img_resize[..., :3],
[0.299, 0.587, 0.114]).reshape(
(self.input_size, self.input_size,
1))
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
test_img_input = np.repeat(test_img_input, 100, 0)
# Inference
predict_heatmap, stage_heatmap_np = self.sess.run(
[
self.model.current_heatmap,
self.model.stage_heatmap,
],
feed_dict={
'input_image:0': test_img_input,
'center_map:0': self.test_center_map
})
# Get finger anchors from detection.
locations = self.get_locations(test_img, stage_heatmap_np)
# filtering the relevant anchor locations.
relevant_locations = [
locations[0], locations[4], locations[8], locations[12],
locations[16], locations[20]
]
# print(locations)
return relevant_locations
def main(argv):
tf_device = '/gpu:0'
with tf.device(tf_device):
"""Build graph
"""
if FLAGS.color_channel == 'RGB':
input_data = tf.placeholder(dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 3],
name='input_image')
else:
input_data = tf.placeholder(dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='input_image')
center_map = tf.placeholder(dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='center_map')
model = cpm_body_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
model.build_model(input_data, center_map, 1)
saver = tf.train.Saver()
"""Create session and restore weights
"""
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, FLAGS.model_path)
test_center_map = cpm_utils.gaussian_img(FLAGS.input_size,
FLAGS.input_size,
FLAGS.input_size / 2,
FLAGS.input_size / 2,
FLAGS.cmap_radius)
test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size,
FLAGS.input_size, 1])
# Check weights
for variable in tf.trainable_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
# Create kalman filters
if FLAGS.KALMAN_ON:
kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0],
[0, 1, 0, 1],
[0, 0, 1, 0],
[0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0],
[0, 1, 0, 0]],
np.float32)
joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, 0],
[0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
# read in video / flow frames
if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')):
# OpenCV can only read in '.avi' files
cam = imageio.get_reader(FLAGS.DEMO_TYPE)
else:
cam = cv2.VideoCapture(FLAGS.cam_num)
# iamge processing
with tf.device(tf_device):
if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')):
ori_fps = cam.get_meta_data()['fps']
print('This video fps is %f' % ori_fps)
video_length = cam.get_length()
writer_path = os.path.join('results', os.path.basename(FLAGS.DEMO_TYPE))
# !! OpenCV can only write in .avi
cv_writer = cv2.VideoWriter(writer_path + '.avi',
# cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'),
cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),
ori_fps,
(FLAGS.input_size, FLAGS.input_size))
# imageio_writer = imageio.get_writer(writer_path, fps=ori_fps)
try:
for it, im in enumerate(cam):
test_img_t = time.time()
test_img = cpm_utils.read_image(im, [], FLAGS.input_size, 'VIDEO')
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
print('img read time %f' % (time.time() - test_img_t))
if FLAGS.color_channel == 'GRAY':
test_img_resize = mgray(test_img_resize, test_img)
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
# Inference
fps_t = time.time()
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap,
],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if (cv2.waitKey(1) == ord('q')): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
cv_writer.write(demo_img.astype(np.uint8))
# imageio_writer.append_data(demo_img[:, :, 1])
except KeyboardInterrupt:
print('Stopped! {}/{} frames captured!'.format(it, video_length))
finally:
cv_writer.release()
# imageio_writer.close()
else:
while True:
test_img_t = time.time()
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE')
else:
test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM')
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
print('img read time %f' % (time.time() - test_img_t))
if FLAGS.color_channel == 'GRAY':
test_img_resize = mgray(test_img_resize, test_img)
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
# Inference
fps_t = time.time()
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap, ],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if cv2.waitKey(0) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
elif FLAGS.DEMO_TYPE == 'MULTI':
# Inference
fps_t = time.time()
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap,
],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
elif FLAGS.DEMO_TYPE == 'SINGLE':
# Inference
fps_t = time.time()
stage_heatmap_np = sess.run([model.stage_heatmap[5]],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('current heatmap', (demo_img).astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
elif FLAGS.DEMO_TYPE == 'HM':
# Inference
fps_t = time.time()
stage_heatmap_np = sess.run([model.stage_heatmap[FLAGS.stages - 1]],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
print('fps: %.2f' % (1 / (time.time() - fps_t)))
# demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape(
# (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints))
demo_stage_heatmap = stage_heatmap_np[-1][0, :, :, 0:FLAGS.joints].reshape(
(FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints))
demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size))
vertical_imgs = []
tmp_img = None
joint_coord_set = np.zeros((FLAGS.joints, 2))
for joint_num in range(FLAGS.joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num]))
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
if PYTHON_VERSION == 3:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]))
else:
joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
else:
if PYTHON_VERSION == 3:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]))
else:
joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot limbs
for limb_num in range(len(limbs)):
if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]) > -2000 and np.min(
demo_stage_heatmap[:, :, limbs[limb_num][1]]) > -2000:
x1 = joint_coord_set[limbs[limb_num][0], 0]
y1 = joint_coord_set[limbs[limb_num][0], 1]
x2 = joint_coord_set[limbs[limb_num][1], 0]
y2 = joint_coord_set[limbs[limb_num][1], 1]
length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg),
0, 360, 1)
color_code_num = limb_num // 4
if PYTHON_VERSION == 3:
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]))
else:
limb_color = map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])
cv2.fillConvexPoly(test_img, polygon, color=limb_color)
if tmp_img is not None:
tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)),
'constant', constant_values=(0, 0))
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
cv2.imshow('hm', output_img)
cv2.moveWindow('hm', 2000, 200)
cv2.imshow('rgb', test_img)
cv2.moveWindow('rgb', 2000, 750)
if cv2.waitKey(1) == ord('q'): break
def main(argv):
""" Initial tracker
"""
tracker = tracking_module.SelfTracker([FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
""" Build network graph
"""
model = cpm_model.CPM_Model(input_size=FLAGS.input_size,
heatmap_size=FLAGS.heatmap_size,
stages=FLAGS.cpm_stages,
joints=FLAGS.num_of_joints,
img_type=FLAGS.color_channel,
is_training=False)
saver = tf.train.Saver()
""" Get output node
"""
output_node = tf.get_default_graph().get_tensor_by_name(name=FLAGS.output_node_names)
device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
sess_config = tf.ConfigProto(device_count=device_count)
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
with tf.Session(config=sess_config) as sess:
model_path_suffix = os.path.join(FLAGS.network_def,
'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size),
'joints_{}'.format(FLAGS.num_of_joints),
'stages_{}'.format(FLAGS.cpm_stages),
'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr, FLAGS.lr_decay_rate,
FLAGS.lr_decay_step)
)
model_save_dir = os.path.join('models',
'weights',
model_path_suffix)
print('Load model from [{}]'.format(os.path.join(model_save_dir, FLAGS.model_path)))
# if the model is a pkl file, then load it, else just restore the pretrained cpm_hand by default
# by here we can see the loading weight structure
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, 'models/weights/cpm_hand')
# Check weights, this seems to be the part that print out weights
for variable in tf.global_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
# Create webcam instance
if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']:
print("REFUSEREFUSE\n\n\n\n\n\n\n\nrefule")
cam = cv2.VideoCapture(FLAGS.cam_id)
# Create kalman filters
if FLAGS.use_kalman:
kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
print("TESTINGTESTING\n\n\n\n\n\n\n\nrTESGING")
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE')
#fixme: I haven't seen how it cut the box, I have put the coordinates here, but not using them currently
b_box, b_height, b_width = dt.bounding_box_from_file(FLAGS.DEMO_TYPE)
b_box = b_box[0][0]
bb_box = dt.normalize_and_centralize_img(b_box[0], b_box[1], b_box[2], b_box[3], 10, b_height, b_width)\
#fixme: this is the end of my code
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
t1 = time.time()
#current_heatmap is the output after going through the last layer
#input_images is a placeholder matrix for iiamge size
# FIXME: I don't know what exactly does this sess run with? the two arguments?
#predict_heatmap.shape = (1,32,32,22)
#stage_heatmap.shape = (1,32,32,22), it seems like, for stage_heatmap, each of the 1*32*32 is something same, a 22 length-array
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
output_node,
],
feed_dict={model.input_images: test_img_input}
)
#frame per second
print('fps: %.2f' % (1 / (time.time() - t1)))
print(stage_heatmap_np[0], "stage heatmap")
print(stage_heatmap_np[0].shape)
#print(predict_heatmap, "predict heatmap")
#print(predict_heatmap.shape)
tmp_img = cv2.resize(stage_heatmap_np[0], (FLAGS.input_size, FLAGS.input_size))
print(tmp_img, "tmp_img")
print(tmp_img.shape)
correct_and_draw_hand(test_img,
cv2.resize(stage_heatmap_np[0], (FLAGS.input_size, FLAGS.input_size)),
kalman_filter_array, tracker, tracker.input_crop_ratio, test_img)
# Show visualized image
#demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array, tracker, tracker.input_crop_ratio, test_img.copy())
# local_img = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker, crop_full_scale, test_img_copy)
cv2.imshow('demo_img', test_img.astype(np.uint8))
cv2.waitKey(0)
elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']:
print("SINGLE\n\n\n\n\n\n\nMULTI")
while True:
# # Prepare input image
_, full_img = cam.read()
# then I kow the misterious joint_detection comes from here! but now it's only zeros???
test_img = tracker.tracking_by_joints(full_img, joint_detections=joint_detections)
crop_full_scale = tracker.input_crop_ratio
test_img_copy = test_img.copy()
# White balance
test_img_wb = utils.img_white_balance(test_img, 5)
test_img_input = normalize_and_centralize_img(test_img_wb)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
local_img = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker, crop_full_scale,
test_img_copy)
cv2.imshow('local_img', local_img.astype(np.uint8))
cv2.imshow('global_img', full_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
elif FLAGS.DEMO_TYPE == 'Joint_HM':
print("JOINTHM\n\n\n\n\n\n\nJOINTHM")
while True:
# Prepare input image
test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM')
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :,
0:FLAGS.num_of_joints].reshape(
(FLAGS.heatmap_size, FLAGS.heatmap_size, FLAGS.num_of_joints))
demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size))
vertical_imgs = []
tmp_img = None
joint_coord_set = np.zeros((FLAGS.num_of_joints, 2))
for joint_num in range(FLAGS.num_of_joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num]))
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
else:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot FLAGS.limbs
for limb_num in range(len(FLAGS.limbs)):
if np.min(demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][0]]) > -2000 and np.min(
demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][1]]) > -2000:
x1 = joint_coord_set[FLAGS.limbs[limb_num][0], 0]
y1 = joint_coord_set[FLAGS.limbs[limb_num][0], 1]
x2 = joint_coord_set[FLAGS.limbs[limb_num][1], 0]
y2 = joint_coord_set[FLAGS.limbs[limb_num][1], 1]
length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg),
0, 360, 1)
color_code_num = limb_num // 4
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.fillConvexPoly(test_img, polygon, color=limb_color)
if tmp_img is not None:
tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)),
'constant', constant_values=(0, 0))
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
cv2.imshow('hm', output_img)
cv2.moveWindow('hm', 2000, 200)
cv2.imshow('rgb', test_img)
cv2.moveWindow('rgb', 2000, 750)
if cv2.waitKey(1) == ord('q'): break
def main(argv):
tf_device = '/gpu:0'
with tf.device(tf_device):
"""Build graph
"""
if FLAGS.color_channel == 'RGB':
input_data = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 3],
name='input_image')
else:
input_data = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='input_image')
center_map = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='center_map')
model = cpm_body_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
model.build_model(input_data, center_map, 1)
saver = tf.train.Saver()
"""Create session and restore weights
"""
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, FLAGS.model_path)
test_center_map = cpm_utils.gaussian_img(FLAGS.input_size,
FLAGS.input_size,
FLAGS.input_size / 2,
FLAGS.input_size / 2,
FLAGS.cmap_radius)
test_center_map = np.reshape(test_center_map,
[1, FLAGS.input_size, FLAGS.input_size, 1])
# Check weights
for variable in tf.trainable_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
# Create kalman filters
if FLAGS.KALMAN_ON:
kalman_filter_array = [
cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)
]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
# read in video / flow frames
if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')):
# OpenCV can only read in '.avi' files
cam = imageio.get_reader(FLAGS.DEMO_TYPE)
else:
cam = cv2.VideoCapture(FLAGS.cam_num)
# iamge processing
with tf.device(tf_device):
test_img_t = time.time()
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [],
FLAGS.input_size, 'IMAGE')
else:
test_img = cpm_utils.read_image([], cam, FLAGS.input_size,
'WEBCAM')
test_img_resize = cv2.resize(test_img,
(FLAGS.input_size, FLAGS.input_size))
print('img read time %f' % (time.time() - test_img_t))
if FLAGS.color_channel == 'GRAY':
test_img_resize = mgray(test_img_resize, test_img)
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
# Inference
fps_t = time.time()
predict_heatmap, stage_heatmap_np = sess.run([
model.current_heatmap,
model.stage_heatmap,
],
feed_dict={
'input_image:0':
test_img_input,
'center_map:0':
test_center_map
})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np,
kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if cv2.waitKey(0) == ord('q'): exit()
print('fps: %.2f' % (1 /
(time.time() - fps_t))) # iamge processing
print(graph)
for op in graph.get_operations():
print(op.name)
with tf.Session(graph=graph) as sess:
input_size = 256
writer = tf.summary.FileWriter('./', sess.graph)
input_tf = graph.get_tensor_by_name('prefix/input_placeholder:0')
output_tf = graph.get_tensor_by_name(
'prefix/stage_6/mid_conv7/BiasAdd:0')
cam = cv2.VideoCapture(0)
while True:
# Prepare input image
t1 = time.time()
test_img = cpm_utils.read_image([], cam, FLAGS.input_size,
'WEBCAM')
test_img_resize = cv2.resize(test_img,
(FLAGS.input_size, FLAGS.input_size))
print('img read time %f' % (time.time() - t1))
test_img_input = normalize_and_centralize_img(test_img_resize)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([output_tf],
feed_dict={input_tf: test_img_input})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np,
None)
cv2.imshow('current heatmap', (demo_img).astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
def main(argv):
tf_device = '/gpu:0'
with tf.device(tf_device):
"""Build graph
"""
if FLAGS.color_channel == 'RGB':
input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3],
name='input_image')
else:
input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='input_image')
center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='center_map')
model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
model.build_model(input_data, center_map, 1)
saver = tf.train.Saver()
"""Create session and restore weights
"""
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, FLAGS.model_path)
test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2,
FLAGS.input_size / 2,
FLAGS.cmap_radius)
test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1])
# Check weights
for variable in tf.trainable_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
if not FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
#if not FLAGS.DEMO_TYPE.endswith(('avi', 'mp4')):
cam = cv2.VideoCapture(FLAGS.cam_num)
# Create kalman filters
if FLAGS.KALMAN_ON:
kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
with tf.device(tf_device):
while True:
t1 = time.time()
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE')
else:
test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM')
#yolo stuff
result = tfnet.return_predict(test_img)
count = 0
if not result:
pass
else:
for i in range(0, len(result)):
if (result[i]['label'] == 'hand_l') & (count == 0):
hand1 = [result[i]['topleft'], result[i]['bottomright']]
count+=1
elif (result[i]['label'] == 'hand_r') & (count == 1):
hand2 = [result[i]['topleft'], result[i]['bottomright']]
count+=1
#elif (count==1):
# hand2 = []
#elif (count==0):
# hand1 = ['NA']
# hand2 = ['NA']
#cropped image
if not hand1:
pass
else:
h1_image = test_img[hand1[0]['y']:hand1[1]['y'], hand1[0]['x']:hand1[1]['x']]
if not hand2:
pass
else:
h2_image = test_img[hand2[0]['y']:hand2[1]['y'], hand2[0]['x']:hand2[1]['x']]
if not h1_image:
pass
else:
h1_image_resize = cv2.resize(h1_image, (FLAGS.input_size, FLAGS.input_size))
h1_img_input = h1_image_resize / 256.0 - 0.5
h1_img_input = np.expand_dims(h1_img_input, axis=0)
if not h2_image:
pass
else:
h2_image_resize = cv2.resize(h2_image, (FLAGS.input_size, FLAGS.input_size))
h2_img_input = h2_image_resize / 256.0 - 0.5
h2_img_input = np.expand_dims(h2_img_input, axis=0)
"""
#test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
#print('img read time %f' % (time.time() - t1))
if FLAGS.color_channel == 'GRAY':
test_img_resize = np.dot(test_img_resize[..., :3], [0.299, 0.587, 0.114]).reshape(
(FLAGS.input_size, FLAGS.input_size, 1))
cv2.imshow('color', test_img.astype(np.uint8))
cv2.imshow('gray', test_img_resize.astype(np.uint8))
cv2.waitKey(1)
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
"""
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
# Inference
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap,
],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if cv2.waitKey(0) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - t1)))
elif FLAGS.DEMO_TYPE == 'MULTI':
# Inference
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap,
],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - t1)))
elif FLAGS.DEMO_TYPE == 'SINGLE':
# Inference
t1 = time.time()
"""
stage_heatmap_np = sess.run([model.stage_heatmap[5]],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
"""
if h1_img_input:
pass
else:
stage_heatmap_np_1 = sess.run([model.stage_heatmap[5]],
feed_dict={'input_image:0': h1_img_input,
'center_map:0': test_center_map})
demo_img_1 = visualize_result(h1_image, FLAGS, stage_heatmap_np_1, kalman_filter_array)
test_img[hand1[0]['y']:hand1[1]['y'], hand1[0]['x']:hand1[1]['x']] = demo_img_1
if h2_img_input:
pass
else:
stage_heatmap_np_2 = sess.run([model.stage_heatmap[5]],
feed_dict={'input_image:0': h2_img_input,
'center_map:0': test_center_map})
demo_img_2 = visualize_result(h2_image, FLAGS, stage_heatmap_np_2, kalman_filter_array)
test_img[hand2[0]['y']:hand2[1]['y'], hand2[0]['x']:hand2[1]['x']] = demo_img_2
# Show visualized image
# X2 times till here -concatenate here
cv2.imshow('current heatmap', (test_img).astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - t1)))
elif FLAGS.DEMO_TYPE == 'HM':
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([model.stage_heatmap[FLAGS.stages - 1]],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
print('fps: %.2f' % (1 / (time.time() - t1)))
demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape(
(FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints))
demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size))
vertical_imgs = []
tmp_img = None
joint_coord_set = np.zeros((FLAGS.joints, 2))
for joint_num in range(FLAGS.joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num]))
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
if PYTHON_VERSION == 3:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]))
else:
joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
else:
if PYTHON_VERSION == 3:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]))
else:
joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot limbs
for limb_num in range(len(limbs)):
if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]) > -2000 and np.min(
demo_stage_heatmap[:, :, limbs[limb_num][1]]) > -2000:
x1 = joint_coord_set[limbs[limb_num][0], 0]
y1 = joint_coord_set[limbs[limb_num][0], 1]
x2 = joint_coord_set[limbs[limb_num][1], 0]
y2 = joint_coord_set[limbs[limb_num][1], 1]
length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg),
0, 360, 1)
color_code_num = limb_num // 4
if PYTHON_VERSION == 3:
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]))
else:
limb_color = map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])
cv2.fillConvexPoly(test_img, polygon, color=limb_color)
if tmp_img is not None:
tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)),
'constant', constant_values=(0, 0))
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
cv2.imshow('hm', output_img)
cv2.moveWindow('hm', 2000, 200)
cv2.imshow('rgb', test_img)
cv2.moveWindow('rgb', 2000, 750)
if cv2.waitKey(1) == ord('q'): break
def main(argv):
global joint_detections
os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)
""" Initial tracker
"""
tracker = tracking_module.SelfTracker([FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
""" Build network graph
"""
model = cpm_model.CPM_Model(input_size=FLAGS.input_size,
heatmap_size=FLAGS.heatmap_size,
stages=FLAGS.cpm_stages,
joints=FLAGS.num_of_joints,
img_type=FLAGS.color_channel,
is_training=False)
saver = tf.train.Saver()
""" Get output node
"""
output_node = tf.get_default_graph().get_tensor_by_name(name=FLAGS.output_node_names)
device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
sess_config = tf.ConfigProto(device_count=device_count)
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
with tf.Session(config=sess_config) as sess:
model_path_suffix = os.path.join(FLAGS.network_def,
'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size),
'joints_{}'.format(FLAGS.num_of_joints),
'stages_{}'.format(FLAGS.cpm_stages),
'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr, FLAGS.lr_decay_rate,
FLAGS.lr_decay_step)
)
model_save_dir = os.path.join('models',
'weights',
model_path_suffix)
print('Load model from [{}]'.format(os.path.join(model_save_dir, FLAGS.model_path)))
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, 'models/weights/cpm_hand')
# Check weights
for variable in tf.global_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
# Create webcam instance
if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']:
cam = cv2.VideoCapture(FLAGS.cam_id)
# Create kalman filters
if FLAGS.use_kalman:
kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE')
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
output_node,
],
feed_dict={model.input_images: test_img_input}
)
print('fps: %.2f' % (1 / (time.time() - t1)))
correct_and_draw_hand(test_img,
cv2.resize(stage_heatmap_np[0], (FLAGS.input_size, FLAGS.input_size)),
kalman_filter_array, tracker, tracker.input_crop_ratio, test_img)
# Show visualized image
# demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', test_img.astype(np.uint8))
cv2.waitKey(0)
elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']:
while True:
# # Prepare input image
_, full_img = cam.read()
test_img = tracker.tracking_by_joints(full_img, joint_detections=joint_detections)
crop_full_scale = tracker.input_crop_ratio
test_img_copy = test_img.copy()
# White balance
test_img_wb = utils.img_white_balance(test_img, 5)
test_img_input = normalize_and_centralize_img(test_img_wb)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
local_img = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker, crop_full_scale,
test_img_copy)
cv2.imshow('local_img', local_img.astype(np.uint8))
cv2.imshow('global_img', full_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
elif FLAGS.DEMO_TYPE == 'Joint_HM':
while True:
# Prepare input image
test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM')
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :,
0:FLAGS.num_of_joints].reshape(
(FLAGS.heatmap_size, FLAGS.heatmap_size, FLAGS.num_of_joints))
demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size))
vertical_imgs = []
tmp_img = None
joint_coord_set = np.zeros((FLAGS.num_of_joints, 2))
for joint_num in range(FLAGS.num_of_joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num]))
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
else:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot FLAGS.limbs
for limb_num in range(len(FLAGS.limbs)):
if np.min(demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][0]]) > -2000 and np.min(
demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][1]]) > -2000:
x1 = joint_coord_set[FLAGS.limbs[limb_num][0], 0]
y1 = joint_coord_set[FLAGS.limbs[limb_num][0], 1]
x2 = joint_coord_set[FLAGS.limbs[limb_num][1], 0]
y2 = joint_coord_set[FLAGS.limbs[limb_num][1], 1]
length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg),
0, 360, 1)
color_code_num = limb_num // 4
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.fillConvexPoly(test_img, polygon, color=limb_color)
if tmp_img is not None:
tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)),
'constant', constant_values=(0, 0))
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
cv2.imshow('hm', output_img)
cv2.moveWindow('hm', 2000, 200)
cv2.imshow('rgb', test_img)
cv2.moveWindow('rgb', 2000, 750)
if cv2.waitKey(1) == ord('q'): break
def main(argv):
tf_device = '/cpu:0'
with tf.device(tf_device):
"""Build graph
"""
if FLAGS.color_channel == 'RGB':
input_data = tf.placeholder(dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 3],
name='input_image')
else:
input_data = tf.placeholder(dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='input_image')
center_map = tf.placeholder(dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='center_map')
model = cpm_body_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
model.build_model(input_data, center_map, 1)
saver = tf.train.Saver()
"""Create session and restore weights
"""
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, FLAGS.model_path)
test_center_map = cpm_utils.gaussian_img(FLAGS.input_size,
FLAGS.input_size,
FLAGS.input_size / 2,
FLAGS.input_size / 2,
FLAGS.cmap_radius)
test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size,
FLAGS.input_size, 1])
# Check weights
for variable in tf.trainable_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
# Create kalman filters
if FLAGS.KALMAN_ON:
kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0],
[0, 1, 0, 1],
[0, 0, 1, 0],
[0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0],
[0, 1, 0, 0]],
np.float32)
joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, 0],
[0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
# read in video / flow frames
if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')):
# OpenCV can only read in '.avi' files
cam = imageio.get_reader(FLAGS.DEMO_TYPE)
else:
cam = cv2.VideoCapture(FLAGS.cam_num)
# iamge processing
with tf.device(tf_device):
if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')):
ori_fps = cam.get_meta_data()['fps']
print('This video fps is %f' % ori_fps)
video_length = cam.get_length()
writer_path = os.path.join('results', os.path.basename(FLAGS.DEMO_TYPE))
# !! OpenCV can only write in .avi
cv_writer = cv2.VideoWriter(writer_path + '.avi',
# cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'),
cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),
ori_fps,
(FLAGS.input_size, FLAGS.input_size))
# imageio_writer = imageio.get_writer(writer_path, fps=ori_fps)
try:
for it, im in enumerate(cam):
test_img_t = time.time()
test_img = cpm_utils.read_image(im, [], FLAGS.input_size, 'VIDEO')
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
print('img read time %f' % (time.time() - test_img_t))
if FLAGS.color_channel == 'GRAY':
test_img_resize = mgray(test_img_resize, test_img)
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
# Inference
fps_t = time.time()
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap,
],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if (cv2.waitKey(1) == ord('q')): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
cv_writer.write(demo_img.astype(np.uint8))
# imageio_writer.append_data(demo_img[:, :, 1])
except KeyboardInterrupt:
print('Stopped! {}/{} frames captured!'.format(it, video_length))
finally:
cv_writer.release()
# imageio_writer.close()
else:
while True:
test_img_t = time.time()
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE')
else:
test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM')
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
print('img read time %f' % (time.time() - test_img_t))
if FLAGS.color_channel == 'GRAY':
test_img_resize = mgray(test_img_resize, test_img)
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
# Inference
fps_t = time.time()
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap, ],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if cv2.waitKey(0) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
elif FLAGS.DEMO_TYPE == 'MULTI':
# Inference
fps_t = time.time()
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap,
],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
elif FLAGS.DEMO_TYPE == 'SINGLE':
# Inference
fps_t = time.time()
stage_heatmap_np = sess.run([model.stage_heatmap[5]],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('current heatmap', (demo_img).astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
elif FLAGS.DEMO_TYPE == 'HM':
# Inference
fps_t = time.time()
stage_heatmap_np = sess.run([model.stage_heatmap[FLAGS.stages - 1]],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
print('fps: %.2f' % (1 / (time.time() - fps_t)))
# demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape(
# (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints))
demo_stage_heatmap = stage_heatmap_np[-1][0, :, :, 0:FLAGS.joints].reshape(
(FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints))
demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size))
vertical_imgs = []
tmp_img = None
joint_coord_set = np.zeros((FLAGS.joints, 2))
for joint_num in range(FLAGS.joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num]))
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
if PYTHON_VERSION == 3:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]))
else:
joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
else:
if PYTHON_VERSION == 3:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]))
else:
joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot limbs
for limb_num in range(len(limbs)):
if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]) > -2000 and np.min(
demo_stage_heatmap[:, :, limbs[limb_num][1]]) > -2000:
x1 = joint_coord_set[limbs[limb_num][0], 0]
y1 = joint_coord_set[limbs[limb_num][0], 1]
x2 = joint_coord_set[limbs[limb_num][1], 0]
y2 = joint_coord_set[limbs[limb_num][1], 1]
length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg),
0, 360, 1)
color_code_num = limb_num // 4
if PYTHON_VERSION == 3:
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]))
else:
limb_color = map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])
cv2.fillConvexPoly(test_img, polygon, color=limb_color)
if tmp_img is not None:
tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)),
'constant', constant_values=(0, 0))
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
cv2.imshow('hm', output_img)
cv2.moveWindow('hm', 2000, 200)
cv2.imshow('rgb', test_img)
cv2.moveWindow('rgb', 2000, 750)
if cv2.waitKey(1) == ord('q'): break
def main(argv):
global ratios, avg_i, m_avg, s
tf_device = '/gpu:0'
with tf.device(tf_device):
#Build graph
input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3],name='input_image')
center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1],name='center_map')
model = cpm_hand_slim.CPM_Model(FLAGS.stages, 21 + 1)
model.build_model(input_data, center_map, 1)
saver = tf.train.Saver()
#Create session and restore weights
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=False))
sess.run(tf.global_variables_initializer())
model.load_weights_from_file(models/weights/cpm_hand.pkl, sess, False)
#get gaussian image
test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2,
FLAGS.input_size / 2,
21)
test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1])
#Starting Video Input
cam = cv2.VideoCapture(0)
# Create kalman filters
kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(21)]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * 3e-2
with tf.device(tf_device):
t0=time.time()
while True:
print(time.time()-t0)
#Read image and resize it according to architecture input size.
test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM')
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
# Starting time of each iteration.
t1 = time.time()
#Run the image through the model and get corresponding heatmap.
stage_heatmap_np = sess.run([model.stage_heatmap[2]],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
#Get the coordinates of each joint and print the image with joints makrked on the figure.
demo_img,coords = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('current heatmap', (demo_img).astype(np.uint8))
#At the beginning of each execution, user gets 30 sec
#for setting reference angles when hand was kept straight.
if ((time.time()-t0)<30):
store_deb(coords, t0)
print('ratios: '+str(ratios))
else:
debug(coords)
print('degrees: '+str(np.mean(m_avg,axis=0)))
#Calculate moving average for past n timesteps.
mavg_deg=np.mean(m_avg,axis=0)
mavg_deg[4]= min(mavg_deg[4],20)
avg_i=(avg_i+1)%15
#Setting the string to be transmitted to Arduino.
transp_str=''
for i in mavg_deg:
transp_str=transp_str+str(i)+','
#Flushing the input and output buffer before writing to Serial.
s.flushInput()
s.flushOutput()
s.write(transp_str.encode())
if cv2.waitKey(1) == ord('q'): break
help='One of image/json')
parser.add_argument('-d',
'--device',
default='gpu',
help='One of cpu or gpu')
args = parser.parse_args()
files = os.listdir(args.input)
if len(files) == 0:
raise Exception("Input directory is empty")
print("Found", len(files), "files")
images = [
cpm_utils.read_image(os.path.join(args.input, f), [], 368, 'IMAGE')
for f in files
]
poses = [
pose for _, pose in hand_pose(
images, right_hand=True, mode="BGR", device=args.device)
]
for f, image, pose in zip(files, images, poses):
save_loc = os.path.join(args.output, f)
if args.format == "json":
json.dump(pose, open(save_loc + ".json", "w"))
else:
new_image = draw_pose(image, pose)
cv2.imwrite(save_loc, new_image)
def main(argv):
global joint_detections
os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)
tracker = tracking_module.SelfTracker(
[FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
model = cpm_model.CPM_Model(input_size=FLAGS.input_size,
heatmap_size=FLAGS.heatmap_size,
stages=FLAGS.cpm_stages,
joints=FLAGS.num_of_joints,
img_type=FLAGS.color_channel,
is_training=False)
saver = tf.train.Saver()
output_node = tf.get_default_graph().get_tensor_by_name(
name=FLAGS.output_node_names)
device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
sess_config = tf.ConfigProto(device_count=device_count)
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
with tf.Session(config=sess_config) as sess:
model_path_suffix = os.path.join(
FLAGS.network_def,
'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size),
'joints_{}'.format(FLAGS.num_of_joints),
'stages_{}'.format(FLAGS.cpm_stages),
'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr,
FLAGS.lr_decay_rate,
FLAGS.lr_decay_step))
model_save_dir = os.path.join('models', 'weights', model_path_suffix)
print('Load model from [{}]'.format(
os.path.join(model_save_dir, FLAGS.model_path)))
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, 'models/weights/cpm_hand')
for variable in tf.global_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']:
cam = cv2.VideoCapture(FLAGS.cam_id)
#cam = pipeline.wait_for_frames()
if FLAGS.use_kalman:
kalman_filter_array = [
cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)
]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [],
FLAGS.input_size, 'IMAGE')
test_img_resize = cv2.resize(test_img,
(FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run(
[
model.current_heatmap,
output_node,
],
feed_dict={model.input_images: test_img_input})
print('fps: %.2f' % (1 / (time.time() - t1)))
correct_and_draw_hand(
test_img,
cv2.resize(stage_heatmap_np[0],
(FLAGS.input_size, FLAGS.input_size)),
kalman_filter_array, tracker, tracker.input_crop_ratio,
test_img)
# Show visualized image
# demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', test_img.astype(np.uint8))
cv2.waitKey(0)
elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']:
while True:
_, full_img = cam.read()
test_img = tracker.tracking_by_joints(
full_img, joint_detections=joint_detections)
crop_full_scale = tracker.input_crop_ratio
test_img_copy = test_img.copy()
# White balance
test_img_wb = utils.img_white_balance(test_img, 5)
test_img_input = normalize_and_centralize_img(test_img_wb)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run(
[output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
local_img = visualize_result(full_img, stage_heatmap_np,
kalman_filter_array, tracker,
crop_full_scale, test_img_copy)
cv2.imshow('local_img', local_img.astype(np.uint8))
cv2.imshow('global_img', full_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
def main(argv):
tf_device = '/gpu:0'
with tf.device(tf_device):
"""Build graph
"""
input_data = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 3],
name='input_image')
center_map = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='center_map')
model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
model.build_model(input_data, center_map, 1)
saver = tf.train.Saver()
"""Create session and restore weights
"""
sess = tf.Session()
sess.run(tf.global_variables_initializer())
model.load_weights_from_file(FLAGS.model_path, sess, False)
#model_vars = tf.trainable_variables()
#slim.model_analyzer.analyze_vars(model_vars, print_info=True)
test_center_map = cpm_utils.gaussian_img(FLAGS.input_size,
FLAGS.input_size,
FLAGS.input_size / 2,
FLAGS.input_size / 2,
FLAGS.cmap_radius)
test_center_map = np.reshape(test_center_map,
[1, FLAGS.input_size, FLAGS.input_size, 1])
if not FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
cam = cv2.VideoCapture(FLAGS.cam_num)
# Create kalman filters
kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
with tf.device(tf_device):
#filedem = open("testfile.txt","w")
#filedem.close()
if FLAGS.DEMO_TYPE == 'SINGLE':
#read default image coords
v = []
with open('test_file.csv') as file:
reader = csv.reader(file, delimiter=',')
for row in reader:
for entries in row:
v.append(entries)
'''
f=open("testfile.txt", 'r')
v=f.readline()
v=v.strip().split(' ')
'''
while True:
t1 = time.time()
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [],
FLAGS.input_size, 'IMAGE')
else:
test_img = cpm_utils.read_image([], cam, FLAGS.input_size,
'WEBCAM')
test_img_resize = cv2.resize(test_img,
(FLAGS.input_size, FLAGS.input_size))
#print('img read time %f' % (time.time() - t1))
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
# Inference
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run(
[
model.current_heatmap,
model.stage_heatmap,
],
feed_dict={
'input_image:0': test_img_input,
'center_map:0': test_center_map
})
# Show visualized image
demo_img, coords = visualize_result(test_img, FLAGS,
stage_heatmap_np,
kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
'''
filedem = open("testfile.txt","w")
for i in [0,5,7,9,11,13,15,17,19]:
for j in [0,1]:
filedem.write(str(coords[i][j])+' ')
filedem.close()
'''
with open('test_file.csv', mode='w') as file:
writer = csv.writer(file, delimiter=',')
for i in [1, 5, 8, 9, 12, 13, 16, 17, 20]:
writer.writerow([coords[i][0], coords[i][1]])
if cv2.waitKey(0) == ord('q'): break
#print('fps: %.2f' % (1 / (time.time() - t1)))
elif FLAGS.DEMO_TYPE == 'SINGLE':
# Inference
t1 = time.time()
stage_heatmap_np = sess.run(
[model.stage_heatmap[2]],
feed_dict={
'input_image:0': test_img_input,
'center_map:0': test_center_map
})
# Show visualized image
demo_img, coords = visualize_result(test_img, FLAGS,
stage_heatmap_np,
kalman_filter_array)
cv2.imshow('current heatmap', (demo_img).astype(np.uint8))
debug(coords, v)
global avg_i, m_avg
avg_i = (avg_i + 1) % 10
print(np.mean(m_avg, axis=0))
'''
filedem = open("testfile.txt","a")
for i in [0,5,7,9,11,13,15,17,19]:
for j in [0,1]:
filedem.write(str(coords[i][j])+' ')
filedem.close()
'''
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - t1)))
def main(argv):
# tf_device = '/gpu:0'
tf_device = '/cpu:0'
with tf.device(tf_device):
"""Build graph
"""
input_data = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 3],
name='input_image')
center_map = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size,
1], #center map的大小也是368x368
name='center_map')
# model = cpm_body.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
model = cpm_body.CPM_Model(
FLAGS.stages,
FLAGS.joints) #这里的stages和joints是传进来的参数，跟训练集的关节点个数应该相等
model.build_model(input_data, center_map, 1)
saver = tf.train.Saver()
"""Create session and restore weights
"""
# gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333)
# sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
# sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, FLAGS.model_path)
test_center_map = cpm_utils.gaussian_img(
FLAGS.input_size,
FLAGS.input_size,
FLAGS.input_size / 2, #预测的时候，需要把人放图像中间，这里构造center map是以图片中心来构造的
FLAGS.input_size / 2,
FLAGS.cmap_radius) #如果预测人没有在中间，就会出现问题
test_center_map = np.reshape(
test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1
]) #增加batch和通道的维数都为1，此刻center map：1x368x368x1
# Check weights
for variable in tf.trainable_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name,
np.mean(sess.run(var))) #这个for循环输出了模型中的参数，包括卷积核值和偏置值
with tf.device(tf_device):
# while True:
test_img_t = time.time()
test_img = cpm_utils.read_image(
FLAGS.img_path, [], FLAGS.input_size,
'IMAGE') #构造预测输入，[]是视频输出，这里没有，test_img为368x368x3
test_img_resize = cv2.resize(
test_img, (FLAGS.input_size, FLAGS.input_size)) #删掉，没用
print('img read time %f' % (time.time() - test_img_t))
test_img_input = test_img_resize / 256.0 - 0.5 #归一化到[-0.5,0.5]
test_img_input = np.expand_dims(
test_img_input,
axis=0) #增加了batch的维数为1，此刻test_img_input： 1x368x368x3
if FLAGS.DEMO_TYPE == 'MULTI':
# Inference
fps_t = time.time()
predict_heatmap, stage_heatmap_np = sess.run([
model.current_heatmap,
model.stage_heatmap,
],
feed_dict={
'input_image:0':
test_img_input,
'center_map:0':
test_center_map
})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np,
None)
# cv2.imshow('demo_img', demo_img.astype(np.uint8))
cv2.imwrite(os.path.join(base_path,
str(time.time()) + "_.jpg"),
demo_img.astype(np.uint8))
# if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
elif FLAGS.DEMO_TYPE == 'SINGLE':
# Inference
fps_t = time.time()
stage_heatmap_np = sess.run([model.stage_heatmap[5]],
feed_dict={
'input_image:0': test_img_input,
'center_map:0': test_center_map
})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np,
None)
# cv2.imshow('current heatmap', (demo_img).astype(np.uint8))
cv2.imwrite(os.path.join(base_path, "current_heatmap.jpg"),
(demo_img).astype(np.uint8))
# if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - fps_t)))
elif FLAGS.DEMO_TYPE == 'HM':
# Inference
fps_t = time.time()
stage_heatmap_np = sess.run(
[model.stage_heatmap[FLAGS.stages - 1]
], #最后一个stage的heatmap，列表长度为1，因为只有一个人
feed_dict={
'input_image:0':
test_img_input, #stage_heatmap_np[0]的shape为 [1, 46, 46, 15]
'center_map:0': test_center_map
})
print('fps: %.2f' % (1 / (time.time() - fps_t)))
# demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape(
# (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints))
demo_stage_heatmap = stage_heatmap_np[-1][
0, :, :,
0:FLAGS.joints].reshape( #将有效的heatmap切出来，把背景排除，结果：46x46x14
(FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints))
demo_stage_heatmap = cv2.resize(
demo_stage_heatmap,
(FLAGS.input_size, FLAGS.input_size)) #heatmap变成了368x368x14
vertical_imgs = [] #下面在拼接heatmap的展示图片
tmp_img = None
joint_coord_set = np.zeros((FLAGS.joints, 2))
for joint_num in range(FLAGS.joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (
255 / np.max(demo_stage_heatmap[:, :, joint_num])
) #将heatmap每层的最大值变成255，共有14层
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(
np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4),
joint_color_code[color_code_num]))
cv2.circle(test_img,
center=(joint_coord[1], joint_coord[0]),
radius=3,
color=joint_color,
thickness=-1)
else:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4),
joint_color_code[color_code_num]))
cv2.circle(test_img,
center=(joint_coord[1], joint_coord[0]),
radius=3,
color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
tmp = cv2.putText(
tmp,
'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20),
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.3,
color=150)
tmp = cv2.putText(
tmp,
'Mean:' +
str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30),
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.3,
color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot limbs
for limb_num in range(len(limbs)):
if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]
) > -2000 and np.min(
demo_stage_heatmap[:, :,
limbs[limb_num][1]]) > -2000:
x1 = joint_coord_set[limbs[limb_num][0],
0] #limbs中存储了关节之间的连接关系，两个节点
y1 = joint_coord_set[limbs[limb_num][0], 1]
x2 = joint_coord_set[limbs[limb_num][1], 0]
y2 = joint_coord_set[limbs[limb_num][1], 1]
length = ((x1 - x2)**2 + (y1 - y2)**2)**0.5 #两个关节的距离，肢体的距离
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2,
y1 - y2)) #肢体的角度
polygon = cv2.ellipse2Poly(
(int(
(y1 + y2) / 2), int(
(x1 + x2) / 2)), (int(length / 2), 3),
int(deg), 0, 360, 1
) #画出肢体的椭圆形状，以两关节的中点为中心，椭圆的a值为肢体长度一半，b值自己定，还要给出椭圆的倾斜角度，也就是上面求出的角度
color_code_num = limb_num // 4 #给当前肢体选颜色
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4),
joint_color_code[color_code_num]))
cv2.fillConvexPoly(
test_img, polygon,
color=limb_color) #test_img是我们的测试输入图片 (368,368,3)
if tmp_img is not None: #上面已经将肢体的椭圆连线画在了测试输入图片上
tmp_img = np.lib.pad(
tmp_img,
((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]),
(0, 0)),
'constant',
constant_values=(
0, 0)) #在test_img的上面(为0)和下面做padding，padding的值为0，这段可以删掉
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None #输出的图片
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
# cv2.imshow('hm', output_img)
print("=======================")
print(base_path)
print(os.path.join(base_path, str(time.time()) + "_.jpg"))
cv2.imwrite(os.path.join(base_path,
str(time.time()) + "_.jpg"),
output_img.astype(np.uint8))
print(output_img)
# cv2.moveWindow('hm', 2000, 200)
# cv2.imshow('rgb', test_img)
cv2.imwrite(os.path.join(base_path,
str(time.time()) + "_.jpg"),
test_img.astype(np.uint8))
# cv2.moveWindow('rgb', 2000, 750)
# if cv2.waitKey(1) == ord('q'): break
time.sleep(5)
def main(argv):
tf_device = '/gpu:0'
with tf.device(tf_device):
"""Build graph
"""
frame_num = 0
input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3],
name='input_image')
center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='center_map')
model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
model.build_model(input_data, center_map, 1)
"""Create session and restore weights
"""
sess = tf.Session()
sess.run(tf.global_variables_initializer())
model.load_weights_from_file(FLAGS.model_path, sess, False)
test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2,
FLAGS.input_size / 2,
FLAGS.cmap_radius)
test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1])
# Check weights
for variable in tf.trainable_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
cam = cv2.VideoCapture(FLAGS.cam_num)
# Create kalman filters
if FLAGS.KALMAN_ON:
kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
# Init Openni2
openni2.initialize("C:\OpenNI\Redist")
dev = openni2.Device.open_any()
depth_stream = dev.create_depth_stream()
depth_stream.start()
depth_stream.set_video_mode(
c_api.OniVideoMode(pixelFormat=c_api.OniPixelFormat.ONI_PIXEL_FORMAT_DEPTH_100_UM, resolutionX=640,
resolutionY=480, fps=30))
with tf.device(tf_device):
while True:
# t1 = time.time()
orig_img, test_img, depth_map = cpm_utils.read_image(cam, FLAGS.input_size, depth_stream)
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
# print('img read time %f' % (time.time() - t1))
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([model.stage_heatmap[5]],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, orig_img, FLAGS, stage_heatmap_np, kalman_filter_array, frame_num,
depth_map)
cv2.imshow('current depth', depth_map)
cv2.imshow('current heatmap', demo_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - t1)))
print(frame_num)
frame_num = frame_num + 1
def main():
with open(sys.argv[1]) as f:
conf = json.load(f)
images_folder = conf['images_folder']
heatmaps_output_folder = conf['heatmaps_output_folder']
masks_output_folder = conf['masks_output_folder']
images_output_folder = conf['images_output_folder']
model_path = conf['model_path']
#1. Read list of images from folder
images_ids = [a for a in os.listdir(images_folder) if a.endswith(".jpg")]
#2.
tf_device = "/gpu:0"
with tf.device(tf_device):
input_data = tf.placeholder(dtype=tf.float32,
shape=[None, input_size, input_size, 3],
name='input_image')
center_map = tf.placeholder(dtype=tf.float32,
shape=[None, input_size, input_size, 1],
name='center_map')
model = cpm_hand.CPM_Model(stages, joints + 1)
model.build_model(input_data, center_map, 1)
saver = tf.train.Saver()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
model.load_weights_from_file(model_path, sess, False)
test_center_map = cpm_utils.gaussian_img(input_size, input_size,
input_size / 2, input_size / 2,
cmap_radius)
test_center_map = np.reshape(test_center_map,
[1, input_size, input_size, 1])
with tf.device(tf_device):
for im_id in images_ids:
print(im_id)
im_path = os.path.join(images_folder, im_id)
test_img = cpm_utils.read_image(im_path, [], input_size, 'IMAGE')
test_img_resize = cv2.resize(test_img, (input_size, input_size))
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
predict_heatmap, stage_heatmap_np = sess.run([
model.current_heatmap,
model.stage_heatmap,
],
feed_dict={
'input_image:0':
test_img_input,
'center_map:0':
test_center_map
})
last_heatmap = stage_heatmap_np[len(stage_heatmap_np) -
1][0, :, :, 0:joints].reshape(
(hmap_size, hmap_size, joints))
heatmap_path = os.path.join(heatmaps_output_folder, im_id + ".npy")
np.save(heatmap_path, last_heatmap)
last_heatmap = cv2.resize(last_heatmap,
(test_img.shape[1], test_img.shape[0]))
demo_img, masked_img = visualize_result(test_img, last_heatmap)
heatmap_path = os.path.join(heatmaps_output_folder, im_id + ".npy")
mask_path = os.path.join(masks_output_folder, im_id)
im_path = os.path.join(images_output_folder, im_id)
imageio.imsave(mask_path, masked_img)
imageio.imsave(im_path, demo_img)
def main(argv):
tf_device = '/gpu:0'
with tf.device(tf_device):
"""
if FLAGS.color_channel == 'RGB':
input_data = tf.placeholder(dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 3],
name='input_image')
else:
input_data = tf.placeholder(dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='input_image')
center_map = tf.placeholder(dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='center_map')
"""
# model = cpm_body_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
# model.build_model(input_data, center_map, 1)
# 没有背景
model = cpm_body_slim.CPM_Model(FLAGS.input_size, FLAGS.hmap_size, FLAGS.stages, FLAGS.joints, 1, img_type = FLAGS.color_channel)
saver = tf.train.Saver()
"""Create session and restore weights
"""
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, FLAGS.model_path)
# 不要centermap, 意义不明
"""
# 创建center_map，正方形的中心放置高斯响应
test_center_map = cpm_utils.gaussian_img(FLAGS.input_size,
FLAGS.input_size,
FLAGS.input_size / 2,
FLAGS.input_size / 2,
FLAGS.cmap_radius)
test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size,
FLAGS.input_size, 1])
"""
# read in video / flow frames
if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')):
# OpenCV can only read in '.avi' files
cam = imageio.get_reader(FLAGS.DEMO_TYPE)
# iamge processing
with tf.device(tf_device):
if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')):
ori_fps = cam.get_meta_data()['fps']
cap = cv2.VideoCapture(FLAGS.DEMO_TYPE)
total_frame = cap.get(cv2.CAP_PROP_FRAME_COUNT)
(W,H) = (int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)))
pbar = tqdm(total=total_frame)
print('This video fps is %f' % ori_fps)
video_length = cam.get_length()
# writer_path = os.path.join('results', os.path.basename(FLAGS.DEMO_TYPE))
# !! OpenCV can only write in .avi
cv_writer = cv2.VideoWriter('results/result.mp4',
# cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'),
cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),
ori_fps,
(W,H))
# imageio_writer = imageio.get_writer(writer_path, fps=ori_fps)
try:
for it, im in enumerate(cam):
test_img_t = time.time()
full_img = im.copy() # 把结果画在原图上
# test_img是原图像resize得到，并且进行了padding
test_img = cpm_utils.read_image(im, [], FLAGS.input_size, 'VIDEO')
# 多余的resize
# test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
# print('img read time %f' % (time.time() - test_img_t))
if FLAGS.color_channel == 'GRAY':
test_img = cv2.cvtColor(test_img, cv2.COLOR_RGB2GRAY)
test_img_input = test_img / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
# Inference
# fps_t = time.time()
"""
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap,
],
feed_dict={'input_placeholder:0': test_img_input,
'cmap_placeholder:0': test_center_map})
"""
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap,
],
feed_dict={'input_placeholder:0': test_img_input})
# Show visualized image
demo_img = visualize_result(test_img, full_img, FLAGS, stage_heatmap_np)
#cv2.putText(demo_img, "FPS: %.1f" % (1 / (time.time() - fps_t)), (20, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 0), 1)
cv_writer.write(demo_img.astype(np.uint8))
pbar.update(1)
# print(1/(time.time()-test_img_t))
cv2.imwrite('results/pics_test/{}.png'.format(str(it).zfill(5)), demo_img[:,:,::-1])
#exit(0)
# imageio_writer.append_data(demo_img[:, :, 1])
except KeyboardInterrupt:
print('Stopped! {}/{} frames captured!'.format(it, video_length))
finally:
cv_writer.release()
# imageio_writer.close()
elif FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE')
test_img_input = test_img / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
# Inference
fps_t = time.time()
"""
stage_heatmap_np = sess.run([model.stage_heatmap[5]],
feed_dict={'input_placeholder:0': test_img_input,
'cmap_placeholder:0': test_center_map})
"""
stage_heatmap_np = sess.run([model.stage_heatmap[5]],
feed_dict={'input_placeholder:0': test_img_input})
# Show visualized image
ori_img = cv2.imread(FLAGS.DEMO_TYPE)
demo_img = visualize_result(test_img, ori_img, FLAGS, stage_heatmap_np)
cv2.imwrite('results/test.jpg', demo_img)
print('fps: %.1f' % (1 / (time.time() - fps_t)))
else:
print('Demo type is not defined, please check it!')
def main(argv):
tf_device = '/gpu:0'
with tf.device(tf_device):
"""Build graph
"""
if FLAGS.color_channel == 'RGB':
input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3],
name='input_image')
else:
input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='input_image')
center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='center_map')
# tensorflow slim model
model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
model.build_model(input_data, center_map, 1)
# keras model
# model = cpm_hand_keras.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
# model.build_model(input_data, center_map, 1)
saver = tf.train.Saver()
"""Create session and restore weights
"""
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, FLAGS.model_path)
test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2,
FLAGS.input_size / 2,
FLAGS.cmap_radius)
test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1])
# Check weights
for variable in tf.trainable_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
if not FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
cam = cv2.VideoCapture(FLAGS.cam_num)
# ------------------------------------------ save model-------------------------------
# save_dir = 'checkpoints/'
# if not os.path.exists(save_dir):
# os.makedirs(save_dir)
# save_path = os.path.join(save_dir, 'best_validation')
# saver.save(sess=sess, save_path=save_path)
# builder = tf.saved_model.builder.SavedModelBuilder('./SavedModel/')
# signature = predict_signature_def(inputs={'input_image:0': input_data,
# 'center_map:0': center_map})
# builder.add_meta_graph_and_variables(sess,
# [tf.saved_model.tag_constants.TRAINING],
# signature_def_map={'predict': signature},
# assets_collection=None,
# strip_default_attrs=True)
# builder.add_meta_graph([tf.saved_model.tag_constants.SERVING], strip_default_attrs=True)
# builder.save()
signature = None
# Create kalman filters
if FLAGS.KALMAN_ON:
kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
with tf.device(tf_device):
while True:
t1 = time.time()
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE')
else:
test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM')
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
print("----------------------------------------start to read img--------------------")
print('img read time %f' % (time.time() - t1))
if FLAGS.color_channel == 'GRAY':
test_img_resize = np.dot(test_img_resize[..., :3], [0.299, 0.587, 0.114]).reshape(
(FLAGS.input_size, FLAGS.input_size, 1))
cv2.imshow('color', test_img.astype(np.uint8))
cv2.imshow('gray', test_img_resize.astype(np.uint8))
cv2.waitKey(1)
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
# Inference
t1 = time.time()
# print("=========================test image============================")
# print(test_img_input)
# print("=========================test image shape============================")
# print(test_img_input.shape)
# slim prediction
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap,
],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# keras prediction
# predict_heatmap, stage_heatmap_np = model.predict(test_img_input)
print("---------------stage_heatmap_np:------------------")
# print(predict_heatmap)
print(np.array(stage_heatmap_np).shape)
# print(stage_heatmap_np)
inputa = tf.saved_model.utils.build_tensor_info(input_data)
predict_heatmap = tf.convert_to_tensor(predict_heatmap)
outputa = tf.saved_model.utils.build_tensor_info(predict_heatmap)
# signatureA = (
# tf.saved_model.signature_def_utils.build_signature_def(
# inputs={"aaa": inputA},
# outputs={"bbb": outputA},
# method_name=tf.saved_model.signature_constants.CLASSIFY_METHOD_NAME)
# )
# print('input:')
# print(input_data)
# print('output:')
# print(predict_heatmap)
# signature = predict_signature_def(inputs={'input_image:0': input_data}
# ,
# outputs={'Const:0': predict_heatmap})
#
# print("signature content:")
# print(signature)
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
# break
if cv2.waitKey(0) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - t1)))
elif FLAGS.DEMO_TYPE == 'MULTI':
# Inference
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap,
],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - t1)))
elif FLAGS.DEMO_TYPE == 'SINGLE':
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([model.stage_heatmap[5]],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
cv2.imshow('current heatmap', (demo_img).astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - t1)))
elif FLAGS.DEMO_TYPE == 'HM':
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([model.stage_heatmap[FLAGS.stages - 1]],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
print('fps: %.2f' % (1 / (time.time() - t1)))
demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape(
(FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints))
demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size))
vertical_imgs = []
tmp_img = None
joint_coord_set = np.zeros((FLAGS.joints, 2))
for joint_num in range(FLAGS.joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num]))
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
if PYTHON_VERSION == 3:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]))
else:
joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
else:
if PYTHON_VERSION == 3:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]))
else:
joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot limbs
for limb_num in range(len(limbs)):
if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]) > -2000 and np.min(
demo_stage_heatmap[:, :, limbs[limb_num][1]]) > -2000:
x1 = joint_coord_set[limbs[limb_num][0], 0]
y1 = joint_coord_set[limbs[limb_num][0], 1]
x2 = joint_coord_set[limbs[limb_num][1], 0]
y2 = joint_coord_set[limbs[limb_num][1], 1]
length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg),
0, 360, 1)
color_code_num = limb_num // 4
if PYTHON_VERSION == 3:
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]))
else:
limb_color = map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])
cv2.fillConvexPoly(test_img, polygon, color=limb_color)
if tmp_img is not None:
tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)),
'constant', constant_values=(0, 0))
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
cv2.imshow('hm', output_img)
cv2.moveWindow('hm', 2000, 200)
cv2.imshow('rgb', test_img)
cv2.moveWindow('rgb', 2000, 750)
if cv2.waitKey(1) == ord('q'): break
def main(argv):
"""
:param argv:
:return:
"""
""" Build network graph
"""
model = cpm_model.CPM_Model(input_size=FLAGS.input_size,
heatmap_size=FLAGS.heatmap_size,
stages=FLAGS.cpm_stages,
joints=FLAGS.num_of_joints,
img_type=FLAGS.color_channel,
is_training=False)
saver = tf.train.Saver()
device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
with tf.Session(config=tf.ConfigProto(device_count=device_count,
allow_soft_placement=True)) as sess:
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, FLAGS.model_path)
tf.train.write_graph(sess.graph, './', 'cpm_hand.pbtxt')
saver.save(sess, 'cpm_hand_frozen')
# Check weights
for variable in tf.trainable_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
# Create webcam instance
if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']:
cam = cv2.VideoCapture(FLAGS.cam_id)
# Create kalman filters
if FLAGS.use_kalman:
kalman_filter_array = [
cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)
]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
t1 = time.time()
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [],
FLAGS.input_size, 'IMAGE')
test_img_resize = cv2.resize(test_img,
(FLAGS.input_size, FLAGS.input_size))
print('img read time %f' % (time.time() - t1))
test_img_input = normalize_and_centralize_img(test_img_resize)
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run(
[
model.current_heatmap,
model.stage_heatmap,
],
feed_dict={model.input_images: test_img_input})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np,
kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
cv2.waitKey(0)
print('fps: %.2f' % (1 / (time.time() - t1)))
elif FLAGS.DEMO_TYPE == 'MULTI':
while True:
# Prepare input image
t1 = time.time()
test_img = cpm_utils.read_image([], cam, FLAGS.input_size,
'WEBCAM')
test_img_resize = cv2.resize(
test_img, (FLAGS.input_size, FLAGS.input_size))
print('img read time %f' % (time.time() - t1))
test_img_input = normalize_and_centralize_img(test_img_resize)
# Inference
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run(
[
model.current_heatmap,
model.stage_heatmap,
],
feed_dict={model.input_images: test_img_input})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np,
kalman_filter_array)
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - t1)))
elif FLAGS.DEMO_TYPE == 'SINGLE':
while True:
# Prepare input image
t1 = time.time()
test_img = cpm_utils.read_image([], cam, FLAGS.input_size,
'WEBCAM')
test_img_resize = cv2.resize(
test_img, (FLAGS.input_size, FLAGS.input_size))
print('img read time %f' % (time.time() - t1))
test_img_input = normalize_and_centralize_img(test_img_resize)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run(
[model.stage_heatmap[-1]],
feed_dict={model.input_images: test_img_input})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np,
kalman_filter_array)
cv2.imshow('current heatmap', (demo_img).astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - t1)))
elif FLAGS.DEMO_TYPE == 'Joint_HM':
while True:
# Prepare input image
t1 = time.time()
test_img = cpm_utils.read_image([], cam, FLAGS.input_size,
'WEBCAM')
test_img_resize = cv2.resize(
test_img, (FLAGS.input_size, FLAGS.input_size))
print('img read time %f' % (time.time() - t1))
test_img_input = normalize_and_centralize_img(test_img_resize)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run(
[model.stage_heatmap[FLAGS.cpm_stages - 1]],
feed_dict={model.input_images: test_img_input})
print('fps: %.2f' % (1 / (time.time() - t1)))
demo_stage_heatmap = stage_heatmap_np[
len(stage_heatmap_np) - 1][0, :, :,
0:FLAGS.num_of_joints].reshape(
(FLAGS.heatmap_size,
FLAGS.heatmap_size,
FLAGS.num_of_joints))
demo_stage_heatmap = cv2.resize(
demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size))
vertical_imgs = []
tmp_img = None
joint_coord_set = np.zeros((FLAGS.num_of_joints, 2))
for joint_num in range(FLAGS.num_of_joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (
255 / np.max(demo_stage_heatmap[:, :, joint_num]))
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(
np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
if PYTHON_VERSION == 3:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4),
joint_color_code[color_code_num]))
else:
joint_color = map(
lambda x: x + 35 * (joint_num % 4),
joint_color_code[color_code_num])
cv2.circle(test_img,
center=(joint_coord[1], joint_coord[0]),
radius=3,
color=joint_color,
thickness=-1)
else:
if PYTHON_VERSION == 3:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4),
joint_color_code[color_code_num]))
else:
joint_color = map(
lambda x: x + 35 * (joint_num % 4),
joint_color_code[color_code_num])
cv2.circle(test_img,
center=(joint_coord[1], joint_coord[0]),
radius=3,
color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
tmp = cv2.putText(
tmp,
'Min:' +
str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20),
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.3,
color=150)
tmp = cv2.putText(
tmp,
'Mean:' +
str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30),
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.3,
color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot limbs
for limb_num in range(len(limbs)):
if np.min(
demo_stage_heatmap[:, :, limbs[limb_num][0]]
) > -2000 and np.min(
demo_stage_heatmap[:, :,
limbs[limb_num][1]]) > -2000:
x1 = joint_coord_set[limbs[limb_num][0], 0]
y1 = joint_coord_set[limbs[limb_num][0], 1]
x2 = joint_coord_set[limbs[limb_num][1], 0]
y2 = joint_coord_set[limbs[limb_num][1], 1]
length = ((x1 - x2)**2 + (y1 - y2)**2)**0.5
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int(
(y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg), 0, 360, 1)
color_code_num = limb_num // 4
if PYTHON_VERSION == 3:
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4),
joint_color_code[color_code_num]))
else:
limb_color = map(
lambda x: x + 35 * (limb_num % 4),
joint_color_code[color_code_num])
cv2.fillConvexPoly(test_img,
polygon,
color=limb_color)
if tmp_img is not None:
tmp_img = np.lib.pad(
tmp_img,
((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]),
(0, 0)),
'constant',
constant_values=(0, 0))
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
cv2.imshow('hm', output_img)
cv2.moveWindow('hm', 2000, 200)
cv2.imshow('rgb', test_img)
cv2.moveWindow('rgb', 2000, 750)
if cv2.waitKey(1) == ord('q'): break
def main(argv):
global joint_detections
os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)
""" Initial tracker
"""
tracker = tracking_module.SelfTracker(
[FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
""" Build network graph
"""
model = cpm_model.CPM_Model(input_size=FLAGS.input_size,
heatmap_size=FLAGS.heatmap_size,
stages=FLAGS.cpm_stages,
joints=FLAGS.num_of_joints,
img_type=FLAGS.color_channel,
is_training=False)
saver = tf.train.Saver()
""" Get output node
"""
output_node = tf.get_default_graph().get_tensor_by_name(
name=FLAGS.output_node_names)
device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
sess_config = tf.ConfigProto(device_count=device_count)
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
with tf.Session(config=sess_config) as sess:
model_path_suffix = os.path.join(
FLAGS.network_def,
'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size),
'joints_{}'.format(FLAGS.num_of_joints),
'stages_{}'.format(FLAGS.cpm_stages),
'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr,
FLAGS.lr_decay_rate,
FLAGS.lr_decay_step))
model_save_dir = os.path.join('models', 'weights', model_path_suffix)
print('Load model from [{}]'.format(
os.path.join(model_save_dir, FLAGS.model_path)))
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, 'models/weights/cpm_hand')
# Check weights
for variable in tf.global_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
# Create webcam instance
if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']:
cam = cv2.VideoCapture(FLAGS.cam_id)
# Create kalman filters
if FLAGS.use_kalman:
kalman_filter_array = [
cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)
]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [],
FLAGS.input_size, 'IMAGE')
test_img_resize = cv2.resize(test_img,
(FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run(
[
model.current_heatmap,
output_node,
],
feed_dict={model.input_images: test_img_input})
print('fps: %.2f' % (1 / (time.time() - t1)))
correct_and_draw_hand(
test_img,
cv2.resize(stage_heatmap_np[0],
(FLAGS.input_size, FLAGS.input_size)),
kalman_filter_array, tracker, tracker.input_crop_ratio,
test_img)
# Show visualized image
# demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', test_img.astype(np.uint8))
cv2.waitKey(0)
elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']:
i = 0
while True:
# Prepare input image
_, full_img = cam.read()
test_img = tracker.tracking_by_joints(
full_img, joint_detections=joint_detections)
crop_full_scale = tracker.input_crop_ratio
test_img_copy = test_img.copy()
# White balance
test_img_wb = utils.img_white_balance(test_img, 5)
test_img_input = normalize_and_centralize_img(test_img_wb)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run(
[output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
local_img = visualize_result(full_img, stage_heatmap_np,
kalman_filter_array, tracker,
crop_full_scale, test_img_copy)
cv2.imshow('local_img', local_img.astype(np.uint8)) # 训练用图
# cv2.imwrite('./storePic/11'+str(i)+'.jpg', local_img.astype(np.uint8), [int(cv2.IMWRITE_JPEG_QUALITY), 90])
cv2.imwrite('./dataset/up/up__' + str(i) + '.jpg',
local_img.astype(np.uint8))
i += 1
print(i)
cv2.imshow('globalq_img', full_img.astype(np.uint8)) # 单人大框
if cv2.waitKey(1) == ord('q'): break
elif FLAGS.DEMO_TYPE == 'Joint_HM':
while True:
# Prepare input image
test_img = cpm_utils.read_image([], cam, FLAGS.input_size,
'WEBCAM')
test_img_resize = cv2.resize(
test_img, (FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run(
[output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
demo_stage_heatmap = stage_heatmap_np[
len(stage_heatmap_np) - 1][0, :, :,
0:FLAGS.num_of_joints].reshape(
(FLAGS.heatmap_size,
FLAGS.heatmap_size,
FLAGS.num_of_joints))
demo_stage_heatmap = cv2.resize(
demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size))
vertical_imgs = []
tmp_img = None
joint_coord_set = np.zeros((FLAGS.num_of_joints, 2))
for joint_num in range(FLAGS.num_of_joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (
255 / np.max(demo_stage_heatmap[:, :, joint_num]))
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(
np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4),
FLAGS.joint_color_code[color_code_num]))
cv2.circle(test_img,
center=(joint_coord[1], joint_coord[0]),
radius=3,
color=joint_color,
thickness=-1)
else:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4),
FLAGS.joint_color_code[color_code_num]))
cv2.circle(test_img,
center=(joint_coord[1], joint_coord[0]),
radius=3,
color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
tmp = cv2.putText(
tmp,
'Min:' +
str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20),
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.3,
color=150)
tmp = cv2.putText(
tmp,
'Mean:' +
str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30),
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.3,
color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot FLAGS.limbs
for limb_num in range(len(FLAGS.limbs)):
if np.min(
demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][0]]
) > -2000 and np.min(
demo_stage_heatmap[:, :, FLAGS.
limbs[limb_num][1]]) > -2000:
x1 = joint_coord_set[FLAGS.limbs[limb_num][0], 0]
y1 = joint_coord_set[FLAGS.limbs[limb_num][0], 1]
x2 = joint_coord_set[FLAGS.limbs[limb_num][1], 0]
y2 = joint_coord_set[FLAGS.limbs[limb_num][1], 1]
length = ((x1 - x2)**2 + (y1 - y2)**2)**0.5
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int(
(y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg), 0, 360, 1)
color_code_num = limb_num // 4
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4),
FLAGS.joint_color_code[color_code_num]))
cv2.fillConvexPoly(test_img,
polygon,
color=limb_color)
if tmp_img is not None:
tmp_img = np.lib.pad(
tmp_img,
((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]),
(0, 0)),
'constant',
constant_values=(0, 0))
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
cv2.imshow('hm', output_img)
cv2.moveWindow('hm', 2000, 200)
cv2.imshow('rgb', test_img)
cv2.moveWindow('rgb', 2000, 750)
if cv2.waitKey(1) == ord('q'): break
def main(argv):
tf_device = '/cpu:0'
with tf.device(tf_device):
"""Build graph
"""
if FLAGS.color_channel == 'RGB':
input_data = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 3],
name='input_image')
else:
input_data = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='input_image')
center_map = tf.placeholder(
dtype=tf.float32,
shape=[None, FLAGS.input_size, FLAGS.input_size, 1],
name='center_map')
model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1)
model.build_model(input_data, center_map, 1)
saver = tf.train.Saver()
"""Create session and restore weights
"""
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, FLAGS.model_path)
test_center_map = cpm_utils.gaussian_img(FLAGS.input_size,
FLAGS.input_size,
FLAGS.input_size / 2,
FLAGS.input_size / 2,
FLAGS.cmap_radius)
test_center_map = np.reshape(test_center_map,
[1, FLAGS.input_size, FLAGS.input_size, 1])
# Check weights
for variable in tf.trainable_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
"""if not FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
cam = cv2.VideoCapture(FLAGS.cam_num)"""
# Create kalman filters
if FLAGS.KALMAN_ON:
kalman_filter_array = [
cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)
]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
#classifier MLP model
model2 = Sequential()
model2.add(InputLayer(input_shape=(21, 2)))
model2.add(Flatten())
model2.add(Dense(21))
model2.add(Dense(500))
model2.add(Dense(1400))
model2.add(Dense(3000))
model2.add(Dense(3000))
model2.add(Dense(1400))
model2.add(Dense(500, activation='relu'))
model2.add(Dense(26, activation='softmax'))
model2.load_weights('models/weights/massey_joint.h5')
notepad_image = 255 * np.ones(shape=[368, 500, 3], dtype=np.uint8)
massey_static = cv2.imread('massey_static.png')
massey_static = cv2.resize(massey_static, (368, 368))
with tf.device(tf_device):
while True:
cam = cv2.VideoCapture(FLAGS.cam_num)
test_img = cpm_utils.read_image([], cam, FLAGS.input_size,
'WEBCAM')
cam.release()
test_img_resize = cv2.resize(test_img,
(FLAGS.input_size, FLAGS.input_size))
t1 = time.time()
#print('img read time %f' % (time.time() - t1))
"""if FLAGS.color_channel == 'GRAY':
test_img_resize = np.dot(test_img_resize[..., :3], [0.299, 0.587, 0.114]).reshape(
(FLAGS.input_size, FLAGS.input_size, 1))
cv2.imshow('color', test_img.astype(np.uint8))
cv2.imshow('gray', test_img_resize.astype(np.uint8))
cv2.waitKey(1)"""
test_img_input = test_img_resize / 256.0 - 0.5
test_img_input = np.expand_dims(test_img_input, axis=0)
"""if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
# Inference
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
model.stage_heatmap,
],
feed_dict={'input_image:0': test_img_input,
'center_map:0': test_center_map})
# Show visualized image
demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array)
#demo_img = cv2.cvtColor(demo_img,cv2.COLOR_BGR2GRAY)
print(demo_img.shape)
#demo_img1 = cv2.GaussianBlur(demo_img, (9, 9), 0)
#demo_img = 2.5*demo_img-demo_img1
cv2.imshow('demo_img', demo_img.astype(np.uint8))
if cv2.waitKey(0) == ord('q'): break
print('fps: %.2f' % (1 / (time.time() - t1)))
elif FLAGS.DEMO_TYPE == 'MULTI':"""
# Inference
t1 = time.time()
#print("chandu")
key1 = cv2.waitKey(1)
#if key1 == 32:
predict_heatmap, stage_heatmap_np = sess.run([
model.current_heatmap,
model.stage_heatmap,
],
feed_dict={
'input_image:0':
test_img_input,
'center_map:0':
test_center_map
})
# Show visualized image
demo_img, joint_points = visualize_result(test_img, FLAGS,
stage_heatmap_np,
kalman_filter_array)
x = joint_points[0][0]
y = joint_points[0][1]
for i in range(21):
joint_points[i][0] = joint_points[i][0] - x
joint_points[i][1] = joint_points[i][1] - y
joint_points = np.array([joint_points])
class_num = model2.predict_classes(joint_points)
if class_num[0] == 26:
character = ' '
else:
character = chr(97 + class_num[0])
global printer_x, printer_y, printer_string
printer_string += character
notepad_image = printer.notepad(notepad_image,
(printer_x, printer_y),
printer_string)
new_note = np.concatenate((notepad_image, demo_img, massey_static),
axis=1)
cv2.imshow('notepad', new_note)
if len(printer_string) == 17:
printer_string = ''
printer_y += 25
print('fps: %.2f' % (1 / (time.time() - t1)))
if key1 == ord('q'):
cv2.destroyAllWindows()
break
