def init_value(self):
self.joint_detections = np.zeros(shape=(21, 2))
# Initial tracker
self.tracker = tracking_module.SelfTracker(
[FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
# Build network graph
self.model = cpm_model.CPM_Model(input_size=FLAGS.input_size,
heatmap_size=FLAGS.heatmap_size,
stages=FLAGS.cpm_stages,
joints=FLAGS.num_of_joints,
img_type=FLAGS.color_channel,
is_training=False)
self.saver = tf.compat.v1.train.Saver()
# Get output node
self.output_node = tf.compat.v1.get_default_graph().get_tensor_by_name(
name=FLAGS.output_node_names)
device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
sess_config = tf.compat.v1.ConfigProto(device_count=device_count)
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.7
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
self.sess = tf.compat.v1.Session(config=sess_config)
model_path_suffix = os.path.join(
FLAGS.network_def,
'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size),
'joints_{}'.format(FLAGS.num_of_joints),
'stages_{}'.format(FLAGS.cpm_stages),
'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr,
FLAGS.lr_decay_rate,
FLAGS.lr_decay_step))
model_save_dir = os.path.join('models', 'weights', model_path_suffix)
print('Load model from [{}]'.format(
os.path.join(model_save_dir, FLAGS.model_path)))
self.saver.restore(self.sess, 'models/weights/cpm_hand')
# new model
self.new_model = keras.models.load_model(model_path)
self.new_model.summary()
self.__CreateKalmanfilters()
self.cam = cv2.VideoCapture(FLAGS.cam_id)
def main(argv):
global joint_detections
os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)
""" Initial tracker
"""
tracker = tracking_module.SelfTracker(
[FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
""" Build network graph
"""
model = cpm_model.CPM_Model(input_size=FLAGS.input_size,
heatmap_size=FLAGS.heatmap_size,
stages=FLAGS.cpm_stages,
joints=FLAGS.num_of_joints,
img_type=FLAGS.color_channel,
is_training=False)
saver = tf.train.Saver()
""" Get output node
"""
output_node = tf.get_default_graph().get_tensor_by_name(
name=FLAGS.output_node_names)
device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
sess_config = tf.ConfigProto(device_count=device_count)
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
with tf.Session(config=sess_config) as sess:
model_path_suffix = os.path.join(
FLAGS.network_def,
'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size),
'joints_{}'.format(FLAGS.num_of_joints),
'stages_{}'.format(FLAGS.cpm_stages),
'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr,
FLAGS.lr_decay_rate,
FLAGS.lr_decay_step))
model_save_dir = os.path.join('models', 'weights', model_path_suffix)
print('Load model from [{}]'.format(
os.path.join(model_save_dir, FLAGS.model_path)))
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, 'models/weights/cpm_hand')
# Check weights
for variable in tf.global_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
# Create webcam instance
if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']:
cam = cv2.VideoCapture(FLAGS.cam_id)
# Create kalman filters
if FLAGS.use_kalman:
kalman_filter_array = [
cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)
]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [],
FLAGS.input_size, 'IMAGE')
test_img_resize = cv2.resize(test_img,
(FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run(
[
model.current_heatmap,
output_node,
],
feed_dict={model.input_images: test_img_input})
print('fps: %.2f' % (1 / (time.time() - t1)))
correct_and_draw_hand(
test_img,
cv2.resize(stage_heatmap_np[0],
(FLAGS.input_size, FLAGS.input_size)),
kalman_filter_array, tracker, tracker.input_crop_ratio,
test_img)
# Show visualized image
# demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', test_img.astype(np.uint8))
cv2.waitKey(0)
elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']:
i = 0
while True:
# Prepare input image
_, full_img = cam.read()
test_img = tracker.tracking_by_joints(
full_img, joint_detections=joint_detections)
crop_full_scale = tracker.input_crop_ratio
test_img_copy = test_img.copy()
# White balance
test_img_wb = utils.img_white_balance(test_img, 5)
test_img_input = normalize_and_centralize_img(test_img_wb)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run(
[output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
local_img = visualize_result(full_img, stage_heatmap_np,
kalman_filter_array, tracker,
crop_full_scale, test_img_copy)
cv2.imshow('local_img', local_img.astype(np.uint8)) # 训练用图
# cv2.imwrite('./storePic/11'+str(i)+'.jpg', local_img.astype(np.uint8), [int(cv2.IMWRITE_JPEG_QUALITY), 90])
cv2.imwrite('./dataset/up/up__' + str(i) + '.jpg',
local_img.astype(np.uint8))
i += 1
print(i)
cv2.imshow('globalq_img', full_img.astype(np.uint8)) # 单人大框
if cv2.waitKey(1) == ord('q'): break
elif FLAGS.DEMO_TYPE == 'Joint_HM':
while True:
# Prepare input image
test_img = cpm_utils.read_image([], cam, FLAGS.input_size,
'WEBCAM')
test_img_resize = cv2.resize(
test_img, (FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run(
[output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
demo_stage_heatmap = stage_heatmap_np[
len(stage_heatmap_np) - 1][0, :, :,
0:FLAGS.num_of_joints].reshape(
(FLAGS.heatmap_size,
FLAGS.heatmap_size,
FLAGS.num_of_joints))
demo_stage_heatmap = cv2.resize(
demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size))
vertical_imgs = []
tmp_img = None
joint_coord_set = np.zeros((FLAGS.num_of_joints, 2))
for joint_num in range(FLAGS.num_of_joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (
255 / np.max(demo_stage_heatmap[:, :, joint_num]))
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(
np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4),
FLAGS.joint_color_code[color_code_num]))
cv2.circle(test_img,
center=(joint_coord[1], joint_coord[0]),
radius=3,
color=joint_color,
thickness=-1)
else:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4),
FLAGS.joint_color_code[color_code_num]))
cv2.circle(test_img,
center=(joint_coord[1], joint_coord[0]),
radius=3,
color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
tmp = cv2.putText(
tmp,
'Min:' +
str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20),
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.3,
color=150)
tmp = cv2.putText(
tmp,
'Mean:' +
str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30),
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.3,
color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot FLAGS.limbs
for limb_num in range(len(FLAGS.limbs)):
if np.min(
demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][0]]
) > -2000 and np.min(
demo_stage_heatmap[:, :, FLAGS.
limbs[limb_num][1]]) > -2000:
x1 = joint_coord_set[FLAGS.limbs[limb_num][0], 0]
y1 = joint_coord_set[FLAGS.limbs[limb_num][0], 1]
x2 = joint_coord_set[FLAGS.limbs[limb_num][1], 0]
y2 = joint_coord_set[FLAGS.limbs[limb_num][1], 1]
length = ((x1 - x2)**2 + (y1 - y2)**2)**0.5
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int(
(y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg), 0, 360, 1)
color_code_num = limb_num // 4
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4),
FLAGS.joint_color_code[color_code_num]))
cv2.fillConvexPoly(test_img,
polygon,
color=limb_color)
if tmp_img is not None:
tmp_img = np.lib.pad(
tmp_img,
((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]),
(0, 0)),
'constant',
constant_values=(0, 0))
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
cv2.imshow('hm', output_img)
cv2.moveWindow('hm', 2000, 200)
cv2.imshow('rgb', test_img)
cv2.moveWindow('rgb', 2000, 750)
if cv2.waitKey(1) == ord('q'): break
def main(argv):
tracker = tracking_module.SelfTracker(
[FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
model = cpm_hand.CPM_Model(
input_size=FLAGS.input_size, # 导入model1的模型结构
heatmap_size=FLAGS.heatmap_size,
stages=FLAGS.cpm_stages,
joints=FLAGS.num_of_joints,
img_type=FLAGS.color_channel,
is_training=False)
print("成功导入1的模型")
saver = tf.train.Saver()
output_node = tf.get_default_graph().get_tensor_by_name(
name=FLAGS.output_node_names)
with tf.Session() as sess:
saver.restore(sess, './models/weights/cpm_hand') # 恢复model1的参数
print("成功导入1的model参数")
tf.train.import_meta_graph(
'./classify/modelSave/model.ckpt.meta') # 导入model3的模型结构
print("成功导入3的model结构")
all_vars = tf.trainable_variables()
# print(all_vars[61:])
saver_fenlei = tf.train.Saver(
all_vars[62:]) # 这个saver1只导入第62个变量及以后变量，是model3的专属变量
saver_fenlei.restore(
sess, tf.train.latest_checkpoint('./classify/modelSave/'))
print("成功导入3的model参数")
x = tf.get_default_graph().get_tensor_by_name("x:0")
logits = tf.get_default_graph().get_tensor_by_name("logits_eval:0")
# Create kalman filters
if FLAGS.use_kalman:
kalman_filter_array = [
cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)
]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
while True:
data = []
global receive_flag, do_again, again_time
files = os.listdir(local_dir + '/storePic')
for f in files:
if f == 'flag_ok': receive_flag = 1
if receive_flag or (do_again and again_time < 2):
os.system('rm ./storePic/flag_ok')
receive_flag = 0
do_again = 0
for i in range(num_of_photo):
full_img = cv2.imread("./storePic/test" + str(i) + ".jpg")
cv2.waitKey(1)
cv2.namedWindow("yuantu")
cv2.imshow("yuantu", full_img)
cv2.moveWindow("yuantu", 0, 0)
test_img = tracker.tracking_by_joints(
full_img, joint_detections=joint_detections)
crop_full_scale = tracker.input_crop_ratio
test_img_copy = test_img.copy()
# White balance
test_img_wb = utils.img_white_balance(test_img, 5)
test_img_input = normalize_and_centralize_img(test_img_wb)
t1 = time.time()
stage_heatmap_np = sess.run(
[output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
local_img = visualize_result(full_img, stage_heatmap_np,
kalman_filter_array, tracker,
crop_full_scale,
test_img_copy)
cv2.namedWindow("local_image")
cv2.namedWindow("global_image")
cv2.imshow('local_img', local_img.astype(np.uint8)) # 训练用图
cv2.imshow('global_img', full_img.astype(np.uint8)) # 单人大框
cv2.moveWindow("local_image", 1000, 0)
cv2.moveWindow("global_image", 0, 2000)
cv2.imwrite("./blackPic/black" + str(i) + ".jpg",
local_img)
local_img = io.imread("./blackPic/black" + str(i) + ".jpg")
img = transform.resize(local_img, (w, h))
data.append(np.asarray(img))
print(len(data))
feed_dict = {x: data}
classification_result = sess.run(logits, feed_dict)
# 打印出预测矩阵
print(classification_result)
# 打印出预测矩阵每一行最大值的索引
print(tf.argmax(classification_result, 1).eval())
# 根据索引通过字典对应花的分类
output = tf.argmax(classification_result, 1).eval()
for k in range(len(output)):
print("第" + str(k + 1) + "帧预测:" + dict[output[k]])
print("#########################################")
output = output.tolist()
num_of_none = find_most(output)
if num_of_none > int((num_of_photo / 2.0)):
again_time += 1
do_again = 1
else:
again_time = 0
do_again = 0
cv2.destroyAllWindows()
print('waiting...')
def main(argv):
global joint_detections
# Create webcam instance
cam = cv2.VideoCapture(FLAGS.cam_id)
# Create kalman filters
kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
cpm_inference = CPMInference()
classify_inference = ClassifyInference()
keyboard = Controller()
tracker = tracking_module.SelfTracker([FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
count = 0
action_count = [5, 10, 20, 40]
last_gesture = -1
action = ['up', 'down', 'left', 'right', 'larger', 'smaller']
labels2key = {0: Key.down, 1: Key.up, 2: Key.right, 3: Key.left, 4: '+', 5: '-'}
while True:
# Prepare input image
_, full_img = cam.read()
test_img = tracker.tracking_by_joints(full_img, joint_detections=joint_detections)
crop_full_scale = tracker.input_crop_ratio
test_img_copy = test_img.copy()
# White balance
test_img_wb = utils.img_white_balance(test_img, 5)
test_img_input = normalize_and_centralize_img(test_img_wb)
# Inference
t1 = time.time()
stage_heatmap_np = cpm_inference.predict(test_img_input)
# print('FPS: %.2f' % (1 / (time.time() - t1)))
local_img, response = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker,
crop_full_scale, test_img_copy)
cv2.imshow('local_img', local_img.astype(np.uint8)) # 训练用图
cv2.imshow('globalq_img', full_img.astype(np.uint8)) # 单人大框
if response > 5.0:
rst = classify_inference.predict([cv2.resize(local_img, (100, 100))])
if rst == last_gesture:
count += 1
else:
count = 1
last_gesture = rst
if count in action_count:
keyboard.press(labels2key[last_gesture])
keyboard.release(labels2key[last_gesture])
print(action[last_gesture])
else:
last_gesture = -1
if cv2.waitKey(1) == ord('q'):
cpm_inference.shutdown()
classify_inference.shutdown()
break
def main(argv):
global joint_detections
os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)
tracker = tracking_module.SelfTracker(
[FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
model = cpm_model.CPM_Model(input_size=FLAGS.input_size,
heatmap_size=FLAGS.heatmap_size,
stages=FLAGS.cpm_stages,
joints=FLAGS.num_of_joints,
img_type=FLAGS.color_channel,
is_training=False)
saver = tf.train.Saver()
output_node = tf.get_default_graph().get_tensor_by_name(
name=FLAGS.output_node_names)
device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
sess_config = tf.ConfigProto(device_count=device_count)
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
with tf.Session(config=sess_config) as sess:
model_path_suffix = os.path.join(
FLAGS.network_def,
'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size),
'joints_{}'.format(FLAGS.num_of_joints),
'stages_{}'.format(FLAGS.cpm_stages),
'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr,
FLAGS.lr_decay_rate,
FLAGS.lr_decay_step))
model_save_dir = os.path.join('models', 'weights', model_path_suffix)
print('Load model from [{}]'.format(
os.path.join(model_save_dir, FLAGS.model_path)))
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, 'models/weights/cpm_hand')
for variable in tf.global_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']:
cam = cv2.VideoCapture(FLAGS.cam_id)
#cam = pipeline.wait_for_frames()
if FLAGS.use_kalman:
kalman_filter_array = [
cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)
]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [],
FLAGS.input_size, 'IMAGE')
test_img_resize = cv2.resize(test_img,
(FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
t1 = time.time()
predict_heatmap, stage_heatmap_np = sess.run(
[
model.current_heatmap,
output_node,
],
feed_dict={model.input_images: test_img_input})
print('fps: %.2f' % (1 / (time.time() - t1)))
correct_and_draw_hand(
test_img,
cv2.resize(stage_heatmap_np[0],
(FLAGS.input_size, FLAGS.input_size)),
kalman_filter_array, tracker, tracker.input_crop_ratio,
test_img)
# Show visualized image
# demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array)
cv2.imshow('demo_img', test_img.astype(np.uint8))
cv2.waitKey(0)
elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']:
while True:
_, full_img = cam.read()
test_img = tracker.tracking_by_joints(
full_img, joint_detections=joint_detections)
crop_full_scale = tracker.input_crop_ratio
test_img_copy = test_img.copy()
# White balance
test_img_wb = utils.img_white_balance(test_img, 5)
test_img_input = normalize_and_centralize_img(test_img_wb)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run(
[output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
local_img = visualize_result(full_img, stage_heatmap_np,
kalman_filter_array, tracker,
crop_full_scale, test_img_copy)
cv2.imshow('local_img', local_img.astype(np.uint8))
cv2.imshow('global_img', full_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
def main(argv):
""" Initial tracker
"""
tracker = tracking_module.SelfTracker([FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
""" Build network graph
"""
model = cpm_model.CPM_Model(input_size=FLAGS.input_size,
heatmap_size=FLAGS.heatmap_size,
stages=FLAGS.cpm_stages,
joints=FLAGS.num_of_joints,
img_type=FLAGS.color_channel,
is_training=False)
saver = tf.train.Saver()
""" Get output node
"""
output_node = tf.get_default_graph().get_tensor_by_name(name=FLAGS.output_node_names)
device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
sess_config = tf.ConfigProto(device_count=device_count)
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
with tf.Session(config=sess_config) as sess:
model_path_suffix = os.path.join(FLAGS.network_def,
'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size),
'joints_{}'.format(FLAGS.num_of_joints),
'stages_{}'.format(FLAGS.cpm_stages),
'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr, FLAGS.lr_decay_rate,
FLAGS.lr_decay_step)
)
model_save_dir = os.path.join('models',
'weights',
model_path_suffix)
print('Load model from [{}]'.format(os.path.join(model_save_dir, FLAGS.model_path)))
# if the model is a pkl file, then load it, else just restore the pretrained cpm_hand by default
# by here we can see the loading weight structure
if FLAGS.model_path.endswith('pkl'):
model.load_weights_from_file(FLAGS.model_path, sess, False)
else:
saver.restore(sess, 'models/weights/cpm_hand')
# Check weights, this seems to be the part that print out weights
for variable in tf.global_variables():
with tf.variable_scope('', reuse=True):
var = tf.get_variable(variable.name.split(':0')[0])
print(variable.name, np.mean(sess.run(var)))
# Create webcam instance
if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']:
print("REFUSEREFUSE\n\n\n\n\n\n\n\nrefule")
cam = cv2.VideoCapture(FLAGS.cam_id)
# Create kalman filters
if FLAGS.use_kalman:
kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')):
print("TESTINGTESTING\n\n\n\n\n\n\n\nrTESGING")
test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE')
#fixme: I haven't seen how it cut the box, I have put the coordinates here, but not using them currently
b_box, b_height, b_width = dt.bounding_box_from_file(FLAGS.DEMO_TYPE)
b_box = b_box[0][0]
bb_box = dt.normalize_and_centralize_img(b_box[0], b_box[1], b_box[2], b_box[3], 10, b_height, b_width)\
#fixme: this is the end of my code
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
t1 = time.time()
#current_heatmap is the output after going through the last layer
#input_images is a placeholder matrix for iiamge size
# FIXME: I don't know what exactly does this sess run with? the two arguments?
#predict_heatmap.shape = (1,32,32,22)
#stage_heatmap.shape = (1,32,32,22), it seems like, for stage_heatmap, each of the 1*32*32 is something same, a 22 length-array
predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap,
output_node,
],
feed_dict={model.input_images: test_img_input}
)
#frame per second
print('fps: %.2f' % (1 / (time.time() - t1)))
print(stage_heatmap_np[0], "stage heatmap")
print(stage_heatmap_np[0].shape)
#print(predict_heatmap, "predict heatmap")
#print(predict_heatmap.shape)
tmp_img = cv2.resize(stage_heatmap_np[0], (FLAGS.input_size, FLAGS.input_size))
print(tmp_img, "tmp_img")
print(tmp_img.shape)
correct_and_draw_hand(test_img,
cv2.resize(stage_heatmap_np[0], (FLAGS.input_size, FLAGS.input_size)),
kalman_filter_array, tracker, tracker.input_crop_ratio, test_img)
# Show visualized image
#demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array, tracker, tracker.input_crop_ratio, test_img.copy())
# local_img = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker, crop_full_scale, test_img_copy)
cv2.imshow('demo_img', test_img.astype(np.uint8))
cv2.waitKey(0)
elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']:
print("SINGLE\n\n\n\n\n\n\nMULTI")
while True:
# # Prepare input image
_, full_img = cam.read()
# then I kow the misterious joint_detection comes from here! but now it's only zeros???
test_img = tracker.tracking_by_joints(full_img, joint_detections=joint_detections)
crop_full_scale = tracker.input_crop_ratio
test_img_copy = test_img.copy()
# White balance
test_img_wb = utils.img_white_balance(test_img, 5)
test_img_input = normalize_and_centralize_img(test_img_wb)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
local_img = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker, crop_full_scale,
test_img_copy)
cv2.imshow('local_img', local_img.astype(np.uint8))
cv2.imshow('global_img', full_img.astype(np.uint8))
if cv2.waitKey(1) == ord('q'): break
elif FLAGS.DEMO_TYPE == 'Joint_HM':
print("JOINTHM\n\n\n\n\n\n\nJOINTHM")
while True:
# Prepare input image
test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM')
test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size))
test_img_input = normalize_and_centralize_img(test_img_resize)
# Inference
t1 = time.time()
stage_heatmap_np = sess.run([output_node],
feed_dict={model.input_images: test_img_input})
print('FPS: %.2f' % (1 / (time.time() - t1)))
demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :,
0:FLAGS.num_of_joints].reshape(
(FLAGS.heatmap_size, FLAGS.heatmap_size, FLAGS.num_of_joints))
demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size))
vertical_imgs = []
tmp_img = None
joint_coord_set = np.zeros((FLAGS.num_of_joints, 2))
for joint_num in range(FLAGS.num_of_joints):
# Concat until 4 img
if (joint_num % 4) == 0 and joint_num != 0:
vertical_imgs.append(tmp_img)
tmp_img = None
demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num]))
# Plot color joints
if np.min(demo_stage_heatmap[:, :, joint_num]) > -50:
joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]),
(FLAGS.input_size, FLAGS.input_size))
joint_coord_set[joint_num, :] = joint_coord
color_code_num = (joint_num // 4)
if joint_num in [0, 4, 8, 12, 16]:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
else:
joint_color = list(
map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color,
thickness=-1)
# Put text
tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8)
tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])),
org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])),
org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150)
tmp_img = np.concatenate((tmp_img, tmp), axis=0) \
if tmp_img is not None else tmp
# Plot FLAGS.limbs
for limb_num in range(len(FLAGS.limbs)):
if np.min(demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][0]]) > -2000 and np.min(
demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][1]]) > -2000:
x1 = joint_coord_set[FLAGS.limbs[limb_num][0], 0]
y1 = joint_coord_set[FLAGS.limbs[limb_num][0], 1]
x2 = joint_coord_set[FLAGS.limbs[limb_num][1], 0]
y2 = joint_coord_set[FLAGS.limbs[limb_num][1], 1]
length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
if length < 10000 and length > 5:
deg = math.degrees(math.atan2(x1 - x2, y1 - y2))
polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)),
(int(length / 2), 3),
int(deg),
0, 360, 1)
color_code_num = limb_num // 4
limb_color = list(
map(lambda x: x + 35 * (limb_num % 4), FLAGS.joint_color_code[color_code_num]))
cv2.fillConvexPoly(test_img, polygon, color=limb_color)
if tmp_img is not None:
tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)),
'constant', constant_values=(0, 0))
vertical_imgs.append(tmp_img)
# Concat horizontally
output_img = None
for col in range(len(vertical_imgs)):
output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \
vertical_imgs[col]
output_img = output_img.astype(np.uint8)
output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET)
test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4)
cv2.imshow('hm', output_img)
cv2.moveWindow('hm', 2000, 200)
cv2.imshow('rgb', test_img)
cv2.moveWindow('rgb', 2000, 750)
if cv2.waitKey(1) == ord('q'): break
def main(argv):
global joint_detections
tracker = tracking_module.SelfTracker(
[FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)
device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
sess_config = tf.ConfigProto(device_count=device_count)
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2
sess_config.gpu_options.allow_growth = True
sess_config.allow_soft_placement = True
graph1 = tf.Graph()
sess1 = tf.Session(graph=graph1)
cam = cv2.VideoCapture(FLAGS.cam_id)
dict1 = {0: '2', 1: '1'}
# Create kalman filters
if FLAGS.use_kalman:
kalman_filter_array = [
cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)
]
for _, joint_kalman_filter in enumerate(kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = \
np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = \
np.array([[1, 0, 0, 0], [0, 1, 0, 0],
[0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
kalman_filter_array = None
# load model
with sess1.as_default():
with graph1.as_default():
tf.global_variables_initializer().run()
saver1 = tf.train.import_meta_graph('../models/joint/check.meta')
saver1.restore(sess1,
tf.train.latest_checkpoint('../models/joint/'))
input_node1 = graph1.get_tensor_by_name("input_placeholder:0")
output_node1 = graph1.get_tensor_by_name(
"stage_3/mid_conv7/BiasAdd:0")
for variable in tf.global_variables():
with tf.variable_scope('', reuse=tf.AUTO_REUSE):
var = graph1.get_tensor_by_name(variable.name)
print(variable.name, np.mean(sess1.run(var)))
# use model
i = 0
while True:
# Prepare input image
_, full_img = cam.read()
test_img = tracker.tracking_by_joints(
full_img, joint_detections=joint_detections)
crop_full_scale = tracker.input_crop_ratio
test_img_copy = test_img.copy()
# White balance
test_img_wb = utils.img_white_balance(test_img, 5)
test_img_input = normalize_and_centralize_img(test_img_wb)
# Inference
t1 = time.time()
with sess1.as_default():
with graph1.as_default():
stage_heatmap_np = sess1.run(
[output_node1], feed_dict={input_node1: test_img_input})
local_img = visualize_result(full_img, stage_heatmap_np,
kalman_filter_array, tracker,
crop_full_scale,
test_img_copy).astype(np.uint8)
if tracker.loss_track is False:
img_array = np.asarray(cv2.resize(local_img, (100, 100))[:, :])
data = [img_array]
print('FPS: %.2f' % (1 / (time.time() - t1)))
cv2.imshow('local_img', local_img.astype(np.uint8))
cv2.imshow('global_img', full_img.astype(np.uint8))
if tracker.loss_track is False:
cv2.imwrite('./handGesturePic/11' + str(i) + '.jpg',
local_img.astype(np.uint8),
[int(cv2.IMWRITE_JPEG_QUALITY), 90])
i += 1
if i >= 300:
break
if cv2.waitKey(1) == ord('q'):
break
def __init__(self):
self.joint_detections = np.zeros(shape=(21, 2))
self.tracker = tracking_module.SelfTracker(
[FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size)
os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id)
device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0}
self.sess_config = tf.ConfigProto(device_count=device_count)
self.sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2
self.sess_config.gpu_options.allow_growth = True
self.sess_config.allow_soft_placement = True
self.graph1 = tf.Graph()
self.graph2 = tf.Graph()
self.sess1 = tf.Session(graph=self.graph1)
self.sess2 = tf.Session(graph=self.graph2)
self.dictList = {
0: 'back',
1: 'left',
2: 'right',
3: 'up',
4: 'down',
5: 'front'
}
self.temp_result = ''
# Create kalman filters
if FLAGS.use_kalman:
self.kalman_filter_array = [
cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)
]
for _, joint_kalman_filter in enumerate(self.kalman_filter_array):
joint_kalman_filter.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
joint_kalman_filter.measurementMatrix = \
np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32)
joint_kalman_filter.processNoiseCov = \
np.array([[1, 0, 0, 0], [0, 1, 0, 0],
[0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * FLAGS.kalman_noise
else:
self.kalman_filter_array = None
# load model
print(" Loading joint model.")
with self.sess1.as_default():
with self.graph1.as_default():
tf.global_variables_initializer().run()
saver1 = tf.train.import_meta_graph(
'./models/joint/check.meta')
saver1.restore(self.sess1,
tf.train.latest_checkpoint('./models/joint/'))
self.input_node1 = self.graph1.get_tensor_by_name(
"input_placeholder:0")
self.output_node1 = self.graph1.get_tensor_by_name(
"stage_3/mid_conv7/BiasAdd:0")
for variable in tf.global_variables():
with tf.variable_scope('', reuse=tf.AUTO_REUSE):
var = self.graph1.get_tensor_by_name(variable.name)
print(variable.name, np.mean(self.sess1.run(var)))
print(" Loading classify model.")
with self.sess2.as_default():
with self.graph2.as_default():
tf.global_variables_initializer().run()
saver2 = tf.train.import_meta_graph(
'./models/classify/model.ckpt.meta')
saver2.restore(self.sess2,
tf.train.latest_checkpoint('models/classify/'))
self.input_node2 = self.graph2.get_tensor_by_name("x:0")
self.output_node2 = self.graph2.get_tensor_by_name(
"logits_eval:0")
for variable in tf.global_variables():
with tf.variable_scope('', reuse=tf.AUTO_REUSE):
var = self.graph2.get_tensor_by_name(variable.name)
print(variable.name, np.mean(self.sess2.run(var)))
