def main(argv): tf_device = '/gpu:0' with tf.device(tf_device): """Build graph """ if FLAGS.color_channel == 'RGB': input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') else: input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='input_image') center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) if not FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): #if not FLAGS.DEMO_TYPE.endswith(('avi', 'mp4')): cam = cv2.VideoCapture(FLAGS.cam_num) # Create kalman filters if FLAGS.KALMAN_ON: kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None with tf.device(tf_device): while True: t1 = time.time() if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') else: test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') #yolo stuff result = tfnet.return_predict(test_img) count = 0 if not result: pass else: for i in range(0, len(result)): if (result[i]['label'] == 'hand_l') & (count == 0): hand1 = [result[i]['topleft'], result[i]['bottomright']] count+=1 elif (result[i]['label'] == 'hand_r') & (count == 1): hand2 = [result[i]['topleft'], result[i]['bottomright']] count+=1 #elif (count==1): # hand2 = [] #elif (count==0): # hand1 = ['NA'] # hand2 = ['NA'] #cropped image if not hand1: pass else: h1_image = test_img[hand1[0]['y']:hand1[1]['y'], hand1[0]['x']:hand1[1]['x']] if not hand2: pass else: h2_image = test_img[hand2[0]['y']:hand2[1]['y'], hand2[0]['x']:hand2[1]['x']] if not h1_image: pass else: h1_image_resize = cv2.resize(h1_image, (FLAGS.input_size, FLAGS.input_size)) h1_img_input = h1_image_resize / 256.0 - 0.5 h1_img_input = np.expand_dims(h1_img_input, axis=0) if not h2_image: pass else: h2_image_resize = cv2.resize(h2_image, (FLAGS.input_size, FLAGS.input_size)) h2_img_input = h2_image_resize / 256.0 - 0.5 h2_img_input = np.expand_dims(h2_img_input, axis=0) """ #test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) #print('img read time %f' % (time.time() - t1)) if FLAGS.color_channel == 'GRAY': test_img_resize = np.dot(test_img_resize[..., :3], [0.299, 0.587, 0.114]).reshape( (FLAGS.input_size, FLAGS.input_size, 1)) cv2.imshow('color', test_img.astype(np.uint8)) cv2.imshow('gray', test_img_resize.astype(np.uint8)) cv2.waitKey(1) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) """ if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): # Inference t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(0) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'MULTI': # Inference t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'SINGLE': # Inference t1 = time.time() """ stage_heatmap_np = sess.run([model.stage_heatmap[5]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) """ if h1_img_input: pass else: stage_heatmap_np_1 = sess.run([model.stage_heatmap[5]], feed_dict={'input_image:0': h1_img_input, 'center_map:0': test_center_map}) demo_img_1 = visualize_result(h1_image, FLAGS, stage_heatmap_np_1, kalman_filter_array) test_img[hand1[0]['y']:hand1[1]['y'], hand1[0]['x']:hand1[1]['x']] = demo_img_1 if h2_img_input: pass else: stage_heatmap_np_2 = sess.run([model.stage_heatmap[5]], feed_dict={'input_image:0': h2_img_input, 'center_map:0': test_center_map}) demo_img_2 = visualize_result(h2_image, FLAGS, stage_heatmap_np_2, kalman_filter_array) test_img[hand2[0]['y']:hand2[1]['y'], hand2[0]['x']:hand2[1]['x']] = demo_img_2 # Show visualized image # X2 times till here -concatenate here cv2.imshow('current heatmap', (test_img).astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'HM': # Inference t1 = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[FLAGS.stages - 1]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) print('fps: %.2f' % (1 / (time.time() - t1))) demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape( (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints)) demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size)) vertical_imgs = [] tmp_img = None joint_coord_set = np.zeros((FLAGS.joints, 2)) for joint_num in range(FLAGS.joints): # Concat until 4 img if (joint_num % 4) == 0 and joint_num != 0: vertical_imgs.append(tmp_img) tmp_img = None demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num])) # Plot color joints if np.min(demo_stage_heatmap[:, :, joint_num]) > -50: joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]), (FLAGS.input_size, FLAGS.input_size)) joint_coord_set[joint_num, :] = joint_coord color_code_num = (joint_num // 4) if joint_num in [0, 4, 8, 12, 16]: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) else: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) # Put text tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8) tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])), org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])), org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp_img = np.concatenate((tmp_img, tmp), axis=0) \ if tmp_img is not None else tmp # Plot limbs for limb_num in range(len(limbs)): if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]) > -2000 and np.min( demo_stage_heatmap[:, :, limbs[limb_num][1]]) > -2000: x1 = joint_coord_set[limbs[limb_num][0], 0] y1 = joint_coord_set[limbs[limb_num][0], 1] x2 = joint_coord_set[limbs[limb_num][1], 0] y2 = joint_coord_set[limbs[limb_num][1], 1] length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5 if length < 10000 and length > 5: deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)), (int(length / 2), 3), int(deg), 0, 360, 1) color_code_num = limb_num // 4 if PYTHON_VERSION == 3: limb_color = list( map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])) else: limb_color = map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]) cv2.fillConvexPoly(test_img, polygon, color=limb_color) if tmp_img is not None: tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)), 'constant', constant_values=(0, 0)) vertical_imgs.append(tmp_img) # Concat horizontally output_img = None for col in range(len(vertical_imgs)): output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \ vertical_imgs[col] output_img = output_img.astype(np.uint8) output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET) test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4) cv2.imshow('hm', output_img) cv2.moveWindow('hm', 2000, 200) cv2.imshow('rgb', test_img) cv2.moveWindow('rgb', 2000, 750) if cv2.waitKey(1) == ord('q'): break
def main(argv): tf_device = '/gpu:0' with tf.device(tf_device): """Build graph """ frame_num = 0 input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) model.load_weights_from_file(FLAGS.model_path, sess, False) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) cam = cv2.VideoCapture(FLAGS.cam_num) # Create kalman filters if FLAGS.KALMAN_ON: kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None # Init Openni2 openni2.initialize("C:\OpenNI\Redist") dev = openni2.Device.open_any() depth_stream = dev.create_depth_stream() depth_stream.start() depth_stream.set_video_mode( c_api.OniVideoMode(pixelFormat=c_api.OniPixelFormat.ONI_PIXEL_FORMAT_DEPTH_100_UM, resolutionX=640, resolutionY=480, fps=30)) with tf.device(tf_device): while True: # t1 = time.time() orig_img, test_img, depth_map = cpm_utils.read_image(cam, FLAGS.input_size, depth_stream) test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) # print('img read time %f' % (time.time() - t1)) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) # Inference t1 = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[5]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, orig_img, FLAGS, stage_heatmap_np, kalman_filter_array, frame_num, depth_map) cv2.imshow('current depth', depth_map) cv2.imshow('current heatmap', demo_img.astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) print(frame_num) frame_num = frame_num + 1
def main(argv): tf_device = '/gpu:0' with tf.device(tf_device): """Build graph """ if FLAGS.color_channel == 'RGB': input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') else: input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='input_image') center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') # tensorflow slim model model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) # keras model # model = cpm_hand_keras.CPM_Model(FLAGS.stages, FLAGS.joints + 1) # model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) if not FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): cam = cv2.VideoCapture(FLAGS.cam_num) # ------------------------------------------ save model------------------------------- # save_dir = 'checkpoints/' # if not os.path.exists(save_dir): # os.makedirs(save_dir) # save_path = os.path.join(save_dir, 'best_validation') # saver.save(sess=sess, save_path=save_path) # builder = tf.saved_model.builder.SavedModelBuilder('./SavedModel/') # signature = predict_signature_def(inputs={'input_image:0': input_data, # 'center_map:0': center_map}) # builder.add_meta_graph_and_variables(sess, # [tf.saved_model.tag_constants.TRAINING], # signature_def_map={'predict': signature}, # assets_collection=None, # strip_default_attrs=True) # builder.add_meta_graph([tf.saved_model.tag_constants.SERVING], strip_default_attrs=True) # builder.save() signature = None # Create kalman filters if FLAGS.KALMAN_ON: kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None with tf.device(tf_device): while True: t1 = time.time() if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') else: test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) print("----------------------------------------start to read img--------------------") print('img read time %f' % (time.time() - t1)) if FLAGS.color_channel == 'GRAY': test_img_resize = np.dot(test_img_resize[..., :3], [0.299, 0.587, 0.114]).reshape( (FLAGS.input_size, FLAGS.input_size, 1)) cv2.imshow('color', test_img.astype(np.uint8)) cv2.imshow('gray', test_img_resize.astype(np.uint8)) cv2.waitKey(1) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): # Inference t1 = time.time() # print("=========================test image============================") # print(test_img_input) # print("=========================test image shape============================") # print(test_img_input.shape) # slim prediction predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # keras prediction # predict_heatmap, stage_heatmap_np = model.predict(test_img_input) print("---------------stage_heatmap_np:------------------") # print(predict_heatmap) print(np.array(stage_heatmap_np).shape) # print(stage_heatmap_np) inputa = tf.saved_model.utils.build_tensor_info(input_data) predict_heatmap = tf.convert_to_tensor(predict_heatmap) outputa = tf.saved_model.utils.build_tensor_info(predict_heatmap) # signatureA = ( # tf.saved_model.signature_def_utils.build_signature_def( # inputs={"aaa": inputA}, # outputs={"bbb": outputA}, # method_name=tf.saved_model.signature_constants.CLASSIFY_METHOD_NAME) # ) # print('input:') # print(input_data) # print('output:') # print(predict_heatmap) # signature = predict_signature_def(inputs={'input_image:0': input_data} # , # outputs={'Const:0': predict_heatmap}) # # print("signature content:") # print(signature) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) # break if cv2.waitKey(0) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'MULTI': # Inference t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'SINGLE': # Inference t1 = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[5]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('current heatmap', (demo_img).astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'HM': # Inference t1 = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[FLAGS.stages - 1]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) print('fps: %.2f' % (1 / (time.time() - t1))) demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape( (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints)) demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size)) vertical_imgs = [] tmp_img = None joint_coord_set = np.zeros((FLAGS.joints, 2)) for joint_num in range(FLAGS.joints): # Concat until 4 img if (joint_num % 4) == 0 and joint_num != 0: vertical_imgs.append(tmp_img) tmp_img = None demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num])) # Plot color joints if np.min(demo_stage_heatmap[:, :, joint_num]) > -50: joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]), (FLAGS.input_size, FLAGS.input_size)) joint_coord_set[joint_num, :] = joint_coord color_code_num = (joint_num // 4) if joint_num in [0, 4, 8, 12, 16]: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) else: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) # Put text tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8) tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])), org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])), org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp_img = np.concatenate((tmp_img, tmp), axis=0) \ if tmp_img is not None else tmp # Plot limbs for limb_num in range(len(limbs)): if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]) > -2000 and np.min( demo_stage_heatmap[:, :, limbs[limb_num][1]]) > -2000: x1 = joint_coord_set[limbs[limb_num][0], 0] y1 = joint_coord_set[limbs[limb_num][0], 1] x2 = joint_coord_set[limbs[limb_num][1], 0] y2 = joint_coord_set[limbs[limb_num][1], 1] length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5 if length < 10000 and length > 5: deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)), (int(length / 2), 3), int(deg), 0, 360, 1) color_code_num = limb_num // 4 if PYTHON_VERSION == 3: limb_color = list( map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])) else: limb_color = map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]) cv2.fillConvexPoly(test_img, polygon, color=limb_color) if tmp_img is not None: tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)), 'constant', constant_values=(0, 0)) vertical_imgs.append(tmp_img) # Concat horizontally output_img = None for col in range(len(vertical_imgs)): output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \ vertical_imgs[col] output_img = output_img.astype(np.uint8) output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET) test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4) cv2.imshow('hm', output_img) cv2.moveWindow('hm', 2000, 200) cv2.imshow('rgb', test_img) cv2.moveWindow('rgb', 2000, 750) if cv2.waitKey(1) == ord('q'): break
def main(argv): tf_device = '/gpu:0' with tf.device(tf_device): """Build graph """ input_data = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') center_map = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) model.load_weights_from_file(FLAGS.model_path, sess, False) #model_vars = tf.trainable_variables() #slim.model_analyzer.analyze_vars(model_vars, print_info=True) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) if not FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): cam = cv2.VideoCapture(FLAGS.cam_num) # Create kalman filters kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array( [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array( [[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array( [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise with tf.device(tf_device): #filedem = open("testfile.txt","w") #filedem.close() if FLAGS.DEMO_TYPE == 'SINGLE': #read default image coords v = [] with open('test_file.csv') as file: reader = csv.reader(file, delimiter=',') for row in reader: for entries in row: v.append(entries) ''' f=open("testfile.txt", 'r') v=f.readline() v=v.strip().split(' ') ''' while True: t1 = time.time() if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') else: test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) #print('img read time %f' % (time.time() - t1)) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): # Inference t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run( [ model.current_heatmap, model.stage_heatmap, ], feed_dict={ 'input_image:0': test_img_input, 'center_map:0': test_center_map }) # Show visualized image demo_img, coords = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) ''' filedem = open("testfile.txt","w") for i in [0,5,7,9,11,13,15,17,19]: for j in [0,1]: filedem.write(str(coords[i][j])+' ') filedem.close() ''' with open('test_file.csv', mode='w') as file: writer = csv.writer(file, delimiter=',') for i in [1, 5, 8, 9, 12, 13, 16, 17, 20]: writer.writerow([coords[i][0], coords[i][1]]) if cv2.waitKey(0) == ord('q'): break #print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'SINGLE': # Inference t1 = time.time() stage_heatmap_np = sess.run( [model.stage_heatmap[2]], feed_dict={ 'input_image:0': test_img_input, 'center_map:0': test_center_map }) # Show visualized image demo_img, coords = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('current heatmap', (demo_img).astype(np.uint8)) debug(coords, v) global avg_i, m_avg avg_i = (avg_i + 1) % 10 print(np.mean(m_avg, axis=0)) ''' filedem = open("testfile.txt","a") for i in [0,5,7,9,11,13,15,17,19]: for j in [0,1]: filedem.write(str(coords[i][j])+' ') filedem.close() ''' if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1)))
def main(argv): global ratios, avg_i, m_avg, s tf_device = '/gpu:0' with tf.device(tf_device): #Build graph input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3],name='input_image') center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1],name='center_map') model = cpm_hand_slim.CPM_Model(FLAGS.stages, 21 + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() #Create session and restore weights sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=False)) sess.run(tf.global_variables_initializer()) model.load_weights_from_file(models/weights/cpm_hand.pkl, sess, False) #get gaussian image test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, 21) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) #Starting Video Input cam = cv2.VideoCapture(0) # Create kalman filters kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(21)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * 3e-2 with tf.device(tf_device): t0=time.time() while True: print(time.time()-t0) #Read image and resize it according to architecture input size. test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) # Starting time of each iteration. t1 = time.time() #Run the image through the model and get corresponding heatmap. stage_heatmap_np = sess.run([model.stage_heatmap[2]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) #Get the coordinates of each joint and print the image with joints makrked on the figure. demo_img,coords = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('current heatmap', (demo_img).astype(np.uint8)) #At the beginning of each execution, user gets 30 sec #for setting reference angles when hand was kept straight. if ((time.time()-t0)<30): store_deb(coords, t0) print('ratios: '+str(ratios)) else: debug(coords) print('degrees: '+str(np.mean(m_avg,axis=0))) #Calculate moving average for past n timesteps. mavg_deg=np.mean(m_avg,axis=0) mavg_deg[4]= min(mavg_deg[4],20) avg_i=(avg_i+1)%15 #Setting the string to be transmitted to Arduino. transp_str='' for i in mavg_deg: transp_str=transp_str+str(i)+',' #Flushing the input and output buffer before writing to Serial. s.flushInput() s.flushOutput() s.write(transp_str.encode()) if cv2.waitKey(1) == ord('q'): break
def __init__(self, model_path, img_size, stages=6, joints=21, use_kalman=True, kalman_noise=3e-2, color_channel='RGB', cmap_radius=21, hmap_size=46, tf_device='/cpu:0'): print("Device", tf_device) self.model_path = model_path self.input_size = img_size self.stages = stages self.joints = joints self.use_kalman = use_kalman self.kalman_noise = kalman_noise self.color_channel = color_channel self.cmap_radius = cmap_radius self.hmap_size = hmap_size self.tf_device = tf_device with tf.device(self.tf_device): """Build graph """ if self.color_channel == 'RGB': input_data = tf.placeholder( dtype=tf.float32, shape=[None, self.input_size, self.input_size, 3], name='input_image') else: input_data = tf.placeholder( dtype=tf.float32, shape=[None, self.input_size, self.input_size, 1], name='input_image') center_map = tf.placeholder( dtype=tf.float32, shape=[None, self.input_size, self.input_size, 1], name='center_map') self.model = cpm_hand_slim.CPM_Model(self.stages, self.joints + 1) self.model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ self.sess = tf.Session(config=tf.ConfigProto( allow_soft_placement=True)) self.sess.run(tf.global_variables_initializer()) if self.model_path.endswith('pkl'): self.model.load_weights_from_file(self.model_path, self.sess, False) else: saver.restore(self.sess, self.model_path) self.test_center_map = cpm_utils.gaussian_img(self.input_size, self.input_size, self.input_size / 2, self.input_size / 2, self.cmap_radius) self.test_center_map = np.reshape( self.test_center_map, [1, self.input_size, self.input_size, 1]) # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) # Create kalman filters if self.use_kalman: self.kalman_filter_array = [ cv2.KalmanFilter(4, 2) for _ in range(self.joints) ] for _, joint_kalman_filter in enumerate(self.kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array( [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array( [[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array( [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * self.kalman_noise else: self.kalman_filter_array = None
input_data = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') else: input_data = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='input_image') center_map = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2,
def main(argv): tf_device = '/cpu:0' with tf.device(tf_device): """Build graph """ if FLAGS.color_channel == 'RGB': input_data = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') else: input_data = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='input_image') center_map = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) """if not FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): cam = cv2.VideoCapture(FLAGS.cam_num)""" # Create kalman filters if FLAGS.KALMAN_ON: kalman_filter_array = [ cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints) ] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array( [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array( [[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array( [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None #classifier MLP model model2 = Sequential() model2.add(InputLayer(input_shape=(21, 2))) model2.add(Flatten()) model2.add(Dense(21)) model2.add(Dense(500)) model2.add(Dense(1400)) model2.add(Dense(3000)) model2.add(Dense(3000)) model2.add(Dense(1400)) model2.add(Dense(500, activation='relu')) model2.add(Dense(26, activation='softmax')) model2.load_weights('models/weights/massey_joint.h5') notepad_image = 255 * np.ones(shape=[368, 500, 3], dtype=np.uint8) massey_static = cv2.imread('massey_static.png') massey_static = cv2.resize(massey_static, (368, 368)) with tf.device(tf_device): while True: cam = cv2.VideoCapture(FLAGS.cam_num) test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') cam.release() test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) t1 = time.time() #print('img read time %f' % (time.time() - t1)) """if FLAGS.color_channel == 'GRAY': test_img_resize = np.dot(test_img_resize[..., :3], [0.299, 0.587, 0.114]).reshape( (FLAGS.input_size, FLAGS.input_size, 1)) cv2.imshow('color', test_img.astype(np.uint8)) cv2.imshow('gray', test_img_resize.astype(np.uint8)) cv2.waitKey(1)""" test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) """if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): # Inference t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) #demo_img = cv2.cvtColor(demo_img,cv2.COLOR_BGR2GRAY) print(demo_img.shape) #demo_img1 = cv2.GaussianBlur(demo_img, (9, 9), 0) #demo_img = 2.5*demo_img-demo_img1 cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(0) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'MULTI':""" # Inference t1 = time.time() #print("chandu") key1 = cv2.waitKey(1) #if key1 == 32: predict_heatmap, stage_heatmap_np = sess.run([ model.current_heatmap, model.stage_heatmap, ], feed_dict={ 'input_image:0': test_img_input, 'center_map:0': test_center_map }) # Show visualized image demo_img, joint_points = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) x = joint_points[0][0] y = joint_points[0][1] for i in range(21): joint_points[i][0] = joint_points[i][0] - x joint_points[i][1] = joint_points[i][1] - y joint_points = np.array([joint_points]) class_num = model2.predict_classes(joint_points) if class_num[0] == 26: character = ' ' else: character = chr(97 + class_num[0]) global printer_x, printer_y, printer_string printer_string += character notepad_image = printer.notepad(notepad_image, (printer_x, printer_y), printer_string) new_note = np.concatenate((notepad_image, demo_img, massey_static), axis=1) cv2.imshow('notepad', new_note) if len(printer_string) == 17: printer_string = '' printer_y += 25 print('fps: %.2f' % (1 / (time.time() - t1))) if key1 == ord('q'): cv2.destroyAllWindows() break