def evaluateHmImageFormdata(): with tf.device(tf_device): if 'file' not in request.files: abort(409) abort(Response('No field `file`')) file = request.files['file'] test_img = cpm_utils.read_image(file.stream.read(), [], FLAGS.input_size, 'BIN') stage_heatmap_np = prepare_heatmap(test_img) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) ret_val, image_binary = cv2.imencode('.png', (demo_img).astype(np.uint8)) if ret_val: response = make_response(image_binary.tobytes()) response.headers.set('Content-Type', 'image/png') response.headers.set('Content-Disposition', 'attachment', filename='%s.png' % 'img') return response else: abort(500) abort(Response('failed to encode image'))
def evaluateHm(): with tf.device(tf_device): t1 = time.time() test_img = cpm_utils.read_image(request.data, [], FLAGS.input_size, 'BIN') stage_heatmap_np = prepare_heatmap(test_img) return jsonify({ 'stage_heatmap_np': stage_heatmap_np[0].tolist(), 'time': time.time() - t1 })
def evaluateHmImage(): with tf.device(tf_device): test_img = cpm_utils.read_image(request.data, [], FLAGS.input_size, 'BIN') stage_heatmap_np = prepare_heatmap(test_img) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) ret_val, image_binary = cv2.imencode('.png', (demo_img).astype(np.uint8)) if ret_val: response = make_response(image_binary.tobytes()) response.headers.set('Content-Type', 'image/png') response.headers.set('Content-Disposition', 'attachment', filename='%s.png' % 'img') return response else: abort(500)
def predict_batch(self, input_test_img): with tf.device(self.tf_device): test_img = cpm_utils.read_image(input_test_img, [], self.input_size, 'IMAGE') test_img_resize = cv2.resize(test_img, (self.input_size, self.input_size)) if self.color_channel == 'GRAY': test_img_resize = np.dot(test_img_resize[..., :3], [0.299, 0.587, 0.114]).reshape( (self.input_size, self.input_size, 1)) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) test_img_input = np.repeat(test_img_input, 100, 0) # Inference predict_heatmap, stage_heatmap_np = self.sess.run( [ self.model.current_heatmap, self.model.stage_heatmap, ], feed_dict={ 'input_image:0': test_img_input, 'center_map:0': self.test_center_map }) # Get finger anchors from detection. locations = self.get_locations(test_img, stage_heatmap_np) # filtering the relevant anchor locations. relevant_locations = [ locations[0], locations[4], locations[8], locations[12], locations[16], locations[20] ] # print(locations) return relevant_locations
def main(argv): tf_device = '/gpu:0' with tf.device(tf_device): """Build graph """ if FLAGS.color_channel == 'RGB': input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') else: input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='input_image') center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') model = cpm_body_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) # Create kalman filters if FLAGS.KALMAN_ON: kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None # read in video / flow frames if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')): # OpenCV can only read in '.avi' files cam = imageio.get_reader(FLAGS.DEMO_TYPE) else: cam = cv2.VideoCapture(FLAGS.cam_num) # iamge processing with tf.device(tf_device): if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')): ori_fps = cam.get_meta_data()['fps'] print('This video fps is %f' % ori_fps) video_length = cam.get_length() writer_path = os.path.join('results', os.path.basename(FLAGS.DEMO_TYPE)) # !! OpenCV can only write in .avi cv_writer = cv2.VideoWriter(writer_path + '.avi', # cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'), cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), ori_fps, (FLAGS.input_size, FLAGS.input_size)) # imageio_writer = imageio.get_writer(writer_path, fps=ori_fps) try: for it, im in enumerate(cam): test_img_t = time.time() test_img = cpm_utils.read_image(im, [], FLAGS.input_size, 'VIDEO') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) print('img read time %f' % (time.time() - test_img_t)) if FLAGS.color_channel == 'GRAY': test_img_resize = mgray(test_img_resize, test_img) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) # Inference fps_t = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if (cv2.waitKey(1) == ord('q')): break print('fps: %.2f' % (1 / (time.time() - fps_t))) cv_writer.write(demo_img.astype(np.uint8)) # imageio_writer.append_data(demo_img[:, :, 1]) except KeyboardInterrupt: print('Stopped! {}/{} frames captured!'.format(it, video_length)) finally: cv_writer.release() # imageio_writer.close() else: while True: test_img_t = time.time() if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') else: test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) print('img read time %f' % (time.time() - test_img_t)) if FLAGS.color_channel == 'GRAY': test_img_resize = mgray(test_img_resize, test_img) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): # Inference fps_t = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(0) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - fps_t))) elif FLAGS.DEMO_TYPE == 'MULTI': # Inference fps_t = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - fps_t))) elif FLAGS.DEMO_TYPE == 'SINGLE': # Inference fps_t = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[5]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('current heatmap', (demo_img).astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - fps_t))) elif FLAGS.DEMO_TYPE == 'HM': # Inference fps_t = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[FLAGS.stages - 1]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) print('fps: %.2f' % (1 / (time.time() - fps_t))) # demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape( # (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints)) demo_stage_heatmap = stage_heatmap_np[-1][0, :, :, 0:FLAGS.joints].reshape( (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints)) demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size)) vertical_imgs = [] tmp_img = None joint_coord_set = np.zeros((FLAGS.joints, 2)) for joint_num in range(FLAGS.joints): # Concat until 4 img if (joint_num % 4) == 0 and joint_num != 0: vertical_imgs.append(tmp_img) tmp_img = None demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num])) # Plot color joints if np.min(demo_stage_heatmap[:, :, joint_num]) > -50: joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]), (FLAGS.input_size, FLAGS.input_size)) joint_coord_set[joint_num, :] = joint_coord color_code_num = (joint_num // 4) if joint_num in [0, 4, 8, 12, 16]: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) else: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) # Put text tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8) tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])), org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])), org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp_img = np.concatenate((tmp_img, tmp), axis=0) \ if tmp_img is not None else tmp # Plot limbs for limb_num in range(len(limbs)): if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]) > -2000 and np.min( demo_stage_heatmap[:, :, limbs[limb_num][1]]) > -2000: x1 = joint_coord_set[limbs[limb_num][0], 0] y1 = joint_coord_set[limbs[limb_num][0], 1] x2 = joint_coord_set[limbs[limb_num][1], 0] y2 = joint_coord_set[limbs[limb_num][1], 1] length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5 if length < 10000 and length > 5: deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)), (int(length / 2), 3), int(deg), 0, 360, 1) color_code_num = limb_num // 4 if PYTHON_VERSION == 3: limb_color = list( map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])) else: limb_color = map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]) cv2.fillConvexPoly(test_img, polygon, color=limb_color) if tmp_img is not None: tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)), 'constant', constant_values=(0, 0)) vertical_imgs.append(tmp_img) # Concat horizontally output_img = None for col in range(len(vertical_imgs)): output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \ vertical_imgs[col] output_img = output_img.astype(np.uint8) output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET) test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4) cv2.imshow('hm', output_img) cv2.moveWindow('hm', 2000, 200) cv2.imshow('rgb', test_img) cv2.moveWindow('rgb', 2000, 750) if cv2.waitKey(1) == ord('q'): break
def main(argv): """ Initial tracker """ tracker = tracking_module.SelfTracker([FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size) """ Build network graph """ model = cpm_model.CPM_Model(input_size=FLAGS.input_size, heatmap_size=FLAGS.heatmap_size, stages=FLAGS.cpm_stages, joints=FLAGS.num_of_joints, img_type=FLAGS.color_channel, is_training=False) saver = tf.train.Saver() """ Get output node """ output_node = tf.get_default_graph().get_tensor_by_name(name=FLAGS.output_node_names) device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0} sess_config = tf.ConfigProto(device_count=device_count) sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2 sess_config.gpu_options.allow_growth = True sess_config.allow_soft_placement = True with tf.Session(config=sess_config) as sess: model_path_suffix = os.path.join(FLAGS.network_def, 'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size), 'joints_{}'.format(FLAGS.num_of_joints), 'stages_{}'.format(FLAGS.cpm_stages), 'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr, FLAGS.lr_decay_rate, FLAGS.lr_decay_step) ) model_save_dir = os.path.join('models', 'weights', model_path_suffix) print('Load model from [{}]'.format(os.path.join(model_save_dir, FLAGS.model_path))) # if the model is a pkl file, then load it, else just restore the pretrained cpm_hand by default # by here we can see the loading weight structure if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, 'models/weights/cpm_hand') # Check weights, this seems to be the part that print out weights for variable in tf.global_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) # Create webcam instance if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']: print("REFUSEREFUSE\n\n\n\n\n\n\n\nrefule") cam = cv2.VideoCapture(FLAGS.cam_id) # Create kalman filters if FLAGS.use_kalman: kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array( [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): print("TESTINGTESTING\n\n\n\n\n\n\n\nrTESGING") test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') #fixme: I haven't seen how it cut the box, I have put the coordinates here, but not using them currently b_box, b_height, b_width = dt.bounding_box_from_file(FLAGS.DEMO_TYPE) b_box = b_box[0][0] bb_box = dt.normalize_and_centralize_img(b_box[0], b_box[1], b_box[2], b_box[3], 10, b_height, b_width)\ #fixme: this is the end of my code test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) test_img_input = normalize_and_centralize_img(test_img_resize) t1 = time.time() #current_heatmap is the output after going through the last layer #input_images is a placeholder matrix for iiamge size # FIXME: I don't know what exactly does this sess run with? the two arguments? #predict_heatmap.shape = (1,32,32,22) #stage_heatmap.shape = (1,32,32,22), it seems like, for stage_heatmap, each of the 1*32*32 is something same, a 22 length-array predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, output_node, ], feed_dict={model.input_images: test_img_input} ) #frame per second print('fps: %.2f' % (1 / (time.time() - t1))) print(stage_heatmap_np[0], "stage heatmap") print(stage_heatmap_np[0].shape) #print(predict_heatmap, "predict heatmap") #print(predict_heatmap.shape) tmp_img = cv2.resize(stage_heatmap_np[0], (FLAGS.input_size, FLAGS.input_size)) print(tmp_img, "tmp_img") print(tmp_img.shape) correct_and_draw_hand(test_img, cv2.resize(stage_heatmap_np[0], (FLAGS.input_size, FLAGS.input_size)), kalman_filter_array, tracker, tracker.input_crop_ratio, test_img) # Show visualized image #demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array, tracker, tracker.input_crop_ratio, test_img.copy()) # local_img = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker, crop_full_scale, test_img_copy) cv2.imshow('demo_img', test_img.astype(np.uint8)) cv2.waitKey(0) elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']: print("SINGLE\n\n\n\n\n\n\nMULTI") while True: # # Prepare input image _, full_img = cam.read() # then I kow the misterious joint_detection comes from here! but now it's only zeros??? test_img = tracker.tracking_by_joints(full_img, joint_detections=joint_detections) crop_full_scale = tracker.input_crop_ratio test_img_copy = test_img.copy() # White balance test_img_wb = utils.img_white_balance(test_img, 5) test_img_input = normalize_and_centralize_img(test_img_wb) # Inference t1 = time.time() stage_heatmap_np = sess.run([output_node], feed_dict={model.input_images: test_img_input}) print('FPS: %.2f' % (1 / (time.time() - t1))) local_img = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker, crop_full_scale, test_img_copy) cv2.imshow('local_img', local_img.astype(np.uint8)) cv2.imshow('global_img', full_img.astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break elif FLAGS.DEMO_TYPE == 'Joint_HM': print("JOINTHM\n\n\n\n\n\n\nJOINTHM") while True: # Prepare input image test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) test_img_input = normalize_and_centralize_img(test_img_resize) # Inference t1 = time.time() stage_heatmap_np = sess.run([output_node], feed_dict={model.input_images: test_img_input}) print('FPS: %.2f' % (1 / (time.time() - t1))) demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.num_of_joints].reshape( (FLAGS.heatmap_size, FLAGS.heatmap_size, FLAGS.num_of_joints)) demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size)) vertical_imgs = [] tmp_img = None joint_coord_set = np.zeros((FLAGS.num_of_joints, 2)) for joint_num in range(FLAGS.num_of_joints): # Concat until 4 img if (joint_num % 4) == 0 and joint_num != 0: vertical_imgs.append(tmp_img) tmp_img = None demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num])) # Plot color joints if np.min(demo_stage_heatmap[:, :, joint_num]) > -50: joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]), (FLAGS.input_size, FLAGS.input_size)) joint_coord_set[joint_num, :] = joint_coord color_code_num = (joint_num // 4) if joint_num in [0, 4, 8, 12, 16]: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num])) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) else: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num])) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) # Put text tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8) tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])), org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])), org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp_img = np.concatenate((tmp_img, tmp), axis=0) \ if tmp_img is not None else tmp # Plot FLAGS.limbs for limb_num in range(len(FLAGS.limbs)): if np.min(demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][0]]) > -2000 and np.min( demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][1]]) > -2000: x1 = joint_coord_set[FLAGS.limbs[limb_num][0], 0] y1 = joint_coord_set[FLAGS.limbs[limb_num][0], 1] x2 = joint_coord_set[FLAGS.limbs[limb_num][1], 0] y2 = joint_coord_set[FLAGS.limbs[limb_num][1], 1] length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5 if length < 10000 and length > 5: deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)), (int(length / 2), 3), int(deg), 0, 360, 1) color_code_num = limb_num // 4 limb_color = list( map(lambda x: x + 35 * (limb_num % 4), FLAGS.joint_color_code[color_code_num])) cv2.fillConvexPoly(test_img, polygon, color=limb_color) if tmp_img is not None: tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)), 'constant', constant_values=(0, 0)) vertical_imgs.append(tmp_img) # Concat horizontally output_img = None for col in range(len(vertical_imgs)): output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \ vertical_imgs[col] output_img = output_img.astype(np.uint8) output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET) test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4) cv2.imshow('hm', output_img) cv2.moveWindow('hm', 2000, 200) cv2.imshow('rgb', test_img) cv2.moveWindow('rgb', 2000, 750) if cv2.waitKey(1) == ord('q'): break
def main(argv): tf_device = '/gpu:0' with tf.device(tf_device): """Build graph """ if FLAGS.color_channel == 'RGB': input_data = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') else: input_data = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='input_image') center_map = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') model = cpm_body_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) # Create kalman filters if FLAGS.KALMAN_ON: kalman_filter_array = [ cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints) ] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array( [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array( [[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array( [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None # read in video / flow frames if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')): # OpenCV can only read in '.avi' files cam = imageio.get_reader(FLAGS.DEMO_TYPE) else: cam = cv2.VideoCapture(FLAGS.cam_num) # iamge processing with tf.device(tf_device): test_img_t = time.time() if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') else: test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) print('img read time %f' % (time.time() - test_img_t)) if FLAGS.color_channel == 'GRAY': test_img_resize = mgray(test_img_resize, test_img) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): # Inference fps_t = time.time() predict_heatmap, stage_heatmap_np = sess.run([ model.current_heatmap, model.stage_heatmap, ], feed_dict={ 'input_image:0': test_img_input, 'center_map:0': test_center_map }) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(0) == ord('q'): exit() print('fps: %.2f' % (1 / (time.time() - fps_t))) # iamge processing
print(graph) for op in graph.get_operations(): print(op.name) with tf.Session(graph=graph) as sess: input_size = 256 writer = tf.summary.FileWriter('./', sess.graph) input_tf = graph.get_tensor_by_name('prefix/input_placeholder:0') output_tf = graph.get_tensor_by_name( 'prefix/stage_6/mid_conv7/BiasAdd:0') cam = cv2.VideoCapture(0) while True: # Prepare input image t1 = time.time() test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) print('img read time %f' % (time.time() - t1)) test_img_input = normalize_and_centralize_img(test_img_resize) # Inference t1 = time.time() stage_heatmap_np = sess.run([output_tf], feed_dict={input_tf: test_img_input}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, None) cv2.imshow('current heatmap', (demo_img).astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break
def main(argv): tf_device = '/gpu:0' with tf.device(tf_device): """Build graph """ if FLAGS.color_channel == 'RGB': input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') else: input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='input_image') center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) if not FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): #if not FLAGS.DEMO_TYPE.endswith(('avi', 'mp4')): cam = cv2.VideoCapture(FLAGS.cam_num) # Create kalman filters if FLAGS.KALMAN_ON: kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None with tf.device(tf_device): while True: t1 = time.time() if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') else: test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') #yolo stuff result = tfnet.return_predict(test_img) count = 0 if not result: pass else: for i in range(0, len(result)): if (result[i]['label'] == 'hand_l') & (count == 0): hand1 = [result[i]['topleft'], result[i]['bottomright']] count+=1 elif (result[i]['label'] == 'hand_r') & (count == 1): hand2 = [result[i]['topleft'], result[i]['bottomright']] count+=1 #elif (count==1): # hand2 = [] #elif (count==0): # hand1 = ['NA'] # hand2 = ['NA'] #cropped image if not hand1: pass else: h1_image = test_img[hand1[0]['y']:hand1[1]['y'], hand1[0]['x']:hand1[1]['x']] if not hand2: pass else: h2_image = test_img[hand2[0]['y']:hand2[1]['y'], hand2[0]['x']:hand2[1]['x']] if not h1_image: pass else: h1_image_resize = cv2.resize(h1_image, (FLAGS.input_size, FLAGS.input_size)) h1_img_input = h1_image_resize / 256.0 - 0.5 h1_img_input = np.expand_dims(h1_img_input, axis=0) if not h2_image: pass else: h2_image_resize = cv2.resize(h2_image, (FLAGS.input_size, FLAGS.input_size)) h2_img_input = h2_image_resize / 256.0 - 0.5 h2_img_input = np.expand_dims(h2_img_input, axis=0) """ #test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) #print('img read time %f' % (time.time() - t1)) if FLAGS.color_channel == 'GRAY': test_img_resize = np.dot(test_img_resize[..., :3], [0.299, 0.587, 0.114]).reshape( (FLAGS.input_size, FLAGS.input_size, 1)) cv2.imshow('color', test_img.astype(np.uint8)) cv2.imshow('gray', test_img_resize.astype(np.uint8)) cv2.waitKey(1) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) """ if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): # Inference t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(0) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'MULTI': # Inference t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'SINGLE': # Inference t1 = time.time() """ stage_heatmap_np = sess.run([model.stage_heatmap[5]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) """ if h1_img_input: pass else: stage_heatmap_np_1 = sess.run([model.stage_heatmap[5]], feed_dict={'input_image:0': h1_img_input, 'center_map:0': test_center_map}) demo_img_1 = visualize_result(h1_image, FLAGS, stage_heatmap_np_1, kalman_filter_array) test_img[hand1[0]['y']:hand1[1]['y'], hand1[0]['x']:hand1[1]['x']] = demo_img_1 if h2_img_input: pass else: stage_heatmap_np_2 = sess.run([model.stage_heatmap[5]], feed_dict={'input_image:0': h2_img_input, 'center_map:0': test_center_map}) demo_img_2 = visualize_result(h2_image, FLAGS, stage_heatmap_np_2, kalman_filter_array) test_img[hand2[0]['y']:hand2[1]['y'], hand2[0]['x']:hand2[1]['x']] = demo_img_2 # Show visualized image # X2 times till here -concatenate here cv2.imshow('current heatmap', (test_img).astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'HM': # Inference t1 = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[FLAGS.stages - 1]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) print('fps: %.2f' % (1 / (time.time() - t1))) demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape( (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints)) demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size)) vertical_imgs = [] tmp_img = None joint_coord_set = np.zeros((FLAGS.joints, 2)) for joint_num in range(FLAGS.joints): # Concat until 4 img if (joint_num % 4) == 0 and joint_num != 0: vertical_imgs.append(tmp_img) tmp_img = None demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num])) # Plot color joints if np.min(demo_stage_heatmap[:, :, joint_num]) > -50: joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]), (FLAGS.input_size, FLAGS.input_size)) joint_coord_set[joint_num, :] = joint_coord color_code_num = (joint_num // 4) if joint_num in [0, 4, 8, 12, 16]: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) else: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) # Put text tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8) tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])), org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])), org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp_img = np.concatenate((tmp_img, tmp), axis=0) \ if tmp_img is not None else tmp # Plot limbs for limb_num in range(len(limbs)): if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]) > -2000 and np.min( demo_stage_heatmap[:, :, limbs[limb_num][1]]) > -2000: x1 = joint_coord_set[limbs[limb_num][0], 0] y1 = joint_coord_set[limbs[limb_num][0], 1] x2 = joint_coord_set[limbs[limb_num][1], 0] y2 = joint_coord_set[limbs[limb_num][1], 1] length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5 if length < 10000 and length > 5: deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)), (int(length / 2), 3), int(deg), 0, 360, 1) color_code_num = limb_num // 4 if PYTHON_VERSION == 3: limb_color = list( map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])) else: limb_color = map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]) cv2.fillConvexPoly(test_img, polygon, color=limb_color) if tmp_img is not None: tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)), 'constant', constant_values=(0, 0)) vertical_imgs.append(tmp_img) # Concat horizontally output_img = None for col in range(len(vertical_imgs)): output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \ vertical_imgs[col] output_img = output_img.astype(np.uint8) output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET) test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4) cv2.imshow('hm', output_img) cv2.moveWindow('hm', 2000, 200) cv2.imshow('rgb', test_img) cv2.moveWindow('rgb', 2000, 750) if cv2.waitKey(1) == ord('q'): break
def main(argv): global joint_detections os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id) """ Initial tracker """ tracker = tracking_module.SelfTracker([FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size) """ Build network graph """ model = cpm_model.CPM_Model(input_size=FLAGS.input_size, heatmap_size=FLAGS.heatmap_size, stages=FLAGS.cpm_stages, joints=FLAGS.num_of_joints, img_type=FLAGS.color_channel, is_training=False) saver = tf.train.Saver() """ Get output node """ output_node = tf.get_default_graph().get_tensor_by_name(name=FLAGS.output_node_names) device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0} sess_config = tf.ConfigProto(device_count=device_count) sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2 sess_config.gpu_options.allow_growth = True sess_config.allow_soft_placement = True with tf.Session(config=sess_config) as sess: model_path_suffix = os.path.join(FLAGS.network_def, 'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size), 'joints_{}'.format(FLAGS.num_of_joints), 'stages_{}'.format(FLAGS.cpm_stages), 'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr, FLAGS.lr_decay_rate, FLAGS.lr_decay_step) ) model_save_dir = os.path.join('models', 'weights', model_path_suffix) print('Load model from [{}]'.format(os.path.join(model_save_dir, FLAGS.model_path))) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, 'models/weights/cpm_hand') # Check weights for variable in tf.global_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) # Create webcam instance if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']: cam = cv2.VideoCapture(FLAGS.cam_id) # Create kalman filters if FLAGS.use_kalman: kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array( [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) test_img_input = normalize_and_centralize_img(test_img_resize) t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, output_node, ], feed_dict={model.input_images: test_img_input} ) print('fps: %.2f' % (1 / (time.time() - t1))) correct_and_draw_hand(test_img, cv2.resize(stage_heatmap_np[0], (FLAGS.input_size, FLAGS.input_size)), kalman_filter_array, tracker, tracker.input_crop_ratio, test_img) # Show visualized image # demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', test_img.astype(np.uint8)) cv2.waitKey(0) elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']: while True: # # Prepare input image _, full_img = cam.read() test_img = tracker.tracking_by_joints(full_img, joint_detections=joint_detections) crop_full_scale = tracker.input_crop_ratio test_img_copy = test_img.copy() # White balance test_img_wb = utils.img_white_balance(test_img, 5) test_img_input = normalize_and_centralize_img(test_img_wb) # Inference t1 = time.time() stage_heatmap_np = sess.run([output_node], feed_dict={model.input_images: test_img_input}) print('FPS: %.2f' % (1 / (time.time() - t1))) local_img = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker, crop_full_scale, test_img_copy) cv2.imshow('local_img', local_img.astype(np.uint8)) cv2.imshow('global_img', full_img.astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break elif FLAGS.DEMO_TYPE == 'Joint_HM': while True: # Prepare input image test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) test_img_input = normalize_and_centralize_img(test_img_resize) # Inference t1 = time.time() stage_heatmap_np = sess.run([output_node], feed_dict={model.input_images: test_img_input}) print('FPS: %.2f' % (1 / (time.time() - t1))) demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.num_of_joints].reshape( (FLAGS.heatmap_size, FLAGS.heatmap_size, FLAGS.num_of_joints)) demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size)) vertical_imgs = [] tmp_img = None joint_coord_set = np.zeros((FLAGS.num_of_joints, 2)) for joint_num in range(FLAGS.num_of_joints): # Concat until 4 img if (joint_num % 4) == 0 and joint_num != 0: vertical_imgs.append(tmp_img) tmp_img = None demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num])) # Plot color joints if np.min(demo_stage_heatmap[:, :, joint_num]) > -50: joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]), (FLAGS.input_size, FLAGS.input_size)) joint_coord_set[joint_num, :] = joint_coord color_code_num = (joint_num // 4) if joint_num in [0, 4, 8, 12, 16]: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num])) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) else: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num])) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) # Put text tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8) tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])), org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])), org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp_img = np.concatenate((tmp_img, tmp), axis=0) \ if tmp_img is not None else tmp # Plot FLAGS.limbs for limb_num in range(len(FLAGS.limbs)): if np.min(demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][0]]) > -2000 and np.min( demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][1]]) > -2000: x1 = joint_coord_set[FLAGS.limbs[limb_num][0], 0] y1 = joint_coord_set[FLAGS.limbs[limb_num][0], 1] x2 = joint_coord_set[FLAGS.limbs[limb_num][1], 0] y2 = joint_coord_set[FLAGS.limbs[limb_num][1], 1] length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5 if length < 10000 and length > 5: deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)), (int(length / 2), 3), int(deg), 0, 360, 1) color_code_num = limb_num // 4 limb_color = list( map(lambda x: x + 35 * (limb_num % 4), FLAGS.joint_color_code[color_code_num])) cv2.fillConvexPoly(test_img, polygon, color=limb_color) if tmp_img is not None: tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)), 'constant', constant_values=(0, 0)) vertical_imgs.append(tmp_img) # Concat horizontally output_img = None for col in range(len(vertical_imgs)): output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \ vertical_imgs[col] output_img = output_img.astype(np.uint8) output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET) test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4) cv2.imshow('hm', output_img) cv2.moveWindow('hm', 2000, 200) cv2.imshow('rgb', test_img) cv2.moveWindow('rgb', 2000, 750) if cv2.waitKey(1) == ord('q'): break
def main(argv): tf_device = '/cpu:0' with tf.device(tf_device): """Build graph """ if FLAGS.color_channel == 'RGB': input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') else: input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='input_image') center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') model = cpm_body_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) # Create kalman filters if FLAGS.KALMAN_ON: kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None # read in video / flow frames if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')): # OpenCV can only read in '.avi' files cam = imageio.get_reader(FLAGS.DEMO_TYPE) else: cam = cv2.VideoCapture(FLAGS.cam_num) # iamge processing with tf.device(tf_device): if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')): ori_fps = cam.get_meta_data()['fps'] print('This video fps is %f' % ori_fps) video_length = cam.get_length() writer_path = os.path.join('results', os.path.basename(FLAGS.DEMO_TYPE)) # !! OpenCV can only write in .avi cv_writer = cv2.VideoWriter(writer_path + '.avi', # cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'), cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), ori_fps, (FLAGS.input_size, FLAGS.input_size)) # imageio_writer = imageio.get_writer(writer_path, fps=ori_fps) try: for it, im in enumerate(cam): test_img_t = time.time() test_img = cpm_utils.read_image(im, [], FLAGS.input_size, 'VIDEO') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) print('img read time %f' % (time.time() - test_img_t)) if FLAGS.color_channel == 'GRAY': test_img_resize = mgray(test_img_resize, test_img) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) # Inference fps_t = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if (cv2.waitKey(1) == ord('q')): break print('fps: %.2f' % (1 / (time.time() - fps_t))) cv_writer.write(demo_img.astype(np.uint8)) # imageio_writer.append_data(demo_img[:, :, 1]) except KeyboardInterrupt: print('Stopped! {}/{} frames captured!'.format(it, video_length)) finally: cv_writer.release() # imageio_writer.close() else: while True: test_img_t = time.time() if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') else: test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) print('img read time %f' % (time.time() - test_img_t)) if FLAGS.color_channel == 'GRAY': test_img_resize = mgray(test_img_resize, test_img) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): # Inference fps_t = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(0) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - fps_t))) elif FLAGS.DEMO_TYPE == 'MULTI': # Inference fps_t = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - fps_t))) elif FLAGS.DEMO_TYPE == 'SINGLE': # Inference fps_t = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[5]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('current heatmap', (demo_img).astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - fps_t))) elif FLAGS.DEMO_TYPE == 'HM': # Inference fps_t = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[FLAGS.stages - 1]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) print('fps: %.2f' % (1 / (time.time() - fps_t))) # demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape( # (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints)) demo_stage_heatmap = stage_heatmap_np[-1][0, :, :, 0:FLAGS.joints].reshape( (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints)) demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size)) vertical_imgs = [] tmp_img = None joint_coord_set = np.zeros((FLAGS.joints, 2)) for joint_num in range(FLAGS.joints): # Concat until 4 img if (joint_num % 4) == 0 and joint_num != 0: vertical_imgs.append(tmp_img) tmp_img = None demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num])) # Plot color joints if np.min(demo_stage_heatmap[:, :, joint_num]) > -50: joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]), (FLAGS.input_size, FLAGS.input_size)) joint_coord_set[joint_num, :] = joint_coord color_code_num = (joint_num // 4) if joint_num in [0, 4, 8, 12, 16]: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) else: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) # Put text tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8) tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])), org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])), org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp_img = np.concatenate((tmp_img, tmp), axis=0) \ if tmp_img is not None else tmp # Plot limbs for limb_num in range(len(limbs)): if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]) > -2000 and np.min( demo_stage_heatmap[:, :, limbs[limb_num][1]]) > -2000: x1 = joint_coord_set[limbs[limb_num][0], 0] y1 = joint_coord_set[limbs[limb_num][0], 1] x2 = joint_coord_set[limbs[limb_num][1], 0] y2 = joint_coord_set[limbs[limb_num][1], 1] length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5 if length < 10000 and length > 5: deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)), (int(length / 2), 3), int(deg), 0, 360, 1) color_code_num = limb_num // 4 if PYTHON_VERSION == 3: limb_color = list( map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])) else: limb_color = map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]) cv2.fillConvexPoly(test_img, polygon, color=limb_color) if tmp_img is not None: tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)), 'constant', constant_values=(0, 0)) vertical_imgs.append(tmp_img) # Concat horizontally output_img = None for col in range(len(vertical_imgs)): output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \ vertical_imgs[col] output_img = output_img.astype(np.uint8) output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET) test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4) cv2.imshow('hm', output_img) cv2.moveWindow('hm', 2000, 200) cv2.imshow('rgb', test_img) cv2.moveWindow('rgb', 2000, 750) if cv2.waitKey(1) == ord('q'): break
def main(argv): global ratios, avg_i, m_avg, s tf_device = '/gpu:0' with tf.device(tf_device): #Build graph input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3],name='input_image') center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1],name='center_map') model = cpm_hand_slim.CPM_Model(FLAGS.stages, 21 + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() #Create session and restore weights sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=False)) sess.run(tf.global_variables_initializer()) model.load_weights_from_file(models/weights/cpm_hand.pkl, sess, False) #get gaussian image test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, 21) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) #Starting Video Input cam = cv2.VideoCapture(0) # Create kalman filters kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(21)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * 3e-2 with tf.device(tf_device): t0=time.time() while True: print(time.time()-t0) #Read image and resize it according to architecture input size. test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) # Starting time of each iteration. t1 = time.time() #Run the image through the model and get corresponding heatmap. stage_heatmap_np = sess.run([model.stage_heatmap[2]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) #Get the coordinates of each joint and print the image with joints makrked on the figure. demo_img,coords = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('current heatmap', (demo_img).astype(np.uint8)) #At the beginning of each execution, user gets 30 sec #for setting reference angles when hand was kept straight. if ((time.time()-t0)<30): store_deb(coords, t0) print('ratios: '+str(ratios)) else: debug(coords) print('degrees: '+str(np.mean(m_avg,axis=0))) #Calculate moving average for past n timesteps. mavg_deg=np.mean(m_avg,axis=0) mavg_deg[4]= min(mavg_deg[4],20) avg_i=(avg_i+1)%15 #Setting the string to be transmitted to Arduino. transp_str='' for i in mavg_deg: transp_str=transp_str+str(i)+',' #Flushing the input and output buffer before writing to Serial. s.flushInput() s.flushOutput() s.write(transp_str.encode()) if cv2.waitKey(1) == ord('q'): break
help='One of image/json') parser.add_argument('-d', '--device', default='gpu', help='One of cpu or gpu') args = parser.parse_args() files = os.listdir(args.input) if len(files) == 0: raise Exception("Input directory is empty") print("Found", len(files), "files") images = [ cpm_utils.read_image(os.path.join(args.input, f), [], 368, 'IMAGE') for f in files ] poses = [ pose for _, pose in hand_pose( images, right_hand=True, mode="BGR", device=args.device) ] for f, image, pose in zip(files, images, poses): save_loc = os.path.join(args.output, f) if args.format == "json": json.dump(pose, open(save_loc + ".json", "w")) else: new_image = draw_pose(image, pose) cv2.imwrite(save_loc, new_image)
def main(argv): global joint_detections os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id) tracker = tracking_module.SelfTracker( [FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size) model = cpm_model.CPM_Model(input_size=FLAGS.input_size, heatmap_size=FLAGS.heatmap_size, stages=FLAGS.cpm_stages, joints=FLAGS.num_of_joints, img_type=FLAGS.color_channel, is_training=False) saver = tf.train.Saver() output_node = tf.get_default_graph().get_tensor_by_name( name=FLAGS.output_node_names) device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0} sess_config = tf.ConfigProto(device_count=device_count) sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2 sess_config.gpu_options.allow_growth = True sess_config.allow_soft_placement = True with tf.Session(config=sess_config) as sess: model_path_suffix = os.path.join( FLAGS.network_def, 'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size), 'joints_{}'.format(FLAGS.num_of_joints), 'stages_{}'.format(FLAGS.cpm_stages), 'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr, FLAGS.lr_decay_rate, FLAGS.lr_decay_step)) model_save_dir = os.path.join('models', 'weights', model_path_suffix) print('Load model from [{}]'.format( os.path.join(model_save_dir, FLAGS.model_path))) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, 'models/weights/cpm_hand') for variable in tf.global_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']: cam = cv2.VideoCapture(FLAGS.cam_id) #cam = pipeline.wait_for_frames() if FLAGS.use_kalman: kalman_filter_array = [ cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints) ] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array( [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array( [[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array( [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) test_img_input = normalize_and_centralize_img(test_img_resize) t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run( [ model.current_heatmap, output_node, ], feed_dict={model.input_images: test_img_input}) print('fps: %.2f' % (1 / (time.time() - t1))) correct_and_draw_hand( test_img, cv2.resize(stage_heatmap_np[0], (FLAGS.input_size, FLAGS.input_size)), kalman_filter_array, tracker, tracker.input_crop_ratio, test_img) # Show visualized image # demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', test_img.astype(np.uint8)) cv2.waitKey(0) elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']: while True: _, full_img = cam.read() test_img = tracker.tracking_by_joints( full_img, joint_detections=joint_detections) crop_full_scale = tracker.input_crop_ratio test_img_copy = test_img.copy() # White balance test_img_wb = utils.img_white_balance(test_img, 5) test_img_input = normalize_and_centralize_img(test_img_wb) # Inference t1 = time.time() stage_heatmap_np = sess.run( [output_node], feed_dict={model.input_images: test_img_input}) print('FPS: %.2f' % (1 / (time.time() - t1))) local_img = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker, crop_full_scale, test_img_copy) cv2.imshow('local_img', local_img.astype(np.uint8)) cv2.imshow('global_img', full_img.astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break
def main(argv): tf_device = '/gpu:0' with tf.device(tf_device): """Build graph """ input_data = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') center_map = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) model.load_weights_from_file(FLAGS.model_path, sess, False) #model_vars = tf.trainable_variables() #slim.model_analyzer.analyze_vars(model_vars, print_info=True) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) if not FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): cam = cv2.VideoCapture(FLAGS.cam_num) # Create kalman filters kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array( [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array( [[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array( [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise with tf.device(tf_device): #filedem = open("testfile.txt","w") #filedem.close() if FLAGS.DEMO_TYPE == 'SINGLE': #read default image coords v = [] with open('test_file.csv') as file: reader = csv.reader(file, delimiter=',') for row in reader: for entries in row: v.append(entries) ''' f=open("testfile.txt", 'r') v=f.readline() v=v.strip().split(' ') ''' while True: t1 = time.time() if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') else: test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) #print('img read time %f' % (time.time() - t1)) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): # Inference t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run( [ model.current_heatmap, model.stage_heatmap, ], feed_dict={ 'input_image:0': test_img_input, 'center_map:0': test_center_map }) # Show visualized image demo_img, coords = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) ''' filedem = open("testfile.txt","w") for i in [0,5,7,9,11,13,15,17,19]: for j in [0,1]: filedem.write(str(coords[i][j])+' ') filedem.close() ''' with open('test_file.csv', mode='w') as file: writer = csv.writer(file, delimiter=',') for i in [1, 5, 8, 9, 12, 13, 16, 17, 20]: writer.writerow([coords[i][0], coords[i][1]]) if cv2.waitKey(0) == ord('q'): break #print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'SINGLE': # Inference t1 = time.time() stage_heatmap_np = sess.run( [model.stage_heatmap[2]], feed_dict={ 'input_image:0': test_img_input, 'center_map:0': test_center_map }) # Show visualized image demo_img, coords = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('current heatmap', (demo_img).astype(np.uint8)) debug(coords, v) global avg_i, m_avg avg_i = (avg_i + 1) % 10 print(np.mean(m_avg, axis=0)) ''' filedem = open("testfile.txt","a") for i in [0,5,7,9,11,13,15,17,19]: for j in [0,1]: filedem.write(str(coords[i][j])+' ') filedem.close() ''' if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1)))
def main(argv): # tf_device = '/gpu:0' tf_device = '/cpu:0' with tf.device(tf_device): """Build graph """ input_data = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') center_map = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], #center map的大小也是368x368 name='center_map') # model = cpm_body.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model = cpm_body.CPM_Model( FLAGS.stages, FLAGS.joints) #这里的stages和joints是传进来的参数,跟训练集的关节点个数应该相等 model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) # sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) # sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) sess = tf.Session() sess.run(tf.global_variables_initializer()) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) test_center_map = cpm_utils.gaussian_img( FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, #预测的时候,需要把人放图像中间,这里构造center map是以图片中心来构造的 FLAGS.input_size / 2, FLAGS.cmap_radius) #如果预测人没有在中间,就会出现问题 test_center_map = np.reshape( test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1 ]) #增加batch和通道的维数都为1,此刻center map:1x368x368x1 # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) #这个for循环输出了模型中的参数,包括卷积核值和偏置值 with tf.device(tf_device): # while True: test_img_t = time.time() test_img = cpm_utils.read_image( FLAGS.img_path, [], FLAGS.input_size, 'IMAGE') #构造预测输入,[]是视频输出,这里没有,test_img为368x368x3 test_img_resize = cv2.resize( test_img, (FLAGS.input_size, FLAGS.input_size)) #删掉,没用 print('img read time %f' % (time.time() - test_img_t)) test_img_input = test_img_resize / 256.0 - 0.5 #归一化到[-0.5,0.5] test_img_input = np.expand_dims( test_img_input, axis=0) #增加了batch的维数为1,此刻test_img_input: 1x368x368x3 if FLAGS.DEMO_TYPE == 'MULTI': # Inference fps_t = time.time() predict_heatmap, stage_heatmap_np = sess.run([ model.current_heatmap, model.stage_heatmap, ], feed_dict={ 'input_image:0': test_img_input, 'center_map:0': test_center_map }) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, None) # cv2.imshow('demo_img', demo_img.astype(np.uint8)) cv2.imwrite(os.path.join(base_path, str(time.time()) + "_.jpg"), demo_img.astype(np.uint8)) # if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - fps_t))) elif FLAGS.DEMO_TYPE == 'SINGLE': # Inference fps_t = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[5]], feed_dict={ 'input_image:0': test_img_input, 'center_map:0': test_center_map }) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, None) # cv2.imshow('current heatmap', (demo_img).astype(np.uint8)) cv2.imwrite(os.path.join(base_path, "current_heatmap.jpg"), (demo_img).astype(np.uint8)) # if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - fps_t))) elif FLAGS.DEMO_TYPE == 'HM': # Inference fps_t = time.time() stage_heatmap_np = sess.run( [model.stage_heatmap[FLAGS.stages - 1] ], #最后一个stage的heatmap,列表长度为1,因为只有一个人 feed_dict={ 'input_image:0': test_img_input, #stage_heatmap_np[0]的shape为 [1, 46, 46, 15] 'center_map:0': test_center_map }) print('fps: %.2f' % (1 / (time.time() - fps_t))) # demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape( # (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints)) demo_stage_heatmap = stage_heatmap_np[-1][ 0, :, :, 0:FLAGS.joints].reshape( #将有效的heatmap切出来,把背景排除,结果:46x46x14 (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints)) demo_stage_heatmap = cv2.resize( demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size)) #heatmap变成了368x368x14 vertical_imgs = [] #下面在拼接heatmap的展示图片 tmp_img = None joint_coord_set = np.zeros((FLAGS.joints, 2)) for joint_num in range(FLAGS.joints): # Concat until 4 img if (joint_num % 4) == 0 and joint_num != 0: vertical_imgs.append(tmp_img) tmp_img = None demo_stage_heatmap[:, :, joint_num] *= ( 255 / np.max(demo_stage_heatmap[:, :, joint_num]) ) #将heatmap每层的最大值变成255,共有14层 # Plot color joints if np.min(demo_stage_heatmap[:, :, joint_num]) > -50: joint_coord = np.unravel_index( np.argmax(demo_stage_heatmap[:, :, joint_num]), (FLAGS.input_size, FLAGS.input_size)) joint_coord_set[joint_num, :] = joint_coord color_code_num = (joint_num // 4) if joint_num in [0, 4, 8, 12, 16]: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) else: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) # Put text tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8) tmp = cv2.putText( tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])), org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp = cv2.putText( tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])), org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp_img = np.concatenate((tmp_img, tmp), axis=0) \ if tmp_img is not None else tmp # Plot limbs for limb_num in range(len(limbs)): if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]] ) > -2000 and np.min( demo_stage_heatmap[:, :, limbs[limb_num][1]]) > -2000: x1 = joint_coord_set[limbs[limb_num][0], 0] #limbs中存储了关节之间的连接关系,两个节点 y1 = joint_coord_set[limbs[limb_num][0], 1] x2 = joint_coord_set[limbs[limb_num][1], 0] y2 = joint_coord_set[limbs[limb_num][1], 1] length = ((x1 - x2)**2 + (y1 - y2)**2)**0.5 #两个关节的距离,肢体的距离 if length < 10000 and length > 5: deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) #肢体的角度 polygon = cv2.ellipse2Poly( (int( (y1 + y2) / 2), int( (x1 + x2) / 2)), (int(length / 2), 3), int(deg), 0, 360, 1 ) #画出肢体的椭圆形状,以两关节的中点为中心,椭圆的a值为肢体长度一半,b值自己定,还要给出椭圆的倾斜角度,也就是上面求出的角度 color_code_num = limb_num // 4 #给当前肢体选颜色 limb_color = list( map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])) cv2.fillConvexPoly( test_img, polygon, color=limb_color) #test_img是我们的测试输入图片 (368,368,3) if tmp_img is not None: #上面已经将肢体的椭圆连线画在了测试输入图片上 tmp_img = np.lib.pad( tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)), 'constant', constant_values=( 0, 0)) #在test_img的上面(为0)和下面做padding,padding的值为0,这段可以删掉 vertical_imgs.append(tmp_img) # Concat horizontally output_img = None #输出的图片 for col in range(len(vertical_imgs)): output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \ vertical_imgs[col] output_img = output_img.astype(np.uint8) output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET) test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4) # cv2.imshow('hm', output_img) print("=======================") print(base_path) print(os.path.join(base_path, str(time.time()) + "_.jpg")) cv2.imwrite(os.path.join(base_path, str(time.time()) + "_.jpg"), output_img.astype(np.uint8)) print(output_img) # cv2.moveWindow('hm', 2000, 200) # cv2.imshow('rgb', test_img) cv2.imwrite(os.path.join(base_path, str(time.time()) + "_.jpg"), test_img.astype(np.uint8)) # cv2.moveWindow('rgb', 2000, 750) # if cv2.waitKey(1) == ord('q'): break time.sleep(5)
def main(argv): tf_device = '/gpu:0' with tf.device(tf_device): """Build graph """ frame_num = 0 input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) model.load_weights_from_file(FLAGS.model_path, sess, False) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) cam = cv2.VideoCapture(FLAGS.cam_num) # Create kalman filters if FLAGS.KALMAN_ON: kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None # Init Openni2 openni2.initialize("C:\OpenNI\Redist") dev = openni2.Device.open_any() depth_stream = dev.create_depth_stream() depth_stream.start() depth_stream.set_video_mode( c_api.OniVideoMode(pixelFormat=c_api.OniPixelFormat.ONI_PIXEL_FORMAT_DEPTH_100_UM, resolutionX=640, resolutionY=480, fps=30)) with tf.device(tf_device): while True: # t1 = time.time() orig_img, test_img, depth_map = cpm_utils.read_image(cam, FLAGS.input_size, depth_stream) test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) # print('img read time %f' % (time.time() - t1)) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) # Inference t1 = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[5]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, orig_img, FLAGS, stage_heatmap_np, kalman_filter_array, frame_num, depth_map) cv2.imshow('current depth', depth_map) cv2.imshow('current heatmap', demo_img.astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) print(frame_num) frame_num = frame_num + 1
def main(): with open(sys.argv[1]) as f: conf = json.load(f) images_folder = conf['images_folder'] heatmaps_output_folder = conf['heatmaps_output_folder'] masks_output_folder = conf['masks_output_folder'] images_output_folder = conf['images_output_folder'] model_path = conf['model_path'] #1. Read list of images from folder images_ids = [a for a in os.listdir(images_folder) if a.endswith(".jpg")] #2. tf_device = "/gpu:0" with tf.device(tf_device): input_data = tf.placeholder(dtype=tf.float32, shape=[None, input_size, input_size, 3], name='input_image') center_map = tf.placeholder(dtype=tf.float32, shape=[None, input_size, input_size, 1], name='center_map') model = cpm_hand.CPM_Model(stages, joints + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() sess = tf.Session() sess.run(tf.global_variables_initializer()) model.load_weights_from_file(model_path, sess, False) test_center_map = cpm_utils.gaussian_img(input_size, input_size, input_size / 2, input_size / 2, cmap_radius) test_center_map = np.reshape(test_center_map, [1, input_size, input_size, 1]) with tf.device(tf_device): for im_id in images_ids: print(im_id) im_path = os.path.join(images_folder, im_id) test_img = cpm_utils.read_image(im_path, [], input_size, 'IMAGE') test_img_resize = cv2.resize(test_img, (input_size, input_size)) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) predict_heatmap, stage_heatmap_np = sess.run([ model.current_heatmap, model.stage_heatmap, ], feed_dict={ 'input_image:0': test_img_input, 'center_map:0': test_center_map }) last_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:joints].reshape( (hmap_size, hmap_size, joints)) heatmap_path = os.path.join(heatmaps_output_folder, im_id + ".npy") np.save(heatmap_path, last_heatmap) last_heatmap = cv2.resize(last_heatmap, (test_img.shape[1], test_img.shape[0])) demo_img, masked_img = visualize_result(test_img, last_heatmap) heatmap_path = os.path.join(heatmaps_output_folder, im_id + ".npy") mask_path = os.path.join(masks_output_folder, im_id) im_path = os.path.join(images_output_folder, im_id) imageio.imsave(mask_path, masked_img) imageio.imsave(im_path, demo_img)
def main(argv): tf_device = '/gpu:0' with tf.device(tf_device): """ if FLAGS.color_channel == 'RGB': input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') else: input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='input_image') center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') """ # model = cpm_body_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) # model.build_model(input_data, center_map, 1) # 没有背景 model = cpm_body_slim.CPM_Model(FLAGS.input_size, FLAGS.hmap_size, FLAGS.stages, FLAGS.joints, 1, img_type = FLAGS.color_channel) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) # 不要centermap, 意义不明 """ # 创建center_map,正方形的中心放置高斯响应 test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) """ # read in video / flow frames if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')): # OpenCV can only read in '.avi' files cam = imageio.get_reader(FLAGS.DEMO_TYPE) # iamge processing with tf.device(tf_device): if FLAGS.DEMO_TYPE.endswith(('avi', 'flv', 'mp4')): ori_fps = cam.get_meta_data()['fps'] cap = cv2.VideoCapture(FLAGS.DEMO_TYPE) total_frame = cap.get(cv2.CAP_PROP_FRAME_COUNT) (W,H) = (int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))) pbar = tqdm(total=total_frame) print('This video fps is %f' % ori_fps) video_length = cam.get_length() # writer_path = os.path.join('results', os.path.basename(FLAGS.DEMO_TYPE)) # !! OpenCV can only write in .avi cv_writer = cv2.VideoWriter('results/result.mp4', # cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'), cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), ori_fps, (W,H)) # imageio_writer = imageio.get_writer(writer_path, fps=ori_fps) try: for it, im in enumerate(cam): test_img_t = time.time() full_img = im.copy() # 把结果画在原图上 # test_img是原图像resize得到,并且进行了padding test_img = cpm_utils.read_image(im, [], FLAGS.input_size, 'VIDEO') # 多余的resize # test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) # print('img read time %f' % (time.time() - test_img_t)) if FLAGS.color_channel == 'GRAY': test_img = cv2.cvtColor(test_img, cv2.COLOR_RGB2GRAY) test_img_input = test_img / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) # Inference # fps_t = time.time() """ predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_placeholder:0': test_img_input, 'cmap_placeholder:0': test_center_map}) """ predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_placeholder:0': test_img_input}) # Show visualized image demo_img = visualize_result(test_img, full_img, FLAGS, stage_heatmap_np) #cv2.putText(demo_img, "FPS: %.1f" % (1 / (time.time() - fps_t)), (20, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 0), 1) cv_writer.write(demo_img.astype(np.uint8)) pbar.update(1) # print(1/(time.time()-test_img_t)) cv2.imwrite('results/pics_test/{}.png'.format(str(it).zfill(5)), demo_img[:,:,::-1]) #exit(0) # imageio_writer.append_data(demo_img[:, :, 1]) except KeyboardInterrupt: print('Stopped! {}/{} frames captured!'.format(it, video_length)) finally: cv_writer.release() # imageio_writer.close() elif FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') test_img_input = test_img / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) # Inference fps_t = time.time() """ stage_heatmap_np = sess.run([model.stage_heatmap[5]], feed_dict={'input_placeholder:0': test_img_input, 'cmap_placeholder:0': test_center_map}) """ stage_heatmap_np = sess.run([model.stage_heatmap[5]], feed_dict={'input_placeholder:0': test_img_input}) # Show visualized image ori_img = cv2.imread(FLAGS.DEMO_TYPE) demo_img = visualize_result(test_img, ori_img, FLAGS, stage_heatmap_np) cv2.imwrite('results/test.jpg', demo_img) print('fps: %.1f' % (1 / (time.time() - fps_t))) else: print('Demo type is not defined, please check it!')
def main(argv): tf_device = '/gpu:0' with tf.device(tf_device): """Build graph """ if FLAGS.color_channel == 'RGB': input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') else: input_data = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='input_image') center_map = tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') # tensorflow slim model model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) # keras model # model = cpm_hand_keras.CPM_Model(FLAGS.stages, FLAGS.joints + 1) # model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) if not FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): cam = cv2.VideoCapture(FLAGS.cam_num) # ------------------------------------------ save model------------------------------- # save_dir = 'checkpoints/' # if not os.path.exists(save_dir): # os.makedirs(save_dir) # save_path = os.path.join(save_dir, 'best_validation') # saver.save(sess=sess, save_path=save_path) # builder = tf.saved_model.builder.SavedModelBuilder('./SavedModel/') # signature = predict_signature_def(inputs={'input_image:0': input_data, # 'center_map:0': center_map}) # builder.add_meta_graph_and_variables(sess, # [tf.saved_model.tag_constants.TRAINING], # signature_def_map={'predict': signature}, # assets_collection=None, # strip_default_attrs=True) # builder.add_meta_graph([tf.saved_model.tag_constants.SERVING], strip_default_attrs=True) # builder.save() signature = None # Create kalman filters if FLAGS.KALMAN_ON: kalman_filter_array = [cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints)] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None with tf.device(tf_device): while True: t1 = time.time() if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') else: test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) print("----------------------------------------start to read img--------------------") print('img read time %f' % (time.time() - t1)) if FLAGS.color_channel == 'GRAY': test_img_resize = np.dot(test_img_resize[..., :3], [0.299, 0.587, 0.114]).reshape( (FLAGS.input_size, FLAGS.input_size, 1)) cv2.imshow('color', test_img.astype(np.uint8)) cv2.imshow('gray', test_img_resize.astype(np.uint8)) cv2.waitKey(1) test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): # Inference t1 = time.time() # print("=========================test image============================") # print(test_img_input) # print("=========================test image shape============================") # print(test_img_input.shape) # slim prediction predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # keras prediction # predict_heatmap, stage_heatmap_np = model.predict(test_img_input) print("---------------stage_heatmap_np:------------------") # print(predict_heatmap) print(np.array(stage_heatmap_np).shape) # print(stage_heatmap_np) inputa = tf.saved_model.utils.build_tensor_info(input_data) predict_heatmap = tf.convert_to_tensor(predict_heatmap) outputa = tf.saved_model.utils.build_tensor_info(predict_heatmap) # signatureA = ( # tf.saved_model.signature_def_utils.build_signature_def( # inputs={"aaa": inputA}, # outputs={"bbb": outputA}, # method_name=tf.saved_model.signature_constants.CLASSIFY_METHOD_NAME) # ) # print('input:') # print(input_data) # print('output:') # print(predict_heatmap) # signature = predict_signature_def(inputs={'input_image:0': input_data} # , # outputs={'Const:0': predict_heatmap}) # # print("signature content:") # print(signature) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) # break if cv2.waitKey(0) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'MULTI': # Inference t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'SINGLE': # Inference t1 = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[5]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('current heatmap', (demo_img).astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'HM': # Inference t1 = time.time() stage_heatmap_np = sess.run([model.stage_heatmap[FLAGS.stages - 1]], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) print('fps: %.2f' % (1 / (time.time() - t1))) demo_stage_heatmap = stage_heatmap_np[len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.joints].reshape( (FLAGS.hmap_size, FLAGS.hmap_size, FLAGS.joints)) demo_stage_heatmap = cv2.resize(demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size)) vertical_imgs = [] tmp_img = None joint_coord_set = np.zeros((FLAGS.joints, 2)) for joint_num in range(FLAGS.joints): # Concat until 4 img if (joint_num % 4) == 0 and joint_num != 0: vertical_imgs.append(tmp_img) tmp_img = None demo_stage_heatmap[:, :, joint_num] *= (255 / np.max(demo_stage_heatmap[:, :, joint_num])) # Plot color joints if np.min(demo_stage_heatmap[:, :, joint_num]) > -50: joint_coord = np.unravel_index(np.argmax(demo_stage_heatmap[:, :, joint_num]), (FLAGS.input_size, FLAGS.input_size)) joint_coord_set[joint_num, :] = joint_coord color_code_num = (joint_num // 4) if joint_num in [0, 4, 8, 12, 16]: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) else: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) # Put text tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8) tmp = cv2.putText(tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])), org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp = cv2.putText(tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])), org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp_img = np.concatenate((tmp_img, tmp), axis=0) \ if tmp_img is not None else tmp # Plot limbs for limb_num in range(len(limbs)): if np.min(demo_stage_heatmap[:, :, limbs[limb_num][0]]) > -2000 and np.min( demo_stage_heatmap[:, :, limbs[limb_num][1]]) > -2000: x1 = joint_coord_set[limbs[limb_num][0], 0] y1 = joint_coord_set[limbs[limb_num][0], 1] x2 = joint_coord_set[limbs[limb_num][1], 0] y2 = joint_coord_set[limbs[limb_num][1], 1] length = ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5 if length < 10000 and length > 5: deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) polygon = cv2.ellipse2Poly((int((y1 + y2) / 2), int((x1 + x2) / 2)), (int(length / 2), 3), int(deg), 0, 360, 1) color_code_num = limb_num // 4 if PYTHON_VERSION == 3: limb_color = list( map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])) else: limb_color = map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]) cv2.fillConvexPoly(test_img, polygon, color=limb_color) if tmp_img is not None: tmp_img = np.lib.pad(tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)), 'constant', constant_values=(0, 0)) vertical_imgs.append(tmp_img) # Concat horizontally output_img = None for col in range(len(vertical_imgs)): output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \ vertical_imgs[col] output_img = output_img.astype(np.uint8) output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET) test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4) cv2.imshow('hm', output_img) cv2.moveWindow('hm', 2000, 200) cv2.imshow('rgb', test_img) cv2.moveWindow('rgb', 2000, 750) if cv2.waitKey(1) == ord('q'): break
def main(argv): """ :param argv: :return: """ """ Build network graph """ model = cpm_model.CPM_Model(input_size=FLAGS.input_size, heatmap_size=FLAGS.heatmap_size, stages=FLAGS.cpm_stages, joints=FLAGS.num_of_joints, img_type=FLAGS.color_channel, is_training=False) saver = tf.train.Saver() device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0} with tf.Session(config=tf.ConfigProto(device_count=device_count, allow_soft_placement=True)) as sess: if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) tf.train.write_graph(sess.graph, './', 'cpm_hand.pbtxt') saver.save(sess, 'cpm_hand_frozen') # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) # Create webcam instance if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']: cam = cv2.VideoCapture(FLAGS.cam_id) # Create kalman filters if FLAGS.use_kalman: kalman_filter_array = [ cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints) ] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array( [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array( [[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array( [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): t1 = time.time() test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) print('img read time %f' % (time.time() - t1)) test_img_input = normalize_and_centralize_img(test_img_resize) t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run( [ model.current_heatmap, model.stage_heatmap, ], feed_dict={model.input_images: test_img_input}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) cv2.waitKey(0) print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'MULTI': while True: # Prepare input image t1 = time.time() test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize( test_img, (FLAGS.input_size, FLAGS.input_size)) print('img read time %f' % (time.time() - t1)) test_img_input = normalize_and_centralize_img(test_img_resize) # Inference t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run( [ model.current_heatmap, model.stage_heatmap, ], feed_dict={model.input_images: test_img_input}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'SINGLE': while True: # Prepare input image t1 = time.time() test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize( test_img, (FLAGS.input_size, FLAGS.input_size)) print('img read time %f' % (time.time() - t1)) test_img_input = normalize_and_centralize_img(test_img_resize) # Inference t1 = time.time() stage_heatmap_np = sess.run( [model.stage_heatmap[-1]], feed_dict={model.input_images: test_img_input}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) cv2.imshow('current heatmap', (demo_img).astype(np.uint8)) if cv2.waitKey(1) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'Joint_HM': while True: # Prepare input image t1 = time.time() test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize( test_img, (FLAGS.input_size, FLAGS.input_size)) print('img read time %f' % (time.time() - t1)) test_img_input = normalize_and_centralize_img(test_img_resize) # Inference t1 = time.time() stage_heatmap_np = sess.run( [model.stage_heatmap[FLAGS.cpm_stages - 1]], feed_dict={model.input_images: test_img_input}) print('fps: %.2f' % (1 / (time.time() - t1))) demo_stage_heatmap = stage_heatmap_np[ len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.num_of_joints].reshape( (FLAGS.heatmap_size, FLAGS.heatmap_size, FLAGS.num_of_joints)) demo_stage_heatmap = cv2.resize( demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size)) vertical_imgs = [] tmp_img = None joint_coord_set = np.zeros((FLAGS.num_of_joints, 2)) for joint_num in range(FLAGS.num_of_joints): # Concat until 4 img if (joint_num % 4) == 0 and joint_num != 0: vertical_imgs.append(tmp_img) tmp_img = None demo_stage_heatmap[:, :, joint_num] *= ( 255 / np.max(demo_stage_heatmap[:, :, joint_num])) # Plot color joints if np.min(demo_stage_heatmap[:, :, joint_num]) > -50: joint_coord = np.unravel_index( np.argmax(demo_stage_heatmap[:, :, joint_num]), (FLAGS.input_size, FLAGS.input_size)) joint_coord_set[joint_num, :] = joint_coord color_code_num = (joint_num // 4) if joint_num in [0, 4, 8, 12, 16]: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map( lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) else: if PYTHON_VERSION == 3: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num])) else: joint_color = map( lambda x: x + 35 * (joint_num % 4), joint_color_code[color_code_num]) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) # Put text tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8) tmp = cv2.putText( tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])), org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp = cv2.putText( tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])), org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp_img = np.concatenate((tmp_img, tmp), axis=0) \ if tmp_img is not None else tmp # Plot limbs for limb_num in range(len(limbs)): if np.min( demo_stage_heatmap[:, :, limbs[limb_num][0]] ) > -2000 and np.min( demo_stage_heatmap[:, :, limbs[limb_num][1]]) > -2000: x1 = joint_coord_set[limbs[limb_num][0], 0] y1 = joint_coord_set[limbs[limb_num][0], 1] x2 = joint_coord_set[limbs[limb_num][1], 0] y2 = joint_coord_set[limbs[limb_num][1], 1] length = ((x1 - x2)**2 + (y1 - y2)**2)**0.5 if length < 10000 and length > 5: deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) polygon = cv2.ellipse2Poly((int( (y1 + y2) / 2), int((x1 + x2) / 2)), (int(length / 2), 3), int(deg), 0, 360, 1) color_code_num = limb_num // 4 if PYTHON_VERSION == 3: limb_color = list( map(lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num])) else: limb_color = map( lambda x: x + 35 * (limb_num % 4), joint_color_code[color_code_num]) cv2.fillConvexPoly(test_img, polygon, color=limb_color) if tmp_img is not None: tmp_img = np.lib.pad( tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)), 'constant', constant_values=(0, 0)) vertical_imgs.append(tmp_img) # Concat horizontally output_img = None for col in range(len(vertical_imgs)): output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \ vertical_imgs[col] output_img = output_img.astype(np.uint8) output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET) test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4) cv2.imshow('hm', output_img) cv2.moveWindow('hm', 2000, 200) cv2.imshow('rgb', test_img) cv2.moveWindow('rgb', 2000, 750) if cv2.waitKey(1) == ord('q'): break
def main(argv): global joint_detections os.environ['CUDA_VISIBLE_DEVICES'] = str(FLAGS.gpu_id) """ Initial tracker """ tracker = tracking_module.SelfTracker( [FLAGS.webcam_height, FLAGS.webcam_width], FLAGS.input_size) """ Build network graph """ model = cpm_model.CPM_Model(input_size=FLAGS.input_size, heatmap_size=FLAGS.heatmap_size, stages=FLAGS.cpm_stages, joints=FLAGS.num_of_joints, img_type=FLAGS.color_channel, is_training=False) saver = tf.train.Saver() """ Get output node """ output_node = tf.get_default_graph().get_tensor_by_name( name=FLAGS.output_node_names) device_count = {'GPU': 1} if FLAGS.use_gpu else {'GPU': 0} sess_config = tf.ConfigProto(device_count=device_count) sess_config.gpu_options.per_process_gpu_memory_fraction = 0.2 sess_config.gpu_options.allow_growth = True sess_config.allow_soft_placement = True with tf.Session(config=sess_config) as sess: model_path_suffix = os.path.join( FLAGS.network_def, 'input_{}_output_{}'.format(FLAGS.input_size, FLAGS.heatmap_size), 'joints_{}'.format(FLAGS.num_of_joints), 'stages_{}'.format(FLAGS.cpm_stages), 'init_{}_rate_{}_step_{}'.format(FLAGS.init_lr, FLAGS.lr_decay_rate, FLAGS.lr_decay_step)) model_save_dir = os.path.join('models', 'weights', model_path_suffix) print('Load model from [{}]'.format( os.path.join(model_save_dir, FLAGS.model_path))) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, 'models/weights/cpm_hand') # Check weights for variable in tf.global_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) # Create webcam instance if FLAGS.DEMO_TYPE in ['MULTI', 'SINGLE', 'Joint_HM']: cam = cv2.VideoCapture(FLAGS.cam_id) # Create kalman filters if FLAGS.use_kalman: kalman_filter_array = [ cv2.KalmanFilter(4, 2) for _ in range(FLAGS.num_of_joints) ] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array( [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array( [[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array( [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): test_img = cpm_utils.read_image(FLAGS.DEMO_TYPE, [], FLAGS.input_size, 'IMAGE') test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) test_img_input = normalize_and_centralize_img(test_img_resize) t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run( [ model.current_heatmap, output_node, ], feed_dict={model.input_images: test_img_input}) print('fps: %.2f' % (1 / (time.time() - t1))) correct_and_draw_hand( test_img, cv2.resize(stage_heatmap_np[0], (FLAGS.input_size, FLAGS.input_size)), kalman_filter_array, tracker, tracker.input_crop_ratio, test_img) # Show visualized image # demo_img = visualize_result(test_img, stage_heatmap_np, kalman_filter_array) cv2.imshow('demo_img', test_img.astype(np.uint8)) cv2.waitKey(0) elif FLAGS.DEMO_TYPE in ['SINGLE', 'MULTI']: i = 0 while True: # Prepare input image _, full_img = cam.read() test_img = tracker.tracking_by_joints( full_img, joint_detections=joint_detections) crop_full_scale = tracker.input_crop_ratio test_img_copy = test_img.copy() # White balance test_img_wb = utils.img_white_balance(test_img, 5) test_img_input = normalize_and_centralize_img(test_img_wb) # Inference t1 = time.time() stage_heatmap_np = sess.run( [output_node], feed_dict={model.input_images: test_img_input}) print('FPS: %.2f' % (1 / (time.time() - t1))) local_img = visualize_result(full_img, stage_heatmap_np, kalman_filter_array, tracker, crop_full_scale, test_img_copy) cv2.imshow('local_img', local_img.astype(np.uint8)) # 训练用图 # cv2.imwrite('./storePic/11'+str(i)+'.jpg', local_img.astype(np.uint8), [int(cv2.IMWRITE_JPEG_QUALITY), 90]) cv2.imwrite('./dataset/up/up__' + str(i) + '.jpg', local_img.astype(np.uint8)) i += 1 print(i) cv2.imshow('globalq_img', full_img.astype(np.uint8)) # 单人大框 if cv2.waitKey(1) == ord('q'): break elif FLAGS.DEMO_TYPE == 'Joint_HM': while True: # Prepare input image test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') test_img_resize = cv2.resize( test_img, (FLAGS.input_size, FLAGS.input_size)) test_img_input = normalize_and_centralize_img(test_img_resize) # Inference t1 = time.time() stage_heatmap_np = sess.run( [output_node], feed_dict={model.input_images: test_img_input}) print('FPS: %.2f' % (1 / (time.time() - t1))) demo_stage_heatmap = stage_heatmap_np[ len(stage_heatmap_np) - 1][0, :, :, 0:FLAGS.num_of_joints].reshape( (FLAGS.heatmap_size, FLAGS.heatmap_size, FLAGS.num_of_joints)) demo_stage_heatmap = cv2.resize( demo_stage_heatmap, (FLAGS.input_size, FLAGS.input_size)) vertical_imgs = [] tmp_img = None joint_coord_set = np.zeros((FLAGS.num_of_joints, 2)) for joint_num in range(FLAGS.num_of_joints): # Concat until 4 img if (joint_num % 4) == 0 and joint_num != 0: vertical_imgs.append(tmp_img) tmp_img = None demo_stage_heatmap[:, :, joint_num] *= ( 255 / np.max(demo_stage_heatmap[:, :, joint_num])) # Plot color joints if np.min(demo_stage_heatmap[:, :, joint_num]) > -50: joint_coord = np.unravel_index( np.argmax(demo_stage_heatmap[:, :, joint_num]), (FLAGS.input_size, FLAGS.input_size)) joint_coord_set[joint_num, :] = joint_coord color_code_num = (joint_num // 4) if joint_num in [0, 4, 8, 12, 16]: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num])) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) else: joint_color = list( map(lambda x: x + 35 * (joint_num % 4), FLAGS.joint_color_code[color_code_num])) cv2.circle(test_img, center=(joint_coord[1], joint_coord[0]), radius=3, color=joint_color, thickness=-1) # Put text tmp = demo_stage_heatmap[:, :, joint_num].astype(np.uint8) tmp = cv2.putText( tmp, 'Min:' + str(np.min(demo_stage_heatmap[:, :, joint_num])), org=(5, 20), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp = cv2.putText( tmp, 'Mean:' + str(np.mean(demo_stage_heatmap[:, :, joint_num])), org=(5, 30), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.3, color=150) tmp_img = np.concatenate((tmp_img, tmp), axis=0) \ if tmp_img is not None else tmp # Plot FLAGS.limbs for limb_num in range(len(FLAGS.limbs)): if np.min( demo_stage_heatmap[:, :, FLAGS.limbs[limb_num][0]] ) > -2000 and np.min( demo_stage_heatmap[:, :, FLAGS. limbs[limb_num][1]]) > -2000: x1 = joint_coord_set[FLAGS.limbs[limb_num][0], 0] y1 = joint_coord_set[FLAGS.limbs[limb_num][0], 1] x2 = joint_coord_set[FLAGS.limbs[limb_num][1], 0] y2 = joint_coord_set[FLAGS.limbs[limb_num][1], 1] length = ((x1 - x2)**2 + (y1 - y2)**2)**0.5 if length < 10000 and length > 5: deg = math.degrees(math.atan2(x1 - x2, y1 - y2)) polygon = cv2.ellipse2Poly((int( (y1 + y2) / 2), int((x1 + x2) / 2)), (int(length / 2), 3), int(deg), 0, 360, 1) color_code_num = limb_num // 4 limb_color = list( map(lambda x: x + 35 * (limb_num % 4), FLAGS.joint_color_code[color_code_num])) cv2.fillConvexPoly(test_img, polygon, color=limb_color) if tmp_img is not None: tmp_img = np.lib.pad( tmp_img, ((0, vertical_imgs[0].shape[0] - tmp_img.shape[0]), (0, 0)), 'constant', constant_values=(0, 0)) vertical_imgs.append(tmp_img) # Concat horizontally output_img = None for col in range(len(vertical_imgs)): output_img = np.concatenate((output_img, vertical_imgs[col]), axis=1) if output_img is not None else \ vertical_imgs[col] output_img = output_img.astype(np.uint8) output_img = cv2.applyColorMap(output_img, cv2.COLORMAP_JET) test_img = cv2.resize(test_img, (300, 300), cv2.INTER_LANCZOS4) cv2.imshow('hm', output_img) cv2.moveWindow('hm', 2000, 200) cv2.imshow('rgb', test_img) cv2.moveWindow('rgb', 2000, 750) if cv2.waitKey(1) == ord('q'): break
def main(argv): tf_device = '/cpu:0' with tf.device(tf_device): """Build graph """ if FLAGS.color_channel == 'RGB': input_data = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 3], name='input_image') else: input_data = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='input_image') center_map = tf.placeholder( dtype=tf.float32, shape=[None, FLAGS.input_size, FLAGS.input_size, 1], name='center_map') model = cpm_hand_slim.CPM_Model(FLAGS.stages, FLAGS.joints + 1) model.build_model(input_data, center_map, 1) saver = tf.train.Saver() """Create session and restore weights """ sess = tf.Session() sess.run(tf.global_variables_initializer()) if FLAGS.model_path.endswith('pkl'): model.load_weights_from_file(FLAGS.model_path, sess, False) else: saver.restore(sess, FLAGS.model_path) test_center_map = cpm_utils.gaussian_img(FLAGS.input_size, FLAGS.input_size, FLAGS.input_size / 2, FLAGS.input_size / 2, FLAGS.cmap_radius) test_center_map = np.reshape(test_center_map, [1, FLAGS.input_size, FLAGS.input_size, 1]) # Check weights for variable in tf.trainable_variables(): with tf.variable_scope('', reuse=True): var = tf.get_variable(variable.name.split(':0')[0]) print(variable.name, np.mean(sess.run(var))) """if not FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): cam = cv2.VideoCapture(FLAGS.cam_num)""" # Create kalman filters if FLAGS.KALMAN_ON: kalman_filter_array = [ cv2.KalmanFilter(4, 2) for _ in range(FLAGS.joints) ] for _, joint_kalman_filter in enumerate(kalman_filter_array): joint_kalman_filter.transitionMatrix = np.array( [[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) joint_kalman_filter.measurementMatrix = np.array( [[1, 0, 0, 0], [0, 1, 0, 0]], np.float32) joint_kalman_filter.processNoiseCov = np.array( [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], np.float32) * FLAGS.kalman_noise else: kalman_filter_array = None #classifier MLP model model2 = Sequential() model2.add(InputLayer(input_shape=(21, 2))) model2.add(Flatten()) model2.add(Dense(21)) model2.add(Dense(500)) model2.add(Dense(1400)) model2.add(Dense(3000)) model2.add(Dense(3000)) model2.add(Dense(1400)) model2.add(Dense(500, activation='relu')) model2.add(Dense(26, activation='softmax')) model2.load_weights('models/weights/massey_joint.h5') notepad_image = 255 * np.ones(shape=[368, 500, 3], dtype=np.uint8) massey_static = cv2.imread('massey_static.png') massey_static = cv2.resize(massey_static, (368, 368)) with tf.device(tf_device): while True: cam = cv2.VideoCapture(FLAGS.cam_num) test_img = cpm_utils.read_image([], cam, FLAGS.input_size, 'WEBCAM') cam.release() test_img_resize = cv2.resize(test_img, (FLAGS.input_size, FLAGS.input_size)) t1 = time.time() #print('img read time %f' % (time.time() - t1)) """if FLAGS.color_channel == 'GRAY': test_img_resize = np.dot(test_img_resize[..., :3], [0.299, 0.587, 0.114]).reshape( (FLAGS.input_size, FLAGS.input_size, 1)) cv2.imshow('color', test_img.astype(np.uint8)) cv2.imshow('gray', test_img_resize.astype(np.uint8)) cv2.waitKey(1)""" test_img_input = test_img_resize / 256.0 - 0.5 test_img_input = np.expand_dims(test_img_input, axis=0) """if FLAGS.DEMO_TYPE.endswith(('png', 'jpg')): # Inference t1 = time.time() predict_heatmap, stage_heatmap_np = sess.run([model.current_heatmap, model.stage_heatmap, ], feed_dict={'input_image:0': test_img_input, 'center_map:0': test_center_map}) # Show visualized image demo_img = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) #demo_img = cv2.cvtColor(demo_img,cv2.COLOR_BGR2GRAY) print(demo_img.shape) #demo_img1 = cv2.GaussianBlur(demo_img, (9, 9), 0) #demo_img = 2.5*demo_img-demo_img1 cv2.imshow('demo_img', demo_img.astype(np.uint8)) if cv2.waitKey(0) == ord('q'): break print('fps: %.2f' % (1 / (time.time() - t1))) elif FLAGS.DEMO_TYPE == 'MULTI':""" # Inference t1 = time.time() #print("chandu") key1 = cv2.waitKey(1) #if key1 == 32: predict_heatmap, stage_heatmap_np = sess.run([ model.current_heatmap, model.stage_heatmap, ], feed_dict={ 'input_image:0': test_img_input, 'center_map:0': test_center_map }) # Show visualized image demo_img, joint_points = visualize_result(test_img, FLAGS, stage_heatmap_np, kalman_filter_array) x = joint_points[0][0] y = joint_points[0][1] for i in range(21): joint_points[i][0] = joint_points[i][0] - x joint_points[i][1] = joint_points[i][1] - y joint_points = np.array([joint_points]) class_num = model2.predict_classes(joint_points) if class_num[0] == 26: character = ' ' else: character = chr(97 + class_num[0]) global printer_x, printer_y, printer_string printer_string += character notepad_image = printer.notepad(notepad_image, (printer_x, printer_y), printer_string) new_note = np.concatenate((notepad_image, demo_img, massey_static), axis=1) cv2.imshow('notepad', new_note) if len(printer_string) == 17: printer_string = '' printer_y += 25 print('fps: %.2f' % (1 / (time.time() - t1))) if key1 == ord('q'): cv2.destroyAllWindows() break