def worker(input_q, output_q, cropped_output_q, inferences_q, cap_params, frame_processed): print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) print(">> loading keras model for worker") try: model, classification_graph, session = classifier.load_KerasGraph( "./cnn/cnn.h5") except Exception as e: print(e) while True: #print("> ===== in worker loop, frame ", frame_processed) frame = input_q.get() if (frame is not None): # Actual detection. Variable boxes contains the bounding box cordinates for hands detected, # while scores contains the confidence for each of these boxes. # Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold) boxes, scores = detector_utils.detect_objects( frame, detection_graph, sess) # get region of interest res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # draw bounding boxes detector_utils.draw_box_on_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # classify hand pose if res is not None: res = cv2.cvtColor(res, cv2.COLOR_RGB2GRAY) res = cv2.resize(res, (28, 28), interpolation=cv2.INTER_AREA) class_res = classifier.classify(model, classification_graph, session, res) inferences_q.put(class_res) # add frame annotated with bounding box to queue cropped_output_q.put(res) output_q.put(frame) frame_processed += 1 else: output_q.put(frame) sess.close()
def worker(input_q, output_q, boxes_q, inferences_q, cap_params, frame_processed): print(">> loading frozen model for worker") detection_graph, sess = detector.load_inference_graph() sess = tf.Session(graph=detection_graph) print(">> loading keras model for worker") try: model, classification_graph, session = classifier.load_KerasGraph( "cnn/models/hand_poses_wGarbage_10.h5") except Exception as e: print(e) try: while True: # print("> ===== in worker loop, frame ", frame_processed) frame = input_q.get() if (frame is not None): # Actual detection. Variable boxes contains the bounding box cordinates for hands detected, # while scores contains the confidence for each of these boxes. # Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold) boxes, scores = detector.detect_objects( frame, detection_graph, sess) # get region of interest box, res = detector.get_box_image( cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # classify hand pose if res is not None: class_res = classifier.classify(model, classification_graph, session, res) inferences_q.put(class_res) output_q.put(frame) boxes_q.put(box) frame_processed += 1 else: output_q.put(frame) except Exception as e: print(e) sess.close()
def worker(input_q, output_q, cap_params, frame_processed, poses): print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) print(">> loading keras model for worker") try: model, classification_graph, session = classifier.load_KerasGraph( "F:\Realtime_Hand_tracking\cnn\models\hand_poses_wGarbage_100.h5") except Exception as e: print(e) detection_centres_x = [] detection_centres_y = [] is_centres_filled = False detected = False index = 0 detection_area = [] start_flag = False flag_start = pause_time = 0 sensitivity = gui_sensitivity area = centre_x = centre_y = 0 detection = "" direction = "" while True: predicted_label = "" frame = input_q.get() if (frame is not None): frame_processed += 1 boxes, scores = detector_utils.detect_objects(frame, detection_graph, sess) # get region of interest res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params['score_thresh'], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # get boundary box if pause_time == 0: centre_x, centre_y, area = detector_utils.draw_box_on_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) if pause_time > 0: pause_time -= 1 if is_centres_filled: detection_centres_x = detection_centres_x[1:10] detection_centres_y = detection_centres_y[1:10] detection_area = detection_area[1:10] detection_centres_x.append(centre_x) detection_centres_y.append(centre_y) detection_area.append(area) else: detection_centres_x.append(centre_x) detection_centres_y.append(centre_y) detection_area.append(area) if pause_time == 0: index += 1 if index >= sensitivity: index = 0 is_centres_filled = True if index == 0: predicted_label = classify(res, model, classification_graph, session, poses) #print(predicted_label) if predicted_label == "Start" and flag_start == 0: #print("Start") detection = "Start tracking" start_flag = True flag_start = 1 if detected: detection_centres_x = [] detection_centres_y = [] is_centres_filled = False index = 0 detected = False detection_area = [] frame_processed = 0 pause_time = 30 centres_x = detection_centres_x.copy() centres_y = detection_centres_y.copy() areas = detection_area.copy() centres_x = [v for v in centres_x if v] centres_y = [v for v in centres_y if v] areas = [a for a in areas if a] # angle_coordinate if len(centres_x) > 3 and is_centres_filled and len(centres_y) > 3 and len(areas) > 3 and start_flag : flag = 0 dX = centres_x[-1] - centres_x[0] dY = centres_y[-1] - centres_y[0] if dX > 20 and dY > 20: m = dY / dX angle = math.degrees(math.atan(m)) if angle < 45: flag = 1 elif angle > 45: flag = 2 if dX > 100 and (abs(dY) < 20 or flag == 1): direction = "Right" keyboard.press_and_release('right') detected = True #print(direction) elif -dX > 100 and (abs(dY) < 20 or flag == 1): direction = "Left" keyboard.press_and_release('left') detected = True #print(direction) elif dY > 50 and (abs(dX) < 10 or flag == 2): direction = "Down" detected = True #print(direction) elif -dY > 50 and (abs(dX) < 10 or flag == 2): direction = "Up" detected = True #print(direction) elif areas[-1] - 3000 > areas[0] and abs(dX) < 30 and abs(dY) < 20: direction = "Zoom in" detected = True #print(direction) elif areas[-1] < areas[0] - 3000 and abs(dX) < 10 and abs(dY) < 20: direction = "Zoom out" detected = True #print(direction) output_q.put((frame, direction,predicted_label)) sess.close()
def worker(input_q, output_q, cropped_output_q, inferences_q, pointX_q, pointY_q, cap_params, frame_processed): print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) print(">> loading keras model for worker") try: model, classification_graph, session = classifier.load_KerasGraph("cnn/models/hand_poses_wGarbage_10.h5") except Exception as e: print(e) while True: #print("> ===== in worker loop, frame ", frame_processed) frame = input_q.get() if (frame is not None): #実際の検出。 変数ボックスには、検出された手の境界ボックスの座標が含まれています。 #スコアには、これらの各ボックスの信頼度が含まれています。 #ヒント:len(boxes)> 1の場合、(スコアのしきい値内で)少なくとも片方の手を見つけたと見なすことができます。 boxes, scores = detector_utils.detect_objects( frame, detection_graph, sess) #関心領域を取得 #フレームの画像サイズを取得 cropped_height,cropped_width,a = frame.shape[:3] res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cropped_width,cropped_height, frame) #手の判定が一定値を超えたとき if (scores[0] > score_thresh): (left, right, top, bottom) = (boxes[0][1] * cropped_width, boxes[0][3] * cropped_width, boxes[0][0] * cropped_height, boxes[0][2] * cropped_height) #ウィンドウサイズを取得 width,height = autopy.screen.size() #画面比率変数設定 # wx = (width + (int(right)-int(left))*(width / cap_params['im_width'])) / cap_params['im_width'] # # hx = (height + (int(bottom)-int(top))*(height / cap_params['im_height'])) / cap_params['im_height'] # wx = (width + (int(right)-int(left))*(width / cropped_width)) / (cropped_width-20) # # hx = (height + (int(bottom)-int(top))*(height / cropped_height)) / (cropped_height-20) # p1 = int(left)*wx # p2 = int(bottom)*hx-(int(bottom)*hx-int(top)*hx) hx = (height / cropped_height)*1.3 wx = (width / cropped_width)*1.3 #手を識別したボックスの固定化処理 hi = 100 - (int(bottom)-int(top)) wi = 70 - (int(right)-int(left)) top = top - hi left = left - wi fp = (int(left),int(top)) ep = (int(right),int(bottom)) p1 = int(left)*wx p2 = int(top)*wx if(abs(hi)>25 or abs(wi)>25): cv2.rectangle(frame, fp, ep, (255, 0, 0), 1, 1) #判定した手の範囲を表示 else: cv2.rectangle(frame, fp, ep, (77, 255, 9), 1, 1) #取得した座標(p1,p2)を挿入 pointX_q.put(p1) pointY_q.put(p2) #手のポーズを分類する if res is not None: class_res = classifier.classify(model, classification_graph, session, res) inferences_q.put(class_res) #バウンディングボックスで注釈が付けられたフレームをキューに追加 cropped_output_q.put(res) output_q.put(frame) frame_processed += 1 else: output_q.put(frame) sess.close()
def worker(input_q, output_q, cap_params, frame_count, poses): centroid = None predicted_label = "" print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) print(">> loading keras model for worker") try: model, classification_graph, session = classifier.load_KerasGraph( "F:/Github/Hand_pose_DKE/cnn/models/hand_poses_wGarbage_10.h5") except Exception as e: print(e) centroid_list = deque(maxlen=buffer) direction = "" (dX, dY) = (0, 0) while True: # print("> ===== in worker loop, frame ", frame_count) frame = input_q.get() if (frame is not None): # Actual detection. Variable boxes contains the bounding box cordinates for hands detected, # while scores contains the confidence for each of these boxes. # Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold) norm_image = cv2.normalize(frame, None, 0, 255, norm_type=cv2.NORM_MINMAX) boxes, scores = detector_utils.detect_objects( norm_image, detection_graph, sess) # print(boxes[0]) # get region of interest res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # get boundary box detector_utils.draw_box_on_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # classify hand pose if res is not None and frame_count == 0: class_res = classifier.classify(model, classification_graph, session, res) class_pred = class_res.argmax(axis=-1) predicted_label = poses[int(class_pred)] #print(predicted_label) if predicted_label == "Start" and frame_count <= frame_end: centroid = detector_utils.get_centroid( cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # elif scores is None: # centroid = None if centroid is not None: centroid_list.appendleft(centroid) #print(centroid_list) sorted(centroid_list) for i in np.arange(1, len(centroid_list)): # if centroid_list[i-1] is None or centroid_list[i] is None: # continue if frame_count == frame_end and centroid_list[ -5] != None and i == 1: dX = centroid_list[-5][0] - centroid_list[i][0] dY = centroid_list[-5][1] - centroid_list[i][1] (dirX, dirY) = ("", "") if np.abs(dX) > 10: dirX = "Right" if np.sign(dX) == 1 else "Left" if np.abs(dY) > 10: dirY = "DOWN" if np.sign(dY) == 1 else "UP" if dirX != "" and dirY != "": direction = "{}-{}".format(dirY, dirX) else: direction = dirX if dirX != "" else dirY thickness = int(np.sqrt(frame_end / float(i + 1)) * 2.5) cv2.line(frame, centroid_list[i - 1], centroid_list[i], (0, 0, 255), thickness) cv2.putText(frame, direction, (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (77, 255, 9), 1) cv2.putText(frame, "dx: {}, dy: {}".format(dX, dY), (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 255), 1) # if direction == "Left": # keyboard.press_and_release('left') # #time.sleep(2) # elif direction == "Right": # keyboard.press_and_release('right') # time.sleep(2) frame_count += 1 if frame_count >= frame_end: frame_count = 0 #centroid_list.clear() direction = "" #flag = 1 output_q.put(frame) # else: # output_q.put(frame) sess.close()
def worker(input_q, output_q, cap_params, frame_processed, poses): print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) print(">> loading keras model for worker") try: model, classification_graph, session = classifier.load_KerasGraph( "F:\Realtime_Hand_tracking\cnn\models\hand_poses_wGarbage_10.h5") except Exception as e: print(e) detection_centres_x = [] detection_centres_y = [] is_centres_filled = False detected = False index = 0 detection_area = [] predicted_label = "" start_flag = False flag_start = 0 pause_time = 0 print("Show Start gesture to start reading") #file = open("eval_list.csv","w") while True: # print("> ===== in worker loop, frame ", frame_count) frame = input_q.get() flag = 0 if (frame is not None): frame_processed += 1 # Actual detection. Variable boxes contains the bounding box cordinates for hands detected, # while scores contains the confidence for each of these boxes. # Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold) # frame = cv2.normalize(frame, None, 0,255, norm_type=cv2.NORM_MINMAX) boxes, scores = detector_utils.detect_objects(frame, detection_graph, sess) # print("Hello") # print(boxes[0]) # get region of interest res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # get boundary box if pause_time == 0: centre_x, centre_y, area = detector_utils.draw_box_on_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) if pause_time > 0: pause_time -= 1 if is_centres_filled: detection_centres_x = detection_centres_x[1:10] detection_centres_y = detection_centres_y[1:10] detection_area = detection_area[1:10] detection_centres_x.append(centre_x) detection_centres_y.append(centre_y) detection_area.append(area) else: detection_centres_x.append(centre_x) detection_centres_y.append(centre_y) detection_area.append(area) if pause_time == 0: index += 1 # print(index) if index >= 20: index = 0 is_centres_filled = True #start_flag = False if index == 0: predicted_label = classify(res, model, classification_graph, session, poses) # print(predicted_label) if predicted_label == "Start" and flag_start == 0: print("Start tracking") start_flag = True flag_start = 1 elif predicted_label == "Stop" and flag_start == 1 and frame_processed > 30: print("Stopped tracking") start_flag = False flag_start = 0 if detected: detection_centres_x = [] detection_centres_y = [] is_centres_filled = False index = 0 detected = False detection_area = [] frame_processed = 0 pause_time = 10 #time.sleep(2) #start_flag = False centres_x = detection_centres_x.copy() centres_y = detection_centres_y.copy() areas = detection_area.copy() centres_x = [v for v in centres_x if v] centres_y = [v for v in centres_y if v] areas = [a for a in areas if a] direction = "" # angle_coordinate # print(areas) if len(centres_x) > 3 and is_centres_filled and len(centres_y) > 3 and len( areas) > 3 and start_flag == True: # centres_asc = centres.copy().sort() # centres_dsc = centres.copy().sort(reverse=True) # print(centres) # print("Last x:", centres_x[-1]) # print("First x: ", centres_x[0]) # print("Last y:", centres_y[-1]) # print("First y:", centres_y[0]) flag = 0 dX = centres_x[-1] - centres_x[0] dY = centres_y[-1] - centres_y[0] if dX > 20 and dY > 20: m = dY / dX angle = math.degrees(math.atan(m)) if angle < 45: flag = 1 elif angle > 45: flag = 2 # print(angle) if dX > 100 and (abs(dY) < 20 or flag == 1): direction = "Right" #keyboard.press_and_release('left') detected = True print(direction) elif -dX > 100 and (abs(dY) < 20 or flag == 1): direction = "Left" #keyboard.press_and_release('right') detected = True print(direction) elif dY > 50 and (abs(dX) < 10 or flag == 2): direction = "Down" detected = True print(direction) elif -dY > 50 and (abs(dX) < 10 or flag == 2): direction = "Up" detected = True print(direction) elif areas[-1] - 3000 > areas[0] and abs(dX) < 30 and abs(dY) < 20: direction = "Zoom in" detected = True print(direction) elif areas[-1] < areas[0] - 3000 and abs(dX) < 10 and abs(dY) < 20: direction = "Zoom out" detected = True print(direction) # print("hello") cv2.putText(frame, direction, (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (77, 255, 9), 1) output_q.put(frame) sess.close()