def worker(input_q, output_q, cap_params, frame_processed): print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() while True: frame = input_q.get() if (frame is not None): im_width = cap_params['im_width'] im_height = cap_params['im_height'] boxes, scores = detector_utils.detect_objects( frame, detection_graph, sess) # draw bounding boxes detector_utils.draw_box_on_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) cropped_img = detector_utils.get_box_image( cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) frame_processed += 1 output_q.put(frame) cropped_output_q.put(cropped_img) else: output_q.put(frame) cropped_output_q.put(cropped_img)
def worker(input_q, output_q, cropped_output_q, inferences_q, cap_params, frame_processed): print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) print(">> loading keras model for worker") try: model, classification_graph, session = classifier.load_KerasGraph( "./cnn/cnn.h5") except Exception as e: print(e) while True: #print("> ===== in worker loop, frame ", frame_processed) frame = input_q.get() if (frame is not None): # Actual detection. Variable boxes contains the bounding box cordinates for hands detected, # while scores contains the confidence for each of these boxes. # Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold) boxes, scores = detector_utils.detect_objects( frame, detection_graph, sess) # get region of interest res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # draw bounding boxes detector_utils.draw_box_on_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # classify hand pose if res is not None: res = cv2.cvtColor(res, cv2.COLOR_RGB2GRAY) res = cv2.resize(res, (28, 28), interpolation=cv2.INTER_AREA) class_res = classifier.classify(model, classification_graph, session, res) inferences_q.put(class_res) # add frame annotated with bounding box to queue cropped_output_q.put(res) output_q.put(frame) frame_processed += 1 else: output_q.put(frame) sess.close()
def worker(input_q, output_q, boxes_q, inferences_q, cap_params, frame_processed): print(">> loading frozen model for worker") detection_graph, sess = detector.load_inference_graph() sess = tf.Session(graph=detection_graph) print(">> loading keras model for worker") try: model, classification_graph, session = classifier.load_KerasGraph( "cnn/models/hand_poses_wGarbage_10.h5") except Exception as e: print(e) try: while True: # print("> ===== in worker loop, frame ", frame_processed) frame = input_q.get() if (frame is not None): # Actual detection. Variable boxes contains the bounding box cordinates for hands detected, # while scores contains the confidence for each of these boxes. # Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold) boxes, scores = detector.detect_objects( frame, detection_graph, sess) # get region of interest box, res = detector.get_box_image( cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # classify hand pose if res is not None: class_res = classifier.classify(model, classification_graph, session, res) inferences_q.put(class_res) output_q.put(frame) boxes_q.put(box) frame_processed += 1 else: output_q.put(frame) except Exception as e: print(e) sess.close()
def main(): currentPath = '' currentExample = '' print('Do you want to : \n 1 - Add new pose \ \n 2 - Add examples to existing pose \ \n 3 - Add garbage examples') menu_choice = input() while menu_choice != '1' and menu_choice != '2' and menu_choice != '3': print('Please enter 1 or 2') menu_choice = input() if menu_choice == '1': print('Enter a name for the pose you want to add :') name_pose = input() # Create folder for pose if not os.path.exists('Poses/' + name_pose): os.makedirs('Poses/' + name_pose + '/' + name_pose + '_1/') currentPath = 'Poses/' + name_pose + '/' + name_pose + '_1/' currentExample = name_pose + '_1_' elif menu_choice == '2': # Display current poses dirs = os.listdir('Poses/') dirs.sort() dirs_choice = '' possible_choices = [] i = 1 for _dir in dirs: dirs_choice += str(i) + ' - ' + str(_dir) + ' / ' possible_choices.append(str(i)) i += 1 # Ask user to choose to which pose to add examples print('Choose one of the following pose:') print(dirs_choice) choice = input() while not choice in possible_choices and dirs[int(choice) - 1] == 'garbage': print('Please enter one of the following (not garbage): ' + str(possible_choices)) choice = input() # Count number of files to increment new example directory subdirs = os.listdir('Poses/' + dirs[int(choice) - 1] + '/') index = len(subdirs) + 1 # Create new example directory if not os.path.exists('Poses/' + dirs[int(choice) - 1] + '/' + dirs[int(choice) - 1] + '_' + str(index) + '/'): os.makedirs('Poses/' + dirs[int(choice) - 1] + '/' + dirs[int(choice) - 1] + '_' + str(index) + '/') # Update current path currentPath = 'Poses/' + dirs[int(choice) - 1] + '/' + dirs[int(choice) - 1] + '_' + str(index) + '/' currentExample = dirs[int(choice) - 1] + '_' + str(index) + '_' name_pose = dirs[int(choice) - 1] elif menu_choice == '3': # Create folder for pose if not os.path.exists('Poses/Garbage/'): os.makedirs('Poses/Garbage/Garbage_1/') currentPath = 'Poses/Garbage/Garbage_1/' currentExample = 'Garbage_1_' name_pose = 'Garbage' else: # Count number of files to increment new example directory subdirs = os.listdir('Poses/Garbage/') index = len(subdirs) + 1 # Create new example directory if not os.path.exists('Poses/Garbage/Garbage_' + str(index) + '/'): os.makedirs('Poses/Garbage/Garbage_' + str(index) + '/') # Update current path currentPath = 'Poses/Garbage/Garbage_' + str(index) + '/' currentExample = 'Garbage_' + str(index) + '_' name_pose = 'Garbage' print('You\'ll now be prompted to record the pose you want to add. \n \ Please place your hand beforehand facing the camera, and press any key when ready. \n \ When finished press \'q\'.') input() # Begin capturing cap = cv2.VideoCapture(0) # Define the codec and create VideoWriter object fourcc = cv2.VideoWriter_fourcc(*'XVID') out = cv2.VideoWriter(currentPath + name_pose + '.avi', fourcc, 25.0, (640, 480)) while cap.isOpened(): ret, frame = cap.read() if ret: # write the frame out.write(frame) cv2.imshow('frame', frame) if cv2.waitKey(1) & 0xFF == ord('q'): break else: break # Release everything if job is finished cap.release() out.release() cv2.destroyAllWindows() vid = cv2.VideoCapture(currentPath + name_pose + '.avi') # Check if the video if not vid.isOpened(): print('Error opening video stream or file') return print('>> loading frozen model..') detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) print('>> model loaded!') _iter = 1 # Read until video is completed while vid.isOpened(): # Capture frame-by-frame ret, frame = vid.read() if ret: print(' Processing frame: ' + str(_iter)) # Resize and convert to RGB for NN to work with frame = cv2.resize(frame, (320, 180), interpolation=cv2.INTER_AREA) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # Detect object boxes, scores = detector_utils.detect_objects(frame, detection_graph, sess) # get region of interest res = detector_utils.get_box_image(1, 0.2, scores, boxes, 320, 180, frame) # Save cropped image if res is not None: cv2.imwrite(currentPath + currentExample + str(_iter) + '.png', cv2.cvtColor(res, cv2.COLOR_RGB2BGR)) _iter += 1 # Break the loop else: break print(' Processed ' + str(_iter) + ' frames!') vid.release()
def worker(input_q, output_q, cap_params, frame_processed, poses): print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) print(">> loading keras model for worker") try: model, classification_graph, session = classifier.load_KerasGraph( "F:\Realtime_Hand_tracking\cnn\models\hand_poses_wGarbage_100.h5") except Exception as e: print(e) detection_centres_x = [] detection_centres_y = [] is_centres_filled = False detected = False index = 0 detection_area = [] start_flag = False flag_start = pause_time = 0 sensitivity = gui_sensitivity area = centre_x = centre_y = 0 detection = "" direction = "" while True: predicted_label = "" frame = input_q.get() if (frame is not None): frame_processed += 1 boxes, scores = detector_utils.detect_objects(frame, detection_graph, sess) # get region of interest res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params['score_thresh'], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # get boundary box if pause_time == 0: centre_x, centre_y, area = detector_utils.draw_box_on_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) if pause_time > 0: pause_time -= 1 if is_centres_filled: detection_centres_x = detection_centres_x[1:10] detection_centres_y = detection_centres_y[1:10] detection_area = detection_area[1:10] detection_centres_x.append(centre_x) detection_centres_y.append(centre_y) detection_area.append(area) else: detection_centres_x.append(centre_x) detection_centres_y.append(centre_y) detection_area.append(area) if pause_time == 0: index += 1 if index >= sensitivity: index = 0 is_centres_filled = True if index == 0: predicted_label = classify(res, model, classification_graph, session, poses) #print(predicted_label) if predicted_label == "Start" and flag_start == 0: #print("Start") detection = "Start tracking" start_flag = True flag_start = 1 if detected: detection_centres_x = [] detection_centres_y = [] is_centres_filled = False index = 0 detected = False detection_area = [] frame_processed = 0 pause_time = 30 centres_x = detection_centres_x.copy() centres_y = detection_centres_y.copy() areas = detection_area.copy() centres_x = [v for v in centres_x if v] centres_y = [v for v in centres_y if v] areas = [a for a in areas if a] # angle_coordinate if len(centres_x) > 3 and is_centres_filled and len(centres_y) > 3 and len(areas) > 3 and start_flag : flag = 0 dX = centres_x[-1] - centres_x[0] dY = centres_y[-1] - centres_y[0] if dX > 20 and dY > 20: m = dY / dX angle = math.degrees(math.atan(m)) if angle < 45: flag = 1 elif angle > 45: flag = 2 if dX > 100 and (abs(dY) < 20 or flag == 1): direction = "Right" keyboard.press_and_release('right') detected = True #print(direction) elif -dX > 100 and (abs(dY) < 20 or flag == 1): direction = "Left" keyboard.press_and_release('left') detected = True #print(direction) elif dY > 50 and (abs(dX) < 10 or flag == 2): direction = "Down" detected = True #print(direction) elif -dY > 50 and (abs(dX) < 10 or flag == 2): direction = "Up" detected = True #print(direction) elif areas[-1] - 3000 > areas[0] and abs(dX) < 30 and abs(dY) < 20: direction = "Zoom in" detected = True #print(direction) elif areas[-1] < areas[0] - 3000 and abs(dX) < 10 and abs(dY) < 20: direction = "Zoom out" detected = True #print(direction) output_q.put((frame, direction,predicted_label)) sess.close()
def worker(input_q, output_q, cropped_output_q, inferences_q, pointX_q, pointY_q, cap_params, frame_processed): print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) print(">> loading keras model for worker") try: model, classification_graph, session = classifier.load_KerasGraph("cnn/models/hand_poses_wGarbage_10.h5") except Exception as e: print(e) while True: #print("> ===== in worker loop, frame ", frame_processed) frame = input_q.get() if (frame is not None): #実際の検出。 変数ボックスには、検出された手の境界ボックスの座標が含まれています。 #スコアには、これらの各ボックスの信頼度が含まれています。 #ヒント:len(boxes)> 1の場合、(スコアのしきい値内で)少なくとも片方の手を見つけたと見なすことができます。 boxes, scores = detector_utils.detect_objects( frame, detection_graph, sess) #関心領域を取得 #フレームの画像サイズを取得 cropped_height,cropped_width,a = frame.shape[:3] res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cropped_width,cropped_height, frame) #手の判定が一定値を超えたとき if (scores[0] > score_thresh): (left, right, top, bottom) = (boxes[0][1] * cropped_width, boxes[0][3] * cropped_width, boxes[0][0] * cropped_height, boxes[0][2] * cropped_height) #ウィンドウサイズを取得 width,height = autopy.screen.size() #画面比率変数設定 # wx = (width + (int(right)-int(left))*(width / cap_params['im_width'])) / cap_params['im_width'] # # hx = (height + (int(bottom)-int(top))*(height / cap_params['im_height'])) / cap_params['im_height'] # wx = (width + (int(right)-int(left))*(width / cropped_width)) / (cropped_width-20) # # hx = (height + (int(bottom)-int(top))*(height / cropped_height)) / (cropped_height-20) # p1 = int(left)*wx # p2 = int(bottom)*hx-(int(bottom)*hx-int(top)*hx) hx = (height / cropped_height)*1.3 wx = (width / cropped_width)*1.3 #手を識別したボックスの固定化処理 hi = 100 - (int(bottom)-int(top)) wi = 70 - (int(right)-int(left)) top = top - hi left = left - wi fp = (int(left),int(top)) ep = (int(right),int(bottom)) p1 = int(left)*wx p2 = int(top)*wx if(abs(hi)>25 or abs(wi)>25): cv2.rectangle(frame, fp, ep, (255, 0, 0), 1, 1) #判定した手の範囲を表示 else: cv2.rectangle(frame, fp, ep, (77, 255, 9), 1, 1) #取得した座標(p1,p2)を挿入 pointX_q.put(p1) pointY_q.put(p2) #手のポーズを分類する if res is not None: class_res = classifier.classify(model, classification_graph, session, res) inferences_q.put(class_res) #バウンディングボックスで注釈が付けられたフレームをキューに追加 cropped_output_q.put(res) output_q.put(frame) frame_processed += 1 else: output_q.put(frame) sess.close()
def worker(input_q, output_q, cropped_output_q, inferences_q, landmark_ouput_q, cap_params, frame_processed): print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor( "landmarks/shape_predictor_68_face_landmarks.dat") print(">> loading keras model for worker") try: model = tf.keras.models.load_model('models/asl_char_model.h5', compile=False) except Exception as e: print(e) while True: frame = input_q.get() if (frame is not None): boxes, scores = detector_utils.detect_objects( frame, detection_graph, sess) # get region of interest res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # draw bounding boxes detector_utils.draw_box_on_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # classify hand if res is not None: class_res = "" try: proba = model.predict(process_image(res))[0] mx = np.argmax(proba) score = proba[mx] * 100 sequence = categories[mx][1] class_res = str(score) + "/" + sequence except: score = 0.0 sequence = "" class_res = "empty" inferences_q.put(class_res) image_np1 = imutils.resize(frame, width=400) gray = cv2.cvtColor(image_np1, cv2.COLOR_BGR2GRAY) #lanmarking # detect faces in the grayscale frame rects = detector(gray, 0) # loop over the face detections for rect in rects: # determine the facial landmarks for the face region, then # convert the facial landmark (x, y)-coordinates to a NumPy # array shape = predictor(gray, rect) shape = face_utils.shape_to_np(shape) # loop over the (x, y)-coordinates for the facial landmarks # and draw them on the image for (x, y) in shape: cv2.circle(image_np1, (x, y), 1, (0, 0, 255), -1) # add frame annotated with bounding box to queue landmark_ouput_q.put(image_np1) cropped_output_q.put(res) output_q.put(frame) frame_processed += 1 else: output_q.put(frame) sess.close()
def worker(input_q, output_q, cap_params, frame_processed): print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() model = tf.keras.models.load_model('./saved_model2/') ################# # all the following variables store data for sliding window sw_size = 30 # sliding window size trajecX = [] # trajectory x coord trajecY = [] # trajectory y coord preds = [] # classified classes preds_freq = np.zeros(5) # frequency of each class ################# screenWd, screenHt = pyautogui.size() frameWd, frameHt = cap_params['im_width'], cap_params['im_height'] scaleX = 1.0 * screenWd / frameWd scaleY = 1.0 * screenHt / frameHt state = -1 rad_in, rad_mid, rad_out= 25, 90, 200 # radii of annulus last_swipe, swipe_cnt = -1, 0 # will provide sanitary check so same swipes are not performed twice consecutively while True: # print("> ===== in worker loop, frame ", frame_processed) frame = input_q.get() if (frame is not None): # print(nm, "Not None") im_width = cap_params['im_width'] im_height = cap_params['im_height'] boxes, scores = detector_utils.detect_objects( frame, detection_graph, sess) # get cropped image cropped_img = detector_utils.get_box_image( cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # draw bounding boxes centroidX, centroidY = detector_utils.draw_box_on_image( cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) ## enable this if only want to visualize the bounding box ## and blur rest of the image # visualize_blurred(cap_params['num_hands_detect'], cap_params['score_thresh'], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # negative centroid means no hand is detected if centroidX < 0: frame_processed += 1 output_q.put(frame) # cropped_output_q.put(cropped_img) continue # Updation of centroid and add to trajectory if needed if centroidX > 0: if len(trajecX) == 0: trajecX.append(centroidX) trajecY.append(centroidY) else: px, py = trajecX[-1], trajecY[-1] d = dist(centroidX, centroidY, px, py) if d <= rad_in or (d > rad_mid and d < rad_out): centroidX, centroidY = px, py trajecX.append(centroidX) trajecY.append(centroidY) # move sliding window if len(trajecX) > sw_size: trajecX = trajecX[-sw_size:] trajecY = trajecY[-sw_size:] # draw centroid if len(trajecX) > 0: x, y = trajecX[-1], trajecY[-1] # frame = visualize_blurred(cap_params['num_hands_detect'], cap_params['score_thresh'], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # visualize_roi(cap_params['num_hands_detect'], (x, y), # cap_params['score_thresh'], scores, # rad_in, rad_mid, rad_out, frame) detector_utils.draw_centroid_on_image( cap_params['num_hands_detect'], (x, y), cap_params["score_thresh"], scores, frame) # Static gesture prediction if cropped_img is not None: cropped_img, idx = classify(cropped_img, model) detector_utils.draw_label_on_image(mp[idx], frame) preds.append(idx) preds_freq[idx] += 1 if len(preds) > sw_size: preds_freq[preds[0]] -= 1 preds = preds[-sw_size:] if preds[-1] == 1 or preds[-1] == 3: state = control_vlc(last_swipe, preds[-1], state) # Control mouse pointer # if len(trajecX) > 1: # x = int((trajecX[-1] - trajecX[-2]) ) # y = int((trajecY[-1] - trajecY[-2]) ) # control_mouse_pointer(x, y, screenWd, screenHt, preds[-1], preds[-2]) # Dynamic gesture prediction if len(trajecX) > 0: draw_trajectory(trajecX, trajecY, frame) p = probab(preds_freq) d = dist(trajecX[-1], trajecY[-1], trajecX[0], trajecY[0]) # print(d, p) if p > 0.5 and d > 110: direc = get_direction(trajecX[-1], trajecY[-1], trajecX[0], trajecY[0]) if not last_swipe == direc: # print("swipe {} count: ".format(mp2[last_swipe]), swipe_cnt) swipe_cnt = 0 else: swipe_cnt += 1 last_swipe = direc if swipe_cnt % 6== 0 and swipe_cnt > 0: print("swipe {} count: ".format(mp2[last_swipe]), swipe_cnt) state = control_vlc(last_swipe, preds[-1], state) detector_utils.draw_label_on_image(mp2[direc], frame, (500, 100)) frame_processed += 1 output_q.put(frame) # cropped_output_q.put(cropped_img) else: output_q.put(frame)
def worker(input_q, output_q, cap_params, frame_count, poses): centroid = None predicted_label = "" print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) print(">> loading keras model for worker") try: model, classification_graph, session = classifier.load_KerasGraph( "F:/Github/Hand_pose_DKE/cnn/models/hand_poses_wGarbage_10.h5") except Exception as e: print(e) centroid_list = deque(maxlen=buffer) direction = "" (dX, dY) = (0, 0) while True: # print("> ===== in worker loop, frame ", frame_count) frame = input_q.get() if (frame is not None): # Actual detection. Variable boxes contains the bounding box cordinates for hands detected, # while scores contains the confidence for each of these boxes. # Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold) norm_image = cv2.normalize(frame, None, 0, 255, norm_type=cv2.NORM_MINMAX) boxes, scores = detector_utils.detect_objects( norm_image, detection_graph, sess) # print(boxes[0]) # get region of interest res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # get boundary box detector_utils.draw_box_on_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # classify hand pose if res is not None and frame_count == 0: class_res = classifier.classify(model, classification_graph, session, res) class_pred = class_res.argmax(axis=-1) predicted_label = poses[int(class_pred)] #print(predicted_label) if predicted_label == "Start" and frame_count <= frame_end: centroid = detector_utils.get_centroid( cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # elif scores is None: # centroid = None if centroid is not None: centroid_list.appendleft(centroid) #print(centroid_list) sorted(centroid_list) for i in np.arange(1, len(centroid_list)): # if centroid_list[i-1] is None or centroid_list[i] is None: # continue if frame_count == frame_end and centroid_list[ -5] != None and i == 1: dX = centroid_list[-5][0] - centroid_list[i][0] dY = centroid_list[-5][1] - centroid_list[i][1] (dirX, dirY) = ("", "") if np.abs(dX) > 10: dirX = "Right" if np.sign(dX) == 1 else "Left" if np.abs(dY) > 10: dirY = "DOWN" if np.sign(dY) == 1 else "UP" if dirX != "" and dirY != "": direction = "{}-{}".format(dirY, dirX) else: direction = dirX if dirX != "" else dirY thickness = int(np.sqrt(frame_end / float(i + 1)) * 2.5) cv2.line(frame, centroid_list[i - 1], centroid_list[i], (0, 0, 255), thickness) cv2.putText(frame, direction, (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (77, 255, 9), 1) cv2.putText(frame, "dx: {}, dy: {}".format(dX, dY), (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 255), 1) # if direction == "Left": # keyboard.press_and_release('left') # #time.sleep(2) # elif direction == "Right": # keyboard.press_and_release('right') # time.sleep(2) frame_count += 1 if frame_count >= frame_end: frame_count = 0 #centroid_list.clear() direction = "" #flag = 1 output_q.put(frame) # else: # output_q.put(frame) sess.close()
def main(): currentPath = '' currentExample = '' #----------------------------------------------------------------------------------------------------- print('\n 1 - Add new pose \ \n 2 - Add examplles to existing pose \ \n 3 - Add garbage examples' ) #tell docker whitch function you want to do #----------------------------------------------------------------------------------------------------- menu_choice = input() while (menu_choice != '1' and menu_choice != '2' and menu_choice != '3'): print('Please enter number 1~3') menu_choice = input() #set user's input function #function 1:Add new pose:--------------------------------------------------------------------------------------- if (menu_choice == '1'): print('Enter a name for the pose you want to add :') name_pose = input( ) #if choice add new pose than set input names name_pose # Create folder for pose if not os.path.exists( 'Poses/' + name_pose ): #check if input pose in the pose folder, if not than do os.makedirs( 'Poses/' + name_pose + '/' + name_pose + '_1/' ) #create a folder named your input pose's name + name_pose_1 under path 'Pose/name_pose' currentPath = 'Poses/' + name_pose + '/' + name_pose + '_1/' #set current path currentExample = name_pose + '_1_' #set hole folder to current exp print('function1 is sucessfully selected!') #---------------------------------------------------------------------------------------------------------------- #function 2:Add new data into exist pose:------------------------------------------------------------------------ elif (menu_choice == '2'): # Display current poses dirs = os.listdir('Poses/') #set pose list as dirs#? dirs_choice = '' possible_choices = [] i = 1 for _dir in dirs: #_dir means form array's dirs 0 to end of order dirs_choice += str(i) + ' - ' + str(_dir) + ' / ' possible_choices.append( str(i)) #set exsist pose name as possible_choice one by one. i += 1 # Ask user to choose to which pose to add examples print('Choose one of the following pose:') print(dirs_choice) #print exist pose choice = input() while (not choice in possible_choices and dirs[int(choice) - 1] == 'garbage'): print('Please enter one of the following: ' + str(possible_choices)) choice = input() # Count number of files to increment new example directory subdirs = os.listdir('Poses/' + dirs[int(choice) - 1] + '/') index = len(subdirs) + 1 # Create new example directory if not os.path.exists('Poses/' + dirs[int(choice) - 1] + '/' + dirs[int(choice) - 1] + '_' + str(index) + '/'): os.makedirs('Poses/' + dirs[int(choice) - 1] + '/' + dirs[int(choice) - 1] + '_' + str(index) + '/') #? #Update current path currentPath = 'Poses/' + dirs[int(choice) - 1] + '/' + dirs[ int(choice) - 1] + '_' + str(index) + '/' currentExample = dirs[int(choice) - 1] + '_' + str(index) + '_' name_pose = dirs[int(choice) - 1] print('function2 is sucessfully selected!') #---------------------------------------------------------------------------------------------------------------- #function 3:Add 'useless' pose to promot real-time acc:------------------------------------------------------------------------ elif (menu_choice == '3'): # Create folder for pose if not os.path.exists('Poses/Garbage/'): os.makedirs('Poses/Garbage/Garbage_1/') currentPath = 'Poses/Garbage/Garbage_1/' currentExample = 'Garbage_1_' name_pose = 'Garbage' else: # Count number of files to increment new example directory subdirs = os.listdir('Poses/Garbage/') index = len(subdirs) + 1 # Create new example directory if not os.path.exists('Poses/Garbage/Garbage_' + str(index) + '/'): os.makedirs('Poses/Garbage/Garbage_' + str(index) + '/') #Update current path currentPath = 'Poses/Garbage/Garbage_' + str(index) + '/' currentExample = 'Garbage_' + str(index) + '_' name_pose = 'Garbage' print('function3 is sucessfully selected!') #-------------------------------------------------------------------------------------------------------------------------------- print('You will now be prompted to record the pose you want to add. \n \ Please place your hand beforehand facing the camera, and press any key when you ready. \n \ If you finished your record, please press q on your keybroad.') input() #record a vedio after function choice--------------------------------------------------------------------------------------------------------- cap = cv2.VideoCapture(0) # Begin capturing # Define the codec and create VideoWriter object fourcc = cv2.VideoWriter_fourcc(*'XVID') #MPEG-4 code out = cv2.VideoWriter( currentPath + name_pose + '.avi', fourcc, 25.0, (640, 480)) #(file name)、(code)、(fps)、(window width and high) detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) while (cap.isOpened()): #check webcam is opened ret, frame = cap.read( ) #capture every frame (return success or not ture/flase)、(every single frame) if ret: # write the frame out.write(frame) #write result into file cv2.imshow('frame', frame) #open a frame that named 'frame' if cv2.waitKey(1) & 0xFF == ord('q'): break else: print('Error:vedio capture is faild') break #----------------------------------------------------------------------------------------------------------------------------------------------- # Release everything if job is finished cap.release() out.release() cv2.destroyAllWindows() vid = cv2.VideoCapture(currentPath + name_pose + '.avi') #finished vedio # Check if the video if (not vid.isOpened()): print('Error opening video stream or file') return #hand detection---------------------------------------------------------------------------------------------------------------------------------- print('>> loading frozen model..') detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) print('>> model loaded!') _iter = 1 # Read until video is completed while (vid.isOpened()): # Capture frame-by-frame ret, frame = vid.read() if ret: print(' Processing frame: ' + str(_iter)) # Resize and convert to RGB for NN to work with frame = cv2.resize(frame, (320, 180), interpolation=cv2.INTER_AREA ) #interpolation is clculater of pixl output frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # Detect object boxes, scores = detector_utils.detect_objects( frame, detection_graph, sess) # get region of interest res = detector_utils.get_box_image(1, 0.2, scores, boxes, 320, 180, frame) # Save cropped image if (res is not None): cv2.imwrite(currentPath + currentExample + str(_iter) + '.png', cv2.cvtColor(res, cv2.COLOR_RGB2BGR)) _iter += 1 # Break the loop else: break print(' Processed ' + str(_iter) + ' frames!') vid.release()
def worker(input_q, output_q, cap_params, frame_processed, poses): print(">> loading frozen model for worker") detection_graph, sess = detector_utils.load_inference_graph() sess = tf.Session(graph=detection_graph) print(">> loading keras model for worker") try: model, classification_graph, session = classifier.load_KerasGraph( "F:\Realtime_Hand_tracking\cnn\models\hand_poses_wGarbage_10.h5") except Exception as e: print(e) detection_centres_x = [] detection_centres_y = [] is_centres_filled = False detected = False index = 0 detection_area = [] predicted_label = "" start_flag = False flag_start = 0 pause_time = 0 print("Show Start gesture to start reading") #file = open("eval_list.csv","w") while True: # print("> ===== in worker loop, frame ", frame_count) frame = input_q.get() flag = 0 if (frame is not None): frame_processed += 1 # Actual detection. Variable boxes contains the bounding box cordinates for hands detected, # while scores contains the confidence for each of these boxes. # Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold) # frame = cv2.normalize(frame, None, 0,255, norm_type=cv2.NORM_MINMAX) boxes, scores = detector_utils.detect_objects(frame, detection_graph, sess) # print("Hello") # print(boxes[0]) # get region of interest res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) # get boundary box if pause_time == 0: centre_x, centre_y, area = detector_utils.draw_box_on_image(cap_params['num_hands_detect'], cap_params["score_thresh"], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame) if pause_time > 0: pause_time -= 1 if is_centres_filled: detection_centres_x = detection_centres_x[1:10] detection_centres_y = detection_centres_y[1:10] detection_area = detection_area[1:10] detection_centres_x.append(centre_x) detection_centres_y.append(centre_y) detection_area.append(area) else: detection_centres_x.append(centre_x) detection_centres_y.append(centre_y) detection_area.append(area) if pause_time == 0: index += 1 # print(index) if index >= 20: index = 0 is_centres_filled = True #start_flag = False if index == 0: predicted_label = classify(res, model, classification_graph, session, poses) # print(predicted_label) if predicted_label == "Start" and flag_start == 0: print("Start tracking") start_flag = True flag_start = 1 elif predicted_label == "Stop" and flag_start == 1 and frame_processed > 30: print("Stopped tracking") start_flag = False flag_start = 0 if detected: detection_centres_x = [] detection_centres_y = [] is_centres_filled = False index = 0 detected = False detection_area = [] frame_processed = 0 pause_time = 10 #time.sleep(2) #start_flag = False centres_x = detection_centres_x.copy() centres_y = detection_centres_y.copy() areas = detection_area.copy() centres_x = [v for v in centres_x if v] centres_y = [v for v in centres_y if v] areas = [a for a in areas if a] direction = "" # angle_coordinate # print(areas) if len(centres_x) > 3 and is_centres_filled and len(centres_y) > 3 and len( areas) > 3 and start_flag == True: # centres_asc = centres.copy().sort() # centres_dsc = centres.copy().sort(reverse=True) # print(centres) # print("Last x:", centres_x[-1]) # print("First x: ", centres_x[0]) # print("Last y:", centres_y[-1]) # print("First y:", centres_y[0]) flag = 0 dX = centres_x[-1] - centres_x[0] dY = centres_y[-1] - centres_y[0] if dX > 20 and dY > 20: m = dY / dX angle = math.degrees(math.atan(m)) if angle < 45: flag = 1 elif angle > 45: flag = 2 # print(angle) if dX > 100 and (abs(dY) < 20 or flag == 1): direction = "Right" #keyboard.press_and_release('left') detected = True print(direction) elif -dX > 100 and (abs(dY) < 20 or flag == 1): direction = "Left" #keyboard.press_and_release('right') detected = True print(direction) elif dY > 50 and (abs(dX) < 10 or flag == 2): direction = "Down" detected = True print(direction) elif -dY > 50 and (abs(dX) < 10 or flag == 2): direction = "Up" detected = True print(direction) elif areas[-1] - 3000 > areas[0] and abs(dX) < 30 and abs(dY) < 20: direction = "Zoom in" detected = True print(direction) elif areas[-1] < areas[0] - 3000 and abs(dX) < 10 and abs(dY) < 20: direction = "Zoom out" detected = True print(direction) # print("hello") cv2.putText(frame, direction, (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (77, 255, 9), 1) output_q.put(frame) sess.close()