Esempio n. 1
0
def main():

    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    max_num_faces = args.max_num_faces
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    use_brect = args.use_brect

    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    mp_face_mesh = mp.solutions.face_mesh
    face_mesh = mp_face_mesh.FaceMesh(
        max_num_faces=max_num_faces,
        min_detection_confidence=min_detection_confidence,
        min_tracking_confidence=min_tracking_confidence,
    )

    cvFpsCalc = CvFpsCalc(buffer_len=10)

    while True:
        display_fps = cvFpsCalc.get()

        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)
        debug_image = copy.deepcopy(image)

        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
        results = face_mesh.process(image)

        if results.multi_face_landmarks is not None:
            for face_landmarks in results.multi_face_landmarks:
                # 外接矩形の計算
                brect = calc_bounding_rect(debug_image, face_landmarks)
                # 描画
                debug_image = draw_landmarks(debug_image, face_landmarks)
                debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        cv.putText(debug_image, "MOsta_FPS:" + str(display_fps), (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)

        # キー処理(ESC:終了) #################################################
        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        # 画面反映 #############################################################
        cv.imshow('MediaPipe Face Mesh Demo', debug_image)

    cap.release()
    cv.destroyAllWindows()
Esempio n. 2
0
def main():
    # init global vars
    global gesture_buffer
    global gesture_id
    global battery_status

    # Argument parsing
    args = get_args()
    KEYBOARD_CONTROL = args.is_keyboard
    WRITE_CONTROL = False
    in_flight = False

    cap_webcam = cv.VideoCapture(0)

    gesture_detector = GestureRecognition(args.use_static_image_mode,
                                          args.min_detection_confidence,
                                          args.min_tracking_confidence)

    gesture_buffer = GestureBuffer(buffer_len=args.buffer_len)

    # FPS Measurement
    cv_fps_calc = CvFpsCalc(buffer_len=10)

    mode = 0
    number = -1

    while True:
        fps = cv_fps_calc.get()

        # Process Key (ESC: end)
        key = cv.waitKey(1) & 0xff
        if key == 27:  # ESC
            break
        elif key == ord('n'):
            mode = 1
            WRITE_CONTROL = True
            KEYBOARD_CONTROL = True

        if WRITE_CONTROL:
            number = -1
            if 48 <= key <= 57:  # 0 ~ 9
                number = key - 48

        # Camera capture
        image = cap_webcam.read()[1]

        try:
            debug_image, gesture_id = gesture_detector.recognize(
                image, number, mode)
            gesture_buffer.add_gesture(gesture_id)

            debug_image = gesture_detector.draw_info(debug_image, fps, mode,
                                                     number)

            cv.imshow('Webcam Gesture Recognition', debug_image)

        except:
            print("exception")

    cv.destroyAllWindows()
def main():

    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    upper_body_only = args.upper_body_only
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    use_brect = args.use_brect

    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    mp_pose = mp.solutions.pose
    pose = mp_pose.Pose(
        upper_body_only=upper_body_only,
        min_detection_confidence=min_detection_confidence,
        min_tracking_confidence=min_tracking_confidence,
    )

    # FPS
    cvFpsCalc = CvFpsCalc(buffer_len=10)

    while True:
        display_fps = cvFpsCalc.get()

        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)
        debug_image = copy.deepcopy(image)

        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
        results = pose.process(image)

        if results.pose_landmarks is not None:

            brect = calc_bounding_rect(debug_image, results.pose_landmarks)

            debug_image = draw_landmarks(debug_image, results.pose_landmarks,
                                         upper_body_only)
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)

        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        cv.imshow('MediaPipe Pose Demo', debug_image)

    cap.release()
    cv.destroyAllWindows()
Esempio n. 4
0
def main():
    # 引数解析 #################################################################
    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    min_detection_confidence = args.min_detection_confidence

    use_brect = args.use_brect

    # カメラ準備 ###############################################################
    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    # モデルロード #############################################################
    mp_face_detection = mp.solutions.face_detection
    face_detection = mp_face_detection.FaceDetection(
        min_detection_confidence=min_detection_confidence)

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc(buffer_len=10)

    while True:
        display_fps = cvFpsCalc.get()

        # カメラキャプチャ #####################################################
        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)  # ミラー表示
        debug_image = copy.deepcopy(image)

        # 検出実施 #############################################################
        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
        results = face_detection.process(image)

        # 描画 ################################################################
        if results.detections is not None:
            for detection in results.detections:
                # 描画
                debug_image = draw_detection(debug_image, detection)

        cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)

        # キー処理(ESC:終了) #################################################
        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        # 画面反映 #############################################################
        cv.imshow('MediaPipe Face Detection Demo', debug_image)

    cap.release()
    cv.destroyAllWindows()
Esempio n. 5
0
def main():
    print("Image Classification Start...\n")

    # カメラ準備 ##############################################################
    cap = cv.VideoCapture(0)
    frame_width = 960
    frame_height = 540
    cap.set(cv.CAP_PROP_FRAME_WIDTH, frame_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, frame_height)

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc()

    # モデルロード ############################################################
    model = tf.keras.applications.EfficientNetB0(
        include_top=True,
        weights='imagenet',
        input_shape=(224, 224, 3),
    )

    # CVUI初期化 ##############################################################
    cvui.init("Demo")

    # メインループ #############################################################
    while True:
        # FPS算出 #############################################################
        display_fps = cvFpsCalc.get()
        if display_fps == 0:
            display_fps = 0.01

        # カメラキャプチャ ####################################################
        ret, frame = cap.read()
        if not ret:
            continue
        debug_image = copy.deepcopy(frame)

        # 検出実施 ############################################################
        trim_x1 = int((frame_width - frame_height) / 2)
        trim_x2 = frame_width - int((frame_width - frame_height) / 2)
        trimming_image = debug_image[0:frame_height, trim_x1:trim_x2]

        classifications = run_classify(model, trimming_image, 10)

        # デバッグ情報描画 ####################################################
        debug_image = draw_demo_image(
            trimming_image,
            classifications,
            display_fps,
        )

        # 画面反映 #######################################################
        cvui.imshow('Demo', debug_image)

        key = cv.waitKey(1)
        if key == 27:  # ESC
            break
Esempio n. 6
0
def main():
    print("Semantic Segmentation Start...\n")

    # カメラ準備 ##############################################################
    cap = cv.VideoCapture(0)
    frame_width = 960
    frame_height = 540
    cap.set(cv.CAP_PROP_FRAME_WIDTH, frame_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, frame_height)

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc()

    # モデルロード ############################################################
    frozen_path = "model/deeplab_v3/deeplabv3_mnv2.pb"
    sess = graph_load(frozen_path)

    # メインループ #############################################################
    while True:
        # FPS算出 #############################################################
        display_fps = cvFpsCalc.get()
        if display_fps == 0:
            display_fps = 0.01

        # カメラキャプチャ ####################################################
        ret, frame = cap.read()
        if not ret:
            continue
        debug_image = copy.deepcopy(frame)

        # 検出実施 ############################################################
        segmentation_map = session_run(sess, debug_image)

        # デバッグ情報描画 ####################################################
        debug_image = draw_demo_image(
            debug_image,
            segmentation_map,
            display_fps,
        )

        # 画面反映 #######################################################
        cv.imshow('Demo', debug_image)

        key = cv.waitKey(1)
        if key == 27:  # ESC
            break
Esempio n. 7
0
def main():
    print("Style Transfer Start...\n")

    # カメラ準備 ###############################################################
    cap = cv.VideoCapture(0)
    frame_width = 960
    frame_height = 540
    cap.set(cv.CAP_PROP_FRAME_WIDTH, frame_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, frame_height)

    # モデルロード #############################################################
    sess, input_photo, final_out = graph_load('model/white_box_cartoonization')

    # CvComparisonSliderWindow準備 ############################################
    cvwindow = CvComparisonSliderWindow(
        window_name='Demo',
        line_color=(255, 255, 255),
        line_thickness=1,
    )

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc()

    while True:
        display_fps = cvFpsCalc.get()

        # カメラキャプチャ #####################################################
        ret, frame = cap.read()
        if not ret:
            continue
        frame_width, frame_height = frame.shape[1], frame.shape[0]
        debug_image = copy.deepcopy(frame)

        # 変換実施 #############################################################
        out = session_run(sess, debug_image, input_photo, final_out)

        # 画面反映 #############################################################
        cvwindow.imshow(frame, out, fps=display_fps)

        # キー処理 #############################################################
        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

    cap.release()
    cv.destroyAllWindows()
Esempio n. 8
0
def main():
    """
    [summary]
        main()
    Parameters
    ----------
    None
    """
    # 引数解析 #################################################################
    args = get_args()
    cap_device = args.device
    cap_width = args.width
    cap_height = args.height
    fps = args.fps

    model_path = args.model
    score_th = args.score_th
    smaller_ratio = args.smaller_ratio

    # GUI準備 #################################################################
    app_gui = AppGui(window_name='FingerFrameLens')
    # 初期設定
    app_gui.set_score_threshold(score_th)

    # カメラ準備 ###############################################################
    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    # モデルロード #############################################################
    # EfficientDet-D0
    DEFAULT_FUNCTION_KEY = 'serving_default'
    effdet_model = tf.saved_model.load(model_path)
    inference_func = effdet_model.signatures[DEFAULT_FUNCTION_KEY]

    # EfficientNet-B0
    effnet_model = tf.keras.applications.EfficientNetB0(
        include_top=True,
        weights='imagenet',
        input_shape=(224, 224, 3),
    )
    tensor = tf.convert_to_tensor(np.zeros((224, 224, 3), np.uint8))
    effnet_model.predict(tensor)
    effnet_model.make_predict_function()

    # FPS計測準備 ##############################################################
    cvFpsCalc = CvFpsCalc(buffer_len=3)

    cropping_image = None
    classifications = None

    while True:
        start_time = time.time()

        # GUI設定取得 #########################################################
        score_th = app_gui.get_score_threshold()

        # カメラキャプチャ #####################################################
        ret, frame = cap.read()
        if not ret:
            continue
        frame_width, frame_height = frame.shape[1], frame.shape[0]
        debug_image = copy.deepcopy(frame)

        # 物体検出実施 #########################################################
        detections = run_od_inference(inference_func, frame)
        x1, y1, x2, y2 = calc_od_bbox(
            detections,
            score_th,
            smaller_ratio,
            frame_width,
            frame_height,
        )

        # cv.putText(debug_image, '{:.3f}'.format(score), (x1, y1 - 10),
        #            cv.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 255), 2,
        #            cv.LINE_AA)
        # cv.rectangle(debug_image, (x1, y1), (x2, y2), (255, 255, 255), 2)

        # クラス分類実施 #######################################################
        if x1 is not None and y1 is not None and \
           x2 is not None and y2 is not None:
            cropping_image = copy.deepcopy(frame[y1:y2, x1:x2])
            classifications = run_classify(effnet_model, cropping_image)

        # GUI描画更新 ##########################################################
        fps_result = cvFpsCalc.get()
        app_gui.update(
            fps_result,
            debug_image,
            cropping_image,
            classifications,
        )
        app_gui.show()

        # キー入力(ESC:プログラム終了) #########################################
        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        # FPS調整 #############################################################
        elapsed_time = time.time() - start_time
        sleep_time = max(0, ((1.0 / fps) - elapsed_time))
        time.sleep(sleep_time)

    cap.release()
    cv.destroyAllWindows()
Esempio n. 9
0
def main():
    # 引数解析 #################################################################
    args = get_args()

    cap_width = args.width
    cap_height = args.height
    cap_device = args.device
    if args.file is not None:  # 動画ファイルを利用する場合
        cap_device = args.file

    fps = args.fps
    skip_frame = args.skip_frame

    model_path = args.model
    score_th = args.score_th

    sign_interval = args.sign_interval
    jutsu_display_time = args.jutsu_display_time

    use_display_score = args.use_display_score
    erase_bbox = args.erase_bbox
    use_jutsu_lang_en = args.use_jutsu_lang_en

    chattering_check = args.chattering_check

    use_fullscreen = args.use_fullscreen

    # カメラ準備 ###############################################################
    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    # モデル読み込み ############################################################
    DEFAULT_FUNCTION_KEY = 'serving_default'
    loaded_model = tf.saved_model.load(model_path)
    inference_func = loaded_model.signatures[DEFAULT_FUNCTION_KEY]

    # FPS計測モジュール #########################################################
    cvFpsCalc = CvFpsCalc()

    # フォント読み込み ##########################################################
    # https://opentype.jp/kouzanmouhitufont.htm
    font_path = './utils/font/衡山毛筆フォント.ttf'

    # ラベル読み込み ###########################################################
    with open('setting/labels.csv') as f:  # 印
        labels = csv.reader(f)
        labels = [row for row in labels]

    with open('setting/jutsu.csv') as f:  # 術
        jutsu = csv.reader(f)
        jutsu = [row for row in jutsu]

    # 印の表示履歴および、検出履歴 ##############################################
    sign_max_display = 18
    sign_max_history = 44
    sign_display_queue = deque(maxlen=sign_max_display)
    sign_history_queue = deque(maxlen=sign_max_history)

    chattering_check_queue = deque(maxlen=chattering_check)
    for index in range(-1, -1 - chattering_check, -1):
        chattering_check_queue.append(index)

    # 術名の言語設定 ###########################################################
    lang_offset = 0
    jutsu_font_size_ratio = sign_max_display
    if use_jutsu_lang_en:
        lang_offset = 1
        jutsu_font_size_ratio = int((sign_max_display / 3) * 4)

    # その他変数初期化 #########################################################
    sign_interval_start = 0  # 印のインターバル開始時間初期化
    jutsu_index = 0  # 術表示名のインデックス
    jutsu_start_time = 0  # 術名表示の開始時間初期化
    frame_count = 0  # フレームナンバーカウンタ

    window_name = 'NARUTO HandSignDetection Ninjutsu Demo'
    if use_fullscreen:
        cv.namedWindow(window_name, cv.WINDOW_NORMAL)

    while True:
        start_time = time.time()

        # カメラキャプチャ #####################################################
        ret, frame = cap.read()
        if not ret:
            continue
        frame_count += 1
        debug_image = copy.deepcopy(frame)

        if (frame_count % (skip_frame + 1)) != 0:
            continue

        # FPS計測 ##############################################################
        fps_result = cvFpsCalc.get()

        # 検出実施 #############################################################
        frame = frame[:, :, [2, 1, 0]]  # BGR2RGB
        image_np_expanded = np.expand_dims(frame, axis=0)
        result_inference = run_inference_single_image(image_np_expanded,
                                                      inference_func)

        # 検出内容の履歴追加 ####################################################
        num_detections = result_inference['num_detections']
        for i in range(num_detections):
            score = result_inference['detection_scores'][i]
            class_id = result_inference['detection_classes'][i].astype(np.int)

            # 検出閾値未満の結果は捨てる
            if score < score_th:
                continue

            # 指定回数以上、同じ印が続いた場合に、印検出とみなす ※瞬間的な誤検出対策
            chattering_check_queue.append(class_id)
            if len(set(chattering_check_queue)) != 1:
                continue

            # 前回と異なる印の場合のみキューに登録
            if len(sign_display_queue) == 0 or \
                sign_display_queue[-1] != class_id:
                sign_display_queue.append(class_id)
                sign_history_queue.append(class_id)
                sign_interval_start = time.time()  # 印の最終検出時間

        # 前回の印検出から指定時間が経過した場合、履歴を消去 ####################
        if (time.time() - sign_interval_start) > sign_interval:
            sign_display_queue.clear()
            sign_history_queue.clear()

        # 術成立判定 #########################################################
        jutsu_index, jutsu_start_time = check_jutsu(
            sign_history_queue,
            labels,
            jutsu,
            jutsu_index,
            jutsu_start_time,
        )

        # キー処理 ###########################################################
        key = cv.waitKey(1)
        if key == 99:  # C:印の履歴を消去
            sign_display_queue.clear()
            sign_history_queue.clear()
        if key == 27:  # ESC:プログラム終了
            break

        # FPS調整 #############################################################
        elapsed_time = time.time() - start_time
        sleep_time = max(0, ((1.0 / fps) - elapsed_time))
        time.sleep(sleep_time)

        # 画面反映 #############################################################
        debug_image = draw_debug_image(
            debug_image,
            font_path,
            fps_result,
            labels,
            result_inference,
            score_th,
            erase_bbox,
            use_display_score,
            jutsu,
            sign_display_queue,
            sign_max_display,
            jutsu_display_time,
            jutsu_font_size_ratio,
            lang_offset,
            jutsu_index,
            jutsu_start_time,
        )
        if use_fullscreen:
            cv.setWindowProperty(window_name, cv.WND_PROP_FULLSCREEN,
                                 cv.WINDOW_FULLSCREEN)
        cv.imshow(window_name, debug_image)
        # cv.moveWindow(window_name, 100, 100)

    cap.release()
    cv.destroyAllWindows()
Esempio n. 10
0
def main():
    # 引数解析 #################################################################
    args = get_args()
    cap_device = args.device
    cap_width = args.width
    cap_height = args.height
    fps = args.fps

    model_path = args.model
    score_th = args.score_th

    # カメラ準備 ###############################################################
    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    video = cv.VideoCapture('utils/map.mp4')

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc()

    # モデルロード #############################################################
    DEFAULT_FUNCTION_KEY = 'serving_default'
    loaded_model = tf.saved_model.load(model_path)
    inference_func = loaded_model.signatures[DEFAULT_FUNCTION_KEY]

    buffer_len = 5
    deque_x1 = deque(maxlen=buffer_len)
    deque_y1 = deque(maxlen=buffer_len)
    deque_x2 = deque(maxlen=buffer_len)
    deque_y2 = deque(maxlen=buffer_len)

    # フォント
    font_path = './utils/font/x12y20pxScanLine.ttf'

    while True:
        # FPS算出 #############################################################
        display_fps = cvFpsCalc.get()

        start_time = time.time()

        # カメラキャプチャ #####################################################
        ret, frame = cap.read()
        if not ret:
            continue
        frame_width, frame_height = frame.shape[1], frame.shape[0]
        debug_image = copy.deepcopy(frame)

        # 検出実施 #############################################################
        frame = frame[:, :, [2, 1, 0]]  # BGR2RGB
        image_np_expanded = np.expand_dims(frame, axis=0)

        output = run_inference_single_image(image_np_expanded, inference_func)

        num_detections = output['num_detections']
        for i in range(num_detections):
            score = output['detection_scores'][i]
            bbox = output['detection_boxes'][i]
            # class_id = output['detection_classes'][i].astype(np.int)

            if score < score_th:
                continue

            # 検出結果可視化 ###################################################
            x1, y1 = int(bbox[1] * frame_width), int(bbox[0] * frame_height)
            x2, y2 = int(bbox[3] * frame_width), int(bbox[2] * frame_height)

            risize_ratio = 0.15
            bbox_width = x2 - x1
            bbox_height = y2 - y1
            x1 = x1 + int(bbox_width * risize_ratio)
            y1 = y1 + int(bbox_height * risize_ratio)
            x2 = x2 - int(bbox_width * risize_ratio)
            y2 = y2 - int(bbox_height * risize_ratio)

            x1 = int((x1 - 5) / 10) * 10
            y1 = int((y1 - 5) / 10) * 10
            x2 = int((x2 + 5) / 10) * 10
            y2 = int((y2 + 5) / 10) * 10

            deque_x1.append(x1)
            deque_y1.append(y1)
            deque_x2.append(x2)
            deque_y2.append(y2)
            x1 = int(sum(deque_x1) / len(deque_x1))
            y1 = int(sum(deque_y1) / len(deque_y1))
            x2 = int(sum(deque_x2) / len(deque_x2))
            y2 = int(sum(deque_y2) / len(deque_y2))

            ret, video_frame = video.read()
            if ret is not False:
                video.grab()
                video.grab()

                debug_add_image = np.zeros((frame_height, frame_width, 3),
                                           np.uint8)
                map_resize_image = cv.resize(video_frame,
                                             ((x2 - x1), (y2 - y1)))
                debug_add_image = CvOverlayImage.overlay(
                    debug_add_image, map_resize_image, (x1, y1))
                debug_add_image = cv.cvtColor(debug_add_image,
                                              cv.COLOR_BGRA2BGR)
                # cv.imshow('1', debug_add_image)
                debug_image = cv.addWeighted(debug_image, 1.0, debug_add_image,
                                             2.0, 0)
            else:
                video = cv.VideoCapture('map.mp4')

        # キー処理(ESC:終了) #################################################
        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        # FPS調整 #############################################################
        elapsed_time = time.time() - start_time
        sleep_time = max(0, ((1.0 / fps) - elapsed_time))
        time.sleep(sleep_time)

        # 画面反映 #############################################################
        fps_string = u"FPS:" + str(display_fps)
        debug_image = CvDrawText.puttext(debug_image, fps_string, (17, 17),
                                         font_path, 32, (255, 255, 255))
        fps_string = u"FPS:" + str(display_fps)
        debug_image = CvDrawText.puttext(debug_image, fps_string, (15, 15),
                                         font_path, 32, (0, 56, 86))
        cv.imshow('Demo', debug_image)

    cap.release()
    cv.destroyAllWindows()
Esempio n. 11
0
File: hand.py Progetto: baho0/HoloPC
def main():

    # Argument parsing #################################################################
    args = get_args()

    cap_device = args.device
    cap_width = 1920
    cap_height = 1080

    use_static_image_mode = args.use_static_image_mode
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    use_brect = True

    # Camera preparation ###############################################################
    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    # Model load #############################################################
    mp_hands = mp.solutions.hands
    hands = mp_hands.Hands(
        static_image_mode=use_static_image_mode,
        max_num_hands=1,
        min_detection_confidence=min_detection_confidence,
        min_tracking_confidence=min_tracking_confidence,
    )

    keypoint_classifier = KeyPointClassifier()

    point_history_classifier = PointHistoryClassifier()

    # Read labels ###########################################################
    with open('model/keypoint_classifier/keypoint_classifier_label.csv',
              encoding='utf-8-sig') as f:
        keypoint_classifier_labels = csv.reader(f)
        keypoint_classifier_labels = [
            row[0] for row in keypoint_classifier_labels
        ]
    with open(
            'model/point_history_classifier/point_history_classifier_label.csv',
            encoding='utf-8-sig') as f:
        point_history_classifier_labels = csv.reader(f)
        point_history_classifier_labels = [
            row[0] for row in point_history_classifier_labels
        ]

    # FPS Measurement ########################################################
    cvFpsCalc = CvFpsCalc(buffer_len=10)

    # Coordinate history #################################################################
    history_length = 16
    point_history = deque(maxlen=history_length)

    # Finger gesture history ################################################
    finger_gesture_history = deque(maxlen=history_length)

    #  ########################################################################
    mode = 0

    while True:
        fps = cvFpsCalc.get()

        # Process Key (ESC: end) #################################################
        key = cv.waitKey(10)
        if key == 27:  # ESC
            break
        number, mode = select_mode(key, mode)

        # Camera capture #####################################################
        ret, image = cap.read()

        if not ret:
            break
        image = cv.flip(image, 1)  # Mirror display
        debug_image = copy.deepcopy(image)

        # Detection implementation #############################################################
        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)

        image.flags.writeable = False
        results = hands.process(image)
        image.flags.writeable = True

        #  ####################################################################
        if results.multi_hand_landmarks is not None:
            for hand_landmarks, handedness in zip(results.multi_hand_landmarks,
                                                  results.multi_handedness):
                # Bounding box calculation
                brect = calc_bounding_rect(debug_image, hand_landmarks)
                # Landmark calculation
                landmark_list = calc_landmark_list(debug_image, hand_landmarks)

                # Conversion to relative coordinates / normalized coordinates
                pre_processed_landmark_list = pre_process_landmark(
                    landmark_list)
                pre_processed_point_history_list = pre_process_point_history(
                    debug_image, point_history)
                # Write to the dataset file
                logging_csv(number, mode, pre_processed_landmark_list,
                            pre_processed_point_history_list)

                # Hand sign classification
                hand_sign_id = keypoint_classifier(pre_processed_landmark_list)
                if hand_sign_id == 2:  # Point gesture
                    point_history.append(landmark_list[8])
                else:
                    point_history.append([0, 0])

                # Finger gesture classification
                finger_gesture_id = 0
                point_history_len = len(pre_processed_point_history_list)
                if point_history_len == (history_length * 2):
                    finger_gesture_id = point_history_classifier(
                        pre_processed_point_history_list)

                # Calculates the gesture IDs in the latest detection
                finger_gesture_history.append(finger_gesture_id)
                most_common_fg_id = Counter(
                    finger_gesture_history).most_common()

                # Drawing part
                debug_image = draw_bounding_rect(use_brect, debug_image, brect)
                debug_image = draw_landmarks(debug_image, landmark_list)
                debug_image = draw_info_text(
                    debug_image,
                    brect,
                    handedness,
                    keypoint_classifier_labels[hand_sign_id],
                    point_history_classifier_labels[most_common_fg_id[0][0]],
                )
        else:
            point_history.append([0, 0])

        debug_image = draw_point_history(debug_image, point_history)
        debug_image = draw_info(debug_image, fps, mode, number)

        # Screen reflection #############################################################
        cv.imshow('Hand Gesture Recognition', debug_image)

    cap.release()
    cv.destroyAllWindows()
Esempio n. 12
0
def main(filename):
    args = get_args()
    
    print("Filename in main is " + filename)

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    upper_body_only = args.upper_body_only
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    use_brect = args.use_brect

    cap = cv.VideoCapture(filename)   
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
    length = int(cap.get(cv.CAP_PROP_FRAME_COUNT))
    print(length)

    mp_holistic = mp.solutions.holistic
    holistic = mp_holistic.Holistic(
        upper_body_only=upper_body_only,
        min_detection_confidence=min_detection_confidence,
        min_tracking_confidence=min_tracking_confidence,
    )

    cvFpsCalc = CvFpsCalc(buffer_len=10)
    d = 0
    real_right_shoulder_hip = 0.57
    velocity_counter = 0
    velocity_15 = 0
    velocity_16 = 0
    velocity_27 = 0
    velocity_28 = 0
    array_counter = 0
    velocity_array_counter = 0
    frame_array = []
    angle_1_array = []
    angle_2_array = []
    angle_3_array = []
    angle_4_array = []
    angle_5_array = []
    angle_6_array = []
    angle_7_array = []
    angle_8_array = []
    velocity_array = []
    velocity_15_array = []
    velocity_16_array = []
    velocity_27_array = []
    velocity_28_array = []
    # to append angles of joint to array
    angle_1_array.append("Right Elbow")
    angle_2_array.append("Left Elbow")
    angle_3_array.append("Right Knee")
    angle_4_array.append("Left Knee")
    angle_5_array.append("Right Hip")
    angle_6_array.append("Left Hip")
    angle_7_array.append("Right Shoulder")
    angle_8_array.append("Left Shoulder")
    velocity_15_array.append("Left Wrist")
    velocity_16_array.append("Right Wrist")
    velocity_27_array.append("Left Ankle")
    velocity_28_array.append("Right Ankle")
    while True:
        velocity_counter+=1
        display_fps = cvFpsCalc.get()

        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)  
        debug_image = copy.deepcopy(image)
        blank_image = np.zeros((cap_height,cap_width,3), np.uint8)

        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)

        image.flags.writeable = False
        results = holistic.process(image)
        image.flags.writeable = True

        # Pose ###############################################################
        pose_landmarks = results.pose_landmarks
        if pose_landmarks is not None:
            brect = calc_bounding_rect(debug_image, pose_landmarks)
            
            debug_image, image_right_shoulder_hip, vel_xparsed_15, vel_yparsed_15, vel_xparsed_16, vel_yparsed_16, vel_xparsed_27, vel_yparsed_27, vel_xparsed_28, vel_yparsed_28, angle_1, angle_2, angle_3, angle_4, angle_5, angle_6, angle_7, angle_8 = draw_pose_landmarks(debug_image, pose_landmarks,
                                              upper_body_only)
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)
            #blank_image = draw_pose_landmarks(blank_image, pose_landmarks,
                                              #upper_body_only)

        # to append angles of joint to array
        angle_1_array.append(angle_1)
        angle_2_array.append(angle_2)
        angle_3_array.append(angle_3)
        angle_4_array.append(angle_4)
        angle_5_array.append(angle_5)
        angle_6_array.append(angle_6)
        angle_7_array.append(angle_7)
        angle_8_array.append(angle_8)

        # to calculate velocity of right wrist (no.16) and left wrist (no.15)
        if velocity_counter == 1:
            # set coordinate of joint no. 15 on 1st frame
            vel_x1_15, vel_y1_15 = vel_xparsed_15, vel_yparsed_15
            # set coordinate of joint no. 16 on 1st frame
            vel_x1_16, vel_y1_16 = vel_xparsed_16, vel_yparsed_16
            # set coordinate of joint no. 27 on 1st frame
            vel_x1_27, vel_y1_27 = vel_xparsed_27, vel_yparsed_27
            # set coordinate of joint no. 28 on 1st frame
            vel_x1_28, vel_y1_28 = vel_xparsed_28, vel_yparsed_28
        if velocity_counter == 60:
            # set coordinate of joint no. 15 on 60th frame
            vel_x60_15, vel_y60_15 = vel_xparsed_15, vel_yparsed_15
            # set coordinate of joint no. 16 on 60th frame
            vel_x60_16, vel_y60_16 = vel_xparsed_16, vel_yparsed_16
            # set coordinate of joint no. 27 on 60th frame
            vel_x60_27, vel_y60_27 = vel_xparsed_27, vel_yparsed_27
            # set coordinate of joint no. 28 on 60th frame
            vel_x60_28, vel_y60_28 = vel_xparsed_28, vel_yparsed_28
            # calculate the ratio between the real length and image length of shoulder to hip
            ratio_shoulder_hip = real_right_shoulder_hip / image_right_shoulder_hip
            # calculate the velocity of joint no. 15
            velocity_15 = math.sqrt(pow(vel_x1_15 - vel_x60_15,2) + pow(vel_y1_15 - vel_y60_15,2)) * ratio_shoulder_hip
            velocity_15_array.append(velocity_15)
            # calculate the velocity of joint no. 16
            velocity_16 = math.sqrt(pow(vel_x1_16 - vel_x60_16,2) + pow(vel_y1_16 - vel_y60_16,2)) * ratio_shoulder_hip
            velocity_16_array.append(velocity_16)
            # calculate the velocity of joint no. 27
            velocity_27 = math.sqrt(pow(vel_x1_27 - vel_x60_27,2) + pow(vel_y1_27 - vel_y60_27,2)) * ratio_shoulder_hip
            velocity_27_array.append(velocity_27)
            # calculate the velocity of joint no. 28
            velocity_28 = math.sqrt(pow(vel_x1_28 - vel_x60_28,2) + pow(vel_y1_28 - vel_y60_28,2)) * ratio_shoulder_hip
            velocity_28_array.append(velocity_28)
            cv.putText(debug_image, "V-[9] Right Wrist:" + "{:.2f}".format(velocity_16) + 'ms-1', (10, 330),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
            cv.putText(debug_image, "V-[10] Left Wrist:" + "{:.2f}".format(velocity_15) + 'ms-1', (10, 360),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
            cv.putText(debug_image, "V-[11] Right Ankle:" + "{:.2f}".format(velocity_28) + 'ms-1', (10, 390),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
            cv.putText(debug_image, "V-[12] Left Ankle:" + "{:.2f}".format(velocity_27) + 'ms-1', (10, 420),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
            velocity_counter = 0
        if velocity_counter != 60:
            cv.putText(debug_image, "V-[9] Right Wrist:" + "{:.2f}".format(velocity_16) + 'ms-1', (10, 330),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
            cv.putText(debug_image, "V-[10] Left Wrist:" + "{:.2f}".format(velocity_15) + 'ms-1', (10, 360),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
            cv.putText(debug_image, "V-[11] Right Ankle:" + "{:.2f}".format(velocity_28) + 'ms-1', (10, 390),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
            cv.putText(debug_image, "V-[12] Left Ankle:" + "{:.2f}".format(velocity_27) + 'ms-1', (10, 420),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
            

        #to calculate and display the frame processed
        processed_frame_percent = (d/length) * 100
        cv.putText(debug_image, "Processing Frame:" + "{:.2f}".format(processed_frame_percent) + '%', (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)

        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        filename = 'output/save_%06d.png'%d
        cv.imshow('MediaPipe Holistic Demo', debug_image)
        cv.imwrite(filename, debug_image)



        #to display the 2d output on blank canvas
        #cv.imshow('Blank image',blank_image)
        
        d+=1

    # to convert frame images into video format
    Video_Writer()
    for array_counter in range(length+1):
        frame_array.append(array_counter)

    with open('anglebyframe.csv', 'w', newline='') as angle_file:
        writer = csv.writer(angle_file)
        writer.writerow(frame_array)
        writer.writerow(angle_1_array)
        writer.writerow(angle_2_array)
        writer.writerow(angle_3_array)
        writer.writerow(angle_4_array)
        writer.writerow(angle_5_array)
        writer.writerow(angle_6_array)
        writer.writerow(angle_7_array)
        writer.writerow(angle_8_array)

    total_seconds = length // 60

    for velocity_array_counter in range(total_seconds + 1):
        velocity_array.append(velocity_array_counter)

    with open('velocitybyframe.csv', 'w', newline='') as velocity_file:
        writer = csv.writer(velocity_file)
        writer.writerow(velocity_array)
        writer.writerow(velocity_15_array)
        writer.writerow(velocity_16_array)
        writer.writerow(velocity_27_array)
        writer.writerow(velocity_28_array)

    folder = 'output'
    for filename in os.listdir(folder):
        file_path = os.path.join(folder, filename)
        try:
            if os.path.isfile(file_path) or os.path.islink(file_path):
                os.unlink(file_path)
            elif os.path.isdir(file_path):
                shutil.rmtree(file_path)
        except Exception as e:
            print('Failed to delete %s. Reason: %s' % (file_path, e))

    cap.release()
    cv.destroyAllWindows()
def main():
    # 引数解析 #################################################################
    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    f_min_detection_confidence = args.f_min_detection_confidence
    f_min_tracking_confidence = args.f_min_tracking_confidence
    h_min_detection_confidence = args.h_min_detection_confidence
    h_min_tracking_confidence = args.h_min_tracking_confidence
    p_min_detection_confidence = args.p_min_detection_confidence
    p_min_tracking_confidence = args.p_min_tracking_confidence

    use_brect = args.use_brect

    # カメラ準備 ###############################################################
    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    # モデルロード #############################################################
    mp_face_mesh = mp.solutions.face_mesh
    face_mesh = mp_face_mesh.FaceMesh(
        min_detection_confidence=f_min_detection_confidence,
        min_tracking_confidence=f_min_tracking_confidence,
    )
    mp_hands = mp.solutions.hands
    hands = mp_hands.Hands(
        min_detection_confidence=h_min_detection_confidence,
        min_tracking_confidence=h_min_tracking_confidence,
    )
    mp_pose = mp.solutions.pose
    pose = mp_pose.Pose(
        min_detection_confidence=p_min_detection_confidence,
        min_tracking_confidence=p_min_tracking_confidence,
    )

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc(buffer_len=10)

    while True:
        display_fps = cvFpsCalc.get()

        # カメラキャプチャ #####################################################
        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)  # ミラー表示
        debug_image = copy.deepcopy(image)

        # 検出実施 #############################################################
        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)

        image.flags.writeable = False
        hands_results = hands.process(image)
        face_results = face_mesh.process(image)
        pose_results = pose.process(image)
        image.flags.writeable = True

        # Face Mesh ###########################################################
        if face_results.multi_face_landmarks is not None:
            for face_landmarks in face_results.multi_face_landmarks:
                # 外接矩形の計算
                brect = calc_bounding_rect(debug_image, face_landmarks)
                # 描画
                debug_image = draw_face_landmarks(debug_image, face_landmarks)
                debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        # Pose ###############################################################
        if pose_results.pose_landmarks is not None:
            # 外接矩形の計算
            brect = calc_bounding_rect(debug_image,
                                       pose_results.pose_landmarks)
            # 描画
            debug_image = draw_pose_landmarks(debug_image,
                                              pose_results.pose_landmarks)
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        # Hands ###############################################################
        if hands_results.multi_hand_landmarks is not None:
            for hand_landmarks, handedness in zip(
                    hands_results.multi_hand_landmarks,
                    hands_results.multi_handedness):
                # 手の平重心計算
                cx, cy = calc_palm_moment(debug_image, hand_landmarks)
                # 外接矩形の計算
                brect = calc_bounding_rect(debug_image, hand_landmarks)
                # 描画
                debug_image = draw_hands_landmarks(debug_image, cx, cy,
                                                   hand_landmarks, handedness)
                debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)

        # キー処理(ESC:終了) #################################################
        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        # 画面反映 #############################################################
        cv.imshow('MediaPipe Demo', debug_image)

    cap.release()
    cv.destroyAllWindows()
Esempio n. 14
0
def main():
    # 引数解析 #################################################################
    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    use_static_image_mode = args.use_static_image_mode
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    use_brect = True

    # カメラ準備 ###############################################################
    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    # モデルロード #############################################################
    mp_hands = mp.solutions.hands
    hands = mp_hands.Hands(
        static_image_mode=use_static_image_mode,
        max_num_hands=1,
        min_detection_confidence=min_detection_confidence,
        min_tracking_confidence=min_tracking_confidence,
    )

    keypoint_classifier = KeyPointClassifier()

    point_history_classifier = PointHistoryClassifier()

    # ラベル読み込み ###########################################################
    with open('model/keypoint_classifier/keypoint_classifier_label.csv',
              encoding='utf-8-sig') as f:
        keypoint_classifier_labels = csv.reader(f)
        keypoint_classifier_labels = [
            row[0] for row in keypoint_classifier_labels
        ]
    with open(
            'model/point_history_classifier/point_history_classifier_label.csv',
            encoding='utf-8-sig') as f:
        point_history_classifier_labels = csv.reader(f)
        point_history_classifier_labels = [
            row[0] for row in point_history_classifier_labels
        ]

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc(buffer_len=10)

    # 座標履歴 #################################################################
    history_length = 16
    point_history = deque(maxlen=history_length)

    # フィンガージェスチャー履歴 ################################################
    finger_gesture_history = deque(maxlen=history_length)

    #  ########################################################################
    mode = 0

    while True:
        fps = cvFpsCalc.get()

        # キー処理(ESC:終了) #################################################
        key = cv.waitKey(10)
        if key == 27:  # ESC
            break
        number, mode = select_mode(key, mode)

        # カメラキャプチャ #####################################################
        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)  # ミラー表示
        debug_image = copy.deepcopy(image)

        # 検出実施 #############################################################
        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)

        image.flags.writeable = False
        results = hands.process(image)
        image.flags.writeable = True

        #  ####################################################################
        if results.multi_hand_landmarks is not None:
            for hand_landmarks, handedness in zip(results.multi_hand_landmarks,
                                                  results.multi_handedness):
                # 外接矩形の計算
                brect = calc_bounding_rect(debug_image, hand_landmarks)
                # ランドマークの計算
                landmark_list = calc_landmark_list(debug_image, hand_landmarks)

                # 相対座標・正規化座標への変換
                pre_processed_landmark_list = pre_process_landmark(
                    landmark_list)
                pre_processed_point_history_list = pre_process_point_history(
                    debug_image, point_history)
                # 学習データ保存
                logging_csv(number, mode, pre_processed_landmark_list,
                            pre_processed_point_history_list)

                # ハンドサイン分類
                hand_sign_id = keypoint_classifier(pre_processed_landmark_list)
                if hand_sign_id == 2:  # 指差しサイン
                    point_history.append(landmark_list[8])  # 人差指座標
                else:
                    point_history.append([0, 0])

                # フィンガージェスチャー分類
                finger_gesture_id = 0
                point_history_len = len(pre_processed_point_history_list)
                if point_history_len == (history_length * 2):
                    finger_gesture_id = point_history_classifier(
                        pre_processed_point_history_list)

                # 直近検出の中で最多のジェスチャーIDを算出
                finger_gesture_history.append(finger_gesture_id)
                most_common_fg_id = Counter(
                    finger_gesture_history).most_common()

                # 描画
                debug_image = draw_bounding_rect(use_brect, debug_image, brect)
                debug_image = draw_landmarks(debug_image, landmark_list)
                debug_image = draw_info_text(
                    debug_image,
                    brect,
                    handedness,
                    keypoint_classifier_labels[hand_sign_id],
                    point_history_classifier_labels[most_common_fg_id[0][0]],
                )
        else:
            point_history.append([0, 0])

        debug_image = draw_point_history(debug_image, point_history)
        debug_image = draw_info(debug_image, fps, mode, number)

        # 画面反映 #############################################################
        cv.imshow('Hand Gesture Recognition', debug_image)

    cap.release()
    cv.destroyAllWindows()
Esempio n. 15
0
def main():
    # init global vars
    global gesture_buffer
    global gesture_id
    global battery_status

    # Argument parsing
    args = get_args()
    KEYBOARD_CONTROL = args.is_keyboard
    WRITE_CONTROL = False
    in_flight = False

    # Camera preparation
    tello = Tello()
    #print(dir(tello))

    tello.connect()
    tello.set_speed(speed["slow"])
    print("\n\n" + tello.get_speed() + "\n\n")

    tello.streamon()

    cap_drone = tello.get_frame_read()
    cap_webcam = cv.VideoCapture(0)

    # Init Tello Controllers
    gesture_controller = TelloGestureController(tello)
    keyboard_controller = TelloKeyboardController(tello)

    gesture_detector = GestureRecognition(args.use_static_image_mode,
                                          args.min_detection_confidence,
                                          args.min_tracking_confidence)
    gesture_buffer = GestureBuffer(buffer_len=args.buffer_len)

    def tello_control(key, keyboard_controller, gesture_controller):
        global gesture_buffer

        if KEYBOARD_CONTROL:
            keyboard_controller.control(key)
        else:
            gesture_controller.gesture_control(gesture_buffer)

    def tello_battery(tello):
        global battery_status
        battery_status = tello.get_battery()

    # FPS Measurement
    cv_fps_calc = CvFpsCalc(buffer_len=10)

    mode = 0
    number = -1
    battery_status = -1

    tello.move_down(20)

    while True:
        fps = cv_fps_calc.get()

        # Process Key (ESC: end)
        key = cv.waitKey(1) & 0xff
        if key == 27:  # ESC
            break
        elif key == 32:  # Space
            if not in_flight:
                # Take-off drone
                tello.takeoff()
                in_flight = True

            elif in_flight:
                # Land tello
                tello.land()
                in_flight = False

        elif key == ord('k'):
            mode = 0
            KEYBOARD_CONTROL = True
            WRITE_CONTROL = False
            tello.send_rc_control(0, 0, 0, 0)  # Stop moving
        elif key == ord('g'):
            KEYBOARD_CONTROL = False
        elif key == ord('n'):
            mode = 1
            WRITE_CONTROL = True
            KEYBOARD_CONTROL = True

        if WRITE_CONTROL:
            number = -1
            if 48 <= key <= 57:  # 0 ~ 9
                number = key - 48

        # Camera capture
        image_drone = cap_drone.frame
        image = cap_webcam.read()[1]

        try:
            debug_image, gesture_id = gesture_detector.recognize(
                image, number, mode)
            gesture_buffer.add_gesture(gesture_id)

            # Start control thread
            threading.Thread(target=tello_control,
                             args=(
                                 key,
                                 keyboard_controller,
                                 gesture_controller,
                             )).start()
            threading.Thread(target=tello_battery, args=(tello, )).start()

            debug_image = gesture_detector.draw_info(debug_image, fps, mode,
                                                     number)

            battery_str_postion = (5, 100)  # dustin webcam
            #battery_str_postion = (5, 720 - 5) # drone camera

            # Battery status and image rendering
            cv.putText(debug_image, "Battery: {}".format(battery_status),
                       battery_str_postion, cv.FONT_HERSHEY_SIMPLEX, 1,
                       (0, 0, 255), 2)

            modeStr = "gestures"
            if KEYBOARD_CONTROL:
                modeStr = "keyboard"

            cv.putText(debug_image, modeStr + " control", (5, 150),
                       cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)

            cv.imshow('Webcam Gesture Recognition', debug_image)
            cv.imshow('Tello drone camera', image_drone)
        except:
            print("exception")

    tello.land()
    tello.end()
    cv.destroyAllWindows()
Esempio n. 16
0
def main():

    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    upper_body_only = args.upper_body_only
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    use_brect = args.use_brect

    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    mp_holistic = mp.solutions.holistic
    holistic = mp_holistic.Holistic(
        upper_body_only=upper_body_only,
        min_detection_confidence=min_detection_confidence,
        min_tracking_confidence=min_tracking_confidence,
    )

    # FPS
    cvFpsCalc = CvFpsCalc(buffer_len=10)

    while True:
        display_fps = cvFpsCalc.get()

        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)
        debug_image = copy.deepcopy(image)

        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)

        image.flags.writeable = False
        results = holistic.process(image)
        image.flags.writeable = True

        # Face Mesh
        face_landmarks = results.face_landmarks
        if face_landmarks is not None:

            brect = calc_bounding_rect(debug_image, face_landmarks)

            debug_image = draw_face_landmarks(debug_image, face_landmarks)
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        # Pose
        pose_landmarks = results.pose_landmarks
        if pose_landmarks is not None:

            brect = calc_bounding_rect(debug_image, pose_landmarks)

            debug_image = draw_pose_landmarks(debug_image, pose_landmarks,
                                              upper_body_only)
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        # Hands
        left_hand_landmarks = results.left_hand_landmarks
        right_hand_landmarks = results.right_hand_landmarks

        if left_hand_landmarks is not None:

            cx, cy = calc_palm_moment(debug_image, left_hand_landmarks)

            brect = calc_bounding_rect(debug_image, left_hand_landmarks)

            debug_image = draw_hands_landmarks(debug_image, cx, cy,
                                               left_hand_landmarks,
                                               upper_body_only, 'R')
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        if right_hand_landmarks is not None:

            cx, cy = calc_palm_moment(debug_image, right_hand_landmarks)

            brect = calc_bounding_rect(debug_image, right_hand_landmarks)

            debug_image = draw_hands_landmarks(debug_image, cx, cy,
                                               right_hand_landmarks,
                                               upper_body_only, 'L')
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)

        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        cv.imshow('MediaPipe Holistic Demo', debug_image)

    cap.release()
    cv.destroyAllWindows()
Esempio n. 17
0
def main():
    print("Image Classification Start...\n")

    # カメラ準備 ##############################################################
    cap = cv.VideoCapture(0)
    frame_width = 960
    frame_height = 540
    cap.set(cv.CAP_PROP_FRAME_WIDTH, frame_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, frame_height)

    # ImageNet 日本語ラベル ###################################################
    jsonfile = open('utils/imagenet_class_index.json',
                    'r',
                    encoding="utf-8_sig")
    imagenet_ja_labels = json.load(jsonfile)
    imagenet_ja_label = {}
    for label_temp in imagenet_ja_labels:
        imagenet_ja_label[label_temp['num']] = label_temp['ja']

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc()

    # モデルロード ############################################################
    model = tf.keras.applications.EfficientNetB0(
        include_top=True,
        weights='imagenet',
        input_shape=(224, 224, 3),
    )

    # メインループ #############################################################
    detection_count = 0
    while True:
        # FPS算出 #############################################################
        display_fps = cvFpsCalc.get()
        if display_fps == 0:
            display_fps = 0.01

        # カメラキャプチャ ####################################################
        ret, frame = cap.read()
        if not ret:
            continue
        debug_image = copy.deepcopy(frame)

        # 検出実施 ############################################################
        trim_x1 = int(frame_width / 4) + 50
        trim_x2 = int(frame_width / 4 * 3) - 50
        trim_y1 = int(frame_height / 8)
        trim_y2 = int(frame_height / 8 * 7)
        trimming_image = debug_image[trim_y1:trim_y2, trim_x1:trim_x2]

        classifications = run_classify(model, trimming_image)

        # デバッグ情報描画 ####################################################
        # 表示名作成
        classification_string = ""
        for classification in classifications:
            if float(classification[2]) > 0.5:
                detection_count += 1

                classification_string = imagenet_ja_label[
                    classification[0]] + ":" + str('{:.1f}'.format(
                        float(classification[2]) * 100)) + "%"
            else:
                detection_count = 0
            break  # 1件のみ

        # 描画
        debug_image = draw_demo_image(
            debug_image,
            detection_count,
            classification_string,
            display_fps,
            (trim_x1, trim_y1, trim_x2, trim_y2),
        )

        # 画面反映 #######################################################
        cv.imshow('Demo', debug_image)

        key = cv.waitKey(1)
        if key == 27:  # ESC
            break
def main():
    # 引数解析 #################################################################
    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    upper_body_only = args.upper_body_only
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    use_brect = args.use_brect

    # カメラ準備 ###############################################################
    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    # モデルロード #############################################################
    mp_holistic = mp.solutions.holistic
    holistic = mp_holistic.Holistic(
        upper_body_only=upper_body_only,
        min_detection_confidence=min_detection_confidence,
        min_tracking_confidence=min_tracking_confidence,
    )

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc(buffer_len=10)

    while True:
        display_fps = cvFpsCalc.get()

        # カメラキャプチャ #####################################################
        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)  # ミラー表示
        debug_image = copy.deepcopy(image)

        # 検出実施 #############################################################
        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)

        image.flags.writeable = False
        results = holistic.process(image)
        image.flags.writeable = True

        # Face Mesh ###########################################################
        face_landmarks = results.face_landmarks
        if face_landmarks is not None:
            # 外接矩形の計算
            brect = calc_bounding_rect(debug_image, face_landmarks)
            # 描画
            debug_image = draw_face_landmarks(debug_image, face_landmarks)
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        # Pose ###############################################################
        pose_landmarks = results.pose_landmarks
        if pose_landmarks is not None:
            # 外接矩形の計算
            brect = calc_bounding_rect(debug_image, pose_landmarks)
            # 描画
            debug_image = draw_pose_landmarks(debug_image, pose_landmarks,
                                              upper_body_only)
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        # Hands ###############################################################
        left_hand_landmarks = results.left_hand_landmarks
        right_hand_landmarks = results.right_hand_landmarks
        # 左手
        if left_hand_landmarks is not None:
            # 手の平重心計算
            cx, cy = calc_palm_moment(debug_image, left_hand_landmarks)
            # 外接矩形の計算
            brect = calc_bounding_rect(debug_image, left_hand_landmarks)
            # 描画
            debug_image = draw_hands_landmarks(debug_image, cx, cy,
                                               left_hand_landmarks,
                                               upper_body_only, 'R')
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)
        # 右手
        if right_hand_landmarks is not None:
            # 手の平重心計算
            cx, cy = calc_palm_moment(debug_image, right_hand_landmarks)
            # 外接矩形の計算
            brect = calc_bounding_rect(debug_image, right_hand_landmarks)
            # 描画
            debug_image = draw_hands_landmarks(debug_image, cx, cy,
                                               right_hand_landmarks,
                                               upper_body_only, 'L')
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 255), 2, cv.LINE_AA)
        cv.putText(debug_image, "Mostafizur Rahman", (250, 450),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 255), 2, cv.LINE_AA)

        # キー処理(ESC:終了) #################################################
        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        # 画面反映 #############################################################
        cv.imshow('MediaPipe Holistic Demo', debug_image)

    cap.release()
    cv.destroyAllWindows()
Esempio n. 19
0
def main():

    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    max_num_hands = args.max_num_hands
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    use_brect = args.use_brect


    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    # モデルロード #############################################################
    mp_hands = mp.solutions.hands
    hands = mp_hands.Hands(
        max_num_hands=max_num_hands,
        min_detection_confidence=min_detection_confidence,
        min_tracking_confidence=min_tracking_confidence,
    )


    cvFpsCalc = CvFpsCalc(buffer_len=10)

    while True:
        display_fps = cvFpsCalc.get()


        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)
        debug_image = copy.deepcopy(image)

        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
        results = hands.process(image)

        # 描画 ################################################################
        if results.multi_hand_landmarks is not None:
            for hand_landmarks, handedness in zip(results.multi_hand_landmarks,
                                                  results.multi_handedness):
                # 手の平重心計算
                cx, cy = calc_palm_moment(debug_image, hand_landmarks)
                # 外接矩形の計算
                brect = calc_bounding_rect(debug_image, hand_landmarks)
                # 描画
                debug_image = draw_landmarks(debug_image, cx, cy,
                                             hand_landmarks, handedness)
                debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        cv.putText(debug_image, "Mosta_FPS:" + str(display_fps), (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)

        key = cv.waitKey(1)
        if key == 27:  # ESC
            break


        cv.imshow('MediaPipe Hand Demo', debug_image)

    cap.release()
    cv.destroyAllWindows()
def main():

    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    static_image_mode = args.static_image_mode
    max_num_objects = args.max_num_objects
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence
    model_name = args.model_name

    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    mp_objectron = mp.solutions.objectron
    objectron = mp_objectron.Objectron(
        static_image_mode=static_image_mode,
        max_num_objects=max_num_objects,
        min_detection_confidence=min_detection_confidence,
        min_tracking_confidence=min_tracking_confidence,
        model_name=model_name,
    )

    mp_drawing = mp.solutions.drawing_utils

    cvFpsCalc = CvFpsCalc(buffer_len=10)

    while True:
        display_fps = cvFpsCalc.get()

        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)  # ミラー表示
        debug_image = copy.deepcopy(image)

        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
        results = objectron.process(image)

        if results.detected_objects is not None:
            for detected_object in results.detected_objects:
                mp_drawing.draw_landmarks(debug_image,
                                          detected_object.landmarks_2d,
                                          mp_objectron.BOX_CONNECTIONS)
                mp_drawing.draw_axis(debug_image, detected_object.rotation,
                                     detected_object.translation)

                draw_landmarks(debug_image, detected_object.landmarks_2d)

        cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)

        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        cv.imshow('MediaPipe Objectron Demo', debug_image)

    cap.release()
    cv.destroyAllWindows()
    def run(self):
        # 重写线程执行的run函数
        # 触发自定义信号
        self.stop_flag = False
        # Argument parsing #################################################################
        args = get_args()

        cap_device = args.device
        cap_width = args.width
        cap_height = args.height

        use_static_image_mode = args.use_static_image_mode
        min_detection_confidence = args.min_detection_confidence
        min_tracking_confidence = args.min_tracking_confidence

        use_brect = True

        # Camera preparation ###############################################################
        cap = cv.VideoCapture(cap_device)
        cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
        cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

        # Model load #############################################################
        mp_hands = mp.solutions.hands
        hands = mp_hands.Hands(
            static_image_mode=use_static_image_mode,
            max_num_hands=1,
            min_detection_confidence=min_detection_confidence,
            min_tracking_confidence=min_tracking_confidence,
        )

        mouse_classifier = MouseClassifier(invalid_value=2, score_th=0.4)
        point_history_classifier = PointHistoryClassifier()

        # Read labels ###########################################################

        with open(
                'model/point_history_classifier/point_history_classifier_label.csv',
                encoding='utf-8-sig') as f:
            point_history_classifier_labels = csv.reader(f)
            point_history_classifier_labels = [
                row[0] for row in point_history_classifier_labels
            ]
        with open('model/mouse_classifier/mouse_classifier_label.csv',
                  encoding='utf-8-sig') as f:
            mouse_classifier_labels = csv.reader(f)
            mouse_classifier_labels = [
                row[0] for row in mouse_classifier_labels
            ]
        # FPS Measurement ########################################################
        cvFpsCalc = CvFpsCalc(buffer_len=3)

        # Coordinate history #################################################################
        history_length = 16
        point_history = deque(maxlen=history_length)

        # Finger gesture history ################################################
        finger_gesture_history = deque(maxlen=history_length)
        mouse_id_history = deque(maxlen=30)
        m_id_history = deque(maxlen=6)

        # ========= 按鍵前置作業 =========
        mode = 0
        presstime = resttime = time.time()

        mode_change = False
        detect_mode = 0
        what_mode = 'Sleep'
        landmark_list = 0
        pyautogui.PAUSE = 0
        i = 0

        # ========= 滑鼠前置作業 =========
        wScr, hScr = pyautogui.size()
        frameR = 100
        smoothening = 10
        plocX, plocY = 0, 0
        clocX, clocY = 0, 0
        # 關閉 滑鼠移至角落啟動保護措施
        pyautogui.FAILSAFE = False

        # ========= 主程式運作 =========
        while self.stop_flag == False:
            mouse_id = -1
            fps = cvFpsCalc.get()

            # Process Key (ESC: end)
            key = cv.waitKey(10)
            if key == 27:  # ESC
                break
            number, mode = select_mode(key, mode)

            # Camera capture
            ret, image = cap.read()
            if not ret:
                break
            image = cv.flip(image, 1)  # Mirror display
            debug_image = copy.deepcopy(image)

            # Detection implementation
            image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
            # image = cv.cvtColor(image, cv.COLOR_BGR2GRAY)

            image.flags.writeable = False
            results = hands.process(image)
            image.flags.writeable = True

            #  ####################################################################
            if results.multi_hand_landmarks is not None:
                for hand_landmarks, handedness in zip(
                        results.multi_hand_landmarks,
                        results.multi_handedness):

                    # Bounding box calculation
                    brect = calc_bounding_rect(debug_image, hand_landmarks)
                    # Landmark calculation
                    landmark_list = calc_landmark_list(debug_image,
                                                       hand_landmarks)
                    # print(landmark_list)

                    # Conversion to relative coordinates / normalized coordinates
                    pre_processed_landmark_list = pre_process_landmark(
                        landmark_list)
                    pre_processed_point_history_list = pre_process_point_history(
                        debug_image, point_history)
                    # Write to the dataset file
                    logging_csv(number, mode, pre_processed_landmark_list,
                                pre_processed_point_history_list)

                    # # 靜態手勢資料預測

                    mouse_id = mouse_classifier(pre_processed_landmark_list)
                    # print(mouse_id)

                    # 手比one 觸發動態資料抓取
                    if mouse_id == 0:
                        point_history.append(landmark_list[8])
                    else:
                        point_history.append([0, 0])

                    # 動態手勢資料預測
                    finger_gesture_id = 0
                    point_history_len = len(pre_processed_point_history_list)
                    if point_history_len == (history_length * 2):
                        finger_gesture_id = point_history_classifier(
                            pre_processed_point_history_list)
                    # print(finger_gesture_id) # 0 = stop, 1 = clockwise, 2 = counterclockwise, 3 = move,偵測出現的動態手勢

                    # 動態手勢最常出現id #########################################
                    # Calculates the gesture IDs in the latest detection
                    finger_gesture_history.append(finger_gesture_id)
                    most_common_fg_id = Counter(
                        finger_gesture_history).most_common()

                    # 滑鼠的deque
                    mouse_id_history.append(mouse_id)
                    most_common_ms_id = Counter(mouse_id_history).most_common()
                    # print(most_common_ms_id)

                    m_id_history.append(mouse_id)
                    m_id = Counter(m_id_history).most_common(2)
                    mouse_id = m_id[0][0] if m_id[0][1] >= 4 else 2

                    # print(f'm_id {m_id}\n, mouse_id {mouse_id}')
                    # ===== 偵測到手時,重製紀錄時間 ==============================
                    resttime = time.time()

                    ###############################################################

                    # Drawing part
                    debug_image = draw_bounding_rect(use_brect, debug_image,
                                                     brect)
                    debug_image = draw_landmarks(debug_image, landmark_list)
                    debug_image = draw_info_text(
                        debug_image, brect, handedness,
                        mouse_classifier_labels[mouse_id],
                        point_history_classifier_labels[most_common_fg_id[0]
                                                        [0]])

            else:
                point_history.append([0, 0])

            debug_image = draw_point_history(debug_image, point_history)
            debug_image = draw_info(debug_image, fps, mode, number)

            # 偵測是否有手勢 #########################################
            if mouse_id > -1:
                # change mode Gesture six changes to the different mode
                if most_common_ms_id[0][0] == 3 and most_common_ms_id[0][
                        1] == 30:
                    if time.time() - presstime > 2:
                        detect_mode = (detect_mode + 1) % 3
                        mode_change = True
                        presstime = time.time()

                    # control keyboard
                elif detect_mode == 2:
                    # 靜態手勢控制
                    presstime = control_keyboard(mouse_id, 1, 'K', presstime)
                    presstime = control_keyboard(mouse_id, 8, 'C', presstime)
                    presstime = control_keyboard(mouse_id, 5, 'up', presstime)
                    presstime = control_keyboard(mouse_id, 6, 'down',
                                                 presstime)
                    # presstime = control_keyboard(most_common_keypoint_id, 0, 'right', presstime)
                    # presstime = control_keyboard(most_common_keypoint_id, 7, 'left', presstime)

                    if mouse_id == 4:
                        # print(i, time.time() - presstime)
                        if i == 3 and time.time() - presstime > 0.3:
                            pyautogui.press('right')
                            i = 0
                            presstime = time.time()
                        elif i == 3 and time.time() - presstime > 0.25:
                            pyautogui.press('right')
                            presstime = time.time()
                        elif time.time() - presstime > 1:
                            pyautogui.press('right')
                            i += 1
                            presstime = time.time()

                    if mouse_id == 7:
                        # print(i, time.time() - presstime)
                        if i == 3 and time.time() - presstime > 0.3:
                            pyautogui.press('left')
                            i = 0
                            presstime = time.time()
                        elif i == 3 and time.time() - presstime > 0.25:
                            pyautogui.press('left')
                            presstime = time.time()
                        elif time.time() - presstime > 1:
                            pyautogui.press('left')
                            i += 1
                            presstime = time.time()

                    # 動態手勢控制
                    if most_common_fg_id[0][
                            0] == 1 and most_common_fg_id[0][1] > 9:
                        if time.time() - presstime > 1.5:
                            pyautogui.hotkey('shift', '>')
                            print('speed up')
                            presstime = time.time()

                    elif most_common_fg_id[0][
                            0] == 2 and most_common_fg_id[0][1] > 12:
                        if time.time() - presstime > 1.5:
                            pyautogui.hotkey('shift', '<')
                            print('speed down')
                            presstime = time.time()

                elif detect_mode == 1:
                    if mouse_id == 0:  # Point gesture
                        x1, y1 = landmark_list[8]
                        cv.rectangle(debug_image, (50, 30),
                                     (cap_width - 50, cap_height - 170),
                                     (255, 0, 255), 2)
                        # 座標轉換
                        # x軸: 鏡頭上50~(cap_width - 50)轉至螢幕寬0~wScr
                        # y軸: 鏡頭上30~(cap_height - 170)轉至螢幕長0~hScr
                        x3 = np.interp(x1, (50, (cap_width - 50)), (0, wScr))
                        y3 = np.interp(y1, (30, (cap_height - 170)), (0, hScr))

                        # 6. Smoothen Values
                        clocX = plocX + (x3 - plocX) / smoothening
                        clocY = plocY + (y3 - plocY) / smoothening

                        # 7. Move Mouse
                        pyautogui.moveTo(clocX, clocY)
                        cv.circle(debug_image, (x1, y1), 15, (255, 0, 255),
                                  cv.FILLED)
                        plocX, plocY = clocX, clocY

                    elif mouse_id == 1:
                        # 10. Click mouse if distance short
                        if time.time() - presstime > 0.5:
                            pyautogui.click()
                            presstime = time.time()

                    if mouse_id == 5:
                        pyautogui.scroll(-20)

                    if mouse_id == 6:
                        pyautogui.scroll(20)

                    if mouse_id == 7:
                        if time.time() - presstime > 1.5:
                            pyautogui.click(clicks=2)
                            presstime = time.time()

                    if mouse_id == 8:
                        if time.time() - presstime > 2:
                            pyautogui.hotkey('alt', 'left')
                            presstime = time.time()

                # 比讚 從休息模式 換成 鍵盤模式
                elif detect_mode == 0:
                    if mouse_id == 5:
                        i += 1
                        if i == 1 or time.time() - presstime > 3:
                            presstime = time.time()
                        elif time.time() - presstime > 2:
                            detect_mode = 2
                            mode_change = True
                            i = 0

            # 距離上次監測到手的時間大於30秒、切回休息模式 =========================
            if time.time() - resttime > 30:
                if detect_mode != 0:
                    detect_mode = 0
                    mode_change = True

            # 檢查模式有沒有更動 ========================
            if mode_change:
                if detect_mode == 0:
                    what_mode = 'Sleep'
                elif detect_mode == 2:
                    what_mode = 'Keyboard'
                elif detect_mode == 1:
                    what_mode = 'Mouse'

                mode_change = False
                print('Mode has changed')
                print(f'Current mode => {what_mode}')

            cv.putText(debug_image, what_mode, (480, 30),
                       cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 4, cv.LINE_AA)
            # Screen reflection ###################################JL##########################
            cv.imshow('Hand Gesture Recognition', debug_image)

            self.trigger.emit(detect_mode)

        cap.release()
        cv.destroyAllWindows()
Esempio n. 22
0
def HandTracking(cap, width, height, conf_flg=0):
    # ×ボタンが押されたかのフラグ(hand_gui.py内の変数、flg_closePush)の初期化
    hand_gui.close_switch_py(0)
    # 引数解析 #################################################################
    args = get_args()

    flg_video = 0  #「1」でカメラが接続されていない
    flg_break = 0  #「1」で最初のループを抜け終了する⇒正常終了
    name_pose = "Unknown"
    focus_flg = 1  #index.html の表示・非表示の切り替え、「0」:Main.pyで開いた場合、「1」:HandTracking.pyで開いた場合
    namePose_flg = 1  #complete_old.htmlの開始・終了フラグ
    potision_flg = 0
    #flg_closePush = 0
    global ShortCutList
    ShortCutList = config_sys_set()
    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    use_static_image_mode = args.use_static_image_mode
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    use_brect = True
    #width,height = autopy.screen.size() #eel で立ち上げた際の表示位置を指定するために取得

    while (True):  #カメラが再度接続するまでループ処理
        #カメラが接続されていないフラグの場合
        if (flg_video == 1):
            #カメラが接続されているか確認
            cap = cv.VideoCapture(cap_device)
            cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
            cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
            ret, frame = cap.read()
            if (ret is True):
                flg_video = 0
                name_pose = "Unknown"
                focus_flg = 1
                namePose_flg = 1
                cap.release()
                eel.overlay_controll(True)
                eel.object_change("demo2.html", True)
                #eel.sleep(1)
                sel_cam_before = 999

                while (True):
                    if (decideFlgHT == 1):
                        if (sel_camHT != sel_cam_before):
                            cap = cv.VideoCapture(sel_camHT)
                            cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
                            cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
                            ret, frame = cap.read(sel_camHT)
                            if (ret is False):
                                eel.alert_mess()
                                cap.release()
                                decide_camHT_py(999)
                                decide_flgHT_py(0)
                                sel_cam_before = sel_camHT
                                continue
                            else:
                                decide_flgHT_py(0)
                                break
                        decide_flgHT_py(0)
                        break
                    if (sel_camHT != 999):
                        eel.sleep(0.01)
                        if (sel_camHT != sel_cam_before):
                            if (sel_cam_before != 999):
                                cap.release()
                            cap = cv.VideoCapture(sel_camHT)
                            cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
                            cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
                            sel_cam_before = sel_camHT
                        ret, frame = cap.read()
                        if (ret is True):
                            # UI側へ転送(画像) #####################################################
                            _, imencode_image = cv.imencode('.jpg', frame)
                            base64_image = base64.b64encode(imencode_image)
                            eel.set_base64image("data:image/jpg;base64," +
                                                base64_image.decode("ascii"))
                            continue
                        else:
                            eel.alert_mess()
                            cap.release()
                            decide_camHT_py(999)
                            sel_cam_before = sel_camHT
                            if (decideFlgHT == 1):
                                decide_flgHT_py(0)
                            continue
                    else:
                        eel.sleep(0.01)
                        continue  #最初の while に戻る
            else:
                #カメラが接続されていない場合
                #print("webcamないよ!!!")
                eel.sleep(0.01)
                time.sleep(0.01)
                continue  #最初の while に戻る
        elif (flg_break == 1):
            decide_camHT_py(999)
            decide_flgHT_py(0)
            break  #最初の while を抜けて正常終了

        # カメラ準備 ###############################################################
        #cap = cv.VideoCapture(cap_device)
        #cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
        #cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

        # モデルロード #############################################################
        mp_hands = mp.solutions.hands
        hands = mp_hands.Hands(
            static_image_mode=use_static_image_mode,
            max_num_hands=1,
            min_detection_confidence=min_detection_confidence,
            min_tracking_confidence=min_tracking_confidence,
        )

        keypoint_classifier = KeyPointClassifier()

        point_history_classifier = PointHistoryClassifier()

        # ラベル読み込み ###########################################################
        with open('model/keypoint_classifier/keypoint_classifier_label.csv',
                  encoding='utf-8-sig') as f:
            keypoint_classifier_labels = csv.reader(f)
            keypoint_classifier_labels = [
                row[0] for row in keypoint_classifier_labels
            ]
        with open(
                'model/point_history_classifier/point_history_classifier_label.csv',
                encoding='utf-8-sig') as f:
            point_history_classifier_labels = csv.reader(f)
            point_history_classifier_labels = [
                row[0] for row in point_history_classifier_labels
            ]

        # FPS計測モジュール ########################################################
        cvFpsCalc = CvFpsCalc(buffer_len=10)

        # 座標履歴 #################################################################
        history_length = 16
        point_history = deque(maxlen=history_length)

        # フィンガージェスチャー履歴 ################################################
        finger_gesture_history = deque(maxlen=history_length)

        #  ########################################################################
        mode = 0
        CountPose = [0, 0, 0, 0, 0, 0, 0]
        CountMotion = [0, 0, 0, 0]  # [Top,Right,Down,Left]
        identification = False
        while True:
            fps = cvFpsCalc.get()
            # キー処理(ESC:終了) #################################################
            key = cv.waitKey(10)
            if key == 27:  # ESC
                break
            number, mode = select_mode(key, mode)

            # カメラキャプチャ #####################################################
            ret, image = cap.read()
            if not ret:
                #それぞれのフラグを立てて、システムを終了させ、最初の while に戻る
                flg_video = 1
                focus_flg = 0

                print("【通知】WebCameraが接続されていません。")
                eel.focusSwitch(width, height, focus_flg)
                eel.overlay_controll(True)
                eel.object_change("connect.html", True)
                cap.release()
                cv.destroyAllWindows()
                break

            image = cv.flip(image, 1)  # ミラー表示
            debug_image = copy.deepcopy(image)

            # 検出実施 #############################################################
            image = cv.cvtColor(image, cv.COLOR_BGR2RGB)

            image.flags.writeable = False
            results = hands.process(image)
            image.flags.writeable = True

            #  ####################################################################
            if results.multi_hand_landmarks is not None:
                #j=1
                for hand_landmarks, handedness in zip(
                        results.multi_hand_landmarks,
                        results.multi_handedness):
                    # 外接矩形の計算
                    brect = calc_bounding_rect(debug_image, hand_landmarks)
                    # ランドマークの計算
                    landmark_list = calc_landmark_list(debug_image,
                                                       hand_landmarks)

                    # 相対座標・正規化座標への変換
                    pre_processed_landmark_list = pre_process_landmark(
                        landmark_list)
                    pre_processed_point_history_list = pre_process_point_history(
                        debug_image, point_history)
                    # 学習データ保存
                    logging_csv(number, mode, pre_processed_landmark_list,
                                pre_processed_point_history_list)

                    # ハンドサイン分類
                    hand_sign_id = keypoint_classifier(
                        pre_processed_landmark_list)

                    if hand_sign_id == 1:  # Dangサイン
                        point_history.append(landmark_list[8])  # 人差指座標
                    else:
                        point_history.append([0, 0])

                    # フィンガージェスチャー分類
                    finger_gesture_id = 0
                    point_history_len = len(pre_processed_point_history_list)
                    if point_history_len == (history_length * 2):
                        finger_gesture_id = point_history_classifier(
                            pre_processed_point_history_list)

                    # 直近検出の中で最多のジェスチャーIDを算出
                    finger_gesture_history.append(finger_gesture_id)
                    most_common_fg_id = Counter(
                        finger_gesture_history).most_common()

                    gesture_name = point_history_classifier_labels[
                        most_common_fg_id[0][0]]

                    # 描画
                    debug_image = draw_bounding_rect(use_brect, debug_image,
                                                     brect)
                    debug_image = draw_landmarks(debug_image, landmark_list)
                    debug_image = draw_info_text(
                        debug_image,
                        brect,
                        handedness,
                        keypoint_classifier_labels[hand_sign_id],
                        gesture_name,
                    )

                    #人差し指の先の座標を取得
                    x, y = landmark_list[8]
                    #座標調整
                    x_width = args.width * 0.05
                    x = x - x_width
                    x = x * 1.5
                    y = y * 1.5
                    #ジェスチャーが判定された回数をカウント
                    if gesture_name == 'Stop':
                        CountMotion = [0, 0, 0, 0]
                    elif gesture_name == 'Move_Top':
                        Count_temp = CountMotion[0]
                        Count_temp += 1
                        CountMotion = [Count_temp, 0, 0, 0]
                    elif gesture_name == 'Move_Right':
                        Count_temp = CountMotion[1]
                        Count_temp += 1
                        CountMotion = [0, Count_temp, 0, 0]
                    elif gesture_name == 'Move_Down':
                        Count_temp = CountMotion[2]
                        Count_temp += 1
                        CountMotion = [0, 0, Count_temp, 0]
                    elif gesture_name == 'Move_Left':
                        Count_temp = CountMotion[3]
                        Count_temp += 1
                        CountMotion = [0, 0, 0, Count_temp]

                    #各種操作の実行
                    CountPose, CountMotion = PoseAction.action(
                        hand_sign_id, x, y, CountPose, CountMotion,
                        ShortCutList)
                    name_pose = keypoint_classifier_labels[hand_sign_id]
                    eel.set_posegauge(str(name_pose))
                    identification = True

            else:
                point_history.append([0, 0])
                if identification == True:
                    eel.set_posegauge('None')
                    identification = False
                    eel.shortcut_overlay(False, 0)

            debug_image = draw_point_history(debug_image, point_history)
            debug_image = draw_info(debug_image, fps, mode, number)

            # 画面反映 #############################################################
            debug_image = cv.resize(debug_image, dsize=(520, 260))
            cv.imshow('FOCUS preview', debug_image)
            if potision_flg == 0:
                cv.moveWindow('FOCUS preview', 0, 0)
                potision_flg = 1

            # eel立ち上げ #############################################################
            #cnt_gui, flg_end, flg_restart, flg_start, keep_flg = hand_gui.start_gui(cnt_gui, name_pose, flg_restart, flg_start, keep_flg)

            if (namePose_flg == 1):
                eel.object_change("complete_old.html", True)
                #eel.sleep(0.01)
                #eel.init("GUI/web")
                #eel.start("開きたい上記のフォルダ下のファイル名",~
                #eel.start("html/keeper.html",
                #            port = 0,
                #            mode='chrome',
                #            size=(4, 2),  #サイズ指定(横, 縦)
                #            position=(width,0), #位置指定(left, top)
                #            block=False
                #            )
                eel.sleep(0.01)
                print("【通知】準備完了")
                namePose_flg = 0
            elif (focus_flg == 1 and name_pose != 'Unknown'):
                eel.object_change("complete_old.html", False)
                eel.overlay_controll(False)
                eel.focusSwitch(width, height, focus_flg)
                print("【実行】index.html")
                eel.sleep(0.01)
                #eel.set_posegauge(name_pose,i)
                focus_flg = 0
                #i+=1

            # eel立ち上げ #############################################################
            flg_end = hand_gui.start_gui()
            if (flg_end == 1):
                #正常に終了する処理(中間のループを抜ける)
                flg_break = 1
                eel.endSwitch()  #flg_end の値をもとに戻す関数
                cap.release()
                cv.destroyAllWindows()
                eel.overlay_controll(False)
                eel.object_change("endpage.html", False)
                #eel.windowclose_keeper()
                break
Esempio n. 23
0
def main():
       
    

    # 引数 #####################################################################
    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    max_num_faces = args.max_num_faces
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    # カメラ準備 ###############################################################
    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    # モデルロード #############################################################
    face_mesh = FaceMesh(
        max_num_faces,
        min_detection_confidence,
        min_tracking_confidence,
    )
    iris_detector = IrisLandmark()

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc(buffer_len=10)
    
    ## parameter ##
    left_list = [] # for calculate thres size of iris
    right_list = []
    prepare = 1 # 1 - preparation mode, 0 - detection mode
    stack = 0 # if stack if enough => sleep
    flag = 0 # flag for terminate preparation mode

    while True:
        display_fps = cvFpsCalc.get()
        # カメラキャプチャ #####################################################
        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)  # ミラー表示
        debug_image = copy.deepcopy(image)

        # 検出実施 #############################################################
        # Face Mesh検出
        face_results = face_mesh(image)
        for face_result in face_results:
            # 目周辺のバウンディングボックス計算
            left_eye, right_eye = face_mesh.calc_around_eye_bbox(face_result)

            # 虹彩検出
            left_iris, right_iris = detect_iris(image, iris_detector, left_eye, right_eye)

            # 虹彩の外接円を計算
            left_center, left_radius = calc_min_enc_losingCircle(left_iris)
            right_center, right_radius = calc_min_enc_losingCircle(right_iris)


            if (flag==150):
                prepare = 0
                print("#############################################################################\nready\n#############################################################################")
                flag += 1
            elif (flag<150): prepare = 1

            if (prepare):
                flag += 1
                left_list.append(left_radius)
                right_list.append(right_radius)
                left_avg = sum(left_list, 0.0) / len(left_list)
                right_avg = sum(right_list, 0.0) / len(right_list)
                left_thres = 0.80 * left_avg
                right_thres = 0.80 * right_avg
                print("calculating avg iris size")
            
            else:
                num_of_small_iris = 0
                if (left_radius<left_thres): num_of_small_iris+=1
                if (right_radius<left_thres): num_of_small_iris+=1
                if num_of_small_iris==2:
                    stack += 1
                else:
                    if stack != 0:
                        stack -= 1
                if stack>= 30:
                    print("########################################################\nWARNING!!!DON'T SLEEP\n########################################################")
                else:
                    print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!NOT SLEEPING!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
            # デバッグ描画
            debug_image = draw_debug_image(
                debug_image,
                left_iris,
                right_iris,
                left_center,
                left_radius,
                right_center,
                right_radius,
            )

        cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)

        # キー処理(ESC:終了) #################################################
        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        # 画面反映 #############################################################
        cv.imshow('Iris(tflite) Demo', debug_image)

    cap.release()
    cv.destroyAllWindows()

    return
Esempio n. 24
0
def main():
    # 引数解析 #################################################################
    args = get_args()
    cap_device = args.device
    cap_width = args.width
    cap_height = args.height
    ceil_num = args.ceil
    image_ratio = args.image_ratio
    x_offset = args.x_offset
    y_offset = args.y_offset

    # カメラ準備 ###############################################################
    cap = cv.VideoCapture(cap_device)

    # 重畳画像準備 #############################################################
    image_pathlist = sorted(glob.glob('utils/image/*.png'))
    images = []
    for image_path in image_pathlist:
        images.append(cv.imread(image_path, cv.IMREAD_UNCHANGED))

    animation_counter = 0

    # CenterFace準備 ###########################################################
    centerface = CenterFace()

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc()

    # フォント
    font_path = './utils/font/x12y20pxScanLine.ttf'

    while True:
        # FPS算出 #############################################################
        display_fps = cvFpsCalc.get()

        # カメラキャプチャ #####################################################
        ret, frame = cap.read()
        if not ret:
            break
        resize_frame = cv.resize(frame, (int(cap_width), int(cap_height)))
        frame_height, frame_width = resize_frame.shape[:2]

        # 顔検出 ##############################################################
        dets, lms = centerface(resize_frame,
                               frame_height,
                               frame_width,
                               threshold=0.35)

        # デバッグ表示 ########################################################
        # バウンディングボックス
        for det in dets:
            bbox, _ = det[:4], det[4]  # BBox, Score
            x1, y1 = int(bbox[0]), int(bbox[1])
            x2, y2 = int(bbox[2]), int(bbox[3])

            # 顔の立幅に合わせて重畳画像をリサイズ
            image_height, image_width = images[0].shape[:2]
            resize_ratio = (y2 - y1) / image_height
            resize_image_height = int(image_height * resize_ratio)
            resize_image_width = int(image_width * resize_ratio)

            resize_image_height = int(
                (resize_image_height + (ceil_num - 1)) / ceil_num * ceil_num)
            resize_image_width = int(
                (resize_image_width + (ceil_num - 1)) / ceil_num * ceil_num)
            resize_image_height = int(resize_image_height * image_ratio)
            resize_image_width = int(resize_image_width * image_ratio)

            resize_image = cv.resize(images[animation_counter],
                                     (resize_image_width, resize_image_height))

            # 画像描画
            overlay_x = int((x2 + x1) / 2) - int(resize_image_width / 2)
            overlay_y = int((y2 + y1) / 2) - int(resize_image_height / 2)
            resize_frame = CvOverlayImage.overlay(
                resize_frame, resize_image,
                (overlay_x + x_offset, overlay_y + y_offset))
        # ランドマーク
        # for lm in lms:
        #     for i in range(0, 5):
        #         cv.circle(resize_frame, (int(lm[i * 2]), int(lm[i * 2 + 1])),
        #                   3, (0, 0, 255), -1)

        animation_counter += 1
        if animation_counter >= len(images):
            animation_counter = 0

        # 画面反映
        fps_string = u"FPS:" + str(display_fps)
        resize_frame = CvDrawText.puttext(resize_frame, fps_string, (15, 15),
                                          font_path, 40, (255, 255, 255))
        cv.imshow('Demo', resize_frame)
        key = cv.waitKey(1)
        if key == 27:  # ESC
            break
    cap.release()
Esempio n. 25
0
    def run(self):
        # Argument parsing #################################################################
        self.stop_flag = False
        args = get_args()

        cap_device = args.device
        cap_width = args.width
        cap_height = args.height

        use_static_image_mode = args.use_static_image_mode
        min_detection_confidence = args.min_detection_confidence
        min_tracking_confidence = args.min_tracking_confidence

        use_brect = True

        # Camera preparation ###############################################################
        cap = cv.VideoCapture(cap_device)
        cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
        cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

        # Model load #############################################################
        mp_hands = mp.solutions.hands
        hands = mp_hands.Hands(
            static_image_mode=use_static_image_mode,
            max_num_hands=1,
            min_detection_confidence=min_detection_confidence,
            min_tracking_confidence=min_tracking_confidence,
        )

        keypoint_classifier_R = KeyPointClassifier_R(invalid_value=8,
                                                     score_th=0.4)
        keypoint_classifier_L = KeyPointClassifier_L(invalid_value=8,
                                                     score_th=0.4)
        mouse_classifier = MouseClassifier(invalid_value=2, score_th=0.4)
        point_history_classifier = PointHistoryClassifier()

        # Read labels ###########################################################
        with open('model/keypoint_classifier/keypoint_classifier_label.csv',
                  encoding='utf-8-sig') as f:
            keypoint_classifier_labels = csv.reader(f)
            keypoint_classifier_labels = [
                row[0] for row in keypoint_classifier_labels
            ]
        with open(
                'model/point_history_classifier/point_history_classifier_label.csv',
                encoding='utf-8-sig') as f:
            point_history_classifier_labels = csv.reader(f)
            point_history_classifier_labels = [
                row[0] for row in point_history_classifier_labels
            ]

        # FPS Measurement ########################################################
        cvFpsCalc = CvFpsCalc(buffer_len=3)

        # Coordinate history #################################################################
        history_length = 16
        point_history = deque(maxlen=history_length)

        # Finger gesture history ################################################
        finger_gesture_history = deque(maxlen=history_length)
        mouse_id_history = deque(maxlen=40)

        # 靜態手勢最常出現參數初始化
        keypoint_length = 5
        keypoint_R = deque(maxlen=keypoint_length)
        keypoint_L = deque(maxlen=keypoint_length)

        # result deque
        rest_length = 300
        rest_result = deque(maxlen=rest_length)
        speed_up_count = deque(maxlen=3)

        # ========= 使用者自訂姿勢、指令區 =========
        # time.sleep(0.5)
        # keepadd = False

        # ========= 按鍵前置作業 =========
        mode = 0
        presstime = presstime_2 = presstime_3 = resttime = presstime_4 = time.time(
        )

        detect_mode = 2
        what_mode = 'mouse'
        landmark_list = 0
        pyautogui.PAUSE = 0

        # ========= 滑鼠前置作業 =========
        wScr, hScr = pyautogui.size()
        frameR = 100
        smoothening = 7
        plocX, plocY = 0, 0
        clocX, clocY = 0, 0
        mousespeed = 1.5
        clicktime = time.time()
        #關閉 滑鼠移至角落啟動保護措施
        pyautogui.FAILSAFE = False

        # ========= google 小姐 =========
        # speech_0 = gTTS(text="スリープモード", lang='ja')
        # speech_0.save('rest.mp3')
        # speech_0 = gTTS(text="キーボードモード", lang='ja')
        # speech_0.save('keyboard.mp3')
        # speech = gTTS(text="マウスモード", lang='ja')
        # speech.save('mouse.mp3')

        # ===============================
        i = 0
        finger_gesture_id = 0

        # ========= 主程式運作 =========
        while (not self.stop_flag):
            left_id = right_id = -1
            fps = cvFpsCalc.get()

            # Process Key (ESC: end)
            key = cv.waitKey(10)
            if key == 27:  # ESC
                break
            number, mode = select_mode(key, mode)

            # Camera capture
            ret, image = cap.read()
            if not ret:
                break
            image = cv.flip(image, 1)  # Mirror display
            debug_image = copy.deepcopy(image)

            # Detection implementation
            image = cv.cvtColor(image, cv.COLOR_BGR2RGB)

            image.flags.writeable = False
            results = hands.process(image)
            image.flags.writeable = True

            ####rest_result####
            if results.multi_hand_landmarks is None:
                rest_id = 0
                rest_result.append(rest_id)
            if results.multi_hand_landmarks is not None:
                rest_id = 1
                rest_result.append(rest_id)
            most_common_rest_result = Counter(rest_result).most_common()

            # old version for 10 sec to rest mode####################
            #print(most_common_rest_result[0])
            # if most_common_rest_result[0][0] == 0 and most_common_rest_result[0][1] == 300:
            #     if detect_mode != 0:
            #
            #         print('Mode has changed')
            #         detect_mode = 0
            #         what_mode = 'Rest'
            #         print(f'Current mode => {what_mode}')

            # new version for 10 sec to rest mode###################
            if time.time() - resttime > 10:
                if detect_mode != 0:
                    detect_mode = 0
                    what_mode = 'Sleep'
                    print(f'Current mode => {what_mode}')

            ####rest_result####

            #  ####################################################################
            # print(most_common_rest_result)
            if results.multi_hand_landmarks is not None:
                for hand_landmarks, handedness in zip(
                        results.multi_hand_landmarks,
                        results.multi_handedness):

                    # Bounding box calculation
                    brect = calc_bounding_rect(debug_image, hand_landmarks)
                    # Landmark calculation
                    landmark_list = calc_landmark_list(debug_image,
                                                       hand_landmarks)
                    # print(landmark_list)

                    # Conversion to relative coordinates / normalized coordinates
                    pre_processed_landmark_list = pre_process_landmark(
                        landmark_list)
                    pre_processed_point_history_list = pre_process_point_history(
                        debug_image, point_history)
                    # Write to the dataset file
                    logging_csv(number, mode, pre_processed_landmark_list,
                                pre_processed_point_history_list)

                    # 靜態手勢資料預測
                    hand_sign_id_R = keypoint_classifier_R(
                        pre_processed_landmark_list)
                    hand_sign_id_L = keypoint_classifier_L(
                        pre_processed_landmark_list)
                    mouse_id = mouse_classifier(pre_processed_landmark_list)
                    # print(mouse_id)
                    if handedness.classification[0].label[0:] == 'Left':
                        left_id = hand_sign_id_L

                    else:
                        right_id = hand_sign_id_R

                    # 手比one 觸發動態資料抓取
                    if right_id == 1 or left_id == 1:
                        point_history.append(landmark_list[8])
                    else:
                        point_history.append([0, 0])

                    # 動態手勢資料預測
                    finger_gesture_id = 0
                    point_history_len = len(pre_processed_point_history_list)
                    if point_history_len == (history_length * 2):
                        finger_gesture_id = point_history_classifier(
                            pre_processed_point_history_list)
                    # print(finger_gesture_id) # 0 = stop, 1 = clockwise, 2 = counterclockwise, 3 = move,偵測出現的動態手勢

                    # 動態手勢最常出現id #########################################
                    # Calculates the gesture IDs in the latest detection
                    finger_gesture_history.append(finger_gesture_id)
                    most_common_fg_id = Counter(
                        finger_gesture_history).most_common()

                    #滑鼠的deque
                    mouse_id_history.append(mouse_id)
                    most_common_ms_id = Counter(mouse_id_history).most_common()
                    # print(f'finger_gesture_history = {finger_gesture_history}')
                    # print(f'most_common_fg_id = {most_common_fg_id}')

                    # 靜態手勢最常出現id #########################################
                    hand_gesture_id = [right_id, left_id]
                    keypoint_R.append(hand_gesture_id[0])
                    keypoint_L.append(hand_gesture_id[1])
                    # print(keypoint_R) # deque右手的靜態id
                    # print(most_common_keypoint_id) # 右手靜態id最大
                    if right_id != -1:
                        most_common_keypoint_id = Counter(
                            keypoint_R).most_common()
                    else:
                        most_common_keypoint_id = Counter(
                            keypoint_L).most_common()

                    # print(f'keypoint = {keypoint}')
                    # print(f'most_common_keypoint_id = {most_common_keypoint_id}')

                    ###############################################################

                    # Drawing part
                    debug_image = draw_bounding_rect(use_brect, debug_image,
                                                     brect)
                    debug_image = draw_landmarks(debug_image, landmark_list)
                    debug_image = draw_info_text(
                        debug_image,
                        brect,
                        handedness,
                        keypoint_classifier_labels[most_common_keypoint_id[0]
                                                   [0]],
                        point_history_classifier_labels[most_common_fg_id[0]
                                                        [0]],
                    )
                    resttime = time.time()
            else:
                point_history.append([0, 0])

            debug_image = draw_point_history(debug_image, point_history)
            debug_image = draw_info(debug_image, fps, mode, number)

            # 偵測是否有手勢 #########################################

            if left_id + right_id > -2:
                if time.time() - presstime > 1:
                    # change mode
                    if most_common_ms_id[0][0] == 3 and most_common_ms_id[0][
                            1] == 40:  #Gesture six changes to the different mode
                        print('Mode has changed')
                        detect_mode = (detect_mode + 1) % 3
                        if detect_mode == 0:
                            what_mode = 'Sleep'
                            playsound('rest.mp3', block=False)
                        if detect_mode == 1:
                            what_mode = 'Keyboard'
                            playsound('keyboard.mp3', block=False)
                        if detect_mode == 2:
                            what_mode = 'Mouse'
                            playsound('mouse.mp3', block=False)
                        print(f'Current mode => {what_mode}')
                        presstime = time.time() + 1

                    # control keyboard
                    elif detect_mode == 1:
                        if time.time() - presstime_2 > 1:
                            # 靜態手勢控制
                            control_keyboard(most_common_keypoint_id,
                                             2,
                                             'K',
                                             keyboard_TF=True,
                                             print_TF=False)
                            # control_keyboard(most_common_keypoint_id, 0, 'right', keyboard_TF=True, print_TF=False)
                            # control_keyboard(most_common_keypoint_id, 7, 'left', keyboard_TF=True, print_TF=False)
                            control_keyboard(most_common_keypoint_id,
                                             9,
                                             'C',
                                             keyboard_TF=True,
                                             print_TF=False)
                            control_keyboard(most_common_keypoint_id,
                                             5,
                                             'up',
                                             keyboard_TF=True,
                                             print_TF=False)
                            control_keyboard(most_common_keypoint_id,
                                             6,
                                             'down',
                                             keyboard_TF=True,
                                             print_TF=False)
                            presstime_2 = time.time()

                        # right右鍵
                        if most_common_keypoint_id[0][
                                0] == 0 and most_common_keypoint_id[0][1] == 5:
                            # print(i, time.time() - presstime_4)
                            if i == 3 and time.time() - presstime_4 > 0.3:
                                pyautogui.press('right')
                                i = 0
                                presstime_4 = time.time()
                            elif i == 3 and time.time() - presstime_4 > 0.25:
                                pyautogui.press('right')
                                presstime_4 = time.time()
                            elif time.time() - presstime_4 > 1:
                                pyautogui.press('right')
                                i += 1
                                presstime_4 = time.time()

                        # left左鍵
                        if most_common_keypoint_id[0][
                                0] == 7 and most_common_keypoint_id[0][1] == 5:
                            # print(i, time.time() - presstime_4)
                            if i == 3 and time.time() - presstime_4 > 0.3:
                                pyautogui.press('left')
                                i = 0
                                presstime_4 = time.time()
                            elif i == 3 and time.time() - presstime_4 > 0.25:
                                pyautogui.press('left')
                                presstime_4 = time.time()
                            elif time.time() - presstime_4 > 1:
                                pyautogui.press('left')
                                i += 1
                                presstime_4 = time.time()

                        # 動態手勢控制
                        if most_common_fg_id[0][
                                0] == 1 and most_common_fg_id[0][1] > 12:
                            if time.time() - presstime_3 > 1.5:
                                #pyautogui.press(['shift', '>'])
                                pyautogui.hotkey('shift', '>')
                                print('speed up')
                                presstime_3 = time.time()
                        elif most_common_fg_id[0][
                                0] == 2 and most_common_fg_id[0][1] > 12:
                            if time.time() - presstime_3 > 1.5:
                                #pyautogui.press(['shift', '<'])
                                pyautogui.hotkey('shift', '<')
                                print('speed down')
                                presstime_3 = time.time()

                if detect_mode == 2:
                    if mouse_id == 0:  # Point gesture
                        # print(landmark_list[8]) #index finger
                        # print(landmark_list[12]) #middle finger
                        x1, y1 = landmark_list[8]
                        # cv.rectangle(debug_image, (frameR, frameR), (cap_width - frameR, cap_height - frameR),
                        #              (255, 0, 255), 2)
                        cv.rectangle(debug_image, (50, 30),
                                     (cap_width - 50, cap_height - 170),
                                     (255, 0, 255), 2)
                        #座標轉換
                        #x軸: 鏡頭上50~(cap_width - 50)轉至螢幕寬0~wScr
                        #y軸: 鏡頭上30~(cap_height - 170)轉至螢幕長0~hScr
                        x3 = np.interp(x1, (50, (cap_width - 50)), (0, wScr))
                        y3 = np.interp(y1, (30, (cap_height - 170)), (0, hScr))
                        # print(x3, y3)

                        # 6. Smoothen Values
                        clocX = plocX + (x3 - plocX) / smoothening
                        clocY = plocY + (y3 - plocY) / smoothening
                        # 7. Move Mouse
                        pyautogui.moveTo(clocX, clocY)
                        cv.circle(debug_image, (x1, y1), 15, (255, 0, 255),
                                  cv.FILLED)
                        plocX, plocY = clocX, clocY

                    if mouse_id == 1:
                        length, img, lineInfo = findDistance(
                            landmark_list[8], landmark_list[12], debug_image)

                        # 10. Click mouse if distance short
                        if time.time() - clicktime > 0.5:
                            if length < 40:
                                cv.circle(img, (lineInfo[4], lineInfo[5]), 15,
                                          (0, 255, 0), cv.FILLED)
                                pyautogui.click()
                                print('click')
                                clicktime = time.time()

                            # if length > 70:
                            #     cv.circle(img, (lineInfo[4], lineInfo[5]),
                            #               15, (0, 255, 0), cv.FILLED)
                            # pyautogui.click(clicks=2)
                            # print('click*2')
                            # clicktime = time.time()

                    if most_common_keypoint_id[0][
                            0] == 5 and most_common_keypoint_id[0][1] == 5:
                        pyautogui.scroll(20)

                    if most_common_keypoint_id[0][
                            0] == 6 and most_common_keypoint_id[0][1] == 5:
                        pyautogui.scroll(-20)

                    #if left_id == 7 or right_id == 7:
                    if most_common_keypoint_id[0][
                            0] == 0 and most_common_keypoint_id[0][1] == 5:
                        if time.time() - clicktime > 1:
                            pyautogui.click(clicks=2)
                            clicktime = time.time()

                    if most_common_keypoint_id[0][
                            0] == 9 and most_common_keypoint_id[0][1] == 5:
                        if time.time() - clicktime > 2:
                            pyautogui.hotkey('alt', 'left')
                            clicktime = time.time()

            cv.putText(debug_image, what_mode, (400, 30),
                       cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 4, cv.LINE_AA)
            # Screen reflection ###################################JL##########################
            cv.imshow('Hand Gesture Recognition', debug_image)

            self.trigger.emit(detect_mode)

        cap.release()
        cv.destroyAllWindows()
def main():
    # 引数 #####################################################################
    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    max_num_faces = args.max_num_faces
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    # カメラ準備 ###############################################################
    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    # モデルロード #############################################################
    face_mesh = FaceMesh(
        max_num_faces,
        min_detection_confidence,
        min_tracking_confidence,
    )
    iris_detector = IrisLandmark()

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc(buffer_len=10)

    while True:
        display_fps = cvFpsCalc.get()

        # カメラキャプチャ #####################################################
        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)  # ミラー表示
        debug_image = copy.deepcopy(image)

        # 検出実施 #############################################################
        # Face Mesh検出
        face_results = face_mesh(image)
        for face_result in face_results:
            # 目周辺のバウンディングボックス計算
            left_eye, right_eye = face_mesh.calc_around_eye_bbox(face_result)

            # 虹彩検出
            left_iris, right_iris = detect_iris(image, iris_detector, left_eye,
                                                right_eye)

            # 虹彩の外接円を計算
            left_center, left_radius = calc_min_enc_losingCircle(left_iris)
            right_center, right_radius = calc_min_enc_losingCircle(right_iris)

            # デバッグ描画
            debug_image = draw_debug_image(
                debug_image,
                left_iris,
                right_iris,
                left_center,
                left_radius,
                right_center,
                right_radius,
            )

        cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)

        # キー処理(ESC:終了) #################################################
        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        # 画面反映 #############################################################
        cv.imshow('Iris(tflite) Demo', debug_image)

    cap.release()
    cv.destroyAllWindows()

    return
Esempio n. 27
0
def main():
    # 引数解析 (Argument parsing) (Parameter analysis) #################################################################
    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    upper_body_only = args.upper_body_only
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    use_brect = args.use_brect

    # カメラ準備 (Camera preparation) ###############################################################
    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    # モデルロード (Model load) #############################################################
    mp_holistic = mp.solutions.holistic
    holistic = mp_holistic.Holistic(
        upper_body_only=upper_body_only,
        min_detection_confidence=min_detection_confidence,
        min_tracking_confidence=min_tracking_confidence,
    )

    # FPS計測モジュール (Measurement module) ########################################################
    cvFpsCalc = CvFpsCalc(buffer_len=10)

    while True:
        display_fps = cvFpsCalc.get()

        # カメラキャプチャ (Camera capture) #####################################################
        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)  # ミラー表示
        debug_image = copy.deepcopy(image)

        # 検出実施 (Detection implementation) #############################################################
        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)

        image.flags.writeable = False
        results = holistic.process(image)
        image.flags.writeable = True

        # Face Mesh ###########################################################
        face_landmarks = results.face_landmarks
        if face_landmarks is not None:
            # 外接矩形の計算 (Calculation of circumscribed rectangle)
            brect = calc_bounding_rect(debug_image, face_landmarks)
            # 描画 (drawing)
            debug_image = draw_face_landmarks(debug_image, face_landmarks)
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        # Pose ###############################################################
        pose_landmarks = results.pose_landmarks
        if pose_landmarks is not None:
            # 外接矩形の計算 (Calculation of circumscribed rectangle)
            brect = calc_bounding_rect(debug_image, pose_landmarks)
            # 描画 (drawing)
            debug_image = draw_pose_landmarks(debug_image, pose_landmarks,
                                              upper_body_only)
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        # Hands ###############################################################
        left_hand_landmarks = results.left_hand_landmarks
        right_hand_landmarks = results.right_hand_landmarks
        # 左手 (left hand)
        if left_hand_landmarks is not None:
            # 手の平重心計算 (Calculation of the center of gravity of the palm)
            cx, cy = calc_palm_moment(debug_image, left_hand_landmarks)
            # 外接矩形の計算 (Calculation of circumscribed rectangle)
            brect = calc_bounding_rect(debug_image, left_hand_landmarks)
            # 描画 (drawing)
            debug_image = draw_hands_landmarks(debug_image, cx, cy,
                                               left_hand_landmarks,
                                               upper_body_only, 'R')
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)
        # 右手 (right hand)
        if right_hand_landmarks is not None:
            # 手の平重心計算 (Calculation of the center of gravity of the palm)
            cx, cy = calc_palm_moment(debug_image, right_hand_landmarks)
            # 外接矩形の計算 (Calculation of circumscribed rectangle)
            brect = calc_bounding_rect(debug_image, right_hand_landmarks)
            # 描画 (drawing)
            debug_image = draw_hands_landmarks(debug_image, cx, cy,
                                               right_hand_landmarks,
                                               upper_body_only, 'L')
            debug_image = draw_bounding_rect(use_brect, debug_image, brect)

        cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)

        # キー処理 (Key processing) (ESC:終了) (ESC: End)#################################################
        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        # 画面反映 (Screen reflection)#############################################################
        cv.imshow('MediaPipe Holistic Demo', debug_image)

    cap.release()
    cv.destroyAllWindows()
Esempio n. 28
0
def main():
    mp_drawing = mp.solutions.drawing_utils
    mp_hands = mp.solutions.hands

    # Argument parsing
    in_flight = False

    # For webcam input:
    hands = mp_hands.Hands(static_image_mode=True,
                           max_num_hands=2,
                           min_detection_confidence=0.8,
                           min_tracking_confidence=0.8)

    # Camera preparation
    tello = Tello()
    tello.connect()
    tello.streamon()

    # FPS Measurement
    cv_fps_calc = CvFpsCalc(buffer_len=5)

    frame_read = tello.get_frame_read()

    while True:
        fps = cv_fps_calc.get()
        battery_status = tello.get_battery()

        # In reality you want to display frames in a seperate thread. Otherwise
        #  they will freeze while the drone moves.
        image = frame_read.frame

        # Flip the image horizontally for a later selfie-view display, and convert
        # the BGR image to RGB.
        image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)

        # To improve performance, optionally mark the image as not writeable to
        # pass by reference.
        image.flags.writeable = False
        results = hands.process(image)

        # Draw the hand annotations on the image.
        image.flags.writeable = True
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
        if results.multi_hand_landmarks:
            for hand_landmarks in results.multi_hand_landmarks:
                mp_drawing.draw_landmarks(image, hand_landmarks,
                                          mp_hands.HAND_CONNECTIONS)

        # Process Key (ESC: end)
        key = cv2.waitKey(1) & 0xff
        if key == 27:  # ESCArgument parsing
            tello.land()
            break
        elif key == 32:
            if not in_flight:
                # Take-off drone
                tello.takeoff()
                in_flight = True
                tello.move_up(50)
            elif in_flight:
                # Land tello
                tello.land()
                in_flight = False
        elif key == ord('w'):
            tello.move_forward(30)
        elif key == ord('s'):
            tello.move_back(30)
        elif key == ord('a'):
            tello.move_left(30)
        elif key == ord('d'):
            tello.move_right(30)
        elif key == ord('e'):
            tello.rotate_clockwise(30)
        elif key == ord('q'):
            tello.rotate_counter_clockwise(30)
        elif key == ord('r'):
            tello.move_up(30)
        elif key == ord('f'):
            tello.move_down(30)

        # Battery status and image rendering
        cv2.putText(image, "Battery: {} / fps:{}".format(battery_status, fps),
                    (5, 720 - 5), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
        cv2.imshow('DK_Drone', image)

    tello.land()
    tello.end()
    cv2.destroyAllWindows()
Esempio n. 29
0
def main():
    # my add variable value ################################################
    # is_concentrated = False
    # frame_length       = 256
    sampling_time      = 0.1
    detecting_time     = 0
    not_detecting_time = 0
    window_size        = 100
    list_zeros_window  = [0 for i in range(window_size)]
    list_detect_window = list_zeros_window
    detect_count       = 0
    border_face_detect = 0.1
    # alert_time = 60 * 30
    alert_time = 20
    alert_title = "Alert"
    alert_message = "Let's move the body.\nYou have been concentrating on your work for more than 25 minutes."

    # Reset Param ############################################################
    # reset_work_boarder = 60 * 3
    reset_work_boarder = 3

    # Exercise ###############################################################
    # exercise_boarder = 60*5
    exercise_boarder = 5
    is_exercise_state = False
    is_init_exercise_state = True
    finish_alert_title = "Congratulation!"
    finish_alert_message= "Finished Your Exercise Time."

    # Repeat Alert ###########################################################
    repeat_alert_time = 1
    # repeat_boarder = 60
    repeat_boarder = 10
    repeat_alert_title = "Repeat Alerm"
    repeat_alert_message = "Let's move the body.\n"

    # investigation param ####################################################
    output_list = [['time[s]', '顔検出値[0 or 1]']]
    output_count = 0

    # plot param #############################################################
    x = np.linspace(0, 10, 100)
    y = np.zeros(x.size)
    plt.ion()
    figure, ax = plt.subplots(figsize=(8,6))
    line1, = ax.plot(x, y)
    plt.title("Dynamic Plot of face detection",fontsize=25)
    plt.xlabel("time",fontsize=18)
    plt.ylabel("is detected value",fontsize=18)
    plt.ylim(-1.1,1.1)
    updated_y = y
    plot_data = 0
    # border prot
    yb = np.full((1,100), border_face_detect)[0]
    plt.plot(x, yb, color = 'red')


    # 引数解析 #################################################################
    args = get_args()

    cap_device = args.device
    cap_width = args.width
    cap_height = args.height

    max_num_faces = args.max_num_faces
    min_detection_confidence = args.min_detection_confidence
    min_tracking_confidence = args.min_tracking_confidence

    use_brect = args.use_brect

    # カメラ準備 ###############################################################
    cap = cv.VideoCapture(cap_device)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)

    # モデルロード #############################################################
    mp_face_mesh = mp.solutions.face_mesh
    face_mesh = mp_face_mesh.FaceMesh(
        max_num_faces=max_num_faces,
        min_detection_confidence=min_detection_confidence,
        min_tracking_confidence=min_tracking_confidence,
    )

    # FPS計測モジュール ########################################################
    cvFpsCalc = CvFpsCalc(buffer_len=10)

    while True:
        display_fps = cvFpsCalc.get()

        # カメラキャプチャ #####################################################
        ret, image = cap.read()
        if not ret:
            break
        image = cv.flip(image, 1)  # ミラー表示
        debug_image = copy.deepcopy(image)

        # 検出実施 #############################################################
        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
        results = face_mesh.process(image)

        # 描画 ################################################################
        if results.multi_face_landmarks is not None:
            for face_landmarks in results.multi_face_landmarks:
                # 外接矩形の計算
                brect = calc_bounding_rect(debug_image, face_landmarks)
                # 描画
                debug_image = draw_landmarks(debug_image, face_landmarks)
                debug_image = draw_bounding_rect(use_brect, debug_image, brect)

            list_detect_window[detect_count]=1
            output_list.append([output_count, 1])
            plot_data=1
        else:
            list_detect_window[detect_count]=0
            output_list.append([output_count, 0])
            plot_data=0

        cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
                   cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)

        # キー処理(ESC:終了) #################################################
        key = cv.waitKey(1)
        if key == 27:  # ESC
            break

        # 画面反映 #############################################################
        cv.imshow('MediaPipe Face Mesh Demo', debug_image)

        # calculate the mean of list_detect_window ############################
        mean_list_detect_window = statistics.mean(list_detect_window)
        if mean_list_detect_window > border_face_detect:
            detecting_time += sampling_time
            not_detecting_time = 0
            if detecting_time >= alert_time:
                is_exercise_state = True
                is_init_exercise_state = True
                if repeat_alert_time % repeat_boarder == 0:
                    print("repeat alert")
                    do_alert(repeat_alert_title, repeat_alert_message)
                elif repeat_alert_time ==1:
                    do_alert(alert_title, alert_message)
                repeat_alert_time = round(repeat_alert_time + sampling_time, 1)
                # print(repeat_alert_time)
        else:
            if is_exercise_state:
                # if is_init_exercise_state:
                #     mean_list_detect_window = list_zeros_window
                #     is_init_exercise_state = False

                not_detecting_time += sampling_time
                if not_detecting_time >= exercise_boarder:
                    do_alert(finish_alert_title, finish_alert_message)
                    # Reset Param ################################################
                    detecting_time = 0
                    not_detecting_time=0
                    repeat_alert_time=1
                    is_exercise_state = False
            else:
                ### Personal function ########################################
                ### 作業していない期間が一定時間を超えたら作業タイマーをリセットする
                ##############################################################
                not_detecting_time += sampling_time
                if not_detecting_time >= reset_work_boarder:
                    # Reset Param ############################################
                    detecting_time = 0
                    not_detecting_time=0
                    repeat_alert_time=1
        
        if is_exercise_state and is_init_exercise_state:
            # print("test test test")
            list_detect_window = list_zeros_window
            is_init_exercise_state = False

        

        detect_count = 0 if detect_count == window_size-1 else detect_count+1
        time.sleep(sampling_time)
        output_count += sampling_time

        # Update Plot ########################################################
        # updated_y = enqueue(updated_y, plot_data) 
        updated_y = enqueue(updated_y, mean_list_detect_window)
        line1.set_xdata(x)
        line1.set_ydata(updated_y)
        figure.canvas.draw()
        figure.canvas.flush_events()

    cap.release()
    cv.destroyAllWindows()
    export_list_csv(output_list, 'face_detected.csv')
Esempio n. 30
0
def main():
    # init global vars
    global gesture_buffer
    global gesture_id

    # Argument parsing
    KEYBOARD_CONTROL = True
    WRITE_CONTROL = False
    in_flight = False

    # Camera preparation
    tello = Tello()
    tello.connect()
    tello.streamon()

    cap = tello.get_frame_read()

    # Init Tello Controllers
    gesture_controller = TelloGestureController(tello)
    keyboard_controller = TelloKeyboardController(tello)

    gesture_detector = GestureRecognition(min_detection_confidence=0.7,
                                          min_tracking_confidence=0.5)
    gesture_buffer = GestureBuffer(buffer_len=5)

    def tello_control(key, keyboard_controller, gesture_controller):
        global gesture_buffer

        if KEYBOARD_CONTROL:
            keyboard_controller.control(key)
        else:
            gesture_controller.gesture_control(gesture_buffer)

    # FPS Measurement
    cv_fps_calc = CvFpsCalc(buffer_len=10)

    mode = 0
    number = -1
    battery_status = -1

    while True:
        fps = cv_fps_calc.get()
        battery_status = tello.get_battery()

        # Process Key (ESC: end)
        key = cv.waitKey(1) & 0xff
        if key == 27:  # ESC
            break
        elif key == 32:  # Space
            if not in_flight:
                # Take-off drone
                tello.takeoff()
                in_flight = True
                tello.move_up(50)
            elif in_flight:
                # Land tello
                tello.land()
                in_flight = False

        elif key == ord('k'):
            mode = 0
            KEYBOARD_CONTROL = True
            WRITE_CONTROL = False
            tello.send_rc_control(0, 0, 0, 0)  # Stop moving
        elif key == ord('g'):
            KEYBOARD_CONTROL = False
        elif key == ord('n'):
            mode = 1
            WRITE_CONTROL = True
            KEYBOARD_CONTROL = True

        if WRITE_CONTROL:
            number = -1
            if 48 <= key <= 57:  # 0 ~ 9
                number = key - 48

        # Camera capture
        image = cap.frame

        debug_image, gesture_id = gesture_detector.recognize(
            image, number, mode)
        gesture_buffer.add_gesture(gesture_id)

        # Start control thread
        threading.Thread(target=tello_control,
                         args=(
                             key,
                             keyboard_controller,
                             gesture_controller,
                         )).start()
        threading.Thread(target=battery_status, args=(tello, )).start()

        debug_image = gesture_detector.draw_info(debug_image, fps, mode,
                                                 number)

        # Battery status and image rendering
        cv.putText(debug_image, "Battery: {}".format(battery_status),
                   (5, 720 - 5), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
        cv.imshow('Tello Gesture Recognition', debug_image)

    tello.land()
    tello.end()
    cv.destroyAllWindows()