def main():
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
max_num_faces = args.max_num_faces
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = args.use_brect
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
mp_face_mesh = mp.solutions.face_mesh
face_mesh = mp_face_mesh.FaceMesh(
max_num_faces=max_num_faces,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
cvFpsCalc = CvFpsCalc(buffer_len=10)
while True:
display_fps = cvFpsCalc.get()
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1)
debug_image = copy.deepcopy(image)
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
results = face_mesh.process(image)
if results.multi_face_landmarks is not None:
for face_landmarks in results.multi_face_landmarks:
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, face_landmarks)
# 描画
debug_image = draw_landmarks(debug_image, face_landmarks)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
cv.putText(debug_image, "MOsta_FPS:" + str(display_fps), (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
# キー処理(ESC：終了) #################################################
key = cv.waitKey(1)
if key == 27: # ESC
break
# 画面反映 #############################################################
cv.imshow('MediaPipe Face Mesh Demo', debug_image)
cap.release()
cv.destroyAllWindows()
def main():
# init global vars
global gesture_buffer
global gesture_id
global battery_status
# Argument parsing
args = get_args()
KEYBOARD_CONTROL = args.is_keyboard
WRITE_CONTROL = False
in_flight = False
cap_webcam = cv.VideoCapture(0)
gesture_detector = GestureRecognition(args.use_static_image_mode,
args.min_detection_confidence,
args.min_tracking_confidence)
gesture_buffer = GestureBuffer(buffer_len=args.buffer_len)
# FPS Measurement
cv_fps_calc = CvFpsCalc(buffer_len=10)
mode = 0
number = -1
while True:
fps = cv_fps_calc.get()
# Process Key (ESC: end)
key = cv.waitKey(1) & 0xff
if key == 27: # ESC
break
elif key == ord('n'):
mode = 1
WRITE_CONTROL = True
KEYBOARD_CONTROL = True
if WRITE_CONTROL:
number = -1
if 48 <= key <= 57: # 0 ~ 9
number = key - 48
# Camera capture
image = cap_webcam.read()[1]
try:
debug_image, gesture_id = gesture_detector.recognize(
image, number, mode)
gesture_buffer.add_gesture(gesture_id)
debug_image = gesture_detector.draw_info(debug_image, fps, mode,
number)
cv.imshow('Webcam Gesture Recognition', debug_image)
except:
print("exception")
cv.destroyAllWindows()
def main():
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
upper_body_only = args.upper_body_only
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = args.use_brect
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
mp_pose = mp.solutions.pose
pose = mp_pose.Pose(
upper_body_only=upper_body_only,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
# FPS
cvFpsCalc = CvFpsCalc(buffer_len=10)
while True:
display_fps = cvFpsCalc.get()
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1)
debug_image = copy.deepcopy(image)
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
results = pose.process(image)
if results.pose_landmarks is not None:
brect = calc_bounding_rect(debug_image, results.pose_landmarks)
debug_image = draw_landmarks(debug_image, results.pose_landmarks,
upper_body_only)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
key = cv.waitKey(1)
if key == 27: # ESC
break
cv.imshow('MediaPipe Pose Demo', debug_image)
cap.release()
cv.destroyAllWindows()
def main():
# 引数解析 #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
min_detection_confidence = args.min_detection_confidence
use_brect = args.use_brect
# カメラ準備 ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード #############################################################
mp_face_detection = mp.solutions.face_detection
face_detection = mp_face_detection.FaceDetection(
min_detection_confidence=min_detection_confidence)
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
while True:
display_fps = cvFpsCalc.get()
# カメラキャプチャ #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
# 検出実施 #############################################################
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
results = face_detection.process(image)
# 描画 ################################################################
if results.detections is not None:
for detection in results.detections:
# 描画
debug_image = draw_detection(debug_image, detection)
cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
# キー処理(ESC：終了) #################################################
key = cv.waitKey(1)
if key == 27: # ESC
break
# 画面反映 #############################################################
cv.imshow('MediaPipe Face Detection Demo', debug_image)
cap.release()
cv.destroyAllWindows()
def main():
print("Image Classification Start...\n")
# カメラ準備 ##############################################################
cap = cv.VideoCapture(0)
frame_width = 960
frame_height = 540
cap.set(cv.CAP_PROP_FRAME_WIDTH, frame_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, frame_height)
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc()
# モデルロード ############################################################
model = tf.keras.applications.EfficientNetB0(
include_top=True,
weights='imagenet',
input_shape=(224, 224, 3),
)
# CVUI初期化 ##############################################################
cvui.init("Demo")
# メインループ #############################################################
while True:
# FPS算出 #############################################################
display_fps = cvFpsCalc.get()
if display_fps == 0:
display_fps = 0.01
# カメラキャプチャ ####################################################
ret, frame = cap.read()
if not ret:
continue
debug_image = copy.deepcopy(frame)
# 検出実施 ############################################################
trim_x1 = int((frame_width - frame_height) / 2)
trim_x2 = frame_width - int((frame_width - frame_height) / 2)
trimming_image = debug_image[0:frame_height, trim_x1:trim_x2]
classifications = run_classify(model, trimming_image, 10)
# デバッグ情報描画 ####################################################
debug_image = draw_demo_image(
trimming_image,
classifications,
display_fps,
)
# 画面反映 #######################################################
cvui.imshow('Demo', debug_image)
key = cv.waitKey(1)
if key == 27: # ESC
break
def main():
print("Semantic Segmentation Start...\n")
# カメラ準備 ##############################################################
cap = cv.VideoCapture(0)
frame_width = 960
frame_height = 540
cap.set(cv.CAP_PROP_FRAME_WIDTH, frame_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, frame_height)
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc()
# モデルロード ############################################################
frozen_path = "model/deeplab_v3/deeplabv3_mnv2.pb"
sess = graph_load(frozen_path)
# メインループ #############################################################
while True:
# FPS算出 #############################################################
display_fps = cvFpsCalc.get()
if display_fps == 0:
display_fps = 0.01
# カメラキャプチャ ####################################################
ret, frame = cap.read()
if not ret:
continue
debug_image = copy.deepcopy(frame)
# 検出実施 ############################################################
segmentation_map = session_run(sess, debug_image)
# デバッグ情報描画 ####################################################
debug_image = draw_demo_image(
debug_image,
segmentation_map,
display_fps,
)
# 画面反映 #######################################################
cv.imshow('Demo', debug_image)
key = cv.waitKey(1)
if key == 27: # ESC
break
def main():
print("Style Transfer Start...\n")
# カメラ準備 ###############################################################
cap = cv.VideoCapture(0)
frame_width = 960
frame_height = 540
cap.set(cv.CAP_PROP_FRAME_WIDTH, frame_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, frame_height)
# モデルロード #############################################################
sess, input_photo, final_out = graph_load('model/white_box_cartoonization')
# CvComparisonSliderWindow準備 ############################################
cvwindow = CvComparisonSliderWindow(
window_name='Demo',
line_color=(255, 255, 255),
line_thickness=1,
)
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc()
while True:
display_fps = cvFpsCalc.get()
# カメラキャプチャ #####################################################
ret, frame = cap.read()
if not ret:
continue
frame_width, frame_height = frame.shape[1], frame.shape[0]
debug_image = copy.deepcopy(frame)
# 変換実施 #############################################################
out = session_run(sess, debug_image, input_photo, final_out)
# 画面反映 #############################################################
cvwindow.imshow(frame, out, fps=display_fps)
# キー処理 #############################################################
key = cv.waitKey(1)
if key == 27: # ESC
break
cap.release()
cv.destroyAllWindows()
def main():
"""
[summary]
main()
Parameters
----------
None
"""
# 引数解析 #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
fps = args.fps
model_path = args.model
score_th = args.score_th
smaller_ratio = args.smaller_ratio
# GUI準備 #################################################################
app_gui = AppGui(window_name='FingerFrameLens')
# 初期設定
app_gui.set_score_threshold(score_th)
# カメラ準備 ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード #############################################################
# EfficientDet-D0
DEFAULT_FUNCTION_KEY = 'serving_default'
effdet_model = tf.saved_model.load(model_path)
inference_func = effdet_model.signatures[DEFAULT_FUNCTION_KEY]
# EfficientNet-B0
effnet_model = tf.keras.applications.EfficientNetB0(
include_top=True,
weights='imagenet',
input_shape=(224, 224, 3),
)
tensor = tf.convert_to_tensor(np.zeros((224, 224, 3), np.uint8))
effnet_model.predict(tensor)
effnet_model.make_predict_function()
# FPS計測準備 ##############################################################
cvFpsCalc = CvFpsCalc(buffer_len=3)
cropping_image = None
classifications = None
while True:
start_time = time.time()
# GUI設定取得 #########################################################
score_th = app_gui.get_score_threshold()
# カメラキャプチャ #####################################################
ret, frame = cap.read()
if not ret:
continue
frame_width, frame_height = frame.shape[1], frame.shape[0]
debug_image = copy.deepcopy(frame)
# 物体検出実施 #########################################################
detections = run_od_inference(inference_func, frame)
x1, y1, x2, y2 = calc_od_bbox(
detections,
score_th,
smaller_ratio,
frame_width,
frame_height,
)
# cv.putText(debug_image, '{:.3f}'.format(score), (x1, y1 - 10),
# cv.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 255), 2,
# cv.LINE_AA)
# cv.rectangle(debug_image, (x1, y1), (x2, y2), (255, 255, 255), 2)
# クラス分類実施 #######################################################
if x1 is not None and y1 is not None and \
x2 is not None and y2 is not None:
cropping_image = copy.deepcopy(frame[y1:y2, x1:x2])
classifications = run_classify(effnet_model, cropping_image)
# GUI描画更新 ##########################################################
fps_result = cvFpsCalc.get()
app_gui.update(
fps_result,
debug_image,
cropping_image,
classifications,
)
app_gui.show()
# キー入力(ESC:プログラム終了) #########################################
key = cv.waitKey(1)
if key == 27: # ESC
break
# FPS調整 #############################################################
elapsed_time = time.time() - start_time
sleep_time = max(0, ((1.0 / fps) - elapsed_time))
time.sleep(sleep_time)
cap.release()
cv.destroyAllWindows()
def main():
# 引数解析 #################################################################
args = get_args()
cap_width = args.width
cap_height = args.height
cap_device = args.device
if args.file is not None: # 動画ファイルを利用する場合
cap_device = args.file
fps = args.fps
skip_frame = args.skip_frame
model_path = args.model
score_th = args.score_th
sign_interval = args.sign_interval
jutsu_display_time = args.jutsu_display_time
use_display_score = args.use_display_score
erase_bbox = args.erase_bbox
use_jutsu_lang_en = args.use_jutsu_lang_en
chattering_check = args.chattering_check
use_fullscreen = args.use_fullscreen
# カメラ準備 ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデル読み込み ############################################################
DEFAULT_FUNCTION_KEY = 'serving_default'
loaded_model = tf.saved_model.load(model_path)
inference_func = loaded_model.signatures[DEFAULT_FUNCTION_KEY]
# FPS計測モジュール #########################################################
cvFpsCalc = CvFpsCalc()
# フォント読み込み ##########################################################
# https://opentype.jp/kouzanmouhitufont.htm
font_path = './utils/font/衡山毛筆フォント.ttf'
# ラベル読み込み ###########################################################
with open('setting/labels.csv') as f: # 印
labels = csv.reader(f)
labels = [row for row in labels]
with open('setting/jutsu.csv') as f: # 術
jutsu = csv.reader(f)
jutsu = [row for row in jutsu]
# 印の表示履歴および、検出履歴 ##############################################
sign_max_display = 18
sign_max_history = 44
sign_display_queue = deque(maxlen=sign_max_display)
sign_history_queue = deque(maxlen=sign_max_history)
chattering_check_queue = deque(maxlen=chattering_check)
for index in range(-1, -1 - chattering_check, -1):
chattering_check_queue.append(index)
# 術名の言語設定 ###########################################################
lang_offset = 0
jutsu_font_size_ratio = sign_max_display
if use_jutsu_lang_en:
lang_offset = 1
jutsu_font_size_ratio = int((sign_max_display / 3) * 4)
# その他変数初期化 #########################################################
sign_interval_start = 0 # 印のインターバル開始時間初期化
jutsu_index = 0 # 術表示名のインデックス
jutsu_start_time = 0 # 術名表示の開始時間初期化
frame_count = 0 # フレームナンバーカウンタ
window_name = 'NARUTO HandSignDetection Ninjutsu Demo'
if use_fullscreen:
cv.namedWindow(window_name, cv.WINDOW_NORMAL)
while True:
start_time = time.time()
# カメラキャプチャ #####################################################
ret, frame = cap.read()
if not ret:
continue
frame_count += 1
debug_image = copy.deepcopy(frame)
if (frame_count % (skip_frame + 1)) != 0:
continue
# FPS計測 ##############################################################
fps_result = cvFpsCalc.get()
# 検出実施 #############################################################
frame = frame[:, :, [2, 1, 0]] # BGR2RGB
image_np_expanded = np.expand_dims(frame, axis=0)
result_inference = run_inference_single_image(image_np_expanded,
inference_func)
# 検出内容の履歴追加 ####################################################
num_detections = result_inference['num_detections']
for i in range(num_detections):
score = result_inference['detection_scores'][i]
class_id = result_inference['detection_classes'][i].astype(np.int)
# 検出閾値未満の結果は捨てる
if score < score_th:
continue
# 指定回数以上、同じ印が続いた場合に、印検出とみなす ※瞬間的な誤検出対策
chattering_check_queue.append(class_id)
if len(set(chattering_check_queue)) != 1:
continue
# 前回と異なる印の場合のみキューに登録
if len(sign_display_queue) == 0 or \
sign_display_queue[-1] != class_id:
sign_display_queue.append(class_id)
sign_history_queue.append(class_id)
sign_interval_start = time.time() # 印の最終検出時間
# 前回の印検出から指定時間が経過した場合、履歴を消去 ####################
if (time.time() - sign_interval_start) > sign_interval:
sign_display_queue.clear()
sign_history_queue.clear()
# 術成立判定 #########################################################
jutsu_index, jutsu_start_time = check_jutsu(
sign_history_queue,
labels,
jutsu,
jutsu_index,
jutsu_start_time,
)
# キー処理 ###########################################################
key = cv.waitKey(1)
if key == 99: # C：印の履歴を消去
sign_display_queue.clear()
sign_history_queue.clear()
if key == 27: # ESC：プログラム終了
break
# FPS調整 #############################################################
elapsed_time = time.time() - start_time
sleep_time = max(0, ((1.0 / fps) - elapsed_time))
time.sleep(sleep_time)
# 画面反映 #############################################################
debug_image = draw_debug_image(
debug_image,
font_path,
fps_result,
labels,
result_inference,
score_th,
erase_bbox,
use_display_score,
jutsu,
sign_display_queue,
sign_max_display,
jutsu_display_time,
jutsu_font_size_ratio,
lang_offset,
jutsu_index,
jutsu_start_time,
)
if use_fullscreen:
cv.setWindowProperty(window_name, cv.WND_PROP_FULLSCREEN,
cv.WINDOW_FULLSCREEN)
cv.imshow(window_name, debug_image)
# cv.moveWindow(window_name, 100, 100)
cap.release()
cv.destroyAllWindows()
def main():
# 引数解析 #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
fps = args.fps
model_path = args.model
score_th = args.score_th
# カメラ準備 ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
video = cv.VideoCapture('utils/map.mp4')
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc()
# モデルロード #############################################################
DEFAULT_FUNCTION_KEY = 'serving_default'
loaded_model = tf.saved_model.load(model_path)
inference_func = loaded_model.signatures[DEFAULT_FUNCTION_KEY]
buffer_len = 5
deque_x1 = deque(maxlen=buffer_len)
deque_y1 = deque(maxlen=buffer_len)
deque_x2 = deque(maxlen=buffer_len)
deque_y2 = deque(maxlen=buffer_len)
# フォント
font_path = './utils/font/x12y20pxScanLine.ttf'
while True:
# FPS算出 #############################################################
display_fps = cvFpsCalc.get()
start_time = time.time()
# カメラキャプチャ #####################################################
ret, frame = cap.read()
if not ret:
continue
frame_width, frame_height = frame.shape[1], frame.shape[0]
debug_image = copy.deepcopy(frame)
# 検出実施 #############################################################
frame = frame[:, :, [2, 1, 0]] # BGR2RGB
image_np_expanded = np.expand_dims(frame, axis=0)
output = run_inference_single_image(image_np_expanded, inference_func)
num_detections = output['num_detections']
for i in range(num_detections):
score = output['detection_scores'][i]
bbox = output['detection_boxes'][i]
# class_id = output['detection_classes'][i].astype(np.int)
if score < score_th:
continue
# 検出結果可視化 ###################################################
x1, y1 = int(bbox[1] * frame_width), int(bbox[0] * frame_height)
x2, y2 = int(bbox[3] * frame_width), int(bbox[2] * frame_height)
risize_ratio = 0.15
bbox_width = x2 - x1
bbox_height = y2 - y1
x1 = x1 + int(bbox_width * risize_ratio)
y1 = y1 + int(bbox_height * risize_ratio)
x2 = x2 - int(bbox_width * risize_ratio)
y2 = y2 - int(bbox_height * risize_ratio)
x1 = int((x1 - 5) / 10) * 10
y1 = int((y1 - 5) / 10) * 10
x2 = int((x2 + 5) / 10) * 10
y2 = int((y2 + 5) / 10) * 10
deque_x1.append(x1)
deque_y1.append(y1)
deque_x2.append(x2)
deque_y2.append(y2)
x1 = int(sum(deque_x1) / len(deque_x1))
y1 = int(sum(deque_y1) / len(deque_y1))
x2 = int(sum(deque_x2) / len(deque_x2))
y2 = int(sum(deque_y2) / len(deque_y2))
ret, video_frame = video.read()
if ret is not False:
video.grab()
video.grab()
debug_add_image = np.zeros((frame_height, frame_width, 3),
np.uint8)
map_resize_image = cv.resize(video_frame,
((x2 - x1), (y2 - y1)))
debug_add_image = CvOverlayImage.overlay(
debug_add_image, map_resize_image, (x1, y1))
debug_add_image = cv.cvtColor(debug_add_image,
cv.COLOR_BGRA2BGR)
# cv.imshow('1', debug_add_image)
debug_image = cv.addWeighted(debug_image, 1.0, debug_add_image,
2.0, 0)
else:
video = cv.VideoCapture('map.mp4')
# キー処理(ESC：終了) #################################################
key = cv.waitKey(1)
if key == 27: # ESC
break
# FPS調整 #############################################################
elapsed_time = time.time() - start_time
sleep_time = max(0, ((1.0 / fps) - elapsed_time))
time.sleep(sleep_time)
# 画面反映 #############################################################
fps_string = u"FPS:" + str(display_fps)
debug_image = CvDrawText.puttext(debug_image, fps_string, (17, 17),
font_path, 32, (255, 255, 255))
fps_string = u"FPS:" + str(display_fps)
debug_image = CvDrawText.puttext(debug_image, fps_string, (15, 15),
font_path, 32, (0, 56, 86))
cv.imshow('Demo', debug_image)
cap.release()
cv.destroyAllWindows()
def main():
# Argument parsing #################################################################
args = get_args()
cap_device = args.device
cap_width = 1920
cap_height = 1080
use_static_image_mode = args.use_static_image_mode
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = True
# Camera preparation ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# Model load #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
static_image_mode=use_static_image_mode,
max_num_hands=1,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
keypoint_classifier = KeyPointClassifier()
point_history_classifier = PointHistoryClassifier()
# Read labels ###########################################################
with open('model/keypoint_classifier/keypoint_classifier_label.csv',
encoding='utf-8-sig') as f:
keypoint_classifier_labels = csv.reader(f)
keypoint_classifier_labels = [
row[0] for row in keypoint_classifier_labels
]
with open(
'model/point_history_classifier/point_history_classifier_label.csv',
encoding='utf-8-sig') as f:
point_history_classifier_labels = csv.reader(f)
point_history_classifier_labels = [
row[0] for row in point_history_classifier_labels
]
# FPS Measurement ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
# Coordinate history #################################################################
history_length = 16
point_history = deque(maxlen=history_length)
# Finger gesture history ################################################
finger_gesture_history = deque(maxlen=history_length)
# ########################################################################
mode = 0
while True:
fps = cvFpsCalc.get()
# Process Key (ESC: end) #################################################
key = cv.waitKey(10)
if key == 27: # ESC
break
number, mode = select_mode(key, mode)
# Camera capture #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # Mirror display
debug_image = copy.deepcopy(image)
# Detection implementation #############################################################
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = hands.process(image)
image.flags.writeable = True
# ####################################################################
if results.multi_hand_landmarks is not None:
for hand_landmarks, handedness in zip(results.multi_hand_landmarks,
results.multi_handedness):
# Bounding box calculation
brect = calc_bounding_rect(debug_image, hand_landmarks)
# Landmark calculation
landmark_list = calc_landmark_list(debug_image, hand_landmarks)
# Conversion to relative coordinates / normalized coordinates
pre_processed_landmark_list = pre_process_landmark(
landmark_list)
pre_processed_point_history_list = pre_process_point_history(
debug_image, point_history)
# Write to the dataset file
logging_csv(number, mode, pre_processed_landmark_list,
pre_processed_point_history_list)
# Hand sign classification
hand_sign_id = keypoint_classifier(pre_processed_landmark_list)
if hand_sign_id == 2: # Point gesture
point_history.append(landmark_list[8])
else:
point_history.append([0, 0])
# Finger gesture classification
finger_gesture_id = 0
point_history_len = len(pre_processed_point_history_list)
if point_history_len == (history_length * 2):
finger_gesture_id = point_history_classifier(
pre_processed_point_history_list)
# Calculates the gesture IDs in the latest detection
finger_gesture_history.append(finger_gesture_id)
most_common_fg_id = Counter(
finger_gesture_history).most_common()
# Drawing part
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
debug_image = draw_landmarks(debug_image, landmark_list)
debug_image = draw_info_text(
debug_image,
brect,
handedness,
keypoint_classifier_labels[hand_sign_id],
point_history_classifier_labels[most_common_fg_id[0][0]],
)
else:
point_history.append([0, 0])
debug_image = draw_point_history(debug_image, point_history)
debug_image = draw_info(debug_image, fps, mode, number)
# Screen reflection #############################################################
cv.imshow('Hand Gesture Recognition', debug_image)
cap.release()
cv.destroyAllWindows()
def main(filename):
args = get_args()
print("Filename in main is " + filename)
cap_device = args.device
cap_width = args.width
cap_height = args.height
upper_body_only = args.upper_body_only
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = args.use_brect
cap = cv.VideoCapture(filename)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
length = int(cap.get(cv.CAP_PROP_FRAME_COUNT))
print(length)
mp_holistic = mp.solutions.holistic
holistic = mp_holistic.Holistic(
upper_body_only=upper_body_only,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
cvFpsCalc = CvFpsCalc(buffer_len=10)
d = 0
real_right_shoulder_hip = 0.57
velocity_counter = 0
velocity_15 = 0
velocity_16 = 0
velocity_27 = 0
velocity_28 = 0
array_counter = 0
velocity_array_counter = 0
frame_array = []
angle_1_array = []
angle_2_array = []
angle_3_array = []
angle_4_array = []
angle_5_array = []
angle_6_array = []
angle_7_array = []
angle_8_array = []
velocity_array = []
velocity_15_array = []
velocity_16_array = []
velocity_27_array = []
velocity_28_array = []
# to append angles of joint to array
angle_1_array.append("Right Elbow")
angle_2_array.append("Left Elbow")
angle_3_array.append("Right Knee")
angle_4_array.append("Left Knee")
angle_5_array.append("Right Hip")
angle_6_array.append("Left Hip")
angle_7_array.append("Right Shoulder")
angle_8_array.append("Left Shoulder")
velocity_15_array.append("Left Wrist")
velocity_16_array.append("Right Wrist")
velocity_27_array.append("Left Ankle")
velocity_28_array.append("Right Ankle")
while True:
velocity_counter+=1
display_fps = cvFpsCalc.get()
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1)
debug_image = copy.deepcopy(image)
blank_image = np.zeros((cap_height,cap_width,3), np.uint8)
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = holistic.process(image)
image.flags.writeable = True
# Pose ###############################################################
pose_landmarks = results.pose_landmarks
if pose_landmarks is not None:
brect = calc_bounding_rect(debug_image, pose_landmarks)
debug_image, image_right_shoulder_hip, vel_xparsed_15, vel_yparsed_15, vel_xparsed_16, vel_yparsed_16, vel_xparsed_27, vel_yparsed_27, vel_xparsed_28, vel_yparsed_28, angle_1, angle_2, angle_3, angle_4, angle_5, angle_6, angle_7, angle_8 = draw_pose_landmarks(debug_image, pose_landmarks,
upper_body_only)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
#blank_image = draw_pose_landmarks(blank_image, pose_landmarks,
#upper_body_only)
# to append angles of joint to array
angle_1_array.append(angle_1)
angle_2_array.append(angle_2)
angle_3_array.append(angle_3)
angle_4_array.append(angle_4)
angle_5_array.append(angle_5)
angle_6_array.append(angle_6)
angle_7_array.append(angle_7)
angle_8_array.append(angle_8)
# to calculate velocity of right wrist (no.16) and left wrist (no.15)
if velocity_counter == 1:
# set coordinate of joint no. 15 on 1st frame
vel_x1_15, vel_y1_15 = vel_xparsed_15, vel_yparsed_15
# set coordinate of joint no. 16 on 1st frame
vel_x1_16, vel_y1_16 = vel_xparsed_16, vel_yparsed_16
# set coordinate of joint no. 27 on 1st frame
vel_x1_27, vel_y1_27 = vel_xparsed_27, vel_yparsed_27
# set coordinate of joint no. 28 on 1st frame
vel_x1_28, vel_y1_28 = vel_xparsed_28, vel_yparsed_28
if velocity_counter == 60:
# set coordinate of joint no. 15 on 60th frame
vel_x60_15, vel_y60_15 = vel_xparsed_15, vel_yparsed_15
# set coordinate of joint no. 16 on 60th frame
vel_x60_16, vel_y60_16 = vel_xparsed_16, vel_yparsed_16
# set coordinate of joint no. 27 on 60th frame
vel_x60_27, vel_y60_27 = vel_xparsed_27, vel_yparsed_27
# set coordinate of joint no. 28 on 60th frame
vel_x60_28, vel_y60_28 = vel_xparsed_28, vel_yparsed_28
# calculate the ratio between the real length and image length of shoulder to hip
ratio_shoulder_hip = real_right_shoulder_hip / image_right_shoulder_hip
# calculate the velocity of joint no. 15
velocity_15 = math.sqrt(pow(vel_x1_15 - vel_x60_15,2) + pow(vel_y1_15 - vel_y60_15,2)) * ratio_shoulder_hip
velocity_15_array.append(velocity_15)
# calculate the velocity of joint no. 16
velocity_16 = math.sqrt(pow(vel_x1_16 - vel_x60_16,2) + pow(vel_y1_16 - vel_y60_16,2)) * ratio_shoulder_hip
velocity_16_array.append(velocity_16)
# calculate the velocity of joint no. 27
velocity_27 = math.sqrt(pow(vel_x1_27 - vel_x60_27,2) + pow(vel_y1_27 - vel_y60_27,2)) * ratio_shoulder_hip
velocity_27_array.append(velocity_27)
# calculate the velocity of joint no. 28
velocity_28 = math.sqrt(pow(vel_x1_28 - vel_x60_28,2) + pow(vel_y1_28 - vel_y60_28,2)) * ratio_shoulder_hip
velocity_28_array.append(velocity_28)
cv.putText(debug_image, "V-[9] Right Wrist:" + "{:.2f}".format(velocity_16) + 'ms-1', (10, 330),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
cv.putText(debug_image, "V-[10] Left Wrist:" + "{:.2f}".format(velocity_15) + 'ms-1', (10, 360),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
cv.putText(debug_image, "V-[11] Right Ankle:" + "{:.2f}".format(velocity_28) + 'ms-1', (10, 390),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
cv.putText(debug_image, "V-[12] Left Ankle:" + "{:.2f}".format(velocity_27) + 'ms-1', (10, 420),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
velocity_counter = 0
if velocity_counter != 60:
cv.putText(debug_image, "V-[9] Right Wrist:" + "{:.2f}".format(velocity_16) + 'ms-1', (10, 330),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
cv.putText(debug_image, "V-[10] Left Wrist:" + "{:.2f}".format(velocity_15) + 'ms-1', (10, 360),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
cv.putText(debug_image, "V-[11] Right Ankle:" + "{:.2f}".format(velocity_28) + 'ms-1', (10, 390),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
cv.putText(debug_image, "V-[12] Left Ankle:" + "{:.2f}".format(velocity_27) + 'ms-1', (10, 420),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
#to calculate and display the frame processed
processed_frame_percent = (d/length) * 100
cv.putText(debug_image, "Processing Frame:" + "{:.2f}".format(processed_frame_percent) + '%', (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
key = cv.waitKey(1)
if key == 27: # ESC
break
filename = 'output/save_%06d.png'%d
cv.imshow('MediaPipe Holistic Demo', debug_image)
cv.imwrite(filename, debug_image)
#to display the 2d output on blank canvas
#cv.imshow('Blank image',blank_image)
d+=1
# to convert frame images into video format
Video_Writer()
for array_counter in range(length+1):
frame_array.append(array_counter)
with open('anglebyframe.csv', 'w', newline='') as angle_file:
writer = csv.writer(angle_file)
writer.writerow(frame_array)
writer.writerow(angle_1_array)
writer.writerow(angle_2_array)
writer.writerow(angle_3_array)
writer.writerow(angle_4_array)
writer.writerow(angle_5_array)
writer.writerow(angle_6_array)
writer.writerow(angle_7_array)
writer.writerow(angle_8_array)
total_seconds = length // 60
for velocity_array_counter in range(total_seconds + 1):
velocity_array.append(velocity_array_counter)
with open('velocitybyframe.csv', 'w', newline='') as velocity_file:
writer = csv.writer(velocity_file)
writer.writerow(velocity_array)
writer.writerow(velocity_15_array)
writer.writerow(velocity_16_array)
writer.writerow(velocity_27_array)
writer.writerow(velocity_28_array)
folder = 'output'
for filename in os.listdir(folder):
file_path = os.path.join(folder, filename)
try:
if os.path.isfile(file_path) or os.path.islink(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception as e:
print('Failed to delete %s. Reason: %s' % (file_path, e))
cap.release()
cv.destroyAllWindows()
def main():
# 引数解析 #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
f_min_detection_confidence = args.f_min_detection_confidence
f_min_tracking_confidence = args.f_min_tracking_confidence
h_min_detection_confidence = args.h_min_detection_confidence
h_min_tracking_confidence = args.h_min_tracking_confidence
p_min_detection_confidence = args.p_min_detection_confidence
p_min_tracking_confidence = args.p_min_tracking_confidence
use_brect = args.use_brect
# カメラ準備 ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード #############################################################
mp_face_mesh = mp.solutions.face_mesh
face_mesh = mp_face_mesh.FaceMesh(
min_detection_confidence=f_min_detection_confidence,
min_tracking_confidence=f_min_tracking_confidence,
)
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
min_detection_confidence=h_min_detection_confidence,
min_tracking_confidence=h_min_tracking_confidence,
)
mp_pose = mp.solutions.pose
pose = mp_pose.Pose(
min_detection_confidence=p_min_detection_confidence,
min_tracking_confidence=p_min_tracking_confidence,
)
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
while True:
display_fps = cvFpsCalc.get()
# カメラキャプチャ #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
# 検出実施 #############################################################
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
hands_results = hands.process(image)
face_results = face_mesh.process(image)
pose_results = pose.process(image)
image.flags.writeable = True
# Face Mesh ###########################################################
if face_results.multi_face_landmarks is not None:
for face_landmarks in face_results.multi_face_landmarks:
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, face_landmarks)
# 描画
debug_image = draw_face_landmarks(debug_image, face_landmarks)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
# Pose ###############################################################
if pose_results.pose_landmarks is not None:
# 外接矩形の計算
brect = calc_bounding_rect(debug_image,
pose_results.pose_landmarks)
# 描画
debug_image = draw_pose_landmarks(debug_image,
pose_results.pose_landmarks)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
# Hands ###############################################################
if hands_results.multi_hand_landmarks is not None:
for hand_landmarks, handedness in zip(
hands_results.multi_hand_landmarks,
hands_results.multi_handedness):
# 手の平重心計算
cx, cy = calc_palm_moment(debug_image, hand_landmarks)
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, hand_landmarks)
# 描画
debug_image = draw_hands_landmarks(debug_image, cx, cy,
hand_landmarks, handedness)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
# キー処理(ESC：終了) #################################################
key = cv.waitKey(1)
if key == 27: # ESC
break
# 画面反映 #############################################################
cv.imshow('MediaPipe Demo', debug_image)
cap.release()
cv.destroyAllWindows()
def main():
# 引数解析 #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
use_static_image_mode = args.use_static_image_mode
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = True
# カメラ準備 ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
static_image_mode=use_static_image_mode,
max_num_hands=1,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
keypoint_classifier = KeyPointClassifier()
point_history_classifier = PointHistoryClassifier()
# ラベル読み込み ###########################################################
with open('model/keypoint_classifier/keypoint_classifier_label.csv',
encoding='utf-8-sig') as f:
keypoint_classifier_labels = csv.reader(f)
keypoint_classifier_labels = [
row[0] for row in keypoint_classifier_labels
]
with open(
'model/point_history_classifier/point_history_classifier_label.csv',
encoding='utf-8-sig') as f:
point_history_classifier_labels = csv.reader(f)
point_history_classifier_labels = [
row[0] for row in point_history_classifier_labels
]
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
# 座標履歴 #################################################################
history_length = 16
point_history = deque(maxlen=history_length)
# フィンガージェスチャー履歴 ################################################
finger_gesture_history = deque(maxlen=history_length)
# ########################################################################
mode = 0
while True:
fps = cvFpsCalc.get()
# キー処理(ESC：終了) #################################################
key = cv.waitKey(10)
if key == 27: # ESC
break
number, mode = select_mode(key, mode)
# カメラキャプチャ #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
# 検出実施 #############################################################
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = hands.process(image)
image.flags.writeable = True
# ####################################################################
if results.multi_hand_landmarks is not None:
for hand_landmarks, handedness in zip(results.multi_hand_landmarks,
results.multi_handedness):
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, hand_landmarks)
# ランドマークの計算
landmark_list = calc_landmark_list(debug_image, hand_landmarks)
# 相対座標・正規化座標への変換
pre_processed_landmark_list = pre_process_landmark(
landmark_list)
pre_processed_point_history_list = pre_process_point_history(
debug_image, point_history)
# 学習データ保存
logging_csv(number, mode, pre_processed_landmark_list,
pre_processed_point_history_list)
# ハンドサイン分類
hand_sign_id = keypoint_classifier(pre_processed_landmark_list)
if hand_sign_id == 2: # 指差しサイン
point_history.append(landmark_list[8]) # 人差指座標
else:
point_history.append([0, 0])
# フィンガージェスチャー分類
finger_gesture_id = 0
point_history_len = len(pre_processed_point_history_list)
if point_history_len == (history_length * 2):
finger_gesture_id = point_history_classifier(
pre_processed_point_history_list)
# 直近検出の中で最多のジェスチャーIDを算出
finger_gesture_history.append(finger_gesture_id)
most_common_fg_id = Counter(
finger_gesture_history).most_common()
# 描画
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
debug_image = draw_landmarks(debug_image, landmark_list)
debug_image = draw_info_text(
debug_image,
brect,
handedness,
keypoint_classifier_labels[hand_sign_id],
point_history_classifier_labels[most_common_fg_id[0][0]],
)
else:
point_history.append([0, 0])
debug_image = draw_point_history(debug_image, point_history)
debug_image = draw_info(debug_image, fps, mode, number)
# 画面反映 #############################################################
cv.imshow('Hand Gesture Recognition', debug_image)
cap.release()
cv.destroyAllWindows()
def main():
# init global vars
global gesture_buffer
global gesture_id
global battery_status
# Argument parsing
args = get_args()
KEYBOARD_CONTROL = args.is_keyboard
WRITE_CONTROL = False
in_flight = False
# Camera preparation
tello = Tello()
#print(dir(tello))
tello.connect()
tello.set_speed(speed["slow"])
print("\n\n" + tello.get_speed() + "\n\n")
tello.streamon()
cap_drone = tello.get_frame_read()
cap_webcam = cv.VideoCapture(0)
# Init Tello Controllers
gesture_controller = TelloGestureController(tello)
keyboard_controller = TelloKeyboardController(tello)
gesture_detector = GestureRecognition(args.use_static_image_mode,
args.min_detection_confidence,
args.min_tracking_confidence)
gesture_buffer = GestureBuffer(buffer_len=args.buffer_len)
def tello_control(key, keyboard_controller, gesture_controller):
global gesture_buffer
if KEYBOARD_CONTROL:
keyboard_controller.control(key)
else:
gesture_controller.gesture_control(gesture_buffer)
def tello_battery(tello):
global battery_status
battery_status = tello.get_battery()
# FPS Measurement
cv_fps_calc = CvFpsCalc(buffer_len=10)
mode = 0
number = -1
battery_status = -1
tello.move_down(20)
while True:
fps = cv_fps_calc.get()
# Process Key (ESC: end)
key = cv.waitKey(1) & 0xff
if key == 27: # ESC
break
elif key == 32: # Space
if not in_flight:
# Take-off drone
tello.takeoff()
in_flight = True
elif in_flight:
# Land tello
tello.land()
in_flight = False
elif key == ord('k'):
mode = 0
KEYBOARD_CONTROL = True
WRITE_CONTROL = False
tello.send_rc_control(0, 0, 0, 0) # Stop moving
elif key == ord('g'):
KEYBOARD_CONTROL = False
elif key == ord('n'):
mode = 1
WRITE_CONTROL = True
KEYBOARD_CONTROL = True
if WRITE_CONTROL:
number = -1
if 48 <= key <= 57: # 0 ~ 9
number = key - 48
# Camera capture
image_drone = cap_drone.frame
image = cap_webcam.read()[1]
try:
debug_image, gesture_id = gesture_detector.recognize(
image, number, mode)
gesture_buffer.add_gesture(gesture_id)
# Start control thread
threading.Thread(target=tello_control,
args=(
key,
keyboard_controller,
gesture_controller,
)).start()
threading.Thread(target=tello_battery, args=(tello, )).start()
debug_image = gesture_detector.draw_info(debug_image, fps, mode,
number)
battery_str_postion = (5, 100) # dustin webcam
#battery_str_postion = (5, 720 - 5) # drone camera
# Battery status and image rendering
cv.putText(debug_image, "Battery: {}".format(battery_status),
battery_str_postion, cv.FONT_HERSHEY_SIMPLEX, 1,
(0, 0, 255), 2)
modeStr = "gestures"
if KEYBOARD_CONTROL:
modeStr = "keyboard"
cv.putText(debug_image, modeStr + " control", (5, 150),
cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
cv.imshow('Webcam Gesture Recognition', debug_image)
cv.imshow('Tello drone camera', image_drone)
except:
print("exception")
tello.land()
tello.end()
cv.destroyAllWindows()
def main():
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
upper_body_only = args.upper_body_only
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = args.use_brect
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
mp_holistic = mp.solutions.holistic
holistic = mp_holistic.Holistic(
upper_body_only=upper_body_only,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
# FPS
cvFpsCalc = CvFpsCalc(buffer_len=10)
while True:
display_fps = cvFpsCalc.get()
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1)
debug_image = copy.deepcopy(image)
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = holistic.process(image)
image.flags.writeable = True
# Face Mesh
face_landmarks = results.face_landmarks
if face_landmarks is not None:
brect = calc_bounding_rect(debug_image, face_landmarks)
debug_image = draw_face_landmarks(debug_image, face_landmarks)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
# Pose
pose_landmarks = results.pose_landmarks
if pose_landmarks is not None:
brect = calc_bounding_rect(debug_image, pose_landmarks)
debug_image = draw_pose_landmarks(debug_image, pose_landmarks,
upper_body_only)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
# Hands
left_hand_landmarks = results.left_hand_landmarks
right_hand_landmarks = results.right_hand_landmarks
if left_hand_landmarks is not None:
cx, cy = calc_palm_moment(debug_image, left_hand_landmarks)
brect = calc_bounding_rect(debug_image, left_hand_landmarks)
debug_image = draw_hands_landmarks(debug_image, cx, cy,
left_hand_landmarks,
upper_body_only, 'R')
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
if right_hand_landmarks is not None:
cx, cy = calc_palm_moment(debug_image, right_hand_landmarks)
brect = calc_bounding_rect(debug_image, right_hand_landmarks)
debug_image = draw_hands_landmarks(debug_image, cx, cy,
right_hand_landmarks,
upper_body_only, 'L')
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
key = cv.waitKey(1)
if key == 27: # ESC
break
cv.imshow('MediaPipe Holistic Demo', debug_image)
cap.release()
cv.destroyAllWindows()
def main():
print("Image Classification Start...\n")
# カメラ準備 ##############################################################
cap = cv.VideoCapture(0)
frame_width = 960
frame_height = 540
cap.set(cv.CAP_PROP_FRAME_WIDTH, frame_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, frame_height)
# ImageNet 日本語ラベル ###################################################
jsonfile = open('utils/imagenet_class_index.json',
'r',
encoding="utf-8_sig")
imagenet_ja_labels = json.load(jsonfile)
imagenet_ja_label = {}
for label_temp in imagenet_ja_labels:
imagenet_ja_label[label_temp['num']] = label_temp['ja']
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc()
# モデルロード ############################################################
model = tf.keras.applications.EfficientNetB0(
include_top=True,
weights='imagenet',
input_shape=(224, 224, 3),
)
# メインループ #############################################################
detection_count = 0
while True:
# FPS算出 #############################################################
display_fps = cvFpsCalc.get()
if display_fps == 0:
display_fps = 0.01
# カメラキャプチャ ####################################################
ret, frame = cap.read()
if not ret:
continue
debug_image = copy.deepcopy(frame)
# 検出実施 ############################################################
trim_x1 = int(frame_width / 4) + 50
trim_x2 = int(frame_width / 4 * 3) - 50
trim_y1 = int(frame_height / 8)
trim_y2 = int(frame_height / 8 * 7)
trimming_image = debug_image[trim_y1:trim_y2, trim_x1:trim_x2]
classifications = run_classify(model, trimming_image)
# デバッグ情報描画 ####################################################
# 表示名作成
classification_string = ""
for classification in classifications:
if float(classification[2]) > 0.5:
detection_count += 1
classification_string = imagenet_ja_label[
classification[0]] + ":" + str('{:.1f}'.format(
float(classification[2]) * 100)) + "%"
else:
detection_count = 0
break # 1件のみ
# 描画
debug_image = draw_demo_image(
debug_image,
detection_count,
classification_string,
display_fps,
(trim_x1, trim_y1, trim_x2, trim_y2),
)
# 画面反映 #######################################################
cv.imshow('Demo', debug_image)
key = cv.waitKey(1)
if key == 27: # ESC
break
def main():
# 引数解析 #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
upper_body_only = args.upper_body_only
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = args.use_brect
# カメラ準備 ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード #############################################################
mp_holistic = mp.solutions.holistic
holistic = mp_holistic.Holistic(
upper_body_only=upper_body_only,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
while True:
display_fps = cvFpsCalc.get()
# カメラキャプチャ #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
# 検出実施 #############################################################
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = holistic.process(image)
image.flags.writeable = True
# Face Mesh ###########################################################
face_landmarks = results.face_landmarks
if face_landmarks is not None:
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, face_landmarks)
# 描画
debug_image = draw_face_landmarks(debug_image, face_landmarks)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
# Pose ###############################################################
pose_landmarks = results.pose_landmarks
if pose_landmarks is not None:
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, pose_landmarks)
# 描画
debug_image = draw_pose_landmarks(debug_image, pose_landmarks,
upper_body_only)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
# Hands ###############################################################
left_hand_landmarks = results.left_hand_landmarks
right_hand_landmarks = results.right_hand_landmarks
# 左手
if left_hand_landmarks is not None:
# 手の平重心計算
cx, cy = calc_palm_moment(debug_image, left_hand_landmarks)
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, left_hand_landmarks)
# 描画
debug_image = draw_hands_landmarks(debug_image, cx, cy,
left_hand_landmarks,
upper_body_only, 'R')
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
# 右手
if right_hand_landmarks is not None:
# 手の平重心計算
cx, cy = calc_palm_moment(debug_image, right_hand_landmarks)
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, right_hand_landmarks)
# 描画
debug_image = draw_hands_landmarks(debug_image, cx, cy,
right_hand_landmarks,
upper_body_only, 'L')
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 255), 2, cv.LINE_AA)
cv.putText(debug_image, "Mostafizur Rahman", (250, 450),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 255), 2, cv.LINE_AA)
# キー処理(ESC：終了) #################################################
key = cv.waitKey(1)
if key == 27: # ESC
break
# 画面反映 #############################################################
cv.imshow('MediaPipe Holistic Demo', debug_image)
cap.release()
cv.destroyAllWindows()
def main():
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
max_num_hands = args.max_num_hands
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = args.use_brect
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
max_num_hands=max_num_hands,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
cvFpsCalc = CvFpsCalc(buffer_len=10)
while True:
display_fps = cvFpsCalc.get()
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1)
debug_image = copy.deepcopy(image)
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
results = hands.process(image)
# 描画 ################################################################
if results.multi_hand_landmarks is not None:
for hand_landmarks, handedness in zip(results.multi_hand_landmarks,
results.multi_handedness):
# 手の平重心計算
cx, cy = calc_palm_moment(debug_image, hand_landmarks)
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, hand_landmarks)
# 描画
debug_image = draw_landmarks(debug_image, cx, cy,
hand_landmarks, handedness)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
cv.putText(debug_image, "Mosta_FPS:" + str(display_fps), (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
key = cv.waitKey(1)
if key == 27: # ESC
break
cv.imshow('MediaPipe Hand Demo', debug_image)
cap.release()
cv.destroyAllWindows()
def main():
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
static_image_mode = args.static_image_mode
max_num_objects = args.max_num_objects
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
model_name = args.model_name
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
mp_objectron = mp.solutions.objectron
objectron = mp_objectron.Objectron(
static_image_mode=static_image_mode,
max_num_objects=max_num_objects,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
model_name=model_name,
)
mp_drawing = mp.solutions.drawing_utils
cvFpsCalc = CvFpsCalc(buffer_len=10)
while True:
display_fps = cvFpsCalc.get()
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
results = objectron.process(image)
if results.detected_objects is not None:
for detected_object in results.detected_objects:
mp_drawing.draw_landmarks(debug_image,
detected_object.landmarks_2d,
mp_objectron.BOX_CONNECTIONS)
mp_drawing.draw_axis(debug_image, detected_object.rotation,
detected_object.translation)
draw_landmarks(debug_image, detected_object.landmarks_2d)
cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
key = cv.waitKey(1)
if key == 27: # ESC
break
cv.imshow('MediaPipe Objectron Demo', debug_image)
cap.release()
cv.destroyAllWindows()
def run(self):
# 重写线程执行的run函数
# 触发自定义信号
self.stop_flag = False
# Argument parsing #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
use_static_image_mode = args.use_static_image_mode
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = True
# Camera preparation ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# Model load #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
static_image_mode=use_static_image_mode,
max_num_hands=1,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
mouse_classifier = MouseClassifier(invalid_value=2, score_th=0.4)
point_history_classifier = PointHistoryClassifier()
# Read labels ###########################################################
with open(
'model/point_history_classifier/point_history_classifier_label.csv',
encoding='utf-8-sig') as f:
point_history_classifier_labels = csv.reader(f)
point_history_classifier_labels = [
row[0] for row in point_history_classifier_labels
]
with open('model/mouse_classifier/mouse_classifier_label.csv',
encoding='utf-8-sig') as f:
mouse_classifier_labels = csv.reader(f)
mouse_classifier_labels = [
row[0] for row in mouse_classifier_labels
]
# FPS Measurement ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=3)
# Coordinate history #################################################################
history_length = 16
point_history = deque(maxlen=history_length)
# Finger gesture history ################################################
finger_gesture_history = deque(maxlen=history_length)
mouse_id_history = deque(maxlen=30)
m_id_history = deque(maxlen=6)
# ========= 按鍵前置作業 =========
mode = 0
presstime = resttime = time.time()
mode_change = False
detect_mode = 0
what_mode = 'Sleep'
landmark_list = 0
pyautogui.PAUSE = 0
i = 0
# ========= 滑鼠前置作業 =========
wScr, hScr = pyautogui.size()
frameR = 100
smoothening = 10
plocX, plocY = 0, 0
clocX, clocY = 0, 0
# 關閉 滑鼠移至角落啟動保護措施
pyautogui.FAILSAFE = False
# ========= 主程式運作 =========
while self.stop_flag == False:
mouse_id = -1
fps = cvFpsCalc.get()
# Process Key (ESC: end)
key = cv.waitKey(10)
if key == 27: # ESC
break
number, mode = select_mode(key, mode)
# Camera capture
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # Mirror display
debug_image = copy.deepcopy(image)
# Detection implementation
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
# image = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
image.flags.writeable = False
results = hands.process(image)
image.flags.writeable = True
# ####################################################################
if results.multi_hand_landmarks is not None:
for hand_landmarks, handedness in zip(
results.multi_hand_landmarks,
results.multi_handedness):
# Bounding box calculation
brect = calc_bounding_rect(debug_image, hand_landmarks)
# Landmark calculation
landmark_list = calc_landmark_list(debug_image,
hand_landmarks)
# print(landmark_list)
# Conversion to relative coordinates / normalized coordinates
pre_processed_landmark_list = pre_process_landmark(
landmark_list)
pre_processed_point_history_list = pre_process_point_history(
debug_image, point_history)
# Write to the dataset file
logging_csv(number, mode, pre_processed_landmark_list,
pre_processed_point_history_list)
# # 靜態手勢資料預測
mouse_id = mouse_classifier(pre_processed_landmark_list)
# print(mouse_id)
# 手比one 觸發動態資料抓取
if mouse_id == 0:
point_history.append(landmark_list[8])
else:
point_history.append([0, 0])
# 動態手勢資料預測
finger_gesture_id = 0
point_history_len = len(pre_processed_point_history_list)
if point_history_len == (history_length * 2):
finger_gesture_id = point_history_classifier(
pre_processed_point_history_list)
# print(finger_gesture_id) # 0 = stop, 1 = clockwise, 2 = counterclockwise, 3 = move,偵測出現的動態手勢
# 動態手勢最常出現id #########################################
# Calculates the gesture IDs in the latest detection
finger_gesture_history.append(finger_gesture_id)
most_common_fg_id = Counter(
finger_gesture_history).most_common()
# 滑鼠的deque
mouse_id_history.append(mouse_id)
most_common_ms_id = Counter(mouse_id_history).most_common()
# print(most_common_ms_id)
m_id_history.append(mouse_id)
m_id = Counter(m_id_history).most_common(2)
mouse_id = m_id[0][0] if m_id[0][1] >= 4 else 2
# print(f'm_id {m_id}\n, mouse_id {mouse_id}')
# ===== 偵測到手時，重製紀錄時間 ==============================
resttime = time.time()
###############################################################
# Drawing part
debug_image = draw_bounding_rect(use_brect, debug_image,
brect)
debug_image = draw_landmarks(debug_image, landmark_list)
debug_image = draw_info_text(
debug_image, brect, handedness,
mouse_classifier_labels[mouse_id],
point_history_classifier_labels[most_common_fg_id[0]
[0]])
else:
point_history.append([0, 0])
debug_image = draw_point_history(debug_image, point_history)
debug_image = draw_info(debug_image, fps, mode, number)
# 偵測是否有手勢 #########################################
if mouse_id > -1:
# change mode Gesture six changes to the different mode
if most_common_ms_id[0][0] == 3 and most_common_ms_id[0][
1] == 30:
if time.time() - presstime > 2:
detect_mode = (detect_mode + 1) % 3
mode_change = True
presstime = time.time()
# control keyboard
elif detect_mode == 2:
# 靜態手勢控制
presstime = control_keyboard(mouse_id, 1, 'K', presstime)
presstime = control_keyboard(mouse_id, 8, 'C', presstime)
presstime = control_keyboard(mouse_id, 5, 'up', presstime)
presstime = control_keyboard(mouse_id, 6, 'down',
presstime)
# presstime = control_keyboard(most_common_keypoint_id, 0, 'right', presstime)
# presstime = control_keyboard(most_common_keypoint_id, 7, 'left', presstime)
if mouse_id == 4:
# print(i, time.time() - presstime)
if i == 3 and time.time() - presstime > 0.3:
pyautogui.press('right')
i = 0
presstime = time.time()
elif i == 3 and time.time() - presstime > 0.25:
pyautogui.press('right')
presstime = time.time()
elif time.time() - presstime > 1:
pyautogui.press('right')
i += 1
presstime = time.time()
if mouse_id == 7:
# print(i, time.time() - presstime)
if i == 3 and time.time() - presstime > 0.3:
pyautogui.press('left')
i = 0
presstime = time.time()
elif i == 3 and time.time() - presstime > 0.25:
pyautogui.press('left')
presstime = time.time()
elif time.time() - presstime > 1:
pyautogui.press('left')
i += 1
presstime = time.time()
# 動態手勢控制
if most_common_fg_id[0][
0] == 1 and most_common_fg_id[0][1] > 9:
if time.time() - presstime > 1.5:
pyautogui.hotkey('shift', '>')
print('speed up')
presstime = time.time()
elif most_common_fg_id[0][
0] == 2 and most_common_fg_id[0][1] > 12:
if time.time() - presstime > 1.5:
pyautogui.hotkey('shift', '<')
print('speed down')
presstime = time.time()
elif detect_mode == 1:
if mouse_id == 0: # Point gesture
x1, y1 = landmark_list[8]
cv.rectangle(debug_image, (50, 30),
(cap_width - 50, cap_height - 170),
(255, 0, 255), 2)
# 座標轉換
# x軸: 鏡頭上50~(cap_width - 50)轉至螢幕寬0~wScr
# y軸: 鏡頭上30~(cap_height - 170)轉至螢幕長0~hScr
x3 = np.interp(x1, (50, (cap_width - 50)), (0, wScr))
y3 = np.interp(y1, (30, (cap_height - 170)), (0, hScr))
# 6. Smoothen Values
clocX = plocX + (x3 - plocX) / smoothening
clocY = plocY + (y3 - plocY) / smoothening
# 7. Move Mouse
pyautogui.moveTo(clocX, clocY)
cv.circle(debug_image, (x1, y1), 15, (255, 0, 255),
cv.FILLED)
plocX, plocY = clocX, clocY
elif mouse_id == 1:
# 10. Click mouse if distance short
if time.time() - presstime > 0.5:
pyautogui.click()
presstime = time.time()
if mouse_id == 5:
pyautogui.scroll(-20)
if mouse_id == 6:
pyautogui.scroll(20)
if mouse_id == 7:
if time.time() - presstime > 1.5:
pyautogui.click(clicks=2)
presstime = time.time()
if mouse_id == 8:
if time.time() - presstime > 2:
pyautogui.hotkey('alt', 'left')
presstime = time.time()
# 比讚 從休息模式 換成 鍵盤模式
elif detect_mode == 0:
if mouse_id == 5:
i += 1
if i == 1 or time.time() - presstime > 3:
presstime = time.time()
elif time.time() - presstime > 2:
detect_mode = 2
mode_change = True
i = 0
# 距離上次監測到手的時間大於30秒、切回休息模式 =========================
if time.time() - resttime > 30:
if detect_mode != 0:
detect_mode = 0
mode_change = True
# 檢查模式有沒有更動 ========================
if mode_change:
if detect_mode == 0:
what_mode = 'Sleep'
elif detect_mode == 2:
what_mode = 'Keyboard'
elif detect_mode == 1:
what_mode = 'Mouse'
mode_change = False
print('Mode has changed')
print(f'Current mode => {what_mode}')
cv.putText(debug_image, what_mode, (480, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 4, cv.LINE_AA)
# Screen reflection ###################################JL##########################
cv.imshow('Hand Gesture Recognition', debug_image)
self.trigger.emit(detect_mode)
cap.release()
cv.destroyAllWindows()
def HandTracking(cap, width, height, conf_flg=0):
# ×ボタンが押されたかのフラグ(hand_gui.py内の変数、flg_closePush)の初期化
hand_gui.close_switch_py(0)
# 引数解析 #################################################################
args = get_args()
flg_video = 0 #「1」でカメラが接続されていない
flg_break = 0 #「1」で最初のループを抜け終了する⇒正常終了
name_pose = "Unknown"
focus_flg = 1 #index.html の表示・非表示の切り替え、「0」:Main.pyで開いた場合、「1」:HandTracking.pyで開いた場合
namePose_flg = 1 #complete_old.htmlの開始・終了フラグ
potision_flg = 0
#flg_closePush = 0
global ShortCutList
ShortCutList = config_sys_set()
cap_device = args.device
cap_width = args.width
cap_height = args.height
use_static_image_mode = args.use_static_image_mode
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = True
#width,height = autopy.screen.size() #eel で立ち上げた際の表示位置を指定するために取得
while (True): #カメラが再度接続するまでループ処理
#カメラが接続されていないフラグの場合
if (flg_video == 1):
#カメラが接続されているか確認
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
ret, frame = cap.read()
if (ret is True):
flg_video = 0
name_pose = "Unknown"
focus_flg = 1
namePose_flg = 1
cap.release()
eel.overlay_controll(True)
eel.object_change("demo2.html", True)
#eel.sleep(1)
sel_cam_before = 999
while (True):
if (decideFlgHT == 1):
if (sel_camHT != sel_cam_before):
cap = cv.VideoCapture(sel_camHT)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
ret, frame = cap.read(sel_camHT)
if (ret is False):
eel.alert_mess()
cap.release()
decide_camHT_py(999)
decide_flgHT_py(0)
sel_cam_before = sel_camHT
continue
else:
decide_flgHT_py(0)
break
decide_flgHT_py(0)
break
if (sel_camHT != 999):
eel.sleep(0.01)
if (sel_camHT != sel_cam_before):
if (sel_cam_before != 999):
cap.release()
cap = cv.VideoCapture(sel_camHT)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
sel_cam_before = sel_camHT
ret, frame = cap.read()
if (ret is True):
# UI側へ転送(画像) #####################################################
_, imencode_image = cv.imencode('.jpg', frame)
base64_image = base64.b64encode(imencode_image)
eel.set_base64image("data:image/jpg;base64," +
base64_image.decode("ascii"))
continue
else:
eel.alert_mess()
cap.release()
decide_camHT_py(999)
sel_cam_before = sel_camHT
if (decideFlgHT == 1):
decide_flgHT_py(0)
continue
else:
eel.sleep(0.01)
continue #最初の while に戻る
else:
#カメラが接続されていない場合
#print("webcamないよ！！！")
eel.sleep(0.01)
time.sleep(0.01)
continue #最初の while に戻る
elif (flg_break == 1):
decide_camHT_py(999)
decide_flgHT_py(0)
break #最初の while を抜けて正常終了
# カメラ準備 ###############################################################
#cap = cv.VideoCapture(cap_device)
#cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
#cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
static_image_mode=use_static_image_mode,
max_num_hands=1,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
keypoint_classifier = KeyPointClassifier()
point_history_classifier = PointHistoryClassifier()
# ラベル読み込み ###########################################################
with open('model/keypoint_classifier/keypoint_classifier_label.csv',
encoding='utf-8-sig') as f:
keypoint_classifier_labels = csv.reader(f)
keypoint_classifier_labels = [
row[0] for row in keypoint_classifier_labels
]
with open(
'model/point_history_classifier/point_history_classifier_label.csv',
encoding='utf-8-sig') as f:
point_history_classifier_labels = csv.reader(f)
point_history_classifier_labels = [
row[0] for row in point_history_classifier_labels
]
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
# 座標履歴 #################################################################
history_length = 16
point_history = deque(maxlen=history_length)
# フィンガージェスチャー履歴 ################################################
finger_gesture_history = deque(maxlen=history_length)
# ########################################################################
mode = 0
CountPose = [0, 0, 0, 0, 0, 0, 0]
CountMotion = [0, 0, 0, 0] # [Top,Right,Down,Left]
identification = False
while True:
fps = cvFpsCalc.get()
# キー処理(ESC：終了) #################################################
key = cv.waitKey(10)
if key == 27: # ESC
break
number, mode = select_mode(key, mode)
# カメラキャプチャ #####################################################
ret, image = cap.read()
if not ret:
#それぞれのフラグを立てて、システムを終了させ、最初の while に戻る
flg_video = 1
focus_flg = 0
print("【通知】WebCameraが接続されていません。")
eel.focusSwitch(width, height, focus_flg)
eel.overlay_controll(True)
eel.object_change("connect.html", True)
cap.release()
cv.destroyAllWindows()
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
# 検出実施 #############################################################
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = hands.process(image)
image.flags.writeable = True
# ####################################################################
if results.multi_hand_landmarks is not None:
#j=1
for hand_landmarks, handedness in zip(
results.multi_hand_landmarks,
results.multi_handedness):
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, hand_landmarks)
# ランドマークの計算
landmark_list = calc_landmark_list(debug_image,
hand_landmarks)
# 相対座標・正規化座標への変換
pre_processed_landmark_list = pre_process_landmark(
landmark_list)
pre_processed_point_history_list = pre_process_point_history(
debug_image, point_history)
# 学習データ保存
logging_csv(number, mode, pre_processed_landmark_list,
pre_processed_point_history_list)
# ハンドサイン分類
hand_sign_id = keypoint_classifier(
pre_processed_landmark_list)
if hand_sign_id == 1: # Dangサイン
point_history.append(landmark_list[8]) # 人差指座標
else:
point_history.append([0, 0])
# フィンガージェスチャー分類
finger_gesture_id = 0
point_history_len = len(pre_processed_point_history_list)
if point_history_len == (history_length * 2):
finger_gesture_id = point_history_classifier(
pre_processed_point_history_list)
# 直近検出の中で最多のジェスチャーIDを算出
finger_gesture_history.append(finger_gesture_id)
most_common_fg_id = Counter(
finger_gesture_history).most_common()
gesture_name = point_history_classifier_labels[
most_common_fg_id[0][0]]
# 描画
debug_image = draw_bounding_rect(use_brect, debug_image,
brect)
debug_image = draw_landmarks(debug_image, landmark_list)
debug_image = draw_info_text(
debug_image,
brect,
handedness,
keypoint_classifier_labels[hand_sign_id],
gesture_name,
)
#人差し指の先の座標を取得
x, y = landmark_list[8]
#座標調整
x_width = args.width * 0.05
x = x - x_width
x = x * 1.5
y = y * 1.5
#ジェスチャーが判定された回数をカウント
if gesture_name == 'Stop':
CountMotion = [0, 0, 0, 0]
elif gesture_name == 'Move_Top':
Count_temp = CountMotion[0]
Count_temp += 1
CountMotion = [Count_temp, 0, 0, 0]
elif gesture_name == 'Move_Right':
Count_temp = CountMotion[1]
Count_temp += 1
CountMotion = [0, Count_temp, 0, 0]
elif gesture_name == 'Move_Down':
Count_temp = CountMotion[2]
Count_temp += 1
CountMotion = [0, 0, Count_temp, 0]
elif gesture_name == 'Move_Left':
Count_temp = CountMotion[3]
Count_temp += 1
CountMotion = [0, 0, 0, Count_temp]
#各種操作の実行
CountPose, CountMotion = PoseAction.action(
hand_sign_id, x, y, CountPose, CountMotion,
ShortCutList)
name_pose = keypoint_classifier_labels[hand_sign_id]
eel.set_posegauge(str(name_pose))
identification = True
else:
point_history.append([0, 0])
if identification == True:
eel.set_posegauge('None')
identification = False
eel.shortcut_overlay(False, 0)
debug_image = draw_point_history(debug_image, point_history)
debug_image = draw_info(debug_image, fps, mode, number)
# 画面反映 #############################################################
debug_image = cv.resize(debug_image, dsize=(520, 260))
cv.imshow('FOCUS preview', debug_image)
if potision_flg == 0:
cv.moveWindow('FOCUS preview', 0, 0)
potision_flg = 1
# eel立ち上げ #############################################################
#cnt_gui, flg_end, flg_restart, flg_start, keep_flg = hand_gui.start_gui(cnt_gui, name_pose, flg_restart, flg_start, keep_flg)
if (namePose_flg == 1):
eel.object_change("complete_old.html", True)
#eel.sleep(0.01)
#eel.init("GUI/web")
#eel.start("開きたい上記のフォルダ下のファイル名",～
#eel.start("html/keeper.html",
# port = 0,
# mode='chrome',
# size=(4, 2), #サイズ指定（横, 縦）
# position=(width,0), #位置指定（left, top）
# block=False
# )
eel.sleep(0.01)
print("【通知】準備完了")
namePose_flg = 0
elif (focus_flg == 1 and name_pose != 'Unknown'):
eel.object_change("complete_old.html", False)
eel.overlay_controll(False)
eel.focusSwitch(width, height, focus_flg)
print("【実行】index.html")
eel.sleep(0.01)
#eel.set_posegauge(name_pose,i)
focus_flg = 0
#i+=1
# eel立ち上げ #############################################################
flg_end = hand_gui.start_gui()
if (flg_end == 1):
#正常に終了する処理(中間のループを抜ける)
flg_break = 1
eel.endSwitch() #flg_end の値をもとに戻す関数
cap.release()
cv.destroyAllWindows()
eel.overlay_controll(False)
eel.object_change("endpage.html", False)
#eel.windowclose_keeper()
break
def main():
# 引数 #####################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
max_num_faces = args.max_num_faces
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
# カメラ準備 ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード #############################################################
face_mesh = FaceMesh(
max_num_faces,
min_detection_confidence,
min_tracking_confidence,
)
iris_detector = IrisLandmark()
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
## parameter ##
left_list = [] # for calculate thres size of iris
right_list = []
prepare = 1 # 1 - preparation mode, 0 - detection mode
stack = 0 # if stack if enough => sleep
flag = 0 # flag for terminate preparation mode
while True:
display_fps = cvFpsCalc.get()
# カメラキャプチャ #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
# 検出実施 #############################################################
# Face Mesh検出
face_results = face_mesh(image)
for face_result in face_results:
# 目周辺のバウンディングボックス計算
left_eye, right_eye = face_mesh.calc_around_eye_bbox(face_result)
# 虹彩検出
left_iris, right_iris = detect_iris(image, iris_detector, left_eye, right_eye)
# 虹彩の外接円を計算
left_center, left_radius = calc_min_enc_losingCircle(left_iris)
right_center, right_radius = calc_min_enc_losingCircle(right_iris)
if (flag==150):
prepare = 0
print("#############################################################################\nready\n#############################################################################")
flag += 1
elif (flag<150): prepare = 1
if (prepare):
flag += 1
left_list.append(left_radius)
right_list.append(right_radius)
left_avg = sum(left_list, 0.0) / len(left_list)
right_avg = sum(right_list, 0.0) / len(right_list)
left_thres = 0.80 * left_avg
right_thres = 0.80 * right_avg
print("calculating avg iris size")
else:
num_of_small_iris = 0
if (left_radius<left_thres): num_of_small_iris+=1
if (right_radius<left_thres): num_of_small_iris+=1
if num_of_small_iris==2:
stack += 1
else:
if stack != 0:
stack -= 1
if stack>= 30:
print("########################################################\nWARNING!!!DON'T SLEEP\n########################################################")
else:
print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!NOT SLEEPING!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
# デバッグ描画
debug_image = draw_debug_image(
debug_image,
left_iris,
right_iris,
left_center,
left_radius,
right_center,
right_radius,
)
cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
# キー処理(ESC：終了) #################################################
key = cv.waitKey(1)
if key == 27: # ESC
break
# 画面反映 #############################################################
cv.imshow('Iris(tflite) Demo', debug_image)
cap.release()
cv.destroyAllWindows()
return
def main():
# 引数解析 #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
ceil_num = args.ceil
image_ratio = args.image_ratio
x_offset = args.x_offset
y_offset = args.y_offset
# カメラ準備 ###############################################################
cap = cv.VideoCapture(cap_device)
# 重畳画像準備 #############################################################
image_pathlist = sorted(glob.glob('utils/image/*.png'))
images = []
for image_path in image_pathlist:
images.append(cv.imread(image_path, cv.IMREAD_UNCHANGED))
animation_counter = 0
# CenterFace準備 ###########################################################
centerface = CenterFace()
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc()
# フォント
font_path = './utils/font/x12y20pxScanLine.ttf'
while True:
# FPS算出 #############################################################
display_fps = cvFpsCalc.get()
# カメラキャプチャ #####################################################
ret, frame = cap.read()
if not ret:
break
resize_frame = cv.resize(frame, (int(cap_width), int(cap_height)))
frame_height, frame_width = resize_frame.shape[:2]
# 顔検出 ##############################################################
dets, lms = centerface(resize_frame,
frame_height,
frame_width,
threshold=0.35)
# デバッグ表示 ########################################################
# バウンディングボックス
for det in dets:
bbox, _ = det[:4], det[4] # BBox, Score
x1, y1 = int(bbox[0]), int(bbox[1])
x2, y2 = int(bbox[2]), int(bbox[3])
# 顔の立幅に合わせて重畳画像をリサイズ
image_height, image_width = images[0].shape[:2]
resize_ratio = (y2 - y1) / image_height
resize_image_height = int(image_height * resize_ratio)
resize_image_width = int(image_width * resize_ratio)
resize_image_height = int(
(resize_image_height + (ceil_num - 1)) / ceil_num * ceil_num)
resize_image_width = int(
(resize_image_width + (ceil_num - 1)) / ceil_num * ceil_num)
resize_image_height = int(resize_image_height * image_ratio)
resize_image_width = int(resize_image_width * image_ratio)
resize_image = cv.resize(images[animation_counter],
(resize_image_width, resize_image_height))
# 画像描画
overlay_x = int((x2 + x1) / 2) - int(resize_image_width / 2)
overlay_y = int((y2 + y1) / 2) - int(resize_image_height / 2)
resize_frame = CvOverlayImage.overlay(
resize_frame, resize_image,
(overlay_x + x_offset, overlay_y + y_offset))
# ランドマーク
# for lm in lms:
# for i in range(0, 5):
# cv.circle(resize_frame, (int(lm[i * 2]), int(lm[i * 2 + 1])),
# 3, (0, 0, 255), -1)
animation_counter += 1
if animation_counter >= len(images):
animation_counter = 0
# 画面反映
fps_string = u"FPS:" + str(display_fps)
resize_frame = CvDrawText.puttext(resize_frame, fps_string, (15, 15),
font_path, 40, (255, 255, 255))
cv.imshow('Demo', resize_frame)
key = cv.waitKey(1)
if key == 27: # ESC
break
cap.release()
def run(self):
# Argument parsing #################################################################
self.stop_flag = False
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
use_static_image_mode = args.use_static_image_mode
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = True
# Camera preparation ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# Model load #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
static_image_mode=use_static_image_mode,
max_num_hands=1,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
keypoint_classifier_R = KeyPointClassifier_R(invalid_value=8,
score_th=0.4)
keypoint_classifier_L = KeyPointClassifier_L(invalid_value=8,
score_th=0.4)
mouse_classifier = MouseClassifier(invalid_value=2, score_th=0.4)
point_history_classifier = PointHistoryClassifier()
# Read labels ###########################################################
with open('model/keypoint_classifier/keypoint_classifier_label.csv',
encoding='utf-8-sig') as f:
keypoint_classifier_labels = csv.reader(f)
keypoint_classifier_labels = [
row[0] for row in keypoint_classifier_labels
]
with open(
'model/point_history_classifier/point_history_classifier_label.csv',
encoding='utf-8-sig') as f:
point_history_classifier_labels = csv.reader(f)
point_history_classifier_labels = [
row[0] for row in point_history_classifier_labels
]
# FPS Measurement ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=3)
# Coordinate history #################################################################
history_length = 16
point_history = deque(maxlen=history_length)
# Finger gesture history ################################################
finger_gesture_history = deque(maxlen=history_length)
mouse_id_history = deque(maxlen=40)
# 靜態手勢最常出現參數初始化
keypoint_length = 5
keypoint_R = deque(maxlen=keypoint_length)
keypoint_L = deque(maxlen=keypoint_length)
# result deque
rest_length = 300
rest_result = deque(maxlen=rest_length)
speed_up_count = deque(maxlen=3)
# ========= 使用者自訂姿勢、指令區 =========
# time.sleep(0.5)
# keepadd = False
# ========= 按鍵前置作業 =========
mode = 0
presstime = presstime_2 = presstime_3 = resttime = presstime_4 = time.time(
)
detect_mode = 2
what_mode = 'mouse'
landmark_list = 0
pyautogui.PAUSE = 0
# ========= 滑鼠前置作業 =========
wScr, hScr = pyautogui.size()
frameR = 100
smoothening = 7
plocX, plocY = 0, 0
clocX, clocY = 0, 0
mousespeed = 1.5
clicktime = time.time()
#關閉 滑鼠移至角落啟動保護措施
pyautogui.FAILSAFE = False
# ========= google 小姐 =========
# speech_0 = gTTS(text="スリープモード", lang='ja')
# speech_0.save('rest.mp3')
# speech_0 = gTTS(text="キーボードモード", lang='ja')
# speech_0.save('keyboard.mp3')
# speech = gTTS(text="マウスモード", lang='ja')
# speech.save('mouse.mp3')
# ===============================
i = 0
finger_gesture_id = 0
# ========= 主程式運作 =========
while (not self.stop_flag):
left_id = right_id = -1
fps = cvFpsCalc.get()
# Process Key (ESC: end)
key = cv.waitKey(10)
if key == 27: # ESC
break
number, mode = select_mode(key, mode)
# Camera capture
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # Mirror display
debug_image = copy.deepcopy(image)
# Detection implementation
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = hands.process(image)
image.flags.writeable = True
####rest_result####
if results.multi_hand_landmarks is None:
rest_id = 0
rest_result.append(rest_id)
if results.multi_hand_landmarks is not None:
rest_id = 1
rest_result.append(rest_id)
most_common_rest_result = Counter(rest_result).most_common()
# old version for 10 sec to rest mode####################
#print(most_common_rest_result[0])
# if most_common_rest_result[0][0] == 0 and most_common_rest_result[0][1] == 300:
# if detect_mode != 0:
#
# print('Mode has changed')
# detect_mode = 0
# what_mode = 'Rest'
# print(f'Current mode => {what_mode}')
# new version for 10 sec to rest mode###################
if time.time() - resttime > 10:
if detect_mode != 0:
detect_mode = 0
what_mode = 'Sleep'
print(f'Current mode => {what_mode}')
####rest_result####
# ####################################################################
# print(most_common_rest_result)
if results.multi_hand_landmarks is not None:
for hand_landmarks, handedness in zip(
results.multi_hand_landmarks,
results.multi_handedness):
# Bounding box calculation
brect = calc_bounding_rect(debug_image, hand_landmarks)
# Landmark calculation
landmark_list = calc_landmark_list(debug_image,
hand_landmarks)
# print(landmark_list)
# Conversion to relative coordinates / normalized coordinates
pre_processed_landmark_list = pre_process_landmark(
landmark_list)
pre_processed_point_history_list = pre_process_point_history(
debug_image, point_history)
# Write to the dataset file
logging_csv(number, mode, pre_processed_landmark_list,
pre_processed_point_history_list)
# 靜態手勢資料預測
hand_sign_id_R = keypoint_classifier_R(
pre_processed_landmark_list)
hand_sign_id_L = keypoint_classifier_L(
pre_processed_landmark_list)
mouse_id = mouse_classifier(pre_processed_landmark_list)
# print(mouse_id)
if handedness.classification[0].label[0:] == 'Left':
left_id = hand_sign_id_L
else:
right_id = hand_sign_id_R
# 手比one 觸發動態資料抓取
if right_id == 1 or left_id == 1:
point_history.append(landmark_list[8])
else:
point_history.append([0, 0])
# 動態手勢資料預測
finger_gesture_id = 0
point_history_len = len(pre_processed_point_history_list)
if point_history_len == (history_length * 2):
finger_gesture_id = point_history_classifier(
pre_processed_point_history_list)
# print(finger_gesture_id) # 0 = stop, 1 = clockwise, 2 = counterclockwise, 3 = move,偵測出現的動態手勢
# 動態手勢最常出現id #########################################
# Calculates the gesture IDs in the latest detection
finger_gesture_history.append(finger_gesture_id)
most_common_fg_id = Counter(
finger_gesture_history).most_common()
#滑鼠的deque
mouse_id_history.append(mouse_id)
most_common_ms_id = Counter(mouse_id_history).most_common()
# print(f'finger_gesture_history = {finger_gesture_history}')
# print(f'most_common_fg_id = {most_common_fg_id}')
# 靜態手勢最常出現id #########################################
hand_gesture_id = [right_id, left_id]
keypoint_R.append(hand_gesture_id[0])
keypoint_L.append(hand_gesture_id[1])
# print(keypoint_R) # deque右手的靜態id
# print(most_common_keypoint_id) # 右手靜態id最大
if right_id != -1:
most_common_keypoint_id = Counter(
keypoint_R).most_common()
else:
most_common_keypoint_id = Counter(
keypoint_L).most_common()
# print(f'keypoint = {keypoint}')
# print(f'most_common_keypoint_id = {most_common_keypoint_id}')
###############################################################
# Drawing part
debug_image = draw_bounding_rect(use_brect, debug_image,
brect)
debug_image = draw_landmarks(debug_image, landmark_list)
debug_image = draw_info_text(
debug_image,
brect,
handedness,
keypoint_classifier_labels[most_common_keypoint_id[0]
[0]],
point_history_classifier_labels[most_common_fg_id[0]
[0]],
)
resttime = time.time()
else:
point_history.append([0, 0])
debug_image = draw_point_history(debug_image, point_history)
debug_image = draw_info(debug_image, fps, mode, number)
# 偵測是否有手勢 #########################################
if left_id + right_id > -2:
if time.time() - presstime > 1:
# change mode
if most_common_ms_id[0][0] == 3 and most_common_ms_id[0][
1] == 40: #Gesture six changes to the different mode
print('Mode has changed')
detect_mode = (detect_mode + 1) % 3
if detect_mode == 0:
what_mode = 'Sleep'
playsound('rest.mp3', block=False)
if detect_mode == 1:
what_mode = 'Keyboard'
playsound('keyboard.mp3', block=False)
if detect_mode == 2:
what_mode = 'Mouse'
playsound('mouse.mp3', block=False)
print(f'Current mode => {what_mode}')
presstime = time.time() + 1
# control keyboard
elif detect_mode == 1:
if time.time() - presstime_2 > 1:
# 靜態手勢控制
control_keyboard(most_common_keypoint_id,
2,
'K',
keyboard_TF=True,
print_TF=False)
# control_keyboard(most_common_keypoint_id, 0, 'right', keyboard_TF=True, print_TF=False)
# control_keyboard(most_common_keypoint_id, 7, 'left', keyboard_TF=True, print_TF=False)
control_keyboard(most_common_keypoint_id,
9,
'C',
keyboard_TF=True,
print_TF=False)
control_keyboard(most_common_keypoint_id,
5,
'up',
keyboard_TF=True,
print_TF=False)
control_keyboard(most_common_keypoint_id,
6,
'down',
keyboard_TF=True,
print_TF=False)
presstime_2 = time.time()
# right右鍵
if most_common_keypoint_id[0][
0] == 0 and most_common_keypoint_id[0][1] == 5:
# print(i, time.time() - presstime_4)
if i == 3 and time.time() - presstime_4 > 0.3:
pyautogui.press('right')
i = 0
presstime_4 = time.time()
elif i == 3 and time.time() - presstime_4 > 0.25:
pyautogui.press('right')
presstime_4 = time.time()
elif time.time() - presstime_4 > 1:
pyautogui.press('right')
i += 1
presstime_4 = time.time()
# left左鍵
if most_common_keypoint_id[0][
0] == 7 and most_common_keypoint_id[0][1] == 5:
# print(i, time.time() - presstime_4)
if i == 3 and time.time() - presstime_4 > 0.3:
pyautogui.press('left')
i = 0
presstime_4 = time.time()
elif i == 3 and time.time() - presstime_4 > 0.25:
pyautogui.press('left')
presstime_4 = time.time()
elif time.time() - presstime_4 > 1:
pyautogui.press('left')
i += 1
presstime_4 = time.time()
# 動態手勢控制
if most_common_fg_id[0][
0] == 1 and most_common_fg_id[0][1] > 12:
if time.time() - presstime_3 > 1.5:
#pyautogui.press(['shift', '>'])
pyautogui.hotkey('shift', '>')
print('speed up')
presstime_3 = time.time()
elif most_common_fg_id[0][
0] == 2 and most_common_fg_id[0][1] > 12:
if time.time() - presstime_3 > 1.5:
#pyautogui.press(['shift', '<'])
pyautogui.hotkey('shift', '<')
print('speed down')
presstime_3 = time.time()
if detect_mode == 2:
if mouse_id == 0: # Point gesture
# print(landmark_list[8]) #index finger
# print(landmark_list[12]) #middle finger
x1, y1 = landmark_list[8]
# cv.rectangle(debug_image, (frameR, frameR), (cap_width - frameR, cap_height - frameR),
# (255, 0, 255), 2)
cv.rectangle(debug_image, (50, 30),
(cap_width - 50, cap_height - 170),
(255, 0, 255), 2)
#座標轉換
#x軸: 鏡頭上50~(cap_width - 50)轉至螢幕寬0~wScr
#y軸: 鏡頭上30~(cap_height - 170)轉至螢幕長0~hScr
x3 = np.interp(x1, (50, (cap_width - 50)), (0, wScr))
y3 = np.interp(y1, (30, (cap_height - 170)), (0, hScr))
# print(x3, y3)
# 6. Smoothen Values
clocX = plocX + (x3 - plocX) / smoothening
clocY = plocY + (y3 - plocY) / smoothening
# 7. Move Mouse
pyautogui.moveTo(clocX, clocY)
cv.circle(debug_image, (x1, y1), 15, (255, 0, 255),
cv.FILLED)
plocX, plocY = clocX, clocY
if mouse_id == 1:
length, img, lineInfo = findDistance(
landmark_list[8], landmark_list[12], debug_image)
# 10. Click mouse if distance short
if time.time() - clicktime > 0.5:
if length < 40:
cv.circle(img, (lineInfo[4], lineInfo[5]), 15,
(0, 255, 0), cv.FILLED)
pyautogui.click()
print('click')
clicktime = time.time()
# if length > 70:
# cv.circle(img, (lineInfo[4], lineInfo[5]),
# 15, (0, 255, 0), cv.FILLED)
# pyautogui.click(clicks=2)
# print('click*2')
# clicktime = time.time()
if most_common_keypoint_id[0][
0] == 5 and most_common_keypoint_id[0][1] == 5:
pyautogui.scroll(20)
if most_common_keypoint_id[0][
0] == 6 and most_common_keypoint_id[0][1] == 5:
pyautogui.scroll(-20)
#if left_id == 7 or right_id == 7:
if most_common_keypoint_id[0][
0] == 0 and most_common_keypoint_id[0][1] == 5:
if time.time() - clicktime > 1:
pyautogui.click(clicks=2)
clicktime = time.time()
if most_common_keypoint_id[0][
0] == 9 and most_common_keypoint_id[0][1] == 5:
if time.time() - clicktime > 2:
pyautogui.hotkey('alt', 'left')
clicktime = time.time()
cv.putText(debug_image, what_mode, (400, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 4, cv.LINE_AA)
# Screen reflection ###################################JL##########################
cv.imshow('Hand Gesture Recognition', debug_image)
self.trigger.emit(detect_mode)
cap.release()
cv.destroyAllWindows()
def main():
# 引数 #####################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
max_num_faces = args.max_num_faces
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
# カメラ準備 ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード #############################################################
face_mesh = FaceMesh(
max_num_faces,
min_detection_confidence,
min_tracking_confidence,
)
iris_detector = IrisLandmark()
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
while True:
display_fps = cvFpsCalc.get()
# カメラキャプチャ #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
# 検出実施 #############################################################
# Face Mesh検出
face_results = face_mesh(image)
for face_result in face_results:
# 目周辺のバウンディングボックス計算
left_eye, right_eye = face_mesh.calc_around_eye_bbox(face_result)
# 虹彩検出
left_iris, right_iris = detect_iris(image, iris_detector, left_eye,
right_eye)
# 虹彩の外接円を計算
left_center, left_radius = calc_min_enc_losingCircle(left_iris)
right_center, right_radius = calc_min_enc_losingCircle(right_iris)
# デバッグ描画
debug_image = draw_debug_image(
debug_image,
left_iris,
right_iris,
left_center,
left_radius,
right_center,
right_radius,
)
cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
# キー処理(ESC：終了) #################################################
key = cv.waitKey(1)
if key == 27: # ESC
break
# 画面反映 #############################################################
cv.imshow('Iris(tflite) Demo', debug_image)
cap.release()
cv.destroyAllWindows()
return
def main():
# 引数解析 (Argument parsing) (Parameter analysis) #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
upper_body_only = args.upper_body_only
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = args.use_brect
# カメラ準備 (Camera preparation) ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード (Model load) #############################################################
mp_holistic = mp.solutions.holistic
holistic = mp_holistic.Holistic(
upper_body_only=upper_body_only,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
# FPS計測モジュール (Measurement module) ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
while True:
display_fps = cvFpsCalc.get()
# カメラキャプチャ (Camera capture) #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
# 検出実施 (Detection implementation) #############################################################
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = holistic.process(image)
image.flags.writeable = True
# Face Mesh ###########################################################
face_landmarks = results.face_landmarks
if face_landmarks is not None:
# 外接矩形の計算 (Calculation of circumscribed rectangle)
brect = calc_bounding_rect(debug_image, face_landmarks)
# 描画 (drawing)
debug_image = draw_face_landmarks(debug_image, face_landmarks)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
# Pose ###############################################################
pose_landmarks = results.pose_landmarks
if pose_landmarks is not None:
# 外接矩形の計算 (Calculation of circumscribed rectangle)
brect = calc_bounding_rect(debug_image, pose_landmarks)
# 描画 (drawing)
debug_image = draw_pose_landmarks(debug_image, pose_landmarks,
upper_body_only)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
# Hands ###############################################################
left_hand_landmarks = results.left_hand_landmarks
right_hand_landmarks = results.right_hand_landmarks
# 左手 (left hand)
if left_hand_landmarks is not None:
# 手の平重心計算 (Calculation of the center of gravity of the palm)
cx, cy = calc_palm_moment(debug_image, left_hand_landmarks)
# 外接矩形の計算 (Calculation of circumscribed rectangle)
brect = calc_bounding_rect(debug_image, left_hand_landmarks)
# 描画 (drawing)
debug_image = draw_hands_landmarks(debug_image, cx, cy,
left_hand_landmarks,
upper_body_only, 'R')
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
# 右手 (right hand)
if right_hand_landmarks is not None:
# 手の平重心計算 (Calculation of the center of gravity of the palm)
cx, cy = calc_palm_moment(debug_image, right_hand_landmarks)
# 外接矩形の計算 (Calculation of circumscribed rectangle)
brect = calc_bounding_rect(debug_image, right_hand_landmarks)
# 描画 (drawing)
debug_image = draw_hands_landmarks(debug_image, cx, cy,
right_hand_landmarks,
upper_body_only, 'L')
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
# キー処理 (Key processing) (ESC：終了) (ESC: End)#################################################
key = cv.waitKey(1)
if key == 27: # ESC
break
# 画面反映 (Screen reflection)#############################################################
cv.imshow('MediaPipe Holistic Demo', debug_image)
cap.release()
cv.destroyAllWindows()
def main():
mp_drawing = mp.solutions.drawing_utils
mp_hands = mp.solutions.hands
# Argument parsing
in_flight = False
# For webcam input:
hands = mp_hands.Hands(static_image_mode=True,
max_num_hands=2,
min_detection_confidence=0.8,
min_tracking_confidence=0.8)
# Camera preparation
tello = Tello()
tello.connect()
tello.streamon()
# FPS Measurement
cv_fps_calc = CvFpsCalc(buffer_len=5)
frame_read = tello.get_frame_read()
while True:
fps = cv_fps_calc.get()
battery_status = tello.get_battery()
# In reality you want to display frames in a seperate thread. Otherwise
# they will freeze while the drone moves.
image = frame_read.frame
# Flip the image horizontally for a later selfie-view display, and convert
# the BGR image to RGB.
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
# To improve performance, optionally mark the image as not writeable to
# pass by reference.
image.flags.writeable = False
results = hands.process(image)
# Draw the hand annotations on the image.
image.flags.writeable = True
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
if results.multi_hand_landmarks:
for hand_landmarks in results.multi_hand_landmarks:
mp_drawing.draw_landmarks(image, hand_landmarks,
mp_hands.HAND_CONNECTIONS)
# Process Key (ESC: end)
key = cv2.waitKey(1) & 0xff
if key == 27: # ESCArgument parsing
tello.land()
break
elif key == 32:
if not in_flight:
# Take-off drone
tello.takeoff()
in_flight = True
tello.move_up(50)
elif in_flight:
# Land tello
tello.land()
in_flight = False
elif key == ord('w'):
tello.move_forward(30)
elif key == ord('s'):
tello.move_back(30)
elif key == ord('a'):
tello.move_left(30)
elif key == ord('d'):
tello.move_right(30)
elif key == ord('e'):
tello.rotate_clockwise(30)
elif key == ord('q'):
tello.rotate_counter_clockwise(30)
elif key == ord('r'):
tello.move_up(30)
elif key == ord('f'):
tello.move_down(30)
# Battery status and image rendering
cv2.putText(image, "Battery: {} / fps:{}".format(battery_status, fps),
(5, 720 - 5), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
cv2.imshow('DK_Drone', image)
tello.land()
tello.end()
cv2.destroyAllWindows()
def main():
# my add variable value ################################################
# is_concentrated = False
# frame_length = 256
sampling_time = 0.1
detecting_time = 0
not_detecting_time = 0
window_size = 100
list_zeros_window = [0 for i in range(window_size)]
list_detect_window = list_zeros_window
detect_count = 0
border_face_detect = 0.1
# alert_time = 60 * 30
alert_time = 20
alert_title = "Alert"
alert_message = "Let's move the body.\nYou have been concentrating on your work for more than 25 minutes."
# Reset Param ############################################################
# reset_work_boarder = 60 * 3
reset_work_boarder = 3
# Exercise ###############################################################
# exercise_boarder = 60*5
exercise_boarder = 5
is_exercise_state = False
is_init_exercise_state = True
finish_alert_title = "Congratulation！"
finish_alert_message= "Finished Your Exercise Time."
# Repeat Alert ###########################################################
repeat_alert_time = 1
# repeat_boarder = 60
repeat_boarder = 10
repeat_alert_title = "Repeat Alerm"
repeat_alert_message = "Let's move the body.\n"
# investigation param ####################################################
output_list = [['time[s]', '顔検出値[0 or 1]']]
output_count = 0
# plot param #############################################################
x = np.linspace(0, 10, 100)
y = np.zeros(x.size)
plt.ion()
figure, ax = plt.subplots(figsize=(8,6))
line1, = ax.plot(x, y)
plt.title("Dynamic Plot of face detection",fontsize=25)
plt.xlabel("time",fontsize=18)
plt.ylabel("is detected value",fontsize=18)
plt.ylim(-1.1,1.1)
updated_y = y
plot_data = 0
# border prot
yb = np.full((1,100), border_face_detect)[0]
plt.plot(x, yb, color = 'red')
# 引数解析 #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
max_num_faces = args.max_num_faces
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = args.use_brect
# カメラ準備 ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード #############################################################
mp_face_mesh = mp.solutions.face_mesh
face_mesh = mp_face_mesh.FaceMesh(
max_num_faces=max_num_faces,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
while True:
display_fps = cvFpsCalc.get()
# カメラキャプチャ #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
# 検出実施 #############################################################
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
results = face_mesh.process(image)
# 描画 ################################################################
if results.multi_face_landmarks is not None:
for face_landmarks in results.multi_face_landmarks:
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, face_landmarks)
# 描画
debug_image = draw_landmarks(debug_image, face_landmarks)
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
list_detect_window[detect_count]=1
output_list.append([output_count, 1])
plot_data=1
else:
list_detect_window[detect_count]=0
output_list.append([output_count, 0])
plot_data=0
cv.putText(debug_image, "FPS:" + str(display_fps), (10, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 255, 0), 2, cv.LINE_AA)
# キー処理(ESC：終了) #################################################
key = cv.waitKey(1)
if key == 27: # ESC
break
# 画面反映 #############################################################
cv.imshow('MediaPipe Face Mesh Demo', debug_image)
# calculate the mean of list_detect_window ############################
mean_list_detect_window = statistics.mean(list_detect_window)
if mean_list_detect_window > border_face_detect:
detecting_time += sampling_time
not_detecting_time = 0
if detecting_time >= alert_time:
is_exercise_state = True
is_init_exercise_state = True
if repeat_alert_time % repeat_boarder == 0:
print("repeat alert")
do_alert(repeat_alert_title, repeat_alert_message)
elif repeat_alert_time ==1:
do_alert(alert_title, alert_message)
repeat_alert_time = round(repeat_alert_time + sampling_time, 1)
# print(repeat_alert_time)
else:
if is_exercise_state:
# if is_init_exercise_state:
# mean_list_detect_window = list_zeros_window
# is_init_exercise_state = False
not_detecting_time += sampling_time
if not_detecting_time >= exercise_boarder:
do_alert(finish_alert_title, finish_alert_message)
# Reset Param ################################################
detecting_time = 0
not_detecting_time=0
repeat_alert_time=1
is_exercise_state = False
else:
### Personal function ########################################
### 作業していない期間が一定時間を超えたら作業タイマーをリセットする
##############################################################
not_detecting_time += sampling_time
if not_detecting_time >= reset_work_boarder:
# Reset Param ############################################
detecting_time = 0
not_detecting_time=0
repeat_alert_time=1
if is_exercise_state and is_init_exercise_state:
# print("test test test")
list_detect_window = list_zeros_window
is_init_exercise_state = False
detect_count = 0 if detect_count == window_size-1 else detect_count+1
time.sleep(sampling_time)
output_count += sampling_time
# Update Plot ########################################################
# updated_y = enqueue(updated_y, plot_data)
updated_y = enqueue(updated_y, mean_list_detect_window)
line1.set_xdata(x)
line1.set_ydata(updated_y)
figure.canvas.draw()
figure.canvas.flush_events()
cap.release()
cv.destroyAllWindows()
export_list_csv(output_list, 'face_detected.csv')
def main():
# init global vars
global gesture_buffer
global gesture_id
# Argument parsing
KEYBOARD_CONTROL = True
WRITE_CONTROL = False
in_flight = False
# Camera preparation
tello = Tello()
tello.connect()
tello.streamon()
cap = tello.get_frame_read()
# Init Tello Controllers
gesture_controller = TelloGestureController(tello)
keyboard_controller = TelloKeyboardController(tello)
gesture_detector = GestureRecognition(min_detection_confidence=0.7,
min_tracking_confidence=0.5)
gesture_buffer = GestureBuffer(buffer_len=5)
def tello_control(key, keyboard_controller, gesture_controller):
global gesture_buffer
if KEYBOARD_CONTROL:
keyboard_controller.control(key)
else:
gesture_controller.gesture_control(gesture_buffer)
# FPS Measurement
cv_fps_calc = CvFpsCalc(buffer_len=10)
mode = 0
number = -1
battery_status = -1
while True:
fps = cv_fps_calc.get()
battery_status = tello.get_battery()
# Process Key (ESC: end)
key = cv.waitKey(1) & 0xff
if key == 27: # ESC
break
elif key == 32: # Space
if not in_flight:
# Take-off drone
tello.takeoff()
in_flight = True
tello.move_up(50)
elif in_flight:
# Land tello
tello.land()
in_flight = False
elif key == ord('k'):
mode = 0
KEYBOARD_CONTROL = True
WRITE_CONTROL = False
tello.send_rc_control(0, 0, 0, 0) # Stop moving
elif key == ord('g'):
KEYBOARD_CONTROL = False
elif key == ord('n'):
mode = 1
WRITE_CONTROL = True
KEYBOARD_CONTROL = True
if WRITE_CONTROL:
number = -1
if 48 <= key <= 57: # 0 ~ 9
number = key - 48
# Camera capture
image = cap.frame
debug_image, gesture_id = gesture_detector.recognize(
image, number, mode)
gesture_buffer.add_gesture(gesture_id)
# Start control thread
threading.Thread(target=tello_control,
args=(
key,
keyboard_controller,
gesture_controller,
)).start()
threading.Thread(target=battery_status, args=(tello, )).start()
debug_image = gesture_detector.draw_info(debug_image, fps, mode,
number)
# Battery status and image rendering
cv.putText(debug_image, "Battery: {}".format(battery_status),
(5, 720 - 5), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
cv.imshow('Tello Gesture Recognition', debug_image)
tello.land()
tello.end()
cv.destroyAllWindows()
