def load_model(self):
# Model load #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
static_image_mode=self.use_static_image_mode,
max_num_hands=1,
min_detection_confidence=self.min_detection_confidence,
min_tracking_confidence=self.min_tracking_confidence,
)
keypoint_classifier = KeyPointClassifier()
point_history_classifier = PointHistoryClassifier()
# Read labels ###########################################################
with open('model/keypoint_classifier/keypoint_classifier_label.csv',
encoding='utf-8-sig') as f:
keypoint_classifier_labels = csv.reader(f)
keypoint_classifier_labels = [
row[0] for row in keypoint_classifier_labels
]
with open(
'model/point_history_classifier/point_history_classifier_label.csv',
encoding='utf-8-sig') as f:
point_history_classifier_labels = csv.reader(f)
point_history_classifier_labels = [
row[0] for row in point_history_classifier_labels
]
return hands, keypoint_classifier, keypoint_classifier_labels, \
point_history_classifier, point_history_classifier_labels
def run(self):
# Argument parsing #################################################################
self.stop_flag = False
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
use_static_image_mode = args.use_static_image_mode
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = True
# Camera preparation ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# Model load #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
static_image_mode=use_static_image_mode,
max_num_hands=1,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
keypoint_classifier_R = KeyPointClassifier_R(invalid_value=8,
score_th=0.4)
keypoint_classifier_L = KeyPointClassifier_L(invalid_value=8,
score_th=0.4)
mouse_classifier = MouseClassifier(invalid_value=2, score_th=0.4)
point_history_classifier = PointHistoryClassifier()
# Read labels ###########################################################
with open('model/keypoint_classifier/keypoint_classifier_label.csv',
encoding='utf-8-sig') as f:
keypoint_classifier_labels = csv.reader(f)
keypoint_classifier_labels = [
row[0] for row in keypoint_classifier_labels
]
with open(
'model/point_history_classifier/point_history_classifier_label.csv',
encoding='utf-8-sig') as f:
point_history_classifier_labels = csv.reader(f)
point_history_classifier_labels = [
row[0] for row in point_history_classifier_labels
]
# FPS Measurement ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=3)
# Coordinate history #################################################################
history_length = 16
point_history = deque(maxlen=history_length)
# Finger gesture history ################################################
finger_gesture_history = deque(maxlen=history_length)
mouse_id_history = deque(maxlen=40)
# 靜態手勢最常出現參數初始化
keypoint_length = 5
keypoint_R = deque(maxlen=keypoint_length)
keypoint_L = deque(maxlen=keypoint_length)
# result deque
rest_length = 300
rest_result = deque(maxlen=rest_length)
speed_up_count = deque(maxlen=3)
# ========= 使用者自訂姿勢、指令區 =========
# time.sleep(0.5)
# keepadd = False
# ========= 按鍵前置作業 =========
mode = 0
presstime = presstime_2 = presstime_3 = resttime = presstime_4 = time.time(
)
detect_mode = 2
what_mode = 'mouse'
landmark_list = 0
pyautogui.PAUSE = 0
# ========= 滑鼠前置作業 =========
wScr, hScr = pyautogui.size()
frameR = 100
smoothening = 7
plocX, plocY = 0, 0
clocX, clocY = 0, 0
mousespeed = 1.5
clicktime = time.time()
#關閉 滑鼠移至角落啟動保護措施
pyautogui.FAILSAFE = False
# ========= google 小姐 =========
# speech_0 = gTTS(text="スリープモード", lang='ja')
# speech_0.save('rest.mp3')
# speech_0 = gTTS(text="キーボードモード", lang='ja')
# speech_0.save('keyboard.mp3')
# speech = gTTS(text="マウスモード", lang='ja')
# speech.save('mouse.mp3')
# ===============================
i = 0
finger_gesture_id = 0
# ========= 主程式運作 =========
while (not self.stop_flag):
left_id = right_id = -1
fps = cvFpsCalc.get()
# Process Key (ESC: end)
key = cv.waitKey(10)
if key == 27: # ESC
break
number, mode = select_mode(key, mode)
# Camera capture
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # Mirror display
debug_image = copy.deepcopy(image)
# Detection implementation
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = hands.process(image)
image.flags.writeable = True
####rest_result####
if results.multi_hand_landmarks is None:
rest_id = 0
rest_result.append(rest_id)
if results.multi_hand_landmarks is not None:
rest_id = 1
rest_result.append(rest_id)
most_common_rest_result = Counter(rest_result).most_common()
# old version for 10 sec to rest mode####################
#print(most_common_rest_result[0])
# if most_common_rest_result[0][0] == 0 and most_common_rest_result[0][1] == 300:
# if detect_mode != 0:
#
# print('Mode has changed')
# detect_mode = 0
# what_mode = 'Rest'
# print(f'Current mode => {what_mode}')
# new version for 10 sec to rest mode###################
if time.time() - resttime > 10:
if detect_mode != 0:
detect_mode = 0
what_mode = 'Sleep'
print(f'Current mode => {what_mode}')
####rest_result####
# ####################################################################
# print(most_common_rest_result)
if results.multi_hand_landmarks is not None:
for hand_landmarks, handedness in zip(
results.multi_hand_landmarks,
results.multi_handedness):
# Bounding box calculation
brect = calc_bounding_rect(debug_image, hand_landmarks)
# Landmark calculation
landmark_list = calc_landmark_list(debug_image,
hand_landmarks)
# print(landmark_list)
# Conversion to relative coordinates / normalized coordinates
pre_processed_landmark_list = pre_process_landmark(
landmark_list)
pre_processed_point_history_list = pre_process_point_history(
debug_image, point_history)
# Write to the dataset file
logging_csv(number, mode, pre_processed_landmark_list,
pre_processed_point_history_list)
# 靜態手勢資料預測
hand_sign_id_R = keypoint_classifier_R(
pre_processed_landmark_list)
hand_sign_id_L = keypoint_classifier_L(
pre_processed_landmark_list)
mouse_id = mouse_classifier(pre_processed_landmark_list)
# print(mouse_id)
if handedness.classification[0].label[0:] == 'Left':
left_id = hand_sign_id_L
else:
right_id = hand_sign_id_R
# 手比one 觸發動態資料抓取
if right_id == 1 or left_id == 1:
point_history.append(landmark_list[8])
else:
point_history.append([0, 0])
# 動態手勢資料預測
finger_gesture_id = 0
point_history_len = len(pre_processed_point_history_list)
if point_history_len == (history_length * 2):
finger_gesture_id = point_history_classifier(
pre_processed_point_history_list)
# print(finger_gesture_id) # 0 = stop, 1 = clockwise, 2 = counterclockwise, 3 = move,偵測出現的動態手勢
# 動態手勢最常出現id #########################################
# Calculates the gesture IDs in the latest detection
finger_gesture_history.append(finger_gesture_id)
most_common_fg_id = Counter(
finger_gesture_history).most_common()
#滑鼠的deque
mouse_id_history.append(mouse_id)
most_common_ms_id = Counter(mouse_id_history).most_common()
# print(f'finger_gesture_history = {finger_gesture_history}')
# print(f'most_common_fg_id = {most_common_fg_id}')
# 靜態手勢最常出現id #########################################
hand_gesture_id = [right_id, left_id]
keypoint_R.append(hand_gesture_id[0])
keypoint_L.append(hand_gesture_id[1])
# print(keypoint_R) # deque右手的靜態id
# print(most_common_keypoint_id) # 右手靜態id最大
if right_id != -1:
most_common_keypoint_id = Counter(
keypoint_R).most_common()
else:
most_common_keypoint_id = Counter(
keypoint_L).most_common()
# print(f'keypoint = {keypoint}')
# print(f'most_common_keypoint_id = {most_common_keypoint_id}')
###############################################################
# Drawing part
debug_image = draw_bounding_rect(use_brect, debug_image,
brect)
debug_image = draw_landmarks(debug_image, landmark_list)
debug_image = draw_info_text(
debug_image,
brect,
handedness,
keypoint_classifier_labels[most_common_keypoint_id[0]
[0]],
point_history_classifier_labels[most_common_fg_id[0]
[0]],
)
resttime = time.time()
else:
point_history.append([0, 0])
debug_image = draw_point_history(debug_image, point_history)
debug_image = draw_info(debug_image, fps, mode, number)
# 偵測是否有手勢 #########################################
if left_id + right_id > -2:
if time.time() - presstime > 1:
# change mode
if most_common_ms_id[0][0] == 3 and most_common_ms_id[0][
1] == 40: #Gesture six changes to the different mode
print('Mode has changed')
detect_mode = (detect_mode + 1) % 3
if detect_mode == 0:
what_mode = 'Sleep'
playsound('rest.mp3', block=False)
if detect_mode == 1:
what_mode = 'Keyboard'
playsound('keyboard.mp3', block=False)
if detect_mode == 2:
what_mode = 'Mouse'
playsound('mouse.mp3', block=False)
print(f'Current mode => {what_mode}')
presstime = time.time() + 1
# control keyboard
elif detect_mode == 1:
if time.time() - presstime_2 > 1:
# 靜態手勢控制
control_keyboard(most_common_keypoint_id,
2,
'K',
keyboard_TF=True,
print_TF=False)
# control_keyboard(most_common_keypoint_id, 0, 'right', keyboard_TF=True, print_TF=False)
# control_keyboard(most_common_keypoint_id, 7, 'left', keyboard_TF=True, print_TF=False)
control_keyboard(most_common_keypoint_id,
9,
'C',
keyboard_TF=True,
print_TF=False)
control_keyboard(most_common_keypoint_id,
5,
'up',
keyboard_TF=True,
print_TF=False)
control_keyboard(most_common_keypoint_id,
6,
'down',
keyboard_TF=True,
print_TF=False)
presstime_2 = time.time()
# right右鍵
if most_common_keypoint_id[0][
0] == 0 and most_common_keypoint_id[0][1] == 5:
# print(i, time.time() - presstime_4)
if i == 3 and time.time() - presstime_4 > 0.3:
pyautogui.press('right')
i = 0
presstime_4 = time.time()
elif i == 3 and time.time() - presstime_4 > 0.25:
pyautogui.press('right')
presstime_4 = time.time()
elif time.time() - presstime_4 > 1:
pyautogui.press('right')
i += 1
presstime_4 = time.time()
# left左鍵
if most_common_keypoint_id[0][
0] == 7 and most_common_keypoint_id[0][1] == 5:
# print(i, time.time() - presstime_4)
if i == 3 and time.time() - presstime_4 > 0.3:
pyautogui.press('left')
i = 0
presstime_4 = time.time()
elif i == 3 and time.time() - presstime_4 > 0.25:
pyautogui.press('left')
presstime_4 = time.time()
elif time.time() - presstime_4 > 1:
pyautogui.press('left')
i += 1
presstime_4 = time.time()
# 動態手勢控制
if most_common_fg_id[0][
0] == 1 and most_common_fg_id[0][1] > 12:
if time.time() - presstime_3 > 1.5:
#pyautogui.press(['shift', '>'])
pyautogui.hotkey('shift', '>')
print('speed up')
presstime_3 = time.time()
elif most_common_fg_id[0][
0] == 2 and most_common_fg_id[0][1] > 12:
if time.time() - presstime_3 > 1.5:
#pyautogui.press(['shift', '<'])
pyautogui.hotkey('shift', '<')
print('speed down')
presstime_3 = time.time()
if detect_mode == 2:
if mouse_id == 0: # Point gesture
# print(landmark_list[8]) #index finger
# print(landmark_list[12]) #middle finger
x1, y1 = landmark_list[8]
# cv.rectangle(debug_image, (frameR, frameR), (cap_width - frameR, cap_height - frameR),
# (255, 0, 255), 2)
cv.rectangle(debug_image, (50, 30),
(cap_width - 50, cap_height - 170),
(255, 0, 255), 2)
#座標轉換
#x軸: 鏡頭上50~(cap_width - 50)轉至螢幕寬0~wScr
#y軸: 鏡頭上30~(cap_height - 170)轉至螢幕長0~hScr
x3 = np.interp(x1, (50, (cap_width - 50)), (0, wScr))
y3 = np.interp(y1, (30, (cap_height - 170)), (0, hScr))
# print(x3, y3)
# 6. Smoothen Values
clocX = plocX + (x3 - plocX) / smoothening
clocY = plocY + (y3 - plocY) / smoothening
# 7. Move Mouse
pyautogui.moveTo(clocX, clocY)
cv.circle(debug_image, (x1, y1), 15, (255, 0, 255),
cv.FILLED)
plocX, plocY = clocX, clocY
if mouse_id == 1:
length, img, lineInfo = findDistance(
landmark_list[8], landmark_list[12], debug_image)
# 10. Click mouse if distance short
if time.time() - clicktime > 0.5:
if length < 40:
cv.circle(img, (lineInfo[4], lineInfo[5]), 15,
(0, 255, 0), cv.FILLED)
pyautogui.click()
print('click')
clicktime = time.time()
# if length > 70:
# cv.circle(img, (lineInfo[4], lineInfo[5]),
# 15, (0, 255, 0), cv.FILLED)
# pyautogui.click(clicks=2)
# print('click*2')
# clicktime = time.time()
if most_common_keypoint_id[0][
0] == 5 and most_common_keypoint_id[0][1] == 5:
pyautogui.scroll(20)
if most_common_keypoint_id[0][
0] == 6 and most_common_keypoint_id[0][1] == 5:
pyautogui.scroll(-20)
#if left_id == 7 or right_id == 7:
if most_common_keypoint_id[0][
0] == 0 and most_common_keypoint_id[0][1] == 5:
if time.time() - clicktime > 1:
pyautogui.click(clicks=2)
clicktime = time.time()
if most_common_keypoint_id[0][
0] == 9 and most_common_keypoint_id[0][1] == 5:
if time.time() - clicktime > 2:
pyautogui.hotkey('alt', 'left')
clicktime = time.time()
cv.putText(debug_image, what_mode, (400, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 4, cv.LINE_AA)
# Screen reflection ###################################JL##########################
cv.imshow('Hand Gesture Recognition', debug_image)
self.trigger.emit(detect_mode)
cap.release()
cv.destroyAllWindows()
def run(self):
# 重写线程执行的run函数
# 触发自定义信号
self.stop_flag = False
# Argument parsing #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
use_static_image_mode = args.use_static_image_mode
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = True
# Camera preparation ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# Model load #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
static_image_mode=use_static_image_mode,
max_num_hands=1,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
mouse_classifier = MouseClassifier(invalid_value=2, score_th=0.4)
point_history_classifier = PointHistoryClassifier()
# Read labels ###########################################################
with open(
'model/point_history_classifier/point_history_classifier_label.csv',
encoding='utf-8-sig') as f:
point_history_classifier_labels = csv.reader(f)
point_history_classifier_labels = [
row[0] for row in point_history_classifier_labels
]
with open('model/mouse_classifier/mouse_classifier_label.csv',
encoding='utf-8-sig') as f:
mouse_classifier_labels = csv.reader(f)
mouse_classifier_labels = [
row[0] for row in mouse_classifier_labels
]
# FPS Measurement ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=3)
# Coordinate history #################################################################
history_length = 16
point_history = deque(maxlen=history_length)
# Finger gesture history ################################################
finger_gesture_history = deque(maxlen=history_length)
mouse_id_history = deque(maxlen=30)
m_id_history = deque(maxlen=6)
# ========= 按鍵前置作業 =========
mode = 0
presstime = resttime = time.time()
mode_change = False
detect_mode = 0
what_mode = 'Sleep'
landmark_list = 0
pyautogui.PAUSE = 0
i = 0
# ========= 滑鼠前置作業 =========
wScr, hScr = pyautogui.size()
frameR = 100
smoothening = 10
plocX, plocY = 0, 0
clocX, clocY = 0, 0
# 關閉 滑鼠移至角落啟動保護措施
pyautogui.FAILSAFE = False
# ========= 主程式運作 =========
while self.stop_flag == False:
mouse_id = -1
fps = cvFpsCalc.get()
# Process Key (ESC: end)
key = cv.waitKey(10)
if key == 27: # ESC
break
number, mode = select_mode(key, mode)
# Camera capture
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # Mirror display
debug_image = copy.deepcopy(image)
# Detection implementation
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
# image = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
image.flags.writeable = False
results = hands.process(image)
image.flags.writeable = True
# ####################################################################
if results.multi_hand_landmarks is not None:
for hand_landmarks, handedness in zip(
results.multi_hand_landmarks,
results.multi_handedness):
# Bounding box calculation
brect = calc_bounding_rect(debug_image, hand_landmarks)
# Landmark calculation
landmark_list = calc_landmark_list(debug_image,
hand_landmarks)
# print(landmark_list)
# Conversion to relative coordinates / normalized coordinates
pre_processed_landmark_list = pre_process_landmark(
landmark_list)
pre_processed_point_history_list = pre_process_point_history(
debug_image, point_history)
# Write to the dataset file
logging_csv(number, mode, pre_processed_landmark_list,
pre_processed_point_history_list)
# # 靜態手勢資料預測
mouse_id = mouse_classifier(pre_processed_landmark_list)
# print(mouse_id)
# 手比one 觸發動態資料抓取
if mouse_id == 0:
point_history.append(landmark_list[8])
else:
point_history.append([0, 0])
# 動態手勢資料預測
finger_gesture_id = 0
point_history_len = len(pre_processed_point_history_list)
if point_history_len == (history_length * 2):
finger_gesture_id = point_history_classifier(
pre_processed_point_history_list)
# print(finger_gesture_id) # 0 = stop, 1 = clockwise, 2 = counterclockwise, 3 = move,偵測出現的動態手勢
# 動態手勢最常出現id #########################################
# Calculates the gesture IDs in the latest detection
finger_gesture_history.append(finger_gesture_id)
most_common_fg_id = Counter(
finger_gesture_history).most_common()
# 滑鼠的deque
mouse_id_history.append(mouse_id)
most_common_ms_id = Counter(mouse_id_history).most_common()
# print(most_common_ms_id)
m_id_history.append(mouse_id)
m_id = Counter(m_id_history).most_common(2)
mouse_id = m_id[0][0] if m_id[0][1] >= 4 else 2
# print(f'm_id {m_id}\n, mouse_id {mouse_id}')
# ===== 偵測到手時，重製紀錄時間 ==============================
resttime = time.time()
###############################################################
# Drawing part
debug_image = draw_bounding_rect(use_brect, debug_image,
brect)
debug_image = draw_landmarks(debug_image, landmark_list)
debug_image = draw_info_text(
debug_image, brect, handedness,
mouse_classifier_labels[mouse_id],
point_history_classifier_labels[most_common_fg_id[0]
[0]])
else:
point_history.append([0, 0])
debug_image = draw_point_history(debug_image, point_history)
debug_image = draw_info(debug_image, fps, mode, number)
# 偵測是否有手勢 #########################################
if mouse_id > -1:
# change mode Gesture six changes to the different mode
if most_common_ms_id[0][0] == 3 and most_common_ms_id[0][
1] == 30:
if time.time() - presstime > 2:
detect_mode = (detect_mode + 1) % 3
mode_change = True
presstime = time.time()
# control keyboard
elif detect_mode == 2:
# 靜態手勢控制
presstime = control_keyboard(mouse_id, 1, 'K', presstime)
presstime = control_keyboard(mouse_id, 8, 'C', presstime)
presstime = control_keyboard(mouse_id, 5, 'up', presstime)
presstime = control_keyboard(mouse_id, 6, 'down',
presstime)
# presstime = control_keyboard(most_common_keypoint_id, 0, 'right', presstime)
# presstime = control_keyboard(most_common_keypoint_id, 7, 'left', presstime)
if mouse_id == 4:
# print(i, time.time() - presstime)
if i == 3 and time.time() - presstime > 0.3:
pyautogui.press('right')
i = 0
presstime = time.time()
elif i == 3 and time.time() - presstime > 0.25:
pyautogui.press('right')
presstime = time.time()
elif time.time() - presstime > 1:
pyautogui.press('right')
i += 1
presstime = time.time()
if mouse_id == 7:
# print(i, time.time() - presstime)
if i == 3 and time.time() - presstime > 0.3:
pyautogui.press('left')
i = 0
presstime = time.time()
elif i == 3 and time.time() - presstime > 0.25:
pyautogui.press('left')
presstime = time.time()
elif time.time() - presstime > 1:
pyautogui.press('left')
i += 1
presstime = time.time()
# 動態手勢控制
if most_common_fg_id[0][
0] == 1 and most_common_fg_id[0][1] > 9:
if time.time() - presstime > 1.5:
pyautogui.hotkey('shift', '>')
print('speed up')
presstime = time.time()
elif most_common_fg_id[0][
0] == 2 and most_common_fg_id[0][1] > 12:
if time.time() - presstime > 1.5:
pyautogui.hotkey('shift', '<')
print('speed down')
presstime = time.time()
elif detect_mode == 1:
if mouse_id == 0: # Point gesture
x1, y1 = landmark_list[8]
cv.rectangle(debug_image, (50, 30),
(cap_width - 50, cap_height - 170),
(255, 0, 255), 2)
# 座標轉換
# x軸: 鏡頭上50~(cap_width - 50)轉至螢幕寬0~wScr
# y軸: 鏡頭上30~(cap_height - 170)轉至螢幕長0~hScr
x3 = np.interp(x1, (50, (cap_width - 50)), (0, wScr))
y3 = np.interp(y1, (30, (cap_height - 170)), (0, hScr))
# 6. Smoothen Values
clocX = plocX + (x3 - plocX) / smoothening
clocY = plocY + (y3 - plocY) / smoothening
# 7. Move Mouse
pyautogui.moveTo(clocX, clocY)
cv.circle(debug_image, (x1, y1), 15, (255, 0, 255),
cv.FILLED)
plocX, plocY = clocX, clocY
elif mouse_id == 1:
# 10. Click mouse if distance short
if time.time() - presstime > 0.5:
pyautogui.click()
presstime = time.time()
if mouse_id == 5:
pyautogui.scroll(-20)
if mouse_id == 6:
pyautogui.scroll(20)
if mouse_id == 7:
if time.time() - presstime > 1.5:
pyautogui.click(clicks=2)
presstime = time.time()
if mouse_id == 8:
if time.time() - presstime > 2:
pyautogui.hotkey('alt', 'left')
presstime = time.time()
# 比讚 從休息模式 換成 鍵盤模式
elif detect_mode == 0:
if mouse_id == 5:
i += 1
if i == 1 or time.time() - presstime > 3:
presstime = time.time()
elif time.time() - presstime > 2:
detect_mode = 2
mode_change = True
i = 0
# 距離上次監測到手的時間大於30秒、切回休息模式 =========================
if time.time() - resttime > 30:
if detect_mode != 0:
detect_mode = 0
mode_change = True
# 檢查模式有沒有更動 ========================
if mode_change:
if detect_mode == 0:
what_mode = 'Sleep'
elif detect_mode == 2:
what_mode = 'Keyboard'
elif detect_mode == 1:
what_mode = 'Mouse'
mode_change = False
print('Mode has changed')
print(f'Current mode => {what_mode}')
cv.putText(debug_image, what_mode, (480, 30),
cv.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), 4, cv.LINE_AA)
# Screen reflection ###################################JL##########################
cv.imshow('Hand Gesture Recognition', debug_image)
self.trigger.emit(detect_mode)
cap.release()
cv.destroyAllWindows()
def main():
# Argument parsing #################################################################
args = get_args()
cap_device = args.device
cap_width = 1920
cap_height = 1080
use_static_image_mode = args.use_static_image_mode
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = True
# Camera preparation ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# Model load #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
static_image_mode=use_static_image_mode,
max_num_hands=1,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
keypoint_classifier = KeyPointClassifier()
point_history_classifier = PointHistoryClassifier()
# Read labels ###########################################################
with open('model/keypoint_classifier/keypoint_classifier_label.csv',
encoding='utf-8-sig') as f:
keypoint_classifier_labels = csv.reader(f)
keypoint_classifier_labels = [
row[0] for row in keypoint_classifier_labels
]
with open(
'model/point_history_classifier/point_history_classifier_label.csv',
encoding='utf-8-sig') as f:
point_history_classifier_labels = csv.reader(f)
point_history_classifier_labels = [
row[0] for row in point_history_classifier_labels
]
# FPS Measurement ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
# Coordinate history #################################################################
history_length = 16
point_history = deque(maxlen=history_length)
# Finger gesture history ################################################
finger_gesture_history = deque(maxlen=history_length)
# ########################################################################
mode = 0
while True:
fps = cvFpsCalc.get()
# Process Key (ESC: end) #################################################
key = cv.waitKey(10)
if key == 27: # ESC
break
number, mode = select_mode(key, mode)
# Camera capture #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # Mirror display
debug_image = copy.deepcopy(image)
# Detection implementation #############################################################
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = hands.process(image)
image.flags.writeable = True
# ####################################################################
if results.multi_hand_landmarks is not None:
for hand_landmarks, handedness in zip(results.multi_hand_landmarks,
results.multi_handedness):
# Bounding box calculation
brect = calc_bounding_rect(debug_image, hand_landmarks)
# Landmark calculation
landmark_list = calc_landmark_list(debug_image, hand_landmarks)
# Conversion to relative coordinates / normalized coordinates
pre_processed_landmark_list = pre_process_landmark(
landmark_list)
pre_processed_point_history_list = pre_process_point_history(
debug_image, point_history)
# Write to the dataset file
logging_csv(number, mode, pre_processed_landmark_list,
pre_processed_point_history_list)
# Hand sign classification
hand_sign_id = keypoint_classifier(pre_processed_landmark_list)
if hand_sign_id == 2: # Point gesture
point_history.append(landmark_list[8])
else:
point_history.append([0, 0])
# Finger gesture classification
finger_gesture_id = 0
point_history_len = len(pre_processed_point_history_list)
if point_history_len == (history_length * 2):
finger_gesture_id = point_history_classifier(
pre_processed_point_history_list)
# Calculates the gesture IDs in the latest detection
finger_gesture_history.append(finger_gesture_id)
most_common_fg_id = Counter(
finger_gesture_history).most_common()
# Drawing part
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
debug_image = draw_landmarks(debug_image, landmark_list)
debug_image = draw_info_text(
debug_image,
brect,
handedness,
keypoint_classifier_labels[hand_sign_id],
point_history_classifier_labels[most_common_fg_id[0][0]],
)
else:
point_history.append([0, 0])
debug_image = draw_point_history(debug_image, point_history)
debug_image = draw_info(debug_image, fps, mode, number)
# Screen reflection #############################################################
cv.imshow('Hand Gesture Recognition', debug_image)
cap.release()
cv.destroyAllWindows()
def main():
# 引数解析 #################################################################
args = get_args()
cap_device = args.device
cap_width = args.width
cap_height = args.height
use_static_image_mode = args.use_static_image_mode
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = True
# カメラ準備 ###############################################################
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
static_image_mode=use_static_image_mode,
max_num_hands=1,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
keypoint_classifier = KeyPointClassifier()
point_history_classifier = PointHistoryClassifier()
# ラベル読み込み ###########################################################
with open('model/keypoint_classifier/keypoint_classifier_label.csv',
encoding='utf-8-sig') as f:
keypoint_classifier_labels = csv.reader(f)
keypoint_classifier_labels = [
row[0] for row in keypoint_classifier_labels
]
with open(
'model/point_history_classifier/point_history_classifier_label.csv',
encoding='utf-8-sig') as f:
point_history_classifier_labels = csv.reader(f)
point_history_classifier_labels = [
row[0] for row in point_history_classifier_labels
]
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
# 座標履歴 #################################################################
history_length = 16
point_history = deque(maxlen=history_length)
# フィンガージェスチャー履歴 ################################################
finger_gesture_history = deque(maxlen=history_length)
# ########################################################################
mode = 0
while True:
fps = cvFpsCalc.get()
# キー処理(ESC：終了) #################################################
key = cv.waitKey(10)
if key == 27: # ESC
break
number, mode = select_mode(key, mode)
# カメラキャプチャ #####################################################
ret, image = cap.read()
if not ret:
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
# 検出実施 #############################################################
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = hands.process(image)
image.flags.writeable = True
# ####################################################################
if results.multi_hand_landmarks is not None:
for hand_landmarks, handedness in zip(results.multi_hand_landmarks,
results.multi_handedness):
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, hand_landmarks)
# ランドマークの計算
landmark_list = calc_landmark_list(debug_image, hand_landmarks)
# 相対座標・正規化座標への変換
pre_processed_landmark_list = pre_process_landmark(
landmark_list)
pre_processed_point_history_list = pre_process_point_history(
debug_image, point_history)
# 学習データ保存
logging_csv(number, mode, pre_processed_landmark_list,
pre_processed_point_history_list)
# ハンドサイン分類
hand_sign_id = keypoint_classifier(pre_processed_landmark_list)
if hand_sign_id == 2: # 指差しサイン
point_history.append(landmark_list[8]) # 人差指座標
else:
point_history.append([0, 0])
# フィンガージェスチャー分類
finger_gesture_id = 0
point_history_len = len(pre_processed_point_history_list)
if point_history_len == (history_length * 2):
finger_gesture_id = point_history_classifier(
pre_processed_point_history_list)
# 直近検出の中で最多のジェスチャーIDを算出
finger_gesture_history.append(finger_gesture_id)
most_common_fg_id = Counter(
finger_gesture_history).most_common()
# 描画
debug_image = draw_bounding_rect(use_brect, debug_image, brect)
debug_image = draw_landmarks(debug_image, landmark_list)
debug_image = draw_info_text(
debug_image,
brect,
handedness,
keypoint_classifier_labels[hand_sign_id],
point_history_classifier_labels[most_common_fg_id[0][0]],
)
else:
point_history.append([0, 0])
debug_image = draw_point_history(debug_image, point_history)
debug_image = draw_info(debug_image, fps, mode, number)
# 画面反映 #############################################################
cv.imshow('Hand Gesture Recognition', debug_image)
cap.release()
cv.destroyAllWindows()
def HandTracking(cap, width, height, conf_flg=0):
# ×ボタンが押されたかのフラグ(hand_gui.py内の変数、flg_closePush)の初期化
hand_gui.close_switch_py(0)
# 引数解析 #################################################################
args = get_args()
flg_video = 0 #「1」でカメラが接続されていない
flg_break = 0 #「1」で最初のループを抜け終了する⇒正常終了
name_pose = "Unknown"
focus_flg = 1 #index.html の表示・非表示の切り替え、「0」:Main.pyで開いた場合、「1」:HandTracking.pyで開いた場合
namePose_flg = 1 #complete_old.htmlの開始・終了フラグ
potision_flg = 0
#flg_closePush = 0
global ShortCutList
ShortCutList = config_sys_set()
cap_device = args.device
cap_width = args.width
cap_height = args.height
use_static_image_mode = args.use_static_image_mode
min_detection_confidence = args.min_detection_confidence
min_tracking_confidence = args.min_tracking_confidence
use_brect = True
#width,height = autopy.screen.size() #eel で立ち上げた際の表示位置を指定するために取得
while (True): #カメラが再度接続するまでループ処理
#カメラが接続されていないフラグの場合
if (flg_video == 1):
#カメラが接続されているか確認
cap = cv.VideoCapture(cap_device)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
ret, frame = cap.read()
if (ret is True):
flg_video = 0
name_pose = "Unknown"
focus_flg = 1
namePose_flg = 1
cap.release()
eel.overlay_controll(True)
eel.object_change("demo2.html", True)
#eel.sleep(1)
sel_cam_before = 999
while (True):
if (decideFlgHT == 1):
if (sel_camHT != sel_cam_before):
cap = cv.VideoCapture(sel_camHT)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
ret, frame = cap.read(sel_camHT)
if (ret is False):
eel.alert_mess()
cap.release()
decide_camHT_py(999)
decide_flgHT_py(0)
sel_cam_before = sel_camHT
continue
else:
decide_flgHT_py(0)
break
decide_flgHT_py(0)
break
if (sel_camHT != 999):
eel.sleep(0.01)
if (sel_camHT != sel_cam_before):
if (sel_cam_before != 999):
cap.release()
cap = cv.VideoCapture(sel_camHT)
cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
sel_cam_before = sel_camHT
ret, frame = cap.read()
if (ret is True):
# UI側へ転送(画像) #####################################################
_, imencode_image = cv.imencode('.jpg', frame)
base64_image = base64.b64encode(imencode_image)
eel.set_base64image("data:image/jpg;base64," +
base64_image.decode("ascii"))
continue
else:
eel.alert_mess()
cap.release()
decide_camHT_py(999)
sel_cam_before = sel_camHT
if (decideFlgHT == 1):
decide_flgHT_py(0)
continue
else:
eel.sleep(0.01)
continue #最初の while に戻る
else:
#カメラが接続されていない場合
#print("webcamないよ！！！")
eel.sleep(0.01)
time.sleep(0.01)
continue #最初の while に戻る
elif (flg_break == 1):
decide_camHT_py(999)
decide_flgHT_py(0)
break #最初の while を抜けて正常終了
# カメラ準備 ###############################################################
#cap = cv.VideoCapture(cap_device)
#cap.set(cv.CAP_PROP_FRAME_WIDTH, cap_width)
#cap.set(cv.CAP_PROP_FRAME_HEIGHT, cap_height)
# モデルロード #############################################################
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
static_image_mode=use_static_image_mode,
max_num_hands=1,
min_detection_confidence=min_detection_confidence,
min_tracking_confidence=min_tracking_confidence,
)
keypoint_classifier = KeyPointClassifier()
point_history_classifier = PointHistoryClassifier()
# ラベル読み込み ###########################################################
with open('model/keypoint_classifier/keypoint_classifier_label.csv',
encoding='utf-8-sig') as f:
keypoint_classifier_labels = csv.reader(f)
keypoint_classifier_labels = [
row[0] for row in keypoint_classifier_labels
]
with open(
'model/point_history_classifier/point_history_classifier_label.csv',
encoding='utf-8-sig') as f:
point_history_classifier_labels = csv.reader(f)
point_history_classifier_labels = [
row[0] for row in point_history_classifier_labels
]
# FPS計測モジュール ########################################################
cvFpsCalc = CvFpsCalc(buffer_len=10)
# 座標履歴 #################################################################
history_length = 16
point_history = deque(maxlen=history_length)
# フィンガージェスチャー履歴 ################################################
finger_gesture_history = deque(maxlen=history_length)
# ########################################################################
mode = 0
CountPose = [0, 0, 0, 0, 0, 0, 0]
CountMotion = [0, 0, 0, 0] # [Top,Right,Down,Left]
identification = False
while True:
fps = cvFpsCalc.get()
# キー処理(ESC：終了) #################################################
key = cv.waitKey(10)
if key == 27: # ESC
break
number, mode = select_mode(key, mode)
# カメラキャプチャ #####################################################
ret, image = cap.read()
if not ret:
#それぞれのフラグを立てて、システムを終了させ、最初の while に戻る
flg_video = 1
focus_flg = 0
print("【通知】WebCameraが接続されていません。")
eel.focusSwitch(width, height, focus_flg)
eel.overlay_controll(True)
eel.object_change("connect.html", True)
cap.release()
cv.destroyAllWindows()
break
image = cv.flip(image, 1) # ミラー表示
debug_image = copy.deepcopy(image)
# 検出実施 #############################################################
image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
image.flags.writeable = False
results = hands.process(image)
image.flags.writeable = True
# ####################################################################
if results.multi_hand_landmarks is not None:
#j=1
for hand_landmarks, handedness in zip(
results.multi_hand_landmarks,
results.multi_handedness):
# 外接矩形の計算
brect = calc_bounding_rect(debug_image, hand_landmarks)
# ランドマークの計算
landmark_list = calc_landmark_list(debug_image,
hand_landmarks)
# 相対座標・正規化座標への変換
pre_processed_landmark_list = pre_process_landmark(
landmark_list)
pre_processed_point_history_list = pre_process_point_history(
debug_image, point_history)
# 学習データ保存
logging_csv(number, mode, pre_processed_landmark_list,
pre_processed_point_history_list)
# ハンドサイン分類
hand_sign_id = keypoint_classifier(
pre_processed_landmark_list)
if hand_sign_id == 1: # Dangサイン
point_history.append(landmark_list[8]) # 人差指座標
else:
point_history.append([0, 0])
# フィンガージェスチャー分類
finger_gesture_id = 0
point_history_len = len(pre_processed_point_history_list)
if point_history_len == (history_length * 2):
finger_gesture_id = point_history_classifier(
pre_processed_point_history_list)
# 直近検出の中で最多のジェスチャーIDを算出
finger_gesture_history.append(finger_gesture_id)
most_common_fg_id = Counter(
finger_gesture_history).most_common()
gesture_name = point_history_classifier_labels[
most_common_fg_id[0][0]]
# 描画
debug_image = draw_bounding_rect(use_brect, debug_image,
brect)
debug_image = draw_landmarks(debug_image, landmark_list)
debug_image = draw_info_text(
debug_image,
brect,
handedness,
keypoint_classifier_labels[hand_sign_id],
gesture_name,
)
#人差し指の先の座標を取得
x, y = landmark_list[8]
#座標調整
x_width = args.width * 0.05
x = x - x_width
x = x * 1.5
y = y * 1.5
#ジェスチャーが判定された回数をカウント
if gesture_name == 'Stop':
CountMotion = [0, 0, 0, 0]
elif gesture_name == 'Move_Top':
Count_temp = CountMotion[0]
Count_temp += 1
CountMotion = [Count_temp, 0, 0, 0]
elif gesture_name == 'Move_Right':
Count_temp = CountMotion[1]
Count_temp += 1
CountMotion = [0, Count_temp, 0, 0]
elif gesture_name == 'Move_Down':
Count_temp = CountMotion[2]
Count_temp += 1
CountMotion = [0, 0, Count_temp, 0]
elif gesture_name == 'Move_Left':
Count_temp = CountMotion[3]
Count_temp += 1
CountMotion = [0, 0, 0, Count_temp]
#各種操作の実行
CountPose, CountMotion = PoseAction.action(
hand_sign_id, x, y, CountPose, CountMotion,
ShortCutList)
name_pose = keypoint_classifier_labels[hand_sign_id]
eel.set_posegauge(str(name_pose))
identification = True
else:
point_history.append([0, 0])
if identification == True:
eel.set_posegauge('None')
identification = False
eel.shortcut_overlay(False, 0)
debug_image = draw_point_history(debug_image, point_history)
debug_image = draw_info(debug_image, fps, mode, number)
# 画面反映 #############################################################
debug_image = cv.resize(debug_image, dsize=(520, 260))
cv.imshow('FOCUS preview', debug_image)
if potision_flg == 0:
cv.moveWindow('FOCUS preview', 0, 0)
potision_flg = 1
# eel立ち上げ #############################################################
#cnt_gui, flg_end, flg_restart, flg_start, keep_flg = hand_gui.start_gui(cnt_gui, name_pose, flg_restart, flg_start, keep_flg)
if (namePose_flg == 1):
eel.object_change("complete_old.html", True)
#eel.sleep(0.01)
#eel.init("GUI/web")
#eel.start("開きたい上記のフォルダ下のファイル名",～
#eel.start("html/keeper.html",
# port = 0,
# mode='chrome',
# size=(4, 2), #サイズ指定（横, 縦）
# position=(width,0), #位置指定（left, top）
# block=False
# )
eel.sleep(0.01)
print("【通知】準備完了")
namePose_flg = 0
elif (focus_flg == 1 and name_pose != 'Unknown'):
eel.object_change("complete_old.html", False)
eel.overlay_controll(False)
eel.focusSwitch(width, height, focus_flg)
print("【実行】index.html")
eel.sleep(0.01)
#eel.set_posegauge(name_pose,i)
focus_flg = 0
#i+=1
# eel立ち上げ #############################################################
flg_end = hand_gui.start_gui()
if (flg_end == 1):
#正常に終了する処理(中間のループを抜ける)
flg_break = 1
eel.endSwitch() #flg_end の値をもとに戻す関数
cap.release()
cv.destroyAllWindows()
eel.overlay_controll(False)
eel.object_change("endpage.html", False)
#eel.windowclose_keeper()
break