def worker(input_q, output_q, cropped_output_q, inferences_q, cap_params,
frame_processed):
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print(">> loading keras model for worker")
try:
model, classification_graph, session = classifier.load_KerasGraph(
"./cnn/cnn.h5")
except Exception as e:
print(e)
while True:
#print("> ===== in worker loop, frame ", frame_processed)
frame = input_q.get()
if (frame is not None):
# Actual detection. Variable boxes contains the bounding box cordinates for hands detected,
# while scores contains the confidence for each of these boxes.
# Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold)
boxes, scores = detector_utils.detect_objects(
frame, detection_graph, sess)
# get region of interest
res = detector_utils.get_box_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes,
cap_params['im_width'],
cap_params['im_height'], frame)
# draw bounding boxes
detector_utils.draw_box_on_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes,
cap_params['im_width'],
cap_params['im_height'], frame)
# classify hand pose
if res is not None:
res = cv2.cvtColor(res, cv2.COLOR_RGB2GRAY)
res = cv2.resize(res, (28, 28), interpolation=cv2.INTER_AREA)
class_res = classifier.classify(model, classification_graph,
session, res)
inferences_q.put(class_res)
# add frame annotated with bounding box to queue
cropped_output_q.put(res)
output_q.put(frame)
frame_processed += 1
else:
output_q.put(frame)
sess.close()
def worker(input_q, output_q, boxes_q, inferences_q, cap_params,
frame_processed):
print(">> loading frozen model for worker")
detection_graph, sess = detector.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print(">> loading keras model for worker")
try:
model, classification_graph, session = classifier.load_KerasGraph(
"cnn/models/hand_poses_wGarbage_10.h5")
except Exception as e:
print(e)
try:
while True:
# print("> ===== in worker loop, frame ", frame_processed)
frame = input_q.get()
if (frame is not None):
# Actual detection. Variable boxes contains the bounding box cordinates for hands detected,
# while scores contains the confidence for each of these boxes.
# Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold)
boxes, scores = detector.detect_objects(
frame, detection_graph, sess)
# get region of interest
box, res = detector.get_box_image(
cap_params['num_hands_detect'], cap_params["score_thresh"],
scores, boxes, cap_params['im_width'],
cap_params['im_height'], frame)
# classify hand pose
if res is not None:
class_res = classifier.classify(model,
classification_graph,
session, res)
inferences_q.put(class_res)
output_q.put(frame)
boxes_q.put(box)
frame_processed += 1
else:
output_q.put(frame)
except Exception as e:
print(e)
sess.close()
def worker(input_q, output_q, cap_params, frame_processed, poses):
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print(">> loading keras model for worker")
try:
model, classification_graph, session = classifier.load_KerasGraph(
"F:\Realtime_Hand_tracking\cnn\models\hand_poses_wGarbage_100.h5")
except Exception as e:
print(e)
detection_centres_x = []
detection_centres_y = []
is_centres_filled = False
detected = False
index = 0
detection_area = []
start_flag = False
flag_start = pause_time = 0
sensitivity = gui_sensitivity
area = centre_x = centre_y = 0
detection = ""
direction = ""
while True:
predicted_label = ""
frame = input_q.get()
if (frame is not None):
frame_processed += 1
boxes, scores = detector_utils.detect_objects(frame, detection_graph, sess)
# get region of interest
res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params['score_thresh'],
scores, boxes, cap_params['im_width'], cap_params['im_height'], frame)
# get boundary box
if pause_time == 0:
centre_x, centre_y, area = detector_utils.draw_box_on_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes, cap_params['im_width'],
cap_params['im_height'],
frame)
if pause_time > 0:
pause_time -= 1
if is_centres_filled:
detection_centres_x = detection_centres_x[1:10]
detection_centres_y = detection_centres_y[1:10]
detection_area = detection_area[1:10]
detection_centres_x.append(centre_x)
detection_centres_y.append(centre_y)
detection_area.append(area)
else:
detection_centres_x.append(centre_x)
detection_centres_y.append(centre_y)
detection_area.append(area)
if pause_time == 0:
index += 1
if index >= sensitivity:
index = 0
is_centres_filled = True
if index == 0:
predicted_label = classify(res, model, classification_graph, session, poses)
#print(predicted_label)
if predicted_label == "Start" and flag_start == 0:
#print("Start")
detection = "Start tracking"
start_flag = True
flag_start = 1
if detected:
detection_centres_x = []
detection_centres_y = []
is_centres_filled = False
index = 0
detected = False
detection_area = []
frame_processed = 0
pause_time = 30
centres_x = detection_centres_x.copy()
centres_y = detection_centres_y.copy()
areas = detection_area.copy()
centres_x = [v for v in centres_x if v]
centres_y = [v for v in centres_y if v]
areas = [a for a in areas if a]
# angle_coordinate
if len(centres_x) > 3 and is_centres_filled and len(centres_y) > 3 and len(areas) > 3 and start_flag :
flag = 0
dX = centres_x[-1] - centres_x[0]
dY = centres_y[-1] - centres_y[0]
if dX > 20 and dY > 20:
m = dY / dX
angle = math.degrees(math.atan(m))
if angle < 45:
flag = 1
elif angle > 45:
flag = 2
if dX > 100 and (abs(dY) < 20 or flag == 1):
direction = "Right"
keyboard.press_and_release('right')
detected = True
#print(direction)
elif -dX > 100 and (abs(dY) < 20 or flag == 1):
direction = "Left"
keyboard.press_and_release('left')
detected = True
#print(direction)
elif dY > 50 and (abs(dX) < 10 or flag == 2):
direction = "Down"
detected = True
#print(direction)
elif -dY > 50 and (abs(dX) < 10 or flag == 2):
direction = "Up"
detected = True
#print(direction)
elif areas[-1] - 3000 > areas[0] and abs(dX) < 30 and abs(dY) < 20:
direction = "Zoom in"
detected = True
#print(direction)
elif areas[-1] < areas[0] - 3000 and abs(dX) < 10 and abs(dY) < 20:
direction = "Zoom out"
detected = True
#print(direction)
output_q.put((frame, direction,predicted_label))
sess.close()
def worker(input_q, output_q, cropped_output_q, inferences_q, pointX_q, pointY_q, cap_params, frame_processed):
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print(">> loading keras model for worker")
try:
model, classification_graph, session = classifier.load_KerasGraph("cnn/models/hand_poses_wGarbage_10.h5")
except Exception as e:
print(e)
while True:
#print("> ===== in worker loop, frame ", frame_processed)
frame = input_q.get()
if (frame is not None):
#実際の検出。 変数ボックスには、検出された手の境界ボックスの座標が含まれています。
#スコアには、これらの各ボックスの信頼度が含まれています。
#ヒント：len（boxes）> 1の場合、（スコアのしきい値内で）少なくとも片方の手を見つけたと見なすことができます。
boxes, scores = detector_utils.detect_objects(
frame, detection_graph, sess)
#関心領域を取得
#フレームの画像サイズを取得
cropped_height,cropped_width,a = frame.shape[:3]
res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"],
scores, boxes, cropped_width,cropped_height, frame)
#手の判定が一定値を超えたとき
if (scores[0] > score_thresh):
(left, right, top, bottom) = (boxes[0][1] * cropped_width, boxes[0][3] * cropped_width,
boxes[0][0] * cropped_height, boxes[0][2] * cropped_height)
#ウィンドウサイズを取得
width,height = autopy.screen.size()
#画面比率変数設定
# wx = (width + (int(right)-int(left))*(width / cap_params['im_width'])) / cap_params['im_width']
#
# hx = (height + (int(bottom)-int(top))*(height / cap_params['im_height'])) / cap_params['im_height']
# wx = (width + (int(right)-int(left))*(width / cropped_width)) / (cropped_width-20)
#
# hx = (height + (int(bottom)-int(top))*(height / cropped_height)) / (cropped_height-20)
# p1 = int(left)*wx
# p2 = int(bottom)*hx-(int(bottom)*hx-int(top)*hx)
hx = (height / cropped_height)*1.3
wx = (width / cropped_width)*1.3
#手を識別したボックスの固定化処理
hi = 100 - (int(bottom)-int(top))
wi = 70 - (int(right)-int(left))
top = top - hi
left = left - wi
fp = (int(left),int(top))
ep = (int(right),int(bottom))
p1 = int(left)*wx
p2 = int(top)*wx
if(abs(hi)>25 or abs(wi)>25):
cv2.rectangle(frame, fp, ep, (255, 0, 0), 1, 1)
#判定した手の範囲を表示
else:
cv2.rectangle(frame, fp, ep, (77, 255, 9), 1, 1)
#取得した座標(p1,p2)を挿入
pointX_q.put(p1)
pointY_q.put(p2)
#手のポーズを分類する
if res is not None:
class_res = classifier.classify(model, classification_graph, session, res)
inferences_q.put(class_res)
#バウンディングボックスで注釈が付けられたフレームをキューに追加
cropped_output_q.put(res)
output_q.put(frame)
frame_processed += 1
else:
output_q.put(frame)
sess.close()
def worker(input_q, output_q, cap_params, frame_count, poses):
centroid = None
predicted_label = ""
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print(">> loading keras model for worker")
try:
model, classification_graph, session = classifier.load_KerasGraph(
"F:/Github/Hand_pose_DKE/cnn/models/hand_poses_wGarbage_10.h5")
except Exception as e:
print(e)
centroid_list = deque(maxlen=buffer)
direction = ""
(dX, dY) = (0, 0)
while True:
# print("> ===== in worker loop, frame ", frame_count)
frame = input_q.get()
if (frame is not None):
# Actual detection. Variable boxes contains the bounding box cordinates for hands detected,
# while scores contains the confidence for each of these boxes.
# Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold)
norm_image = cv2.normalize(frame,
None,
0,
255,
norm_type=cv2.NORM_MINMAX)
boxes, scores = detector_utils.detect_objects(
norm_image, detection_graph, sess)
# print(boxes[0])
# get region of interest
res = detector_utils.get_box_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes,
cap_params['im_width'],
cap_params['im_height'], frame)
# get boundary box
detector_utils.draw_box_on_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes,
cap_params['im_width'],
cap_params['im_height'], frame)
# classify hand pose
if res is not None and frame_count == 0:
class_res = classifier.classify(model, classification_graph,
session, res)
class_pred = class_res.argmax(axis=-1)
predicted_label = poses[int(class_pred)]
#print(predicted_label)
if predicted_label == "Start" and frame_count <= frame_end:
centroid = detector_utils.get_centroid(
cap_params['num_hands_detect'], cap_params["score_thresh"],
scores, boxes, cap_params['im_width'],
cap_params['im_height'], frame)
# elif scores is None:
# centroid = None
if centroid is not None:
centroid_list.appendleft(centroid)
#print(centroid_list)
sorted(centroid_list)
for i in np.arange(1, len(centroid_list)):
# if centroid_list[i-1] is None or centroid_list[i] is None:
# continue
if frame_count == frame_end and centroid_list[
-5] != None and i == 1:
dX = centroid_list[-5][0] - centroid_list[i][0]
dY = centroid_list[-5][1] - centroid_list[i][1]
(dirX, dirY) = ("", "")
if np.abs(dX) > 10:
dirX = "Right" if np.sign(dX) == 1 else "Left"
if np.abs(dY) > 10:
dirY = "DOWN" if np.sign(dY) == 1 else "UP"
if dirX != "" and dirY != "":
direction = "{}-{}".format(dirY, dirX)
else:
direction = dirX if dirX != "" else dirY
thickness = int(np.sqrt(frame_end / float(i + 1)) * 2.5)
cv2.line(frame, centroid_list[i - 1], centroid_list[i],
(0, 0, 255), thickness)
cv2.putText(frame, direction, (20, 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.65, (77, 255, 9), 1)
cv2.putText(frame, "dx: {}, dy: {}".format(dX, dY),
(10, frame.shape[0] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 255), 1)
# if direction == "Left":
# keyboard.press_and_release('left')
# #time.sleep(2)
# elif direction == "Right":
# keyboard.press_and_release('right')
# time.sleep(2)
frame_count += 1
if frame_count >= frame_end:
frame_count = 0
#centroid_list.clear()
direction = ""
#flag = 1
output_q.put(frame)
# else:
# output_q.put(frame)
sess.close()
def worker(input_q, output_q, cap_params, frame_processed, poses):
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print(">> loading keras model for worker")
try:
model, classification_graph, session = classifier.load_KerasGraph(
"F:\Realtime_Hand_tracking\cnn\models\hand_poses_wGarbage_10.h5")
except Exception as e:
print(e)
detection_centres_x = []
detection_centres_y = []
is_centres_filled = False
detected = False
index = 0
detection_area = []
predicted_label = ""
start_flag = False
flag_start = 0
pause_time = 0
print("Show Start gesture to start reading")
#file = open("eval_list.csv","w")
while True:
# print("> ===== in worker loop, frame ", frame_count)
frame = input_q.get()
flag = 0
if (frame is not None):
frame_processed += 1
# Actual detection. Variable boxes contains the bounding box cordinates for hands detected,
# while scores contains the confidence for each of these boxes.
# Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold)
# frame = cv2.normalize(frame, None, 0,255, norm_type=cv2.NORM_MINMAX)
boxes, scores = detector_utils.detect_objects(frame, detection_graph, sess)
# print("Hello")
# print(boxes[0])
# get region of interest
res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"],
scores, boxes, cap_params['im_width'], cap_params['im_height'], frame)
# get boundary box
if pause_time == 0:
centre_x, centre_y, area = detector_utils.draw_box_on_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes, cap_params['im_width'],
cap_params['im_height'],
frame)
if pause_time > 0:
pause_time -= 1
if is_centres_filled:
detection_centres_x = detection_centres_x[1:10]
detection_centres_y = detection_centres_y[1:10]
detection_area = detection_area[1:10]
detection_centres_x.append(centre_x)
detection_centres_y.append(centre_y)
detection_area.append(area)
else:
detection_centres_x.append(centre_x)
detection_centres_y.append(centre_y)
detection_area.append(area)
if pause_time == 0:
index += 1
# print(index)
if index >= 20:
index = 0
is_centres_filled = True
#start_flag = False
if index == 0:
predicted_label = classify(res, model, classification_graph, session, poses)
# print(predicted_label)
if predicted_label == "Start" and flag_start == 0:
print("Start tracking")
start_flag = True
flag_start = 1
elif predicted_label == "Stop" and flag_start == 1 and frame_processed > 30:
print("Stopped tracking")
start_flag = False
flag_start = 0
if detected:
detection_centres_x = []
detection_centres_y = []
is_centres_filled = False
index = 0
detected = False
detection_area = []
frame_processed = 0
pause_time = 10
#time.sleep(2)
#start_flag = False
centres_x = detection_centres_x.copy()
centres_y = detection_centres_y.copy()
areas = detection_area.copy()
centres_x = [v for v in centres_x if v]
centres_y = [v for v in centres_y if v]
areas = [a for a in areas if a]
direction = ""
# angle_coordinate
# print(areas)
if len(centres_x) > 3 and is_centres_filled and len(centres_y) > 3 and len(
areas) > 3 and start_flag == True:
# centres_asc = centres.copy().sort()
# centres_dsc = centres.copy().sort(reverse=True)
# print(centres)
# print("Last x:", centres_x[-1])
# print("First x: ", centres_x[0])
# print("Last y:", centres_y[-1])
# print("First y:", centres_y[0])
flag = 0
dX = centres_x[-1] - centres_x[0]
dY = centres_y[-1] - centres_y[0]
if dX > 20 and dY > 20:
m = dY / dX
angle = math.degrees(math.atan(m))
if angle < 45:
flag = 1
elif angle > 45:
flag = 2
# print(angle)
if dX > 100 and (abs(dY) < 20 or flag == 1):
direction = "Right"
#keyboard.press_and_release('left')
detected = True
print(direction)
elif -dX > 100 and (abs(dY) < 20 or flag == 1):
direction = "Left"
#keyboard.press_and_release('right')
detected = True
print(direction)
elif dY > 50 and (abs(dX) < 10 or flag == 2):
direction = "Down"
detected = True
print(direction)
elif -dY > 50 and (abs(dX) < 10 or flag == 2):
direction = "Up"
detected = True
print(direction)
elif areas[-1] - 3000 > areas[0] and abs(dX) < 30 and abs(dY) < 20:
direction = "Zoom in"
detected = True
print(direction)
elif areas[-1] < areas[0] - 3000 and abs(dX) < 10 and abs(dY) < 20:
direction = "Zoom out"
detected = True
print(direction)
# print("hello")
cv2.putText(frame, direction, (20, 50), cv2.FONT_HERSHEY_SIMPLEX,
0.65, (77, 255, 9), 1)
output_q.put(frame)
sess.close()