def worker(input_q, output_q, cap_params, frame_processed):
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
while True:
frame = input_q.get()
if (frame is not None):
im_width = cap_params['im_width']
im_height = cap_params['im_height']
boxes, scores = detector_utils.detect_objects(
frame, detection_graph, sess)
# draw bounding boxes
detector_utils.draw_box_on_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes,
cap_params['im_width'],
cap_params['im_height'], frame)
cropped_img = detector_utils.get_box_image(
cap_params['num_hands_detect'], cap_params["score_thresh"],
scores, boxes, cap_params['im_width'], cap_params['im_height'],
frame)
frame_processed += 1
output_q.put(frame)
cropped_output_q.put(cropped_img)
else:
output_q.put(frame)
cropped_output_q.put(cropped_img)
def worker(input_q, output_q, cropped_output_q, inferences_q, cap_params,
frame_processed):
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print(">> loading keras model for worker")
try:
model, classification_graph, session = classifier.load_KerasGraph(
"./cnn/cnn.h5")
except Exception as e:
print(e)
while True:
#print("> ===== in worker loop, frame ", frame_processed)
frame = input_q.get()
if (frame is not None):
# Actual detection. Variable boxes contains the bounding box cordinates for hands detected,
# while scores contains the confidence for each of these boxes.
# Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold)
boxes, scores = detector_utils.detect_objects(
frame, detection_graph, sess)
# get region of interest
res = detector_utils.get_box_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes,
cap_params['im_width'],
cap_params['im_height'], frame)
# draw bounding boxes
detector_utils.draw_box_on_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes,
cap_params['im_width'],
cap_params['im_height'], frame)
# classify hand pose
if res is not None:
res = cv2.cvtColor(res, cv2.COLOR_RGB2GRAY)
res = cv2.resize(res, (28, 28), interpolation=cv2.INTER_AREA)
class_res = classifier.classify(model, classification_graph,
session, res)
inferences_q.put(class_res)
# add frame annotated with bounding box to queue
cropped_output_q.put(res)
output_q.put(frame)
frame_processed += 1
else:
output_q.put(frame)
sess.close()
def worker(input_q, output_q, boxes_q, inferences_q, cap_params,
frame_processed):
print(">> loading frozen model for worker")
detection_graph, sess = detector.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print(">> loading keras model for worker")
try:
model, classification_graph, session = classifier.load_KerasGraph(
"cnn/models/hand_poses_wGarbage_10.h5")
except Exception as e:
print(e)
try:
while True:
# print("> ===== in worker loop, frame ", frame_processed)
frame = input_q.get()
if (frame is not None):
# Actual detection. Variable boxes contains the bounding box cordinates for hands detected,
# while scores contains the confidence for each of these boxes.
# Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold)
boxes, scores = detector.detect_objects(
frame, detection_graph, sess)
# get region of interest
box, res = detector.get_box_image(
cap_params['num_hands_detect'], cap_params["score_thresh"],
scores, boxes, cap_params['im_width'],
cap_params['im_height'], frame)
# classify hand pose
if res is not None:
class_res = classifier.classify(model,
classification_graph,
session, res)
inferences_q.put(class_res)
output_q.put(frame)
boxes_q.put(box)
frame_processed += 1
else:
output_q.put(frame)
except Exception as e:
print(e)
sess.close()
def main():
currentPath = ''
currentExample = ''
print('Do you want to : \n 1 - Add new pose \
\n 2 - Add examples to existing pose \
\n 3 - Add garbage examples')
menu_choice = input()
while menu_choice != '1' and menu_choice != '2' and menu_choice != '3':
print('Please enter 1 or 2')
menu_choice = input()
if menu_choice == '1':
print('Enter a name for the pose you want to add :')
name_pose = input()
# Create folder for pose
if not os.path.exists('Poses/' + name_pose):
os.makedirs('Poses/' + name_pose + '/' + name_pose + '_1/')
currentPath = 'Poses/' + name_pose + '/' + name_pose + '_1/'
currentExample = name_pose + '_1_'
elif menu_choice == '2':
# Display current poses
dirs = os.listdir('Poses/')
dirs.sort()
dirs_choice = ''
possible_choices = []
i = 1
for _dir in dirs:
dirs_choice += str(i) + ' - ' + str(_dir) + ' / '
possible_choices.append(str(i))
i += 1
# Ask user to choose to which pose to add examples
print('Choose one of the following pose:')
print(dirs_choice)
choice = input()
while not choice in possible_choices and dirs[int(choice) - 1] == 'garbage':
print('Please enter one of the following (not garbage): ' + str(possible_choices))
choice = input()
# Count number of files to increment new example directory
subdirs = os.listdir('Poses/' + dirs[int(choice) - 1] + '/')
index = len(subdirs) + 1
# Create new example directory
if not os.path.exists('Poses/' + dirs[int(choice) - 1] + '/' + dirs[int(choice) - 1] + '_' + str(index) + '/'):
os.makedirs('Poses/' + dirs[int(choice) - 1] + '/' + dirs[int(choice) - 1] + '_' + str(index) + '/')
# Update current path
currentPath = 'Poses/' + dirs[int(choice) - 1] + '/' + dirs[int(choice) - 1] + '_' + str(index) + '/'
currentExample = dirs[int(choice) - 1] + '_' + str(index) + '_'
name_pose = dirs[int(choice) - 1]
elif menu_choice == '3':
# Create folder for pose
if not os.path.exists('Poses/Garbage/'):
os.makedirs('Poses/Garbage/Garbage_1/')
currentPath = 'Poses/Garbage/Garbage_1/'
currentExample = 'Garbage_1_'
name_pose = 'Garbage'
else:
# Count number of files to increment new example directory
subdirs = os.listdir('Poses/Garbage/')
index = len(subdirs) + 1
# Create new example directory
if not os.path.exists('Poses/Garbage/Garbage_' + str(index) + '/'):
os.makedirs('Poses/Garbage/Garbage_' + str(index) + '/')
# Update current path
currentPath = 'Poses/Garbage/Garbage_' + str(index) + '/'
currentExample = 'Garbage_' + str(index) + '_'
name_pose = 'Garbage'
print('You\'ll now be prompted to record the pose you want to add. \n \
Please place your hand beforehand facing the camera, and press any key when ready. \n \
When finished press \'q\'.')
input()
# Begin capturing
cap = cv2.VideoCapture(0)
# Define the codec and create VideoWriter object
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter(currentPath + name_pose + '.avi', fourcc, 25.0, (640, 480))
while cap.isOpened():
ret, frame = cap.read()
if ret:
# write the frame
out.write(frame)
cv2.imshow('frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
break
# Release everything if job is finished
cap.release()
out.release()
cv2.destroyAllWindows()
vid = cv2.VideoCapture(currentPath + name_pose + '.avi')
# Check if the video
if not vid.isOpened():
print('Error opening video stream or file')
return
print('>> loading frozen model..')
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print('>> model loaded!')
_iter = 1
# Read until video is completed
while vid.isOpened():
# Capture frame-by-frame
ret, frame = vid.read()
if ret:
print(' Processing frame: ' + str(_iter))
# Resize and convert to RGB for NN to work with
frame = cv2.resize(frame, (320, 180), interpolation=cv2.INTER_AREA)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# Detect object
boxes, scores = detector_utils.detect_objects(frame, detection_graph, sess)
# get region of interest
res = detector_utils.get_box_image(1, 0.2, scores, boxes, 320, 180, frame)
# Save cropped image
if res is not None:
cv2.imwrite(currentPath + currentExample + str(_iter) + '.png', cv2.cvtColor(res, cv2.COLOR_RGB2BGR))
_iter += 1
# Break the loop
else:
break
print(' Processed ' + str(_iter) + ' frames!')
vid.release()
def worker(input_q, output_q, cap_params, frame_processed, poses):
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print(">> loading keras model for worker")
try:
model, classification_graph, session = classifier.load_KerasGraph(
"F:\Realtime_Hand_tracking\cnn\models\hand_poses_wGarbage_100.h5")
except Exception as e:
print(e)
detection_centres_x = []
detection_centres_y = []
is_centres_filled = False
detected = False
index = 0
detection_area = []
start_flag = False
flag_start = pause_time = 0
sensitivity = gui_sensitivity
area = centre_x = centre_y = 0
detection = ""
direction = ""
while True:
predicted_label = ""
frame = input_q.get()
if (frame is not None):
frame_processed += 1
boxes, scores = detector_utils.detect_objects(frame, detection_graph, sess)
# get region of interest
res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params['score_thresh'],
scores, boxes, cap_params['im_width'], cap_params['im_height'], frame)
# get boundary box
if pause_time == 0:
centre_x, centre_y, area = detector_utils.draw_box_on_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes, cap_params['im_width'],
cap_params['im_height'],
frame)
if pause_time > 0:
pause_time -= 1
if is_centres_filled:
detection_centres_x = detection_centres_x[1:10]
detection_centres_y = detection_centres_y[1:10]
detection_area = detection_area[1:10]
detection_centres_x.append(centre_x)
detection_centres_y.append(centre_y)
detection_area.append(area)
else:
detection_centres_x.append(centre_x)
detection_centres_y.append(centre_y)
detection_area.append(area)
if pause_time == 0:
index += 1
if index >= sensitivity:
index = 0
is_centres_filled = True
if index == 0:
predicted_label = classify(res, model, classification_graph, session, poses)
#print(predicted_label)
if predicted_label == "Start" and flag_start == 0:
#print("Start")
detection = "Start tracking"
start_flag = True
flag_start = 1
if detected:
detection_centres_x = []
detection_centres_y = []
is_centres_filled = False
index = 0
detected = False
detection_area = []
frame_processed = 0
pause_time = 30
centres_x = detection_centres_x.copy()
centres_y = detection_centres_y.copy()
areas = detection_area.copy()
centres_x = [v for v in centres_x if v]
centres_y = [v for v in centres_y if v]
areas = [a for a in areas if a]
# angle_coordinate
if len(centres_x) > 3 and is_centres_filled and len(centres_y) > 3 and len(areas) > 3 and start_flag :
flag = 0
dX = centres_x[-1] - centres_x[0]
dY = centres_y[-1] - centres_y[0]
if dX > 20 and dY > 20:
m = dY / dX
angle = math.degrees(math.atan(m))
if angle < 45:
flag = 1
elif angle > 45:
flag = 2
if dX > 100 and (abs(dY) < 20 or flag == 1):
direction = "Right"
keyboard.press_and_release('right')
detected = True
#print(direction)
elif -dX > 100 and (abs(dY) < 20 or flag == 1):
direction = "Left"
keyboard.press_and_release('left')
detected = True
#print(direction)
elif dY > 50 and (abs(dX) < 10 or flag == 2):
direction = "Down"
detected = True
#print(direction)
elif -dY > 50 and (abs(dX) < 10 or flag == 2):
direction = "Up"
detected = True
#print(direction)
elif areas[-1] - 3000 > areas[0] and abs(dX) < 30 and abs(dY) < 20:
direction = "Zoom in"
detected = True
#print(direction)
elif areas[-1] < areas[0] - 3000 and abs(dX) < 10 and abs(dY) < 20:
direction = "Zoom out"
detected = True
#print(direction)
output_q.put((frame, direction,predicted_label))
sess.close()
def worker(input_q, output_q, cropped_output_q, inferences_q, pointX_q, pointY_q, cap_params, frame_processed):
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print(">> loading keras model for worker")
try:
model, classification_graph, session = classifier.load_KerasGraph("cnn/models/hand_poses_wGarbage_10.h5")
except Exception as e:
print(e)
while True:
#print("> ===== in worker loop, frame ", frame_processed)
frame = input_q.get()
if (frame is not None):
#実際の検出。 変数ボックスには、検出された手の境界ボックスの座標が含まれています。
#スコアには、これらの各ボックスの信頼度が含まれています。
#ヒント：len（boxes）> 1の場合、（スコアのしきい値内で）少なくとも片方の手を見つけたと見なすことができます。
boxes, scores = detector_utils.detect_objects(
frame, detection_graph, sess)
#関心領域を取得
#フレームの画像サイズを取得
cropped_height,cropped_width,a = frame.shape[:3]
res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"],
scores, boxes, cropped_width,cropped_height, frame)
#手の判定が一定値を超えたとき
if (scores[0] > score_thresh):
(left, right, top, bottom) = (boxes[0][1] * cropped_width, boxes[0][3] * cropped_width,
boxes[0][0] * cropped_height, boxes[0][2] * cropped_height)
#ウィンドウサイズを取得
width,height = autopy.screen.size()
#画面比率変数設定
# wx = (width + (int(right)-int(left))*(width / cap_params['im_width'])) / cap_params['im_width']
#
# hx = (height + (int(bottom)-int(top))*(height / cap_params['im_height'])) / cap_params['im_height']
# wx = (width + (int(right)-int(left))*(width / cropped_width)) / (cropped_width-20)
#
# hx = (height + (int(bottom)-int(top))*(height / cropped_height)) / (cropped_height-20)
# p1 = int(left)*wx
# p2 = int(bottom)*hx-(int(bottom)*hx-int(top)*hx)
hx = (height / cropped_height)*1.3
wx = (width / cropped_width)*1.3
#手を識別したボックスの固定化処理
hi = 100 - (int(bottom)-int(top))
wi = 70 - (int(right)-int(left))
top = top - hi
left = left - wi
fp = (int(left),int(top))
ep = (int(right),int(bottom))
p1 = int(left)*wx
p2 = int(top)*wx
if(abs(hi)>25 or abs(wi)>25):
cv2.rectangle(frame, fp, ep, (255, 0, 0), 1, 1)
#判定した手の範囲を表示
else:
cv2.rectangle(frame, fp, ep, (77, 255, 9), 1, 1)
#取得した座標(p1,p2)を挿入
pointX_q.put(p1)
pointY_q.put(p2)
#手のポーズを分類する
if res is not None:
class_res = classifier.classify(model, classification_graph, session, res)
inferences_q.put(class_res)
#バウンディングボックスで注釈が付けられたフレームをキューに追加
cropped_output_q.put(res)
output_q.put(frame)
frame_processed += 1
else:
output_q.put(frame)
sess.close()
def worker(input_q, output_q, cropped_output_q, inferences_q, landmark_ouput_q,
cap_params, frame_processed):
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(
"landmarks/shape_predictor_68_face_landmarks.dat")
print(">> loading keras model for worker")
try:
model = tf.keras.models.load_model('models/asl_char_model.h5',
compile=False)
except Exception as e:
print(e)
while True:
frame = input_q.get()
if (frame is not None):
boxes, scores = detector_utils.detect_objects(
frame, detection_graph, sess)
# get region of interest
res = detector_utils.get_box_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes,
cap_params['im_width'],
cap_params['im_height'], frame)
# draw bounding boxes
detector_utils.draw_box_on_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes,
cap_params['im_width'],
cap_params['im_height'], frame)
# classify hand
if res is not None:
class_res = ""
try:
proba = model.predict(process_image(res))[0]
mx = np.argmax(proba)
score = proba[mx] * 100
sequence = categories[mx][1]
class_res = str(score) + "/" + sequence
except:
score = 0.0
sequence = ""
class_res = "empty"
inferences_q.put(class_res)
image_np1 = imutils.resize(frame, width=400)
gray = cv2.cvtColor(image_np1, cv2.COLOR_BGR2GRAY)
#lanmarking
# detect faces in the grayscale frame
rects = detector(gray, 0)
# loop over the face detections
for rect in rects:
# determine the facial landmarks for the face region, then
# convert the facial landmark (x, y)-coordinates to a NumPy
# array
shape = predictor(gray, rect)
shape = face_utils.shape_to_np(shape)
# loop over the (x, y)-coordinates for the facial landmarks
# and draw them on the image
for (x, y) in shape:
cv2.circle(image_np1, (x, y), 1, (0, 0, 255), -1)
# add frame annotated with bounding box to queue
landmark_ouput_q.put(image_np1)
cropped_output_q.put(res)
output_q.put(frame)
frame_processed += 1
else:
output_q.put(frame)
sess.close()
def worker(input_q, output_q, cap_params, frame_processed):
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
model = tf.keras.models.load_model('./saved_model2/')
#################
# all the following variables store data for sliding window
sw_size = 30 # sliding window size
trajecX = [] # trajectory x coord
trajecY = [] # trajectory y coord
preds = [] # classified classes
preds_freq = np.zeros(5) # frequency of each class
#################
screenWd, screenHt = pyautogui.size()
frameWd, frameHt = cap_params['im_width'], cap_params['im_height']
scaleX = 1.0 * screenWd / frameWd
scaleY = 1.0 * screenHt / frameHt
state = -1
rad_in, rad_mid, rad_out= 25, 90, 200 # radii of annulus
last_swipe, swipe_cnt = -1, 0 # will provide sanitary check so same swipes are not performed twice consecutively
while True:
# print("> ===== in worker loop, frame ", frame_processed)
frame = input_q.get()
if (frame is not None):
# print(nm, "Not None")
im_width = cap_params['im_width']
im_height = cap_params['im_height']
boxes, scores = detector_utils.detect_objects(
frame, detection_graph, sess)
# get cropped image
cropped_img = detector_utils.get_box_image(
cap_params['num_hands_detect'], cap_params["score_thresh"],
scores, boxes, cap_params['im_width'], cap_params['im_height'],
frame)
# draw bounding boxes
centroidX, centroidY = detector_utils.draw_box_on_image(
cap_params['num_hands_detect'], cap_params["score_thresh"],
scores, boxes, cap_params['im_width'], cap_params['im_height'],
frame)
## enable this if only want to visualize the bounding box
## and blur rest of the image
# visualize_blurred(cap_params['num_hands_detect'], cap_params['score_thresh'], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame)
# negative centroid means no hand is detected
if centroidX < 0:
frame_processed += 1
output_q.put(frame)
# cropped_output_q.put(cropped_img)
continue
# Updation of centroid and add to trajectory if needed
if centroidX > 0:
if len(trajecX) == 0:
trajecX.append(centroidX)
trajecY.append(centroidY)
else:
px, py = trajecX[-1], trajecY[-1]
d = dist(centroidX, centroidY, px, py)
if d <= rad_in or (d > rad_mid and d < rad_out):
centroidX, centroidY = px, py
trajecX.append(centroidX)
trajecY.append(centroidY)
# move sliding window
if len(trajecX) > sw_size:
trajecX = trajecX[-sw_size:]
trajecY = trajecY[-sw_size:]
# draw centroid
if len(trajecX) > 0:
x, y = trajecX[-1], trajecY[-1]
# frame = visualize_blurred(cap_params['num_hands_detect'], cap_params['score_thresh'], scores, boxes, cap_params['im_width'], cap_params['im_height'], frame)
# visualize_roi(cap_params['num_hands_detect'], (x, y),
# cap_params['score_thresh'], scores,
# rad_in, rad_mid, rad_out, frame)
detector_utils.draw_centroid_on_image(
cap_params['num_hands_detect'], (x, y), cap_params["score_thresh"],
scores, frame)
# Static gesture prediction
if cropped_img is not None:
cropped_img, idx = classify(cropped_img, model)
detector_utils.draw_label_on_image(mp[idx], frame)
preds.append(idx)
preds_freq[idx] += 1
if len(preds) > sw_size:
preds_freq[preds[0]] -= 1
preds = preds[-sw_size:]
if preds[-1] == 1 or preds[-1] == 3:
state = control_vlc(last_swipe, preds[-1], state)
# Control mouse pointer
# if len(trajecX) > 1:
# x = int((trajecX[-1] - trajecX[-2]) )
# y = int((trajecY[-1] - trajecY[-2]) )
# control_mouse_pointer(x, y, screenWd, screenHt, preds[-1], preds[-2])
# Dynamic gesture prediction
if len(trajecX) > 0:
draw_trajectory(trajecX, trajecY, frame)
p = probab(preds_freq)
d = dist(trajecX[-1], trajecY[-1], trajecX[0], trajecY[0])
# print(d, p)
if p > 0.5 and d > 110:
direc = get_direction(trajecX[-1], trajecY[-1], trajecX[0], trajecY[0])
if not last_swipe == direc:
# print("swipe {} count: ".format(mp2[last_swipe]), swipe_cnt)
swipe_cnt = 0
else:
swipe_cnt += 1
last_swipe = direc
if swipe_cnt % 6== 0 and swipe_cnt > 0:
print("swipe {} count: ".format(mp2[last_swipe]), swipe_cnt)
state = control_vlc(last_swipe, preds[-1], state)
detector_utils.draw_label_on_image(mp2[direc], frame, (500, 100))
frame_processed += 1
output_q.put(frame)
# cropped_output_q.put(cropped_img)
else:
output_q.put(frame)
def worker(input_q, output_q, cap_params, frame_count, poses):
centroid = None
predicted_label = ""
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print(">> loading keras model for worker")
try:
model, classification_graph, session = classifier.load_KerasGraph(
"F:/Github/Hand_pose_DKE/cnn/models/hand_poses_wGarbage_10.h5")
except Exception as e:
print(e)
centroid_list = deque(maxlen=buffer)
direction = ""
(dX, dY) = (0, 0)
while True:
# print("> ===== in worker loop, frame ", frame_count)
frame = input_q.get()
if (frame is not None):
# Actual detection. Variable boxes contains the bounding box cordinates for hands detected,
# while scores contains the confidence for each of these boxes.
# Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold)
norm_image = cv2.normalize(frame,
None,
0,
255,
norm_type=cv2.NORM_MINMAX)
boxes, scores = detector_utils.detect_objects(
norm_image, detection_graph, sess)
# print(boxes[0])
# get region of interest
res = detector_utils.get_box_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes,
cap_params['im_width'],
cap_params['im_height'], frame)
# get boundary box
detector_utils.draw_box_on_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes,
cap_params['im_width'],
cap_params['im_height'], frame)
# classify hand pose
if res is not None and frame_count == 0:
class_res = classifier.classify(model, classification_graph,
session, res)
class_pred = class_res.argmax(axis=-1)
predicted_label = poses[int(class_pred)]
#print(predicted_label)
if predicted_label == "Start" and frame_count <= frame_end:
centroid = detector_utils.get_centroid(
cap_params['num_hands_detect'], cap_params["score_thresh"],
scores, boxes, cap_params['im_width'],
cap_params['im_height'], frame)
# elif scores is None:
# centroid = None
if centroid is not None:
centroid_list.appendleft(centroid)
#print(centroid_list)
sorted(centroid_list)
for i in np.arange(1, len(centroid_list)):
# if centroid_list[i-1] is None or centroid_list[i] is None:
# continue
if frame_count == frame_end and centroid_list[
-5] != None and i == 1:
dX = centroid_list[-5][0] - centroid_list[i][0]
dY = centroid_list[-5][1] - centroid_list[i][1]
(dirX, dirY) = ("", "")
if np.abs(dX) > 10:
dirX = "Right" if np.sign(dX) == 1 else "Left"
if np.abs(dY) > 10:
dirY = "DOWN" if np.sign(dY) == 1 else "UP"
if dirX != "" and dirY != "":
direction = "{}-{}".format(dirY, dirX)
else:
direction = dirX if dirX != "" else dirY
thickness = int(np.sqrt(frame_end / float(i + 1)) * 2.5)
cv2.line(frame, centroid_list[i - 1], centroid_list[i],
(0, 0, 255), thickness)
cv2.putText(frame, direction, (20, 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.65, (77, 255, 9), 1)
cv2.putText(frame, "dx: {}, dy: {}".format(dX, dY),
(10, frame.shape[0] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 255), 1)
# if direction == "Left":
# keyboard.press_and_release('left')
# #time.sleep(2)
# elif direction == "Right":
# keyboard.press_and_release('right')
# time.sleep(2)
frame_count += 1
if frame_count >= frame_end:
frame_count = 0
#centroid_list.clear()
direction = ""
#flag = 1
output_q.put(frame)
# else:
# output_q.put(frame)
sess.close()
def main():
currentPath = ''
currentExample = ''
#-----------------------------------------------------------------------------------------------------
print('\n 1 - Add new pose \
\n 2 - Add examplles to existing pose \
\n 3 - Add garbage examples'
) #tell docker whitch function you want to do
#-----------------------------------------------------------------------------------------------------
menu_choice = input()
while (menu_choice != '1' and menu_choice != '2' and menu_choice != '3'):
print('Please enter number 1~3')
menu_choice = input() #set user's input function
#function 1:Add new pose:---------------------------------------------------------------------------------------
if (menu_choice == '1'):
print('Enter a name for the pose you want to add :')
name_pose = input(
) #if choice add new pose than set input names name_pose
# Create folder for pose
if not os.path.exists(
'Poses/' + name_pose
): #check if input pose in the pose folder, if not than do
os.makedirs(
'Poses/' + name_pose + '/' + name_pose + '_1/'
) #create a folder named your input pose's name + name_pose_1 under path 'Pose/name_pose'
currentPath = 'Poses/' + name_pose + '/' + name_pose + '_1/' #set current path
currentExample = name_pose + '_1_' #set hole folder to current exp
print('function1 is sucessfully selected!')
#----------------------------------------------------------------------------------------------------------------
#function 2:Add new data into exist pose:------------------------------------------------------------------------
elif (menu_choice == '2'):
# Display current poses
dirs = os.listdir('Poses/') #set pose list as dirs#?
dirs_choice = ''
possible_choices = []
i = 1
for _dir in dirs: #_dir means form array's dirs 0 to end of order
dirs_choice += str(i) + ' - ' + str(_dir) + ' / '
possible_choices.append(
str(i)) #set exsist pose name as possible_choice one by one.
i += 1
# Ask user to choose to which pose to add examples
print('Choose one of the following pose:')
print(dirs_choice) #print exist pose
choice = input()
while (not choice in possible_choices
and dirs[int(choice) - 1] == 'garbage'):
print('Please enter one of the following: ' +
str(possible_choices))
choice = input()
# Count number of files to increment new example directory
subdirs = os.listdir('Poses/' + dirs[int(choice) - 1] + '/')
index = len(subdirs) + 1
# Create new example directory
if not os.path.exists('Poses/' + dirs[int(choice) - 1] + '/' +
dirs[int(choice) - 1] + '_' + str(index) + '/'):
os.makedirs('Poses/' + dirs[int(choice) - 1] + '/' +
dirs[int(choice) - 1] + '_' + str(index) + '/') #?
#Update current path
currentPath = 'Poses/' + dirs[int(choice) - 1] + '/' + dirs[
int(choice) - 1] + '_' + str(index) + '/'
currentExample = dirs[int(choice) - 1] + '_' + str(index) + '_'
name_pose = dirs[int(choice) - 1]
print('function2 is sucessfully selected!')
#----------------------------------------------------------------------------------------------------------------
#function 3:Add 'useless' pose to promot real-time acc:------------------------------------------------------------------------
elif (menu_choice == '3'):
# Create folder for pose
if not os.path.exists('Poses/Garbage/'):
os.makedirs('Poses/Garbage/Garbage_1/')
currentPath = 'Poses/Garbage/Garbage_1/'
currentExample = 'Garbage_1_'
name_pose = 'Garbage'
else:
# Count number of files to increment new example directory
subdirs = os.listdir('Poses/Garbage/')
index = len(subdirs) + 1
# Create new example directory
if not os.path.exists('Poses/Garbage/Garbage_' + str(index) + '/'):
os.makedirs('Poses/Garbage/Garbage_' + str(index) + '/')
#Update current path
currentPath = 'Poses/Garbage/Garbage_' + str(index) + '/'
currentExample = 'Garbage_' + str(index) + '_'
name_pose = 'Garbage'
print('function3 is sucessfully selected!')
#--------------------------------------------------------------------------------------------------------------------------------
print('You will now be prompted to record the pose you want to add. \n \
Please place your hand beforehand facing the camera, and press any key when you ready. \n \
If you finished your record, please press q on your keybroad.')
input()
#record a vedio after function choice---------------------------------------------------------------------------------------------------------
cap = cv2.VideoCapture(0) # Begin capturing
# Define the codec and create VideoWriter object
fourcc = cv2.VideoWriter_fourcc(*'XVID') #MPEG-4 code
out = cv2.VideoWriter(
currentPath + name_pose + '.avi', fourcc, 25.0,
(640, 480)) #(file name)、(code)、(fps)、(window width and high)
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
while (cap.isOpened()): #check webcam is opened
ret, frame = cap.read(
) #capture every frame (return success or not ture/flase)、(every single frame)
if ret:
# write the frame
out.write(frame) #write result into file
cv2.imshow('frame', frame) #open a frame that named 'frame'
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
print('Error:vedio capture is faild')
break
#-----------------------------------------------------------------------------------------------------------------------------------------------
# Release everything if job is finished
cap.release()
out.release()
cv2.destroyAllWindows()
vid = cv2.VideoCapture(currentPath + name_pose + '.avi') #finished vedio
# Check if the video
if (not vid.isOpened()):
print('Error opening video stream or file')
return
#hand detection----------------------------------------------------------------------------------------------------------------------------------
print('>> loading frozen model..')
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print('>> model loaded!')
_iter = 1
# Read until video is completed
while (vid.isOpened()):
# Capture frame-by-frame
ret, frame = vid.read()
if ret:
print(' Processing frame: ' + str(_iter))
# Resize and convert to RGB for NN to work with
frame = cv2.resize(frame, (320, 180), interpolation=cv2.INTER_AREA
) #interpolation is clculater of pixl output
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# Detect object
boxes, scores = detector_utils.detect_objects(
frame, detection_graph, sess)
# get region of interest
res = detector_utils.get_box_image(1, 0.2, scores, boxes, 320, 180,
frame)
# Save cropped image
if (res is not None):
cv2.imwrite(currentPath + currentExample + str(_iter) + '.png',
cv2.cvtColor(res, cv2.COLOR_RGB2BGR))
_iter += 1
# Break the loop
else:
break
print(' Processed ' + str(_iter) + ' frames!')
vid.release()
def worker(input_q, output_q, cap_params, frame_processed, poses):
print(">> loading frozen model for worker")
detection_graph, sess = detector_utils.load_inference_graph()
sess = tf.Session(graph=detection_graph)
print(">> loading keras model for worker")
try:
model, classification_graph, session = classifier.load_KerasGraph(
"F:\Realtime_Hand_tracking\cnn\models\hand_poses_wGarbage_10.h5")
except Exception as e:
print(e)
detection_centres_x = []
detection_centres_y = []
is_centres_filled = False
detected = False
index = 0
detection_area = []
predicted_label = ""
start_flag = False
flag_start = 0
pause_time = 0
print("Show Start gesture to start reading")
#file = open("eval_list.csv","w")
while True:
# print("> ===== in worker loop, frame ", frame_count)
frame = input_q.get()
flag = 0
if (frame is not None):
frame_processed += 1
# Actual detection. Variable boxes contains the bounding box cordinates for hands detected,
# while scores contains the confidence for each of these boxes.
# Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold)
# frame = cv2.normalize(frame, None, 0,255, norm_type=cv2.NORM_MINMAX)
boxes, scores = detector_utils.detect_objects(frame, detection_graph, sess)
# print("Hello")
# print(boxes[0])
# get region of interest
res = detector_utils.get_box_image(cap_params['num_hands_detect'], cap_params["score_thresh"],
scores, boxes, cap_params['im_width'], cap_params['im_height'], frame)
# get boundary box
if pause_time == 0:
centre_x, centre_y, area = detector_utils.draw_box_on_image(cap_params['num_hands_detect'],
cap_params["score_thresh"],
scores, boxes, cap_params['im_width'],
cap_params['im_height'],
frame)
if pause_time > 0:
pause_time -= 1
if is_centres_filled:
detection_centres_x = detection_centres_x[1:10]
detection_centres_y = detection_centres_y[1:10]
detection_area = detection_area[1:10]
detection_centres_x.append(centre_x)
detection_centres_y.append(centre_y)
detection_area.append(area)
else:
detection_centres_x.append(centre_x)
detection_centres_y.append(centre_y)
detection_area.append(area)
if pause_time == 0:
index += 1
# print(index)
if index >= 20:
index = 0
is_centres_filled = True
#start_flag = False
if index == 0:
predicted_label = classify(res, model, classification_graph, session, poses)
# print(predicted_label)
if predicted_label == "Start" and flag_start == 0:
print("Start tracking")
start_flag = True
flag_start = 1
elif predicted_label == "Stop" and flag_start == 1 and frame_processed > 30:
print("Stopped tracking")
start_flag = False
flag_start = 0
if detected:
detection_centres_x = []
detection_centres_y = []
is_centres_filled = False
index = 0
detected = False
detection_area = []
frame_processed = 0
pause_time = 10
#time.sleep(2)
#start_flag = False
centres_x = detection_centres_x.copy()
centres_y = detection_centres_y.copy()
areas = detection_area.copy()
centres_x = [v for v in centres_x if v]
centres_y = [v for v in centres_y if v]
areas = [a for a in areas if a]
direction = ""
# angle_coordinate
# print(areas)
if len(centres_x) > 3 and is_centres_filled and len(centres_y) > 3 and len(
areas) > 3 and start_flag == True:
# centres_asc = centres.copy().sort()
# centres_dsc = centres.copy().sort(reverse=True)
# print(centres)
# print("Last x:", centres_x[-1])
# print("First x: ", centres_x[0])
# print("Last y:", centres_y[-1])
# print("First y:", centres_y[0])
flag = 0
dX = centres_x[-1] - centres_x[0]
dY = centres_y[-1] - centres_y[0]
if dX > 20 and dY > 20:
m = dY / dX
angle = math.degrees(math.atan(m))
if angle < 45:
flag = 1
elif angle > 45:
flag = 2
# print(angle)
if dX > 100 and (abs(dY) < 20 or flag == 1):
direction = "Right"
#keyboard.press_and_release('left')
detected = True
print(direction)
elif -dX > 100 and (abs(dY) < 20 or flag == 1):
direction = "Left"
#keyboard.press_and_release('right')
detected = True
print(direction)
elif dY > 50 and (abs(dX) < 10 or flag == 2):
direction = "Down"
detected = True
print(direction)
elif -dY > 50 and (abs(dX) < 10 or flag == 2):
direction = "Up"
detected = True
print(direction)
elif areas[-1] - 3000 > areas[0] and abs(dX) < 30 and abs(dY) < 20:
direction = "Zoom in"
detected = True
print(direction)
elif areas[-1] < areas[0] - 3000 and abs(dX) < 10 and abs(dY) < 20:
direction = "Zoom out"
detected = True
print(direction)
# print("hello")
cv2.putText(frame, direction, (20, 50), cv2.FONT_HERSHEY_SIMPLEX,
0.65, (77, 255, 9), 1)
output_q.put(frame)
sess.close()