def main():
args = parse_args()
if args.input_file is not None:
cap = cv2.VideoCapture(args.input_file)
else:
cap = cv2.VideoCapture(0)
tracker = FaceTracker(args.tracker)
n_frames = 0
face = None
t_start = time.time()
# Detect face.
while cap.isOpened():
success, frame = cap.read()
if not success: break # No more frames?
n_frames += 1
frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
face = tracker.process(frame_gray)
# Viz.
if args.viz:
if face is not None:
(x, y, w, h) = face
cv2.rectangle(frame_gray, (x, y), (x + w, y + h), (0, 255, 0), 2)
cv2.imshow('Frame', frame_gray)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# Track face.
t_end = time.time()
fps = n_frames / (t_end - t_start)
print("FPS: {}".format(fps))
cap.release()
cv2.destroyAllWindows()
def move_mouse(self):
mouse = Controller()
screenCenter = [2560 / 2, 1080 / 2]
# mouse.position = tuple(screenCenter)
scaleFactor = 1.2
pid = PID(.2, .2, 0.05, setpoint=1)
eyeStateLIST = []
scaledChange = [0, 0]
webcam = cv2.VideoCapture(0)
while True:
# set pid controls
controlChangeX = pid((mouse.position[0] - screenCenter[0]) -
scaledChange[0])
controlChangeY = pid((screenCenter[1] - mouse.position[1]) -
scaledChange[1])
_, frame = webcam.read()
FaceTracker()
face_center = FaceTracker().trackface(frame)
print(face_center)
if face_center is not None:
############# YAAAAAAASSSSSS THE PID WORKS ON MULTIPLE FILES ###############33
coarse_newCoord = face_center
changeX = self.imageCenter[0] - coarse_newCoord[0]
changeY = coarse_newCoord[1] - self.imageCenter[1]
# if changex > changeBuffer or changey > changeBuffer:
change = [changeX, changeY]
# else:
scaledChange = np.average([[controlChangeX, controlChangeY],
[change[0] * 35, change[1] * 20]],
axis=0)
newPos = np.add(screenCenter, np.multiply(scaledChange, 1))
# print(newPos)
if newPos[0] > 10 and newPos[0] < 2550 and newPos[
1] > 10 and newPos[1] < 1070:
mouse.position = newPos
else:
break
# show full view
cv2.imshow('full', frame)
#close up
if cv2.waitKey(1) & 0xFF == ord('q'):
break
webcam.release()
cv2.destroyAllWindows()
def __init__(self, capture=None, face_img_path=None, should_mirror=False):
self._window_manager = WindowManager(self.on_keypress)
self._capture_manager = CaptureManager(cv2.VideoCapture(0)) \
if capture is None else CaptureManager(capture)
self._window_name = 'FaceOff'
self._should_mirror = should_mirror
self._face_tracker = FaceTracker()
self._show_face_rect = False
self._swap_face = True
self._template_face = None
if face_img_path is not None:
self._template_face = cv2.imread(face_img_path)
def analyse_slice(start, end):
# slice the subject video to the correct size
command = 'ffmpeg -loglevel panic -y -an' + ' -ss ' + str(
start - 2) + ' -i \"' + path_to_video + '\" -to ' + str(
(end - start) + 2) + ' -c copy -avoid_negative_ts 1 slice.avi'
print command
subprocess.call(shlex.split(command))
# setup video analysis
tracker = FaceTracker(path_to_opencv_cascades, gui=gui)
heartrates_for_slice = subjects_heartrates[subject_state][
'heartrates'][start:end]
inspector = FrameInspector(heartrates_for_slice)
video_capture = cv2.VideoCapture('slice.avi')
i = 0
while True:
i += 1
ret, frame = video_capture.read()
if not ret or frame is None:
print(str((start, end)) + "processed successfuly")
break
roi = None
try:
roi = tracker.detect_face(frame)
except RuntimeError as e:
raise RuntimeError(e.args[0] + " at time:" + str((start, end)))
if roi is None:
raise RuntimeError('face_tracker failed to find face at ' +
str(start) + '-' + str(end))
else:
if (i > 2 * FRAME_RATE):
inspector.extract(roi)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
inspector.done()
# When everything is done, release the capture
video_capture.release()
if gui:
cv2.destroyAllWindows()
data = inspector.get_data()
inspector.flush()
return data
def __init__(self):
'''
calibrate: get center
'''
print("Calibrating: look at center of screen")
time.sleep(2)
print("Calibrating...")
webcam = cv2.VideoCapture(0)
imageWidth = webcam.get(3) # float
imageHeight = webcam.get(4) # float
self.imageCenter = (imageWidth / 2, imageHeight / 2)
print("image center", self.imageCenter)
mouse = Controller()
screenCenter = [2560 / 2, 1080 / 2]
mouse.position = tuple(screenCenter) # set mouse to center of screen
while True:
# We get a new frame from the webcam
_, frame = webcam.read()
self.coarse_faceCenter = FaceTracker().trackface(frame)
if self.coarse_faceCenter is not None:
print(self.coarse_faceCenter)
break
if cv2.waitKey(1) == 27:
break
def analyse_slice(start, end):
# slice the subject video to the correct size
command = 'ffmpeg -loglevel panic -y -an' + ' -ss ' + str(start - 2) + ' -i \"' + path_to_video + '\" -to ' + str((end-start) + 2) + ' -c copy -avoid_negative_ts 1 slice.avi'
print command
subprocess.call(shlex.split(command))
# setup video analysis
tracker = FaceTracker(path_to_opencv_cascades, gui=gui)
heartrates_for_slice = subjects_heartrates[subject_state]['heartrates'][start:end]
inspector = FrameInspector(heartrates_for_slice)
video_capture = cv2.VideoCapture('slice.avi')
i = 0
while True:
i += 1
ret, frame = video_capture.read()
if not ret or frame is None:
print(str((start, end)) + "processed successfuly")
break
roi = None
try:
roi = tracker.detect_face(frame)
except RuntimeError as e:
raise RuntimeError(e.args[0] + " at time:" + str((start, end)))
if roi is None:
raise RuntimeError('face_tracker failed to find face at ' + str(start) + '-' + str(end))
else:
if (i > 2 * FRAME_RATE):
inspector.extract(roi)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
inspector.done()
# When everything is done, release the capture
video_capture.release()
if gui:
cv2.destroyAllWindows()
data = inspector.get_data()
inspector.flush()
return data
def main():
x_model = load_model('data/x_model.hdf5')
y_model = load_model('data/y_model.hdf5')
X_scalar, Y = load_data(False)
normaliser = np.amax(X_scalar, axis=0)
tracker = FaceTracker()
blink_detector = BlinkDetector()
clamp = lambda n, minn, maxn: max(min(maxn, n), minn)
last_x = deque([None] * 3)
last_y = deque([None] * 3)
while True:
frame = tracker.tick()
if tracker.face_type > 0:
if tracker.blinked:
pyautogui.click()
if tracker.left_eye is not None and tracker.right_eye is not None and tracker.rvec is not None and tracker.tvec is not None:
(blink_l,
blink_r) = blink_detector.detect(tracker.left_eye,
tracker.right_eye)
print(str(blink_l) + " : " + str(blink_r))
X_scalar = np.concatenate(
(tracker.rvec, tracker.tvec)).flatten() / normaliser
left = np.expand_dims(tracker.left_eye, axis=3) / 255.0
right = np.expand_dims(tracker.right_eye, axis=3) / 255.0
x_pred = x_model.predict([[left], [right], [X_scalar]])
y_pred = y_model.predict([[left], [right], [X_scalar]])
x = clamp(x_pred[0] * 1920, 0, 1920)
y = clamp(y_pred[0] * 1080, 0, 1080)
last_x[next((i for i, v in enumerate(last_x)
if v is None), -1) if None in last_x else 0] = x
last_y[next((i for i, v in enumerate(last_y)
if v is None), -1) if None in last_y else 0] = y
if None not in last_x: last_x.rotate(1)
if None not in last_y: last_y.rotate(1)
if x_pred is not None and y_pred is not None and None not in last_x and None not in last_y:
pyautogui.moveTo(
sum(last_x) / len(last_x),
sum(last_y) / len(last_y))
if cv2.waitKey(1) & 0xFF == ord('q'): break
def main():
model = load_model('data/model.hdf5')
X_scalar, Y = load_data(False)
normaliser = np.amax(X_scalar, axis=0)
tracker = FaceTracker()
clamp = lambda n, minn, maxn: max(min(maxn, n), minn)
while True:
frame = tracker.tick()
if tracker.face_type > 0:
if tracker.left_eye is not None and tracker.right_eye is not None and tracker.rvec is not None and tracker.tvec is not None:
X_scalar = np.concatenate(
(tracker.rvec, tracker.tvec)).flatten() / normaliser
left = np.expand_dims(tracker.left_eye, axis=3) / 255.0
right = np.expand_dims(tracker.right_eye, axis=3) / 255.0
pred = model.predict([[left], [right], [X_scalar]])
print(pred)
x = clamp(pred[0][0] * 1920, 0, 1920)
y = clamp(pred[0][1] * 1080, 0, 1080)
#print(str(x) + " : " + str(y))
if pred is not None:
pyautogui.moveTo(x, y)
#cv2.imshow('Video', frame)
if cv2.waitKey(1) & 0xFF == ord('q'): break
EYE_DIST = 60
input_cards_path = 'input_cards/'
rotated_cards_path = 'outputs/rotated_cards'
cards_cropped_by_eyes_path = 'outputs/cards_cropped_by_eyes'
if __name__ == '__main__':
import sys, getopt
cascade_fn = "data_harr/haarcascade_frontalface_alt2.xml"
scale = 1
scaleFactor = 1.3
tracker = FaceTracker(cascade_fn, scale, scaleFactor)
files = [
f for f in listdir(input_cards_path)
if isfile(join(input_cards_path, f))
]
#====================#
#read ou image files#
#====================#
for file in files:
img = cv2.imread(input_cards_path + file)
print(file)
#save path
save_rotated_card = os.path.join(rotated_cards_path, file)
class VideoCaptureApp(object):
def __init__(self, capture=None, face_img_path=None, should_mirror=False):
self._window_manager = WindowManager(self.on_keypress)
self._capture_manager = CaptureManager(cv2.VideoCapture(0)) \
if capture is None else CaptureManager(capture)
self._window_name = 'FaceOff'
self._should_mirror = should_mirror
self._face_tracker = FaceTracker()
self._show_face_rect = False
self._swap_face = True
self._template_face = None
if face_img_path is not None:
self._template_face = cv2.imread(face_img_path)
def run(self):
self._window_manager.create_window(self._window_name)
while self._window_manager.is_window_created(self._window_name):
self._capture_manager.enter_frame()
frame = self._capture_manager.frame
if frame is None:
print "get None frame!"
break
# process frame
# detect face
self._face_tracker.update(frame)
face_num = len(self._face_tracker.faces)
face_rect = None if face_num == 0 else \
self._face_tracker.faces[0].face_rect
if self._show_face_rect:
txt_str = 'face_num: {}'.format(face_num)
for face in self._face_tracker.faces:
x, y, w, h = face.face_rect
cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0),
2)
# only show the 1st face
break
if face_rect is not None and self._swap_face and \
self._template_face is not None:
x, y, w, h = face_rect
template_face = self._template_face.copy()
template_face = cv2.resize(template_face, (w, h))
# simply paste
# frame[y:y+h,x:x+w] = template_face
# or use seamless clone
mask = 255 * np.ones(template_face.shape, template_face.dtype)
center = (x + w / 2, y + h / 2)
frame = cv2.seamlessClone(template_face, frame, mask, center, \
cv2.MIXED_CLONE)
# frame = cv2.seamlessClone(template_face, frame, mask, center, \
# cv2.NORMAL_CLONE)
# show frame window
if self._should_mirror:
# horizontal flipping
frame = cv2.flip(frame, 1)
# draw text
if self._show_face_rect:
cv2.putText(frame, txt_str, (50, 100),
cv2.FONT_HERSHEY_SIMPLEX, 1.2, (0, 255, 0), 2)
self._window_manager.show_window(self._window_name, frame)
self._capture_manager.exit_frame()
self._window_manager.process_event()
def on_keypress(self, keycode):
if keycode == ord('m'):
self._should_mirror = not self._should_mirror
elif keycode == ord('s'):
self._swap_face = not self._swap_face
elif keycode == ord('f'):
self._show_face_rect = not self._show_face_rect
elif keycode == 27:
# Escape
self._window_manager.destroy_all_window()
def main(video_src=2):
# Initiate Face Tracker
# Todo 190207
# os.path에 해당 폴더 없으면 만들기
face_tracker = FaceTracker(video_device_id=int(video_src),
enable_age_gender=True,
age_gender_model_path=os.path.join(
'pretrained_models', 'age_gender',
'weights-wkfd.hdf5').replace("\\", "/"),
age_type="min")
# Initiate some variables
_var = None
if sys.argv[2] == "0":
robot_ip = None
client_socket = None
else:
robot_ip = "192.168.0.53"
print("Connecting to robot", robot_ip)
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.bind(("0.0.0.0", 0))
client_socket.connect((robot_ip, 8250))
print("Connected to the robot.")
robot_control = RobotControl(robot_ip, client_socket)
robot_face = '05'
target_face_index = None
target_face_id_in_db = None
if sys.argv[3] == "1":
_update_flag = True
else:
_update_flag = False
social_relation_estimator = SocialRelationEstimator(
robot_control, update_flag=_update_flag, enable_speaker=True)
while True:
s_time = time.time()
# Grab a single frame of video
ret, frame = face_tracker.video_capture.read()
process_frame = face_tracker.run(frame)
move_flag = None
if process_frame:
# Select largest face which is the closest one from the camera
# Todo 190131
# Kalman filter Target ID가 특정(짧은) 안에 다시 잡힐경우 얼굴 크기가 달라져도 계속 추적 -> embedding group
if target_face_id_in_db is None and len(
face_tracker.index_in_known_data) > 0 and len(
face_tracker.index_in_known_data) == len(
face_tracker.face_locations):
# 현재 face location들을 기반으로 가장 큰 얼굴의 index를 찾는다.
target_face_index = face_tracker.select_largest_face(
) # location 기반
# 이 값은 DB에서의 ID값 -> 고유값
target_face_id_in_db = face_tracker.index_in_known_data[
target_face_index] # Data 기반
target_det_time = time.time()
if target_face_id_in_db is not None:
'''
1. 대상이 없을 경우 가장 큰 얼굴을 잡는다. # face_tracker.select_largest_face()
2. 대상이 가장 큰 얼굴이 아니어도 기존에 추적중이었다면 쫓아간다.
3. 대상이 사라졌을 경우 (5)초 동안은 대상을 유지하고 가던 방향으로 속도를 줄인다.
'''
random_utter_flag = robot_control.random_utterance.flag
if random_utter_flag == 2:
# 랜덤 멘트 재생중
move_flag = 2
else:
if len(face_tracker.face_locations) > 0:
# print(target_face_index, face_tracker.index_in_known_data)
if len(face_tracker.index_in_known_data) > 0 and len(
face_tracker.index_in_known_data) == len(
face_tracker.face_locations):
try:
# 현재 찾아낸 얼굴들의 순서를 기준으로 계속 타겟중인 얼굴의 ID가 몇번째에 위치하는지 찾기.
target_face_index = face_tracker.index_in_known_data.index(
target_face_id_in_db)
# 현재 타겟의 이름
target_face_id = face_tracker.known_face_names[
target_face_id_in_db].split(
"ID:")[1].split(",")[0]
print('Target Face Name Id:', target_face_id)
# 현재 보이는 얼굴들의 순서에 DB의 ID값을 대입
# visible_face_index_in_db과 face_tracker.index_in_known_data는 같은듯..? Todo 190208 (완) - 같음
# visible_face_index_in_db = [face_tracker.index_in_known_data[i] for i, _ in enumerate(face_tracker.face_locations)]
# print(visible_face_index_in_db)
if target_face_id_in_db in face_tracker.index_in_known_data:
# 현재 보이는 얼굴들에 기존 타겟이 있음.
'''
move_flag
0: 목표가 그대로 보임
1: 목표가 보이지 않음 -> Slow Down
2: 목표가 보이지 않고 새로운 대상 없음 -> Stop
'''
move_flag = 0
target_face_location = face_tracker.face_locations[
target_face_index]
target_det_time = time.time()
else:
# 목표가 현재 보이지 않음.
move_flag = 1
except:
# print("target_face_id_in_db", target_face_id_in_db, face_tracker.index_in_known_data)
target_face_id = face_tracker.known_face_names[
target_face_id_in_db].split(
"ID:")[1].split(",")[0]
# print('Target Face Name Id:',target_face_id)
for _vi, _visible_index in enumerate(
face_tracker.index_in_known_data):
visible_face_id = face_tracker.known_face_names[
_visible_index].split("ID:")[1].split(
",")[0]
if visible_face_id == target_face_id:
# print('Visible Face Name Id:',visible_face_id)
move_flag = 0
target_face_location = face_tracker.face_locations[
_vi]
# print(face_tracker.known_face_groups)
if move_flag != 0:
# 목표가 사라졌고, 새로운 얼굴이 나타남
if time.time() - target_det_time > 10:
move_flag = 1
target_face_index = None
target_face_id_in_db = None
else:
move_flag = 1
elif time.time() - target_det_time > 10:
move_flag = 2
target_face_index = None
target_face_id_in_db = None
else:
print("Detecting..")
move_flag = 1
elif time.time() - target_det_time > 5 and len(
face_tracker.face_locations) == 0:
# 5초 이상 목표가 보이지 않을 시
move_flag = 2
target_face_index = None
target_face_id_in_db = None
else:
move_flag = 1
print("Move flag", move_flag)
try:
target_name = face_tracker.known_face_names[
target_face_id_in_db]
except:
target_name = None
if move_flag == 0:
# The actual robot part
# print(target_face_location, type(target_face_location))
if face_tracker.center_location is not None:
print("두명 탐지됨")
target_face_location = face_tracker.center_location
_var = robot_control.run(_var, robot_face, target_name,
target_face_location, frame,
move_flag,
social_relation_estimator)
else:
# The actual robot part
_var = robot_control.run(_var, robot_face, target_name,
None, frame, move_flag,
social_relation_estimator)
# 관계 추정 부분
# print(robot_control.status)
if robot_control.status == 0 and len(
face_tracker.known_face_ages) == len(
face_tracker.known_face_names):
# 거리가 일정 거리 이하고, Detect된 얼굴 면적 차이가 일정 크기 이하일 경우 Select
relevant_face_index = face_tracker.get_relevant_faces(
target_face_index)
if len(relevant_face_index) == 2:
face_tracker.center_location = face_tracker.get_center_location(
relevant_face_index)
social_relation_estimator.couple_not_cnt = 0
elif social_relation_estimator.couple_not_cnt is not None:
social_relation_estimator.couple_not_cnt += 1
if social_relation_estimator.couple_not_cnt > 15:
face_tracker.center_location = None
social_relation_estimator.couple_not_cnt = None
ages = [
face_tracker.known_face_ages[
face_tracker.index_in_known_data[i]]
for i in relevant_face_index
]
genders = [
face_tracker.known_face_genders[
face_tracker.index_in_known_data[i]]
for i in relevant_face_index
]
names = [
face_tracker.known_face_names[
face_tracker.index_in_known_data[i]]
for i in relevant_face_index
]
emotions = [
face_tracker.known_face_emotions[
face_tracker.index_in_known_data[i]]
for i in relevant_face_index
]
emotion_probs = [
face_tracker.known_face_emotion_probs[
face_tracker.index_in_known_data[i]]
for i in relevant_face_index
]
detect_cnts = [
face_tracker.known_face_detect_count[
face_tracker.index_in_known_data[i]]
for i in relevant_face_index
]
# print(detect_cnts)
# Todo 190209
# 특정 얼굴 크기 이상일때만 작동하게.
social_relation_estimator.run(detect_cnts, ages, genders,
emotions, emotion_probs,
target_face_id)
# print(min(detect_cnts))
# if done:
# Release handle to the webcam
# face_tracker.video_capture.release()
# except Exception as ex:
# print("ex at main loop:",ex)
else:
_var = robot_control.run(_var, robot_face, None, None, frame,
2, social_relation_estimator)
cv2.rectangle(image, cv_rect[0], cv_rect[1], (0, 255, 0), 2)
cv2.putText(image, text, (cv_rect[0][0], cv_rect[0][1] - 5),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (
0,
255,
0,
), 2)
if __name__ == '__main__':
argv = arg_parser.parse_args(sys.argv[1:])
video_capture = cv2.VideoCapture(argv.video_path)
face_rect_detector = FaceDetector()
ret = True
ratio = argv.scale_view
tracker = FaceTracker()
while ret:
ret, frame = video_capture.read()
if ret:
rects = face_rect_detector(frame)
faces = tracker.track(rects, frame)
frame = imutils.resize(frame, width=int(frame.shape[1] * ratio))
for face in faces:
rect = face.rectangle
rect = rect.scale(ratio).expand(10)
draw_labeled_rectangle(frame, rect, str(face.id))
cv2.imshow('face tracking', frame)
cv2.waitKey(1)
if __name__ == '__main__':
import sys, getopt
video_src = 0
cascade_fn = "data/haarcascade_frontalface_alt2.xml"
cam = create_capture(video_src,
fallback='synth:bg=../cpp/lena.jpg:noise=0.05')
scale = 0.25
scaleFactor = 1.3
tracker = FaceTracker(cascade_fn, scale, scaleFactor)
while True:
ret, img = cam.read()
t = clock()
npoints = tracker.detect(img)
if npoints is not None:
img = tracker.draw_rectangle(img, npoints)
dt = clock() - t
def main():
tracker = FaceTracker()
capturing = False
captured_eye_data = []
captured_head_data = []
captured_mouse_data = []
while True:
frame = tracker.tick()
if DEBUG:
text_x1 = tracker.face_x1
text_y1 = tracker.face_y1 - 3
if text_y1 < 0: text_y1 = 0
cv2.putText(frame, "FACE", (text_x1,text_y1), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,0), 1);
cv2.rectangle(frame, (tracker.face_x1, tracker.face_y1), (tracker.face_x2, tracker.face_y2), (0, 255, 0), 2)
if tracker.face_type > 0:
if DEBUG:
for point in tracker.landmarks_2D:
cv2.circle(frame,( point[0], point[1] ), 2, (0,0,255), -1)
(mx, my) = mouse_pos()
axis = np.float32([[50,0,0], [0,50,0], [0,0,50]])
imgpts, jac = cv2.projectPoints(axis, tracker.rvec, tracker.tvec, tracker.camera_matrix, tracker.camera_distortion)
sellion_xy = (tracker.landmarks_2D[7][0], tracker.landmarks_2D[7][1])
cv2.line(frame, sellion_xy, tuple(imgpts[1].ravel()), (0,255,0), 3) #GREEN
cv2.line(frame, sellion_xy, tuple(imgpts[2].ravel()), (255,0,0), 3) #BLUE
cv2.line(frame, sellion_xy, tuple(imgpts[0].ravel()), (0,0,255), 3) #RED
if tracker.left_eye is not None and tracker.right_eye is not None:
cv2.imshow('left eye', tracker.left_eye)
cv2.imshow('right eye', tracker.right_eye)
if capturing:
captured_eye_data.append((tracker.left_eye, tracker.right_eye))
captured_head_data.append((tracker.rvec, tracker.tvec))
captured_mouse_data.append((mx, my))
cv2.circle(frame, (10, 10), 5, (0, 0, 255), -1)
if DEBUG:
text_x1 = tracker.roi_x1
text_y1 = tracker.roi_y1 - 3
if text_y1 < 0: text_y1 = 0
cv2.putText(frame, "ROI", (text_x1,text_y1), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,255), 1);
cv2.rectangle(frame,
(tracker.roi_x1, tracker.roi_y1),
(tracker.roi_x2, tracker.roi_y2),
(0, 255, 255), 2)
cv2.imshow('Video', frame)
key = cv2.waitKey(1) & 0xFF
capturing = key == ord('c')
if key == ord('w'):
num_existing_pairs = len(os.listdir('./data/imgs/')) / 2
for idx, (left, right) in enumerate(captured_eye_data):
(left_gauss, right_gauss) = add_gaussian_noise([left, right])
cv2.imwrite('data/imgs/l/'+str(int((idx*2+1)+num_existing_pairs))+'.png', left_gauss)
cv2.imwrite('data/imgs/r/'+str(int((idx*2+1)+num_existing_pairs))+'.png', right_gauss)
cv2.imwrite('data/imgs/l/'+str(int((idx*2)+num_existing_pairs))+'.png', left)
cv2.imwrite('data/imgs/r/'+str(int((idx*2)+num_existing_pairs))+'.png', right)
with open('data/data.csv', 'a') as file:
for idx, (mx, my) in enumerate(captured_mouse_data):
(rvec, tvec) = captured_head_data[idx]
file.write(str(rvec[0][0])+','+str(rvec[1][0])+','+str(rvec[2][0])+','+str(tvec[0][0])+','+str(tvec[1][0])+','+str(tvec[2][0])+','+str(mx)+','+str(my))
file.write('\n')
file.write(str(rvec[0][0])+','+str(rvec[1][0])+','+str(rvec[2][0])+','+str(tvec[0][0])+','+str(tvec[1][0])+','+str(tvec[2][0])+','+str(mx)+','+str(my)) # write twice due to gauss augmentation
file.write('\n')
print("Wrote " + str(len(captured_eye_data ) * 2) + " data points to disk.")
captured_eye_data.clear()
captured_mouse_data.clear()
captured_head_data.clear()
if key == ord('q'): break
def main():
tracker = FaceTracker()
capturing = False
captured_eye_data = []
captured_head_data = []
captured_mouse_data = []
aoi_size = 512
aoi_x = random.randrange(0, 1920 - aoi_size)
aoi_y = random.randrange(0, 1080 - aoi_size)
follow_x = random.randrange(aoi_x, aoi_x + aoi_size)
follow_y = random.randrange(aoi_y, aoi_y + aoi_size)
follow_dx = 0.0
follow_dy = 0.0
pause = True
first = True
while True:
follow_x += follow_dx
follow_y += follow_dy
img = np.ones((1080, 1920, 3), np.uint8) * 255
if follow_x > aoi_x + aoi_size or follow_x < aoi_x or follow_y > aoi_y + aoi_size or follow_y < aoi_y or first:
aoi_x = random.randrange(0, 1920 - aoi_size)
aoi_y = random.randrange(0, 1080 - aoi_size)
follow_x = random.randrange(aoi_x, aoi_x + aoi_size)
follow_y = random.randrange(aoi_y, aoi_y + aoi_size)
x_cond = follow_x > aoi_x + aoi_size / 2
y_cond = follow_y > aoi_y + aoi_size / 2
follow_dx = random.randrange(
-9.0, -1.0) if x_cond else random.randrange(1.0, 9.0)
follow_dy = random.randrange(
-9.0, -1.0) if y_cond else random.randrange(1.0, 9.0)
cv2.circle(img, (int(follow_x), int(follow_y)), 12, (0, 255, 0),
-1)
cv2.line(img, (int(follow_x), int(follow_y)),
(int(follow_x + follow_dx * 10),
int(follow_y + follow_dy * 10)), (0, 255, 0), 1)
pause = True
cv2.circle(img, (int(follow_x), int(follow_y)), 8, (0, 0, 255), -1)
cv2.rectangle(img, (aoi_x, aoi_y),
(aoi_x + aoi_size, aoi_y + aoi_size), (0, 255, 0), 2)
cv2.imshow('trainer', img)
cv2.setWindowProperty('trainer', cv2.WND_PROP_FULLSCREEN,
cv2.WINDOW_FULLSCREEN)
cv2.moveWindow('trainer', 0, 0)
frame = tracker.tick()
if DEBUG:
text_x1 = tracker.face_x1
text_y1 = tracker.face_y1 - 3
if text_y1 < 0: text_y1 = 0
cv2.putText(frame, "FACE", (text_x1, text_y1),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)
cv2.rectangle(frame, (tracker.face_x1, tracker.face_y1),
(tracker.face_x2, tracker.face_y2), (0, 255, 0), 2)
if tracker.face_type > 0:
if DEBUG:
for point in tracker.landmarks_2D:
cv2.circle(frame, (point[0], point[1]), 2, (0, 0, 255), -1)
(mx, my) = (follow_x, follow_y)
axis = np.float32([[50, 0, 0], [0, 50, 0], [0, 0, 50]])
imgpts, jac = cv2.projectPoints(axis, tracker.rvec, tracker.tvec,
tracker.camera_matrix,
tracker.camera_distortion)
sellion_xy = (tracker.landmarks_2D[7][0],
tracker.landmarks_2D[7][1])
cv2.line(frame, sellion_xy, tuple(imgpts[1].ravel()), (0, 255, 0),
3) #GREEN
cv2.line(frame, sellion_xy, tuple(imgpts[2].ravel()), (255, 0, 0),
3) #BLUE
cv2.line(frame, sellion_xy, tuple(imgpts[0].ravel()), (0, 0, 255),
3) #RED
if tracker.left_eye is not None and tracker.right_eye is not None:
cv2.imshow('left eye', tracker.left_eye)
cv2.imshow('right eye', tracker.right_eye)
cv2.moveWindow('left eye', 2225, 0)
cv2.moveWindow('right eye', 2580, 0)
captured_eye_data.append((tracker.left_eye, tracker.right_eye))
captured_head_data.append((tracker.rvec, tracker.tvec))
captured_mouse_data.append((mx, my))
cv2.circle(frame, (10, 10), 5, (0, 0, 255), -1)
first = False
if DEBUG:
text_x1 = tracker.roi_x1
text_y1 = tracker.roi_y1 - 3
if text_y1 < 0: text_y1 = 0
cv2.putText(frame, "ROI", (text_x1, text_y1),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 255), 1)
cv2.rectangle(frame, (tracker.roi_x1, tracker.roi_y1),
(tracker.roi_x2, tracker.roi_y2), (0, 255, 255), 2)
cv2.imshow('Video', frame)
cv2.moveWindow('Video', 2980, 0)
#if pause:
# pause = False
# time.sleep(0.5)
cv2.waitKey(1)
if pause:
pause = False
while True:
key = cv2.waitKey(1) & 0xFF
if key == ord('c'):
break
elif key == ord('w'):
num_existing_pairs = len(os.listdir('./data/imgs/r'))
if len(captured_mouse_data) != len(
captured_head_data) or len(
captured_mouse_data) != len(captured_eye_data):
print("Differing array sizes.")
sys.exit(0)
for idx, (left, right) in enumerate(captured_eye_data):
(left_gauss,
right_gauss) = add_gaussian_noise([left, right])
cv2.imwrite(
'data/imgs/l/' +
str(int((idx * 2 + 1) + num_existing_pairs)) +
'.png', left_gauss)
cv2.imwrite(
'data/imgs/r/' +
str(int((idx * 2 + 1) + num_existing_pairs)) +
'.png', right_gauss)
cv2.imwrite(
'data/imgs/l/' +
str(int((idx * 2) + num_existing_pairs)) + '.png',
left)
cv2.imwrite(
'data/imgs/r/' +
str(int((idx * 2) + num_existing_pairs)) + '.png',
right)
with open('data/data.csv', 'a') as file:
for idx, (mx, my) in enumerate(captured_mouse_data):
(rvec, tvec) = captured_head_data[idx]
file.write(
str(rvec[0][0]) + ',' + str(rvec[1][0]) + ',' +
str(rvec[2][0]) + ',' + str(tvec[0][0]) + ',' +
str(tvec[1][0]) + ',' + str(tvec[2][0]) + ',' +
str(mx) + ',' + str(my))
file.write('\n')
file.write(
str(rvec[0][0]) + ',' + str(rvec[1][0]) + ',' +
str(rvec[2][0]) + ',' + str(tvec[0][0]) + ',' +
str(tvec[1][0]) + ',' + str(tvec[2][0]) + ',' +
str(mx) + ',' + str(my)
) # write twice due to gauss augmentation
file.write('\n')
print("Wrote " + str(len(captured_eye_data) * 2) +
" data points to disk.")
captured_eye_data.clear()
captured_mouse_data.clear()
captured_head_data.clear()
sys.exit(0)
elif key == ord('q'):
sys.exit(0)
def MoveMouse(self):
gaze = GazeTracking()
webcam = cv2.VideoCapture(0)
# webcam.set(3, 2560);
# webcam.set(4, 1080);
mouse = Controller()
screenCenter = [2560/2, 1080/2]
# mouse.position = tuple(screenCenter)
scaleFactor = 1.2
pid = PID(.2, .2, 0.01, setpoint=1)
eyeStateLIST = []
scaledChange = [0,0]
while True:
# print(changeX)
controlChangeX = pid((mouse.position[0] - screenCenter[0]) - scaledChange[0])
controlChangeY = pid((screenCenter[1] - mouse.position[1]) - scaledChange[1])
# We get a new frame from the webcam
_, webcam_frame = webcam.read()
FaceTracker()
face_center = FaceTracker().trackface(webcam_frame)
# print(face_center)
# FaceTracker()
# face_frame = FaceTracker().trackeyes(webcam_frame)
# face_frame = FaceTracker().get_face_frame()
frame = cv2.flip(webcam_frame, 1)
# We send this frame to GazeTracking to analyze it
gaze.refresh(frame)
frame = gaze.annotated_frame()
text = ""
eyeState = ""
if gaze.is_blinking():
eyeState = "Blinking"
eyeStateNum = 1
else:
eyeStateNum = 0
eyeStateLIST.append(eyeStateNum)
if len(eyeStateLIST) > 6:
eyeStateAvg = np.rint(np.mean(eyeStateLIST[-5:-1]))
del eyeStateLIST[0]
else:
eyeStateAvg = 0
# elif gaze.is_right():
# text = "Looking right"
# elif gaze.is_left():
# text = "Looking left"
# elif gaze.is_center():
# text = "Looking center"
# print(eyeStateLIST)
# print(eyeStateAvg)
cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (147, 58, 31), 2)
left_pupil = gaze.pupil_left_coords()
right_pupil = gaze.pupil_right_coords()
cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1)
cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1)
cv2.imshow("Demo", frame)
if left_pupil is not None:
coarse_newCoord = left_pupil
if right_pupil is not None:
coarse_newCoord = right_pupil
if left_pupil is not None or right_pupil is not None:
# coarse_newCoord = np.average([left_pupil, right_pupil], axis=0)
changeX = coarse_newCoord[0]-self.imageCenter[0]
changeY = coarse_newCoord[1]-self.imageCenter[1]
# if changex > changeBuffer or changey > changeBuffer:
change = [changeX, changeY]
# else:
scaledChange = np.average([[controlChangeX, controlChangeY], [change[0]*25, change[1]*10]], axis=0)
newPos = np.add(screenCenter, np.multiply(scaledChange,1))
# print(newPos)
if newPos[0] > 0 and newPos[0] < 2560 and newPos[1] > 0 and newPos[1] < 1080:
mouse.position = newPos
else:
# break
pass
if eyeStateAvg == 1:
mouse.click(Button.left, 1)
print(mouse.position)
else:
########################3
# fine control pupil follower
pass
# EyeTracker()
# fine_newCoord = EyeTracker().trackeyes(frame)
# print(fine_newCoord)
if cv2.waitKey(1) == 27:
break