class face_detector():
def __init__(self):
# Load the parameters
self.conf = config()
# initialize dlib's face detector (HOG-based) and then create the
# facial landmark predictor
print("[INFO] loading facial landmark predictor...")
self.detector = dlib.get_frontal_face_detector()
self.predictor = dlib.shape_predictor(self.conf.shape_predictor_path)
# grab the indexes of the facial landmarks for the left and
# right eye, respectively
(self.lStart, self.lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"]
(self.rStart, self.rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"]
# initialize the video stream and sleep for a bit, allowing the
# camera sensor to warm up
self.cap = cv2.VideoCapture(0)
if self.conf.vedio_path == 0:
self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
_, sample_frame = self.cap.read()
# Introduce mark_detector to detect landmarks.
self.mark_detector = MarkDetector()
# Setup process and queues for multiprocessing.
self.img_queue = Queue()
self.box_queue = Queue()
self.img_queue.put(sample_frame)
self.box_process = Process(target=get_face, args=(
self.mark_detector, self.img_queue, self.box_queue,))
self.box_process.start()
# Introduce pose estimator to solve pose. Get one frame to setup the
# estimator according to the image size.
self.height, self.width = sample_frame.shape[:2]
self.pose_estimator = PoseEstimator(img_size=(self.height, self.width))
# Introduce scalar stabilizers for pose.
self.pose_stabilizers = [Stabilizer(
state_num=2,
measure_num=1,
cov_process=0.1,
cov_measure=0.1) for _ in range(6)]
self.tm = cv2.TickMeter()
# Gaze tracking
self.gaze = GazeTracking()
def detect(self):
# loop over the frames from the video stream
temp_steady_pose = 0
while True:
# grab the frame from the threaded video stream, resize it to
# have a maximum width of 400 pixels, and convert it to
# grayscale
frame_got, frame = self.cap.read()
# Empty frame
frame_empty = np.zeros(frame.shape)
# frame = imutils.rotate(frame, 90)
frame = imutils.resize(frame, width=400)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# detect faces in the grayscale frame
rects = self.detector(gray, 0)
# initialize the frame counters and the total number of blinks
TOTAL = 0
COUNTER = 0
# loop over the face detections
for (i, rect) in enumerate(rects):
# determine the facial landmarks for the face region, then
# convert the facial landmark (x, y)-coordinates to a NumPy
# array
self.shape = self.predictor(gray, rect)
self.shape = face_utils.shape_to_np(self.shape)
# ********************************
# Blink detection
# extract the left and right eye coordinates, then use the
# coordinates to compute the eye aspect ratio for both eyes
self.leftEye = self.shape[self.lStart:self.lEnd]
self.rightEye = self.shape[self.rStart:self.rEnd]
self.leftEAR = eye_aspect_ratio(self.leftEye)
self.rightEAR = eye_aspect_ratio(self.rightEye)
# average the eye aspect ratio together for both eyes
ear = (self.leftEAR + self.rightEAR) / 2.0
# check to see if the eye aspect ratio is below the blink
# threshold, and if so, increment the blink frame counter
if ear < self.conf.EYE_AR_THRESH:
COUNTER += 1
# otherwise, the eye aspect ratio is not below the blink
# threshold
else:
# if the eyes were closed for a sufficient number of
# then increment the total number of blinks
if COUNTER >= self.conf.EYE_AR_CONSEC_FRAMES:
TOTAL += 1
# reset the eye frame counter
COUNTER = 0
# Frame empty
cv2.putText(frame_empty, "Blinks: {}".format(TOTAL), (30, 60),
cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 0, 255), 2)
cv2.putText(frame_empty, "EAR: {:.2f}".format(ear), (30, 90),
cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 0, 255), 2)
# ********************************
# convert dlib's rectangle to a OpenCV-style bounding box
# [i.e., (x, y, w, h)], then draw the face bounding box
(x, y, w, h) = face_utils.rect_to_bb(rect)
self.bounding_box = (x, y, w, h)
# cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
# Frame empty
cv2.rectangle(frame_empty, (x, y), (x + w, y + h), (0, 255, 0), 2)
# show the face number
cv2.putText(frame_empty, "Face #{}".format(i + 1), (30, 120),
cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 0, 255), 2)
# loop over the (x, y)-coordinates for the facial landmarks
# and draw them on the image
for (x, y) in self.shape:
# cv2.circle(frame, (x, y), 1, (0, 255, 255), -1)
cv2.circle(frame_empty, (x, y), 1, (0, 255, 255), -1)
# **********************************************************
if frame_got is False:
break
# If frame comes from webcam, flip it so it looks like a mirror.
if self.conf.vedio_path == 0:
frame = cv2.flip(frame, 2)
# Pose estimation by 3 steps:
# 1. detect face;
# 2. detect landmarks;
# 3. estimate pose
# Feed frame to image queue.
self.img_queue.put(frame)
# Get face from box queue.
self.facebox = self.box_queue.get()
if self.facebox is not None:
# Detect landmarks from image of 128x128.
face_img = frame[self.facebox[1]: self.facebox[3],
self.facebox[0]: self.facebox[2]]
face_img = cv2.resize(face_img, (self.conf.CNN_INPUT_SIZE, self.conf.CNN_INPUT_SIZE))
face_img = cv2.cvtColor(face_img, cv2.COLOR_BGR2RGB)
self.tm.start()
# marks = self.mark_detector.detect_marks([face_img])
self.tm.stop()
# Convert the marks locations from local CNN to global image.
self.shape *= (self.facebox[2] - self.facebox[0])
self.shape[:, 0] += self.facebox[0]
self.shape[:, 1] += self.facebox[1]
# Uncomment following line to show raw marks.
# mark_detector.draw_marks(
# frame, marks, color=(0, 255, 0))
# Uncomment following line to show facebox.
# mark_detector.draw_box(frame, [facebox])
# Try pose estimation with 68 points.
self.pose = self.pose_estimator.solve_pose_by_68_points(self.shape)
# Stabilize the pose.
self.steady_pose = []
pose_np = np.array(self.pose).flatten()
for value, ps_stb in zip(pose_np, self.pose_stabilizers):
ps_stb.update([value])
self.steady_pose.append(ps_stb.state[0])
self.steady_pose = np.reshape(self.steady_pose, (-1, 3))
# Uncomment following line to draw pose annotation on frame.
# pose_estimator.draw_annotation_box(
# frame, pose[0], pose[1], color=(255, 128, 128))
# Uncomment following line to draw stabile pose annotation on frame.
# pose_estimator.draw_annotation_box(frame, steady_pose[0], steady_pose[1], color=(128, 255, 128))
# Uncomment following line to draw head axes on frame.
# pose_estimator.draw_axes(frame, steady_pose[0], steady_pose[1])
self.pose_estimator.draw_axes(frame_empty, self.steady_pose[0], self.steady_pose[1])
print('steady pose vector: {}'.format(self.steady_pose[0], self.steady_pose[1]))
else:
# cv2.putText(frame, "Signal loss", (200, 200),
# cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.putText(frame_empty, "Signal loss", (200, 200),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
# ******************************************************************
# We send this frame to GazeTracking to analyze it
self.gaze.refresh(frame)
frame = self.gaze.annotated_frame()
text = ""
if self.gaze.is_blinking():
text = "Blinking"
elif self.gaze.is_right():
text = "Looking right"
elif self.gaze.is_left():
text = "Looking left"
elif self.gaze.is_center():
text = "Looking center"
cv2.putText(frame_empty, text, (250, 250), cv2.FONT_HERSHEY_DUPLEX, 0.5, (147, 58, 31), 2)
left_pupil = self.gaze.pupil_left_coords()
right_pupil = self.gaze.pupil_right_coords()
cv2.putText(frame_empty, "Left pupil: " + str(left_pupil), (250, 280), cv2.FONT_HERSHEY_DUPLEX, 0.5,
(147, 58, 31), 1)
cv2.putText(frame_empty, "Right pupil: " + str(right_pupil), (250, 310), cv2.FONT_HERSHEY_DUPLEX, 0.5,
(147, 58, 31), 1)
# ********************************************************************
# show the frame
# cv2.imshow("Frame", frame)
cv2.imshow("Frame", frame_empty)
key = cv2.waitKey(1) & 0xFF
self.pass_variable = np.array(1)
try:
self._listener(self.pass_variable)
except:
pass
# if the `q` key was pressed, break from the loop
if key == ord("q"):
break
# do a bit of cleanup
cv2.destroyAllWindows()
# self.cap.stop()
def set_listener(self, listener):
self._listener = listener
def main():
"""MAIN"""
# Video source from webcam or video file.
video_src = args.cam if args.cam is not None else args.video
if video_src is None:
print(
"Warning: video source not assigned, default webcam will be used.")
video_src = 0
cap = cv2.VideoCapture(video_src)
if video_src == 0:
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
_, sample_frame = cap.read()
# Introduce mark_detector to detect landmarks.
mark_detector = MarkDetector()
# Setup process and queues for multiprocessing.
img_queue = Queue()
box_queue = Queue()
img_queue.put(sample_frame)
box_process = Process(target=get_face,
args=(
mark_detector,
img_queue,
box_queue,
))
box_process.start()
# Introduce pose estimator to solve pose. Get one frame to setup the
# estimator according to the image size.
height, width = sample_frame.shape[:2]
pose_estimator = PoseEstimator(img_size=(height, width))
# Introduce scalar stabilizers for pose.
pose_stabilizers = [
Stabilizer(state_num=2,
measure_num=1,
cov_process=0.1,
cov_measure=0.1) for _ in range(6)
]
tm = cv2.TickMeter()
while True:
# Read frame, crop it, flip it, suits your needs.
frame_got, frame = cap.read()
if frame_got is False:
break
# Crop it if frame is larger than expected.
# frame = frame[0:480, 300:940]
# If frame comes from webcam, flip it so it looks like a mirror.
if video_src == 0:
frame = cv2.flip(frame, 2)
# Pose estimation by 3 steps:
# 1. detect face;
# 2. detect landmarks;
# 3. estimate pose
# Feed frame to image queue.
img_queue.put(frame)
# Get face from box queue.
facebox = box_queue.get()
if facebox is not None:
# Detect landmarks from image of 128x128.
face_img = frame[facebox[1]:facebox[3], facebox[0]:facebox[2]]
face_img = cv2.resize(face_img, (CNN_INPUT_SIZE, CNN_INPUT_SIZE))
face_img = cv2.cvtColor(face_img, cv2.COLOR_BGR2RGB)
tm.start()
marks = mark_detector.detect_marks(face_img)
tm.stop()
# Convert the marks locations from local CNN to global image.
marks *= (facebox[2] - facebox[0])
marks[:, 0] += facebox[0]
marks[:, 1] += facebox[1]
# Uncomment following line to show raw marks.
# mark_detector.draw_marks(frame, marks, color=(0, 255, 0))
# Uncomment following line to show facebox.
# mark_detector.draw_box(frame, [facebox])
# Try pose estimation with 68 points.
pose = pose_estimator.solve_pose_by_68_points(marks)
# Stabilize the pose.
steady_pose = []
pose_np = np.array(pose).flatten()
for value, ps_stb in zip(pose_np, pose_stabilizers):
ps_stb.update([value])
steady_pose.append(ps_stb.state[0])
steady_pose = np.reshape(steady_pose, (-1, 3))
# Uncomment following line to draw pose annotation on frame.
# pose_estimator.draw_annotation_box(
# frame, pose[0], pose[1], color=(255, 128, 128))
# Uncomment following line to draw stabile pose annotation on frame.
pose_estimator.draw_annotation_box(frame,
steady_pose[0],
steady_pose[1],
color=(128, 255, 128))
# Uncomment following line to draw head axes on frame.
pose_estimator.draw_axes(frame, steady_pose[0], steady_pose[1])
# Show preview.
cv2.imshow("Preview", frame)
if cv2.waitKey(10) == 27:
break
# Clean up the multiprocessing process.
box_process.terminate()
box_process.join()
def main():
"""MAIN"""
# Video source from webcam or video file.
video_src = args.cam if args.cam is not None else args.video
if video_src is None:
engine.say(
"Warning: video source not assigned, default webcam will be used")
engine.runAndWait()
video_src = 0
cap = cv2.VideoCapture(video_src)
if video_src == 0:
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
_, sample_frame = cap.read()
# Introduce mark_detector to detect landmarks.
mark_detector = MarkDetector()
# Setup process and queues for multiprocessing.
img_queue = Queue()
box_queue = Queue()
img_queue.put(sample_frame)
box_process = Process(target=get_face,
args=(
mark_detector,
img_queue,
box_queue,
))
box_process.start()
gaze = GazeTracking()
# Introduce pose estimator to solve pose. Get one frame to setup the
# estimator according to the image size.
height, width = sample_frame.shape[:2]
pose_estimator = PoseEstimator(img_size=(height, width))
# Introduce scalar stabilizers for pose.
pose_stabilizers = [
Stabilizer(state_num=2,
measure_num=1,
cov_process=0.1,
cov_measure=0.1) for _ in range(6)
]
tm = cv2.TickMeter()
head_flag = 0
gaze_flag = 0
while True:
# Read frame, crop it, flip it, suits your needs.
frame_got, frame = cap.read()
if frame_got is False:
break
# Crop it if frame is larger than expected.
# frame = frame[0:480, 300:940]
#audio_record(AUDIO_OUTPUT, 3)
#sphinx_recog(AUDIO_OUTPUT)
# If frame comes from webcam, flip it so it looks like a mirror.
if video_src == 0:
frame = cv2.flip(frame, 2)
# Pose estimation by 3 steps:
# 1. detect face;
# 2. detect landmarks;
# 3. estimate pose
# Feed frame to image queue.
img_queue.put(frame)
# Get face from box queue.
facebox = box_queue.get()
gaze.refresh(frame)
frame = gaze.annotated_frame()
text = ""
if facebox is not None:
# Detect landmarks from image of 128x128.
face_img = frame[facebox[1]:facebox[3], facebox[0]:facebox[2]]
face_img = cv2.resize(face_img, (CNN_INPUT_SIZE, CNN_INPUT_SIZE))
gray = cv2.cvtColor(face_img, cv2.COLOR_BGR2GRAY)
face_img = cv2.cvtColor(face_img, cv2.COLOR_BGR2RGB)
rects = detector(gray, 0)
tm.start()
marks = mark_detector.detect_marks([face_img])
tm.stop()
#
for rect in rects:
# determine the facial landmarks for the face region, then
# convert the facial landmark (x, y)-coordinates to a NumPy
# array
shape = predictor(gray, rect)
shape = face_utils.shape_to_np(
shape) #converting to NumPy Array矩阵运算
mouth = shape[Start:End]
leftEye = shape[lStart:lEnd]
rightEye = shape[rStart:rEnd]
leftEAR = eye_aspect_ratio(leftEye) #眼睛长宽比
rightEAR = eye_aspect_ratio(rightEye)
ear = (leftEAR + rightEAR) / 2.0 #长宽比平均值
lipdistance = lip_distance(shape)
if (lipdistance > YAWN_THRESH):
#print(lipdistance)
flag0 += 1
print("yawning time: ", flag0)
if flag0 >= 40:
cv2.putText(frame, "Yawn Alert", (10, 150),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255),
2)
cv2.putText(frame, "Yawn Alert", (220, 150),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255),
2)
engine.say("don't yawn")
engine.runAndWait()
flag0 = 0
else:
flag0 = 0
if (ear < thresh):
flag += 1
print("eyes closing time: ", flag)
if flag >= frame_check:
cv2.putText(frame,
"****************ALERT!****************",
(10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7,
(0, 0, 255), 2)
cv2.putText(frame,
"****************ALERT!****************",
(10, 250), cv2.FONT_HERSHEY_SIMPLEX, 0.7,
(0, 0, 255), 2)
engine.say("open your eyes")
engine.runAndWait()
flag = 0
else:
flag = 0
if gaze.is_right():
print("Looking right")
text = "Looking right"
elif gaze.is_left():
print("Looking left")
text = "Looking left"
elif gaze.is_up():
text = "Looking up"
else:
text = "Looking center"
if text is not "Looking center":
gaze_flag += 1
if gaze_flag >= 20:
engine.say("look forward")
engine.runAndWait()
gaze_flag = 0
else:
gaze_flag = 0
marks *= (facebox[2] - facebox[0])
marks[:, 0] += facebox[0]
marks[:, 1] += facebox[1]
# Uncomment following line to show raw marks.
# mark_detector.draw_marks(
# frame, marks, color=(0, 255, 0))
# Uncomment following line to show facebox.
# mark_detector.draw_box(frame, [facebox])
# Try pose estimation with 68 points.
pose = pose_estimator.solve_pose_by_68_points(marks)
# get angles
angles = pose_estimator.get_angles(pose[0], pose[1])
if ((-8 > angles[0] or angles[0] > 8)
or (-8 > angles[1] or angles[1] > 8)):
head_flag += 1
if head_flag >= 40:
print(angles[0])
engine.say("please look ahead")
engine.runAndWait()
else:
head_flag = 0
# pose_estimator.draw_info(frame, angles)
# Stabilize the pose.
steady_pose = []
pose_np = np.array(pose).flatten()
for value, ps_stb in zip(pose_np, pose_stabilizers):
ps_stb.update([value])
steady_pose.append(ps_stb.state[0])
steady_pose = np.reshape(steady_pose, (-1, 3))
# Uncomment following line to draw pose annotation on frame.
pose_estimator.draw_annotation_box(frame,
pose[0],
pose[1],
color=(255, 128, 128))
# Uncomment following line to draw stabile pose annotation on frame.
pose_estimator.draw_annotation_box(frame,
steady_pose[0],
steady_pose[1],
color=(128, 255, 128))
# Uncomment following line to draw head axes on frame.
pose_estimator.draw_axes(frame, steady_pose[0], steady_pose[1])
#pose_estimator.show_3d_model
# Show preview.
cv2.imshow("Preview", frame)
if cv2.waitKey(1) & 0xFF == ord("q"):
break
# Clean up the multiprocessing process.
box_process.terminate()
box_process.join()
