def __init__(self,
frame,
EYE_AR_THRESH=0.3,
ROLL_THRESH=20,
TIME_THRESH=10):
self.EYE_AR_THRESH = EYE_AR_THRESH
self.ROLL_THRESH = ROLL_THRESH
self.TIME_THRESH = TIME_THRESH
self.ALARM_ON = False
self.T = None
self.faceDetector = FaceDetector()
self.eyeDetector = EyeDetector()
self.markDetector = MarkDetector(self.faceDetector)
# Setup process and queues for multiprocessing.
self.img_queue = Queue()
self.box_queue = Queue()
self.img_queue.put(frame)
self.box_process = Process(target=self.get_face,
args=(self.markDetector, ))
self.box_process.start()
h, w = frame.shape[:2]
self.poseEstimator = PoseEstimator(img_size=(h, w))
self.pose_stabilizers = [
Stabilizer(state_num=2,
measure_num=1,
cov_process=0.1,
cov_measure=0.1) for _ in range(6)
]
def __init__(self, IOU_threshold, img_size=(720, 1280)):
'''Initialize the tracking system for multiple tracks
Input: IOU_threshold --- the threshold of IOU score to correlate detections on a image to
the ones previous images.
img_size --- the image size for the image or video'''
self.IOU_threshold = IOU_threshold # threshold to say detection on 1 image is the same object as the ones on previous one
#self.max_frame_skipped = max_frame_skipped
#self.max_trace_length = max_trace_length
self.IdCount = 0 # count the total number of Tracks used in this Tracking system
self.tracks = [] # a list holding all Tracks
self.markDetector = MarkDetector() # initialize the landmark detector
self.poseDetector = PoseEstimator(
img_size=img_size) # initialize head pose detector
def main():
from MarkDetector import MarkDetector
markDetector = MarkDetector()
pose = PoseEstimator()
start_time = time.time()
filepath = '/home/eric/Documents/face_analysis/data/photos/group.jpg'
img = cv2.imread(filepath)
boxes = markDetector.extract_cnn_facebox(
image=img) # extract bonding boxes
face_imgs = markDetector.get_face_for_boxes(
img, boxes) # extract face images by bonding boxes
marks = markDetector.detect_marks(
face_imgs) # detect landmarks on the cropped images
marks = markDetector.fit_markers_in_image(
marks, boxes) # get the landmark coordinates on the original images
### 2 different function: choose first or last 2
#r_vect, t_vect = pose.solve_pose_by_68_points(marks)
marks = pose.get_pose_marks(marks) # select 6 points out of 68
pp = pose.solve_pose(marks) # get the mapping from 3D points to 2D points
# tt = pose.solve_single_pose(marks[0]) # debug single face function
# print(tt)
# print(np.array(tt).flatten())
r_vect, t_vect = pp
pose_np = np.array(pp).flatten()
#print(pp)
#print(pose_np)
# stabile_pose = np.reshape(pose_np, (-1, 3))
#print(stabile_pose)
pose.draw_boxes(img, r_vect,
t_vect) # draw annotation for head pose on image
#MarkDetector.draw_marks(image=img, marks=marks)
#detector.draw_all_results(img)
cv2.imshow('image', img)
#print(time.time()-start_time)
cv2.waitKey(0)
cv2.destroyAllWindows()
def main():
# get video from webcam or video file
draw_landmarks = 1
draw_head_pose = 1
video_src = 0 # 0 means webcam; set file name if video
cam = cv2.VideoCapture(video_src)
_, sample_frame = cam.read() # get 1 sample frame to setup codes
# Initialize MarkDetector
mark_detector = MarkDetector()
# Setup process and queues for multiprocessing
img_queue = Queue()
box_queue = Queue()
img_queue.put(sample_frame)
box_process = Process(target=get_face, args=(mark_detector,
img_queue,
box_queue))
box_process.start()
# Initialize PoseEstimator to solve pose. Use 1 sampe frame to setup
height, width = sample_frame.shape[:2]
pose_estimator = PoseEstimator(img_size=(height, width))
# Initiate the landmark smoother by the Kalman filter
# 6 initializers for 6 landmarks per a face, since 6 are used to estimate pose
# if use 68 landmarks to estimate pose, use 68 stabilizers
if draw_head_pose == 1:
pose_stabilizers = [MarkStabilizer(initial_state=[0,0,0,0],
input_dim=2,
cov_process=0.1,
cov_measure=1.5) for _ in range(6)]
if draw_landmarks == 1:
mark_stabilizers68 = [MarkStabilizer(initial_state=[0,0,0,0],
input_dim=2,
cov_process=0.1,
cov_measure=2) for _ in range(68)]
while True:
frame_got, frame = cam.read()
if frame_got is False:
break
# crop image if needed
# frame = cv2.resize(frame, [500, 300])
# flip image if needed in webcam
# if video_src ==0:
# frame = cv2.flip(frame, 2)
# 3 Steps to estimate pose
# 1. detect face
# 2. detect landmarks
# 3. estimate pose
# Feed frame to image queue
img_queue.put(frame)
# Get facebox from box queue
facebox = box_queue.get()
if facebox is not None:
# get face images from the face boxes and images
faces = mark_detector.get_face_for_boxes(frame, facebox)
# get markers from face images
marks = mark_detector.detect_marks(faces)
marks = mark_detector.fit_markers_in_image(marks, facebox)
if draw_landmarks == 1:
# Draw markers if necessary (stabilized marks)
if(len(marks)>0):
#print(marks)
single_face_marks68 = marks[0]
stabilized_marks68 = []
for mark, stb in zip(single_face_marks68, mark_stabilizers68):
stb.update(mark)
stabilized_marks68.append(stb.get_results())
#stabilized_marks.append(mark)
stabilized_marks68 = np.array([stabilized_marks68])
#print(stabilized_marks)
else:
stabilized_marks68 = marks
MarkDetector.draw_marks(image=frame, marksFace=stabilized_marks68)
if draw_head_pose == 1:
# Solve pose by 68 points
# r_vect, t_vect = pose_estimator.solve_pose_by_68_points(marks)
# Solve pose by 6 points
marks = pose_estimator.get_pose_marks(marks)
# take marks from only 1 person (for kalman filter)
if(len(marks)>0):
#print(marks)
single_face_marks = marks[0]
stabilized_marks = []
for mark, stb in zip(single_face_marks, pose_stabilizers):
stb.update(mark)
stabilized_marks.append(stb.get_results())
#stabilized_marks.append(mark)
stabilized_marks = np.array([stabilized_marks])
#print(stabilized_marks)
else:
stabilized_marks = marks
########## stablize
r_vect, t_vect = pose_estimator.solve_pose(stabilized_marks)
# Draw pose boxes on the image
pose_estimator.draw_boxes(frame, r_vect, t_vect)
cv2.imshow("preview", frame)
if cv2.waitKey(10) == 27:
break
# clean up the multiprocessing processes
box_process.terminate()
box_process.join()
class Tracker():
''' A system with multiple Tracks for tracking multiple faces in a video or image'''
def __init__(self, IOU_threshold, img_size=(720, 1280)):
'''Initialize the tracking system for multiple tracks
Input: IOU_threshold --- the threshold of IOU score to correlate detections on a image to
the ones previous images.
img_size --- the image size for the image or video'''
self.IOU_threshold = IOU_threshold # threshold to say detection on 1 image is the same object as the ones on previous one
#self.max_frame_skipped = max_frame_skipped
#self.max_trace_length = max_trace_length
self.IdCount = 0 # count the total number of Tracks used in this Tracking system
self.tracks = [] # a list holding all Tracks
self.markDetector = MarkDetector() # initialize the landmark detector
self.poseDetector = PoseEstimator(
img_size=img_size) # initialize head pose detector
def assign_detections_to_trackers(self, detections):
''' with new detections, try to assign new measurements to existing tracks. Which
detection is already in the tracks, which is newly found (not in tracks), which
tracks are not detected in current detection.
Input: detections --- a list of bonding boxes of faces detected.
Output: matches --- list of the matched detections and tracks
unmatched_dets --- list of newly found detections without matched tracks
unmatched_tracks --- list tracks that has no corresponding detections found.
'''
# intialize a IOU score matrix and calculate the IOU score of each detection with each track
IOU_mat = np.zeros((len(self.tracks), len(detections)),
dtype=np.float32)
for t, trk in enumerate(self.tracks):
for d, det in enumerate(detections):
IOU_mat[t, d] = box_iou2(trk.box, det)
# find the matches of tracks and detections, using Hungarian algorithm
assign_row_ind, assign_col_ind = linear_sum_assignment(-IOU_mat)
unmatched_tracks = [
] # list to store id of tracks without matched detections,
# not trackID, just the ID in the self.tracks list
unmatched_dets = [
] # list to store id of the detections without matched tracks.
for t, trk in enumerate(self.tracks):
if t not in assign_row_ind:
unmatched_tracks.append(t)
for d, det in enumerate(detections):
if d not in assign_col_ind:
unmatched_dets.append(d)
matches = [] # list to store id of tracks and detection that matched
for r, c in zip(assign_row_ind, assign_col_ind):
# if matched detection and track has too small IOU, add them to unmatched tracks or detections
if IOU_mat[r][c] < self.IOU_threshold:
if r not in unmatched_tracks:
unmatched_tracks.append(r)
if c not in unmatched_dets:
unmatched_dets.append(c)
else:
matches.append([r, c])
# keep the format of matches list even if no matched one found.
if len(matches) == 0:
matches = np.empty((0, 2), dtype=int)
else:
matches = np.array(matches)
return matches, np.array(unmatched_dets), unmatched_tracks
def update(self, image, detections, matches, unmatched_dets,
unmatched_tracks):
'''update the results (all bonding boxes, landmarks, headpose) for all matched,
unmatched tracks/detections.
Input: image --- the current image the new detections are from
detections --- the detected bonding boxes on current image
matches --- ids of the detectes that matches with existing tracks, [track id, detection id].
unmatched_dets --- the ids of detections not matched with existing tracks
unmatched_tracks --- the ids of existing tracks not found in current detections'''
# updating the results with matched and non-matched results
# Deal with matched detections
if len(matches) > 0:
for trk_idx, det_idx in matches:
# Update face bonding box
det = detections[det_idx]
tmp_trk = self.tracks[trk_idx]
tmp_trk.predict()
tmp_trk.correct(det)
tmp_trk.hits += 1 # number of matched detections increase 1
# add center of the bonding box into trace.
tmp_trk.num_loss = 0 # once found, reset of number of loss detection to 0
tmp_trk.trace.append([
0.5 * (tmp_trk.box[0] + tmp_trk.box[2]),
0.5 * (tmp_trk.box[1] + tmp_trk.box[3])
])
if FaceVar.LADNMARK_ON or FaceVar.HEADPOSE_ON:
# Face landmarks update if Choose to use landmars in FaceVar file
# get a square bonding box from the ones from detector
single_square_box = self.markDetector.square_single_facebox(
tmp_trk.box)
# crop single face image from original image
single_face_img = self.markDetector.get_single_face_from_boxes(
image, single_square_box)
# find landmarks from the cropped image
singe_face_marks = self.markDetector.detect_marks_on_single_image(
single_face_img)
# fit the landmarks into the original image
mark_detection = self.markDetector.fit_markers_in_single_image(
singe_face_marks, single_square_box)
# update landmarks with kalman filter
tmp_trk.predict_marks()
tmp_trk.correct_marks(mark_detection)
if FaceVar.HEADPOSE_ON:
# get single face poses
# first get 68 landmarks --> get 6 landmarks --> update with kalman filter
# Also get landmarks here to make sure head pose could ran separately
'''
# debug only
single_square_box = self.markDetector.square_single_facebox(tmp_trk.box)
single_face_img = self.markDetector.get_single_face_from_boxes(image, single_square_box)
singe_face_marks = self.markDetector.detect_marks_on_single_image(single_face_img)
mark_detection = self.markDetector.fit_markers_in_single_image(singe_face_marks,
single_square_box)'''
# obtain 6 landmarks from 68 landmarks
marks6_for_pose = self.poseDetector.get_single_face_pose_marks(
mark_detection)
# solve the translation and rotation vecotrs with 6 landmarks
tmp_pose = self.poseDetector.solve_single_pose(
marks6_for_pose)
# update the head pose with kalman filter
tmp_pose_np = np.array(tmp_pose).flatten()
tmp_trk.predict_headPose()
tmp_trk.correct_headPose(tmp_pose_np)
tmp_trk.pose_trace.append(tmp_trk.pose_state)
# Deal with unmatched detections
if len(unmatched_dets) > 0:
for idx in unmatched_dets:
# Face bonding box not matched with existing tracks
det = detections[idx]
self.IdCount += 1 # increase the number of total tracks
tmp_trk = Track(det, self.IdCount) # initialize a new track
tmp_trk.predict()
self.tracks.append(
tmp_trk) # append new track in tracking system
tmp_trk.trace.append([
0.5 * (tmp_trk.box[0] + tmp_trk.box[2]),
0.5 * (tmp_trk.box[1] + tmp_trk.box[3])
])
# Face landmarks
if FaceVar.LADNMARK_ON or FaceVar.HEADPOSE_ON:
single_square_box = self.markDetector.square_single_facebox(
tmp_trk.box)
single_face_img = self.markDetector.get_single_face_from_boxes(
image, single_square_box)
singe_face_marks = self.markDetector.detect_marks_on_single_image(
single_face_img)
mark_detection = self.markDetector.fit_markers_in_single_image(
singe_face_marks, single_square_box)
tmp_trk.initialize_landmarks(mark_detection)
tmp_trk.marks_trace.append(tmp_trk.marks_state)
if FaceVar.HEADPOSE_ON:
# get single face poses
# first get 68 landmarks --> get 6 landmarks --> update with kalman filter
# Also get landmarks here to make sure head pose could ran separately
'''
# debug only
single_square_box = self.markDetector.square_single_facebox(tmp_trk.box)
single_face_img = self.markDetector.get_single_face_from_boxes(image, single_square_box)
singe_face_marks = self.markDetector.detect_marks_on_single_image(single_face_img)
mark_detection = self.markDetector.fit_markers_in_single_image(singe_face_marks,
single_square_box)'''
marks6_for_pose = self.poseDetector.get_single_face_pose_marks(
mark_detection)
tmp_pose = self.poseDetector.solve_single_pose(
marks6_for_pose)
tmp_pose_np = np.array(tmp_pose).flatten()
tmp_trk.initialize_headPose(tmp_pose_np)
tmp_trk.pose_trace.append(tmp_trk.pose_state)
# Deal with unmatched tracks
if len(unmatched_tracks) > 0:
for trk_idx in unmatched_tracks:
# Face bonding box that in existing track but not matched with detections
tmp_trk = self.tracks[trk_idx]
tmp_trk.num_loss += 1 # count the number of images it is not detected
tmp_trk.predict(
) # update Bbox with kalman filter without correction
tmp_trk.trace.append([
0.5 * (tmp_trk.box[0] + tmp_trk.box[2]),
0.5 * (tmp_trk.box[1] + tmp_trk.box[3])
])
# Face landmarks
if FaceVar.LADNMARK_ON:
tmp_trk.predict_marks(
) # update landmarks with kalman filter without correction
tmp_trk.marks_trace.append(tmp_trk.marks_state)
if FaceVar.HEADPOSE_ON:
tmp_trk.predict_headPose(
) # update landmarks with kalman filter without correction
tmp_trk.pose_trace.append(tmp_trk.pose_state)
# delete no useful lists
# tracks_to_delete = filter(lambda x: x.num_loss>FaceVar.MAX_AGE, self.tracks)
# update the list of tracks. If the num_loss is too large, delete it from list of tracks
self.tracks = [x for x in self.tracks if x.num_loss <= FaceVar.MAX_AGE]
def annotate_BBox(self, img):
''' annotate the image with bonding boxes, landmarks, and headpose. Also annotate
the trackId in the image'''
# good_track_list = []
for trk in self.tracks:
if trk.hits >= FaceVar.MIN_HITS and trk.num_loss <= FaceVar.MAX_AGE:
# good_track_list.append(trk)
if FaceVar.DRAW_DETECTION_BOX:
img = draw_box_on_image(img, trk)
if FaceVar.LADNMARK_ON:
img = draw_marks_on_image(img, trk)
if FaceVar.HEADPOSE_ON:
### draw head pose on images
#print(trk.pose_state)
tmp_vect = np.array(trk.pose_state).reshape(-1, 3)
img = self.poseDetector._draw_annotation_box(img,
tmp_vect[0],
tmp_vect[1],
color=(255, 0,
0),
line_width=1)
return img
class Detector:
def __init__(self,
frame,
EYE_AR_THRESH=0.3,
ROLL_THRESH=20,
TIME_THRESH=10):
self.EYE_AR_THRESH = EYE_AR_THRESH
self.ROLL_THRESH = ROLL_THRESH
self.TIME_THRESH = TIME_THRESH
self.ALARM_ON = False
self.T = None
self.faceDetector = FaceDetector()
self.eyeDetector = EyeDetector()
self.markDetector = MarkDetector(self.faceDetector)
# Setup process and queues for multiprocessing.
self.img_queue = Queue()
self.box_queue = Queue()
self.img_queue.put(frame)
self.box_process = Process(target=self.get_face,
args=(self.markDetector, ))
self.box_process.start()
h, w = frame.shape[:2]
self.poseEstimator = PoseEstimator(img_size=(h, w))
self.pose_stabilizers = [
Stabilizer(state_num=2,
measure_num=1,
cov_process=0.1,
cov_measure=0.1) for _ in range(6)
]
def get_face(self, detector):
"""Get face from image queue. This function is used for multiprocessing"""
while True:
image = self.img_queue.get()
box = detector.extract_cnn_facebox(image)
self.box_queue.put(box)
@staticmethod
def show_text(frame, roll, pitch, yaw, ear, ALARM_ON):
"""Display the computed Eye Aspect Ratio, Euler angles, and alert message."""
cv2.putText(frame, "Roll: {}".format(roll),
(frame.shape[1] - 235, frame.shape[0] - 5),
cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 255), 1)
cv2.putText(frame, "Pitch: {}".format(pitch),
(frame.shape[1] - 155, frame.shape[0] - 5),
cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 255), 1)
cv2.putText(frame, "Yaw: {}".format(yaw),
(frame.shape[1] - 75, frame.shape[0] - 5),
cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 255), 1)
cv2.putText(frame, "Eye Aspect Ratio: {:.2f}".format(ear),
(5, frame.shape[0] - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.45,
(0, 0, 255), 1)
if ALARM_ON:
cv2.putText(frame, "ALERT!", (frame.shape[1] - 116, 30),
cv2.FONT_HERSHEY_DUPLEX, 1.1, (0, 0, 255), 2)
def detect_drowsiness(self, frame):
# get frame from Server and feed frame to img_queue
self.img_queue.put(frame)
# Get face from box queue.
facebox = self.box_queue.get()
if facebox is not None:
# Detect landmarks from image of 128x128.
face_img = frame[facebox[1]:facebox[3], facebox[0]:facebox[2]]
face_img = cv2.resize(face_img, (CNN_INPUT_SIZE, CNN_INPUT_SIZE))
face_img = cv2.cvtColor(face_img, cv2.COLOR_BGR2RGB)
marks = self.markDetector.detect_marks(face_img)
# Convert the marks locations from local CNN to global image.
marks *= (facebox[2] - facebox[0])
marks[:, 0] += facebox[0]
marks[:, 1] += facebox[1]
# Uncomment following line to show raw marks.
self.markDetector.draw(frame, marks, color=(0, 255, 0))
# Try pose estimation with 68 points.
unstable_pose = self.poseEstimator.solve_pose_by_68_points(marks)
# Stabilize the pose.
pose = []
pose_np = np.array(unstable_pose).flatten()
for value, ps_stb in zip(pose_np, self.pose_stabilizers):
ps_stb.update([value])
pose.append(ps_stb.state[0])
pose = np.reshape(pose, (-1, 3))
# Uncomment following line to draw stabile pose annotaion on frame.
self.poseEstimator.draw_annotation_box(frame,
pose[0],
pose[1],
color=(0, 255, 0))
# Calculate Eye Aspect Ratio
ear = self.eyeDetector.get_eye(frame, facebox)
# Calculate Euler Angles
roll, pitch, yaw = self.poseEstimator.get_Euler_Angles(
np.array([[pose[0][0]], [pose[0][1]], [pose[0][2]]]),
np.array([[pose[1][0]], [pose[1][1]], [pose[1][2]]]))
if ear < self.EYE_AR_THRESH or abs(roll) > self.ROLL_THRESH:
if self.T is None:
self.T = time.time()
elif time.time() - self.T > self.TIME_THRESH:
self.ALARM_ON = True
else:
self.T = None
self.ALARM_ON = False
self.show_text(frame, roll, pitch, yaw, ear, self.ALARM_ON)
return self.ALARM_ON
# Clean up the multiprocessing process.
self.box_process.terminate()
self.box_process.join()
def main():
# get video from webcam or video file
video_src = 0 # 0 means webcam; set file name if video
cam = cv2.VideoCapture(video_src)
_, sample_frame = cam.read() # get 1 sample frame to setup codes
# Initialize MarkDetector
mark_detector = MarkDetector()
# Setup process and queues for multiprocessing
img_queue = Queue()
box_queue = Queue()
img_queue.put(sample_frame)
box_process = Process(target=get_face, args=(mark_detector,
img_queue,
box_queue))
box_process.start()
# Initialize PoseEstimator to solve pose. Use 1 sampe frame to setup
height, width = sample_frame.shape[:2]
pose_estimator = PoseEstimator(img_size=(height, width))
while True:
frame_got, frame = cam.read()
if frame_got is False:
break
#crop image if needed
# frame = cv2.resize(frame, (500, 300))
# flip image if needed in webcam
# if video_src ==0:
# frame = cv2.flip(frame, 2)
# 3 Steps to estimate pose
# 1. detect face
# 2. detect landmarks
# 3. estimate pose
# Feed frame to image queue
img_queue.put(frame)
# Get facebox from box queue
facebox = box_queue.get()
if facebox is not None:
# get face images from the face boxes and images
faces = mark_detector.get_face_for_boxes(frame, facebox)
# # get markers from face images
marks = mark_detector.detect_marks(faces)
marks = mark_detector.fit_markers_in_image(marks, facebox)
#
# # Draw markers if necessary
# MarkDetector.draw_marks(image=frame, marksFace=marks)
#
# # Solve pose by 68 points
## r_vect, t_vect = pose_estimator.solve_pose_by_68_points(marks)
#
# # Solve pose by 6 points
marks = pose_estimator.get_pose_marks(marks)
r_vect, t_vect = pose_estimator.solve_pose(marks)
# # Draw pose boxes on the image
pose_estimator.draw_boxes(frame, r_vect, t_vect)
cv2.imshow("preview", frame)
if cv2.waitKey(10) == 27:
break
# clean up the multiprocessing processes
box_process.terminate()
box_process.join()