def mouth_opening_detection(img):
rects = find_faces(img, face_model)
isOpen = 0
try:
for rect in rects:
shape = detect_marks(img, landmark_model, rect)
cnt_outer = 0
cnt_inner = 0
draw_marks(img, shape[48:])
for i, (p1, p2) in enumerate(outer_points):
if d_outer[i] + 3 < shape[p2][1] - shape[p1][1]:
cnt_outer += 1
for i, (p1, p2) in enumerate(inner_points):
if d_inner[i] + 2 < shape[p2][1] - shape[p1][1]:
cnt_inner += 1
if cnt_outer > 3 and cnt_inner > 2:
isOpen = 1
else:
isOpen = 0
# print('Mouth open')
# cv2.putText(img, 'Mouth open', (30, 30), font,
# 1, (0, 255, 255), 2)
# show the output image with the face detections + facial landmarks
# cv2.imshow("Output", img)
return isOpen
# if cv2.waitKey(1) & 0xFF == ord('q'):
# break
except:
print('Exception Catched')
def initialize_mouth_model(img, ret, face_model, landmark_model):
if not ret:
return
rects = find_faces(img, face_model)
shape = None
for rect in rects:
shape = detect_marks(img, landmark_model, rect)
draw_marks(img, shape)
return img, shape
def find_distance(img):
while (True):
rects = find_faces(img, face_model)
try:
for rect in rects:
shape = detect_marks(img, landmark_model, rect)
draw_marks(img, shape)
# cv2.putText(img, 'Press r to record Mouth distances', (30, 30), font,
# 1, (0, 255, 255), 2)
# cv2.imshow("Output", img)
# if cv2.waitKey(1) & 0xFF == ord('r'):
for i in range(100):
for i, (p1, p2) in enumerate(outer_points):
d_outer[i] += shape[p2][1] - shape[p1][1]
for i, (p1, p2) in enumerate(inner_points):
d_inner[i] += shape[p2][1] - shape[p1][1]
d_outer[:] = [x / 100 for x in d_outer]
d_inner[:] = [x / 100 for x in d_inner]
break
except:
print('Exception Catched')
face_model = get_face_detector()
landmark_model = get_landmark_model()
outer_points = [[49, 59], [50, 58], [51, 57], [52, 56], [53, 55]]
d_outer = [0] * 5
inner_points = [[61, 67], [62, 66], [63, 65]]
d_inner = [0] * 3
font = cv2.FONT_HERSHEY_SIMPLEX
cap = cv2.VideoCapture(0)
while (True):
ret, img = cap.read()
rects = find_faces(img, face_model)
for rect in rects:
shape = detect_marks(img, landmark_model, rect)
draw_marks(img, shape)
cv2.putText(img, 'Press r to record Mouth distances', (30, 30), font,
1, (0, 255, 255), 2)
cv2.imshow("Output", img)
if cv2.waitKey(1) & 0xFF == ord('r'):
for i in range(100):
for i, (p1, p2) in enumerate(outer_points):
d_outer[i] += shape[p2][1] - shape[p1][1]
for i, (p1, p2) in enumerate(inner_points):
d_inner[i] += shape[p2][1] - shape[p1][1]
break
cv2.destroyAllWindows()
d_outer[:] = [x / 100 for x in d_outer]
d_inner[:] = [x / 100 for x in d_inner]
mouth_open = 0
speeking_count = 0
def main(debug=False):
# Load face detect model & face landmark model
face_model = face_detector.get_face_detector()
landmark_model = face_landmarks.get_landmark_model()
cv2.namedWindow('debug_window')
# Create camera instance for capture picture, speed up processing on windows platform by using cv2.CAP_DSHOW.
if platform.system().lower() == 'windows':
camera = cv2.VideoCapture(0 + cv2.CAP_DSHOW)
else:
camera = cv2.VideoCapture(0)
_, sample_img = camera.read() # grab one frame, for getting resolution
height, width, channel = sample_img.shape # resolution & color channel (1 or None = GrayScale, 3 = BGR colors)
# 3D model points.
model_points = np.array([
(0.0, 0.0, 0.0), # Nose tip
(0.0, -330.0, -65.0), # Chin
(-225.0, 170.0, -135.0), # Left eye left corner
(225.0, 170.0, -135.0), # Right eye right corne
(-150.0, -150.0, -125.0), # Left Mouth corner
(150.0, -150.0, -125.0) # Right mouth corner
])
# Camera internals
focal_length = width
center = (width/2, height/2)
camera_matrix = np.array([
[focal_length, 0, center[0]],
[0, focal_length, center[1]],
[0, 0, 1]
], dtype='double')
# Lens distance coefficient: assuming no lens distortion
dist_coefficient = np.zeros((4, 1))
while True:
# start time stamp this frame
time_start = datetime.now()
# 1. capture image (1 frame)
ret, frame = camera.read()
if ret:
# 2. detect faces
faces = face_detector.find_faces(frame, face_model)
if faces:
if debug:
face_detector.draw_faces(frame, faces)
for face in faces:
# 3. detect face landmarks
try:
landmarks = face_landmarks.detect_marks(frame, landmark_model, face)
except cv2.error:
# Skip this round if failed to detect landmarks.
break
if debug:
face_landmarks.draw_marks(frame, landmarks, color=(0, 255, 0))
frame_points = np.array([
landmarks[30], # Nose tip
landmarks[8], # Chin
landmarks[36], # Left eye left corner
landmarks[45], # Right eye right corne
landmarks[48], # Left Mouth corner
landmarks[54] # Right mouth corner
], dtype='double')
# 4. get rotation & translation vector
success, rotation_vector, translation_vector = cv2.solvePnP(
model_points, frame_points, camera_matrix, dist_coefficient, flags=cv2.SOLVEPNP_UPNP
)
if debug:
print(' ' * 13, 'solvePnP:',
success, rotation_vector.tolist(), translation_vector.tolist(), end='\r')
# TODO: calculate iris, mouth and head's roll pitch yaw
# TODO: create socket client, send calculated data to unity
cv2.imshow('debug_window', frame)
# end time stamp of this frame
time_delta = datetime.now() - time_start
# calculate Frame Per Second
fps = 1e6 / time_delta.microseconds
print(' FPS:', fps, end='\r')
if cv2.waitKey(1) & 0xFF in (27, ord('q')):
break
if debug:
cv2.destroyAllWindows()
camera.release()