def mouth_opening_detection(img):
rects = find_faces(img, face_model)
isOpen = 0
try:
for rect in rects:
shape = detect_marks(img, landmark_model, rect)
cnt_outer = 0
cnt_inner = 0
draw_marks(img, shape[48:])
for i, (p1, p2) in enumerate(outer_points):
if d_outer[i] + 3 < shape[p2][1] - shape[p1][1]:
cnt_outer += 1
for i, (p1, p2) in enumerate(inner_points):
if d_inner[i] + 2 < shape[p2][1] - shape[p1][1]:
cnt_inner += 1
if cnt_outer > 3 and cnt_inner > 2:
isOpen = 1
else:
isOpen = 0
# print('Mouth open')
# cv2.putText(img, 'Mouth open', (30, 30), font,
# 1, (0, 255, 255), 2)
# show the output image with the face detections + facial landmarks
# cv2.imshow("Output", img)
return isOpen
# if cv2.waitKey(1) & 0xFF == ord('q'):
# break
except:
print('Exception Catched')
def initialize_mouth_model(img, ret, face_model, landmark_model):
if not ret:
return
rects = find_faces(img, face_model)
shape = None
for rect in rects:
shape = detect_marks(img, landmark_model, rect)
draw_marks(img, shape)
return img, shape
def run_mouth_open(img, ret, face_model, landmark_model, font, outer_points,
d_outer, inner_points, d_inner):
if not ret:
return
rects = find_faces(img, face_model)
for rect in rects:
shape = detect_marks(img, landmark_model, rect)
cnt_outer = 0
cnt_inner = 0
for i, (p1, p2) in enumerate(outer_points):
if d_outer[i] + 3 < shape[p2][1] - shape[p1][1]:
cnt_outer += 1
for i, (p1, p2) in enumerate(inner_points):
if d_inner[i] + 2 < shape[p2][1] - shape[p1][1]:
cnt_inner += 1
if cnt_outer > 3 and cnt_inner > 2:
return True
return False
def find_distance(img):
while (True):
rects = find_faces(img, face_model)
try:
for rect in rects:
shape = detect_marks(img, landmark_model, rect)
draw_marks(img, shape)
# cv2.putText(img, 'Press r to record Mouth distances', (30, 30), font,
# 1, (0, 255, 255), 2)
# cv2.imshow("Output", img)
# if cv2.waitKey(1) & 0xFF == ord('r'):
for i in range(100):
for i, (p1, p2) in enumerate(outer_points):
d_outer[i] += shape[p2][1] - shape[p1][1]
for i, (p1, p2) in enumerate(inner_points):
d_inner[i] += shape[p2][1] - shape[p1][1]
d_outer[:] = [x / 100 for x in d_outer]
d_inner[:] = [x / 100 for x in d_inner]
break
except:
print('Exception Catched')
(-150.0, -150.0, -125.0), # Left Mouth corner
(150.0, -150.0, -125.0) # Right mouth corner
])
# Camera internals
focal_length = size[1]
center = (size[1] / 2, size[0] / 2)
camera_matrix = np.array(
[[focal_length, 0, center[0]], [0, focal_length, center[1]], [0, 0, 1]],
dtype="double")
while True:
ret, img = cap.read()
if ret == True:
faces = find_faces(img, face_model)
for face in faces:
marks = detect_marks(img, landmark_model, face)
# mark_detector.draw_marks(img, marks, color=(0, 255, 0))
image_points = np.array(
[
marks[30], # Nose tip
marks[8], # Chin
marks[36], # Left eye left corner
marks[45], # Right eye right corne
marks[48], # Left Mouth corner
marks[54] # Right mouth corner
],
dtype="double")
dist_coeffs = np.zeros((4, 1)) # Assuming no lens distortion
(success, rotation_vector,
translation_vector) = cv2.solvePnP(model_points,
image_points,
from face_landmarks import get_landmark_model, detect_marks, draw_marks
face_model = get_face_detector()
landmark_model = get_landmark_model()
outer_points = [[49, 59], [50, 58], [51, 57], [52, 56], [53, 55]]
d_outer = [0] * 5
inner_points = [[61, 67], [62, 66], [63, 65]]
d_inner = [0] * 3
font = cv2.FONT_HERSHEY_SIMPLEX
cap = cv2.VideoCapture(0)
while (True):
ret, img = cap.read()
rects = find_faces(img, face_model)
for rect in rects:
shape = detect_marks(img, landmark_model, rect)
draw_marks(img, shape)
cv2.putText(img, 'Press r to record Mouth distances', (30, 30), font,
1, (0, 255, 255), 2)
cv2.imshow("Output", img)
if cv2.waitKey(1) & 0xFF == ord('r'):
for i in range(100):
for i, (p1, p2) in enumerate(outer_points):
d_outer[i] += shape[p2][1] - shape[p1][1]
for i, (p1, p2) in enumerate(inner_points):
d_inner[i] += shape[p2][1] - shape[p1][1]
break
cv2.destroyAllWindows()
d_outer[:] = [x / 100 for x in d_outer]
d_inner[:] = [x / 100 for x in d_inner]
mouth_open = 0
def main(debug=False):
# Load face detect model & face landmark model
face_model = face_detector.get_face_detector()
landmark_model = face_landmarks.get_landmark_model()
cv2.namedWindow('debug_window')
# Create camera instance for capture picture, speed up processing on windows platform by using cv2.CAP_DSHOW.
if platform.system().lower() == 'windows':
camera = cv2.VideoCapture(0 + cv2.CAP_DSHOW)
else:
camera = cv2.VideoCapture(0)
_, sample_img = camera.read() # grab one frame, for getting resolution
height, width, channel = sample_img.shape # resolution & color channel (1 or None = GrayScale, 3 = BGR colors)
# 3D model points.
model_points = np.array([
(0.0, 0.0, 0.0), # Nose tip
(0.0, -330.0, -65.0), # Chin
(-225.0, 170.0, -135.0), # Left eye left corner
(225.0, 170.0, -135.0), # Right eye right corne
(-150.0, -150.0, -125.0), # Left Mouth corner
(150.0, -150.0, -125.0) # Right mouth corner
])
# Camera internals
focal_length = width
center = (width/2, height/2)
camera_matrix = np.array([
[focal_length, 0, center[0]],
[0, focal_length, center[1]],
[0, 0, 1]
], dtype='double')
# Lens distance coefficient: assuming no lens distortion
dist_coefficient = np.zeros((4, 1))
while True:
# start time stamp this frame
time_start = datetime.now()
# 1. capture image (1 frame)
ret, frame = camera.read()
if ret:
# 2. detect faces
faces = face_detector.find_faces(frame, face_model)
if faces:
if debug:
face_detector.draw_faces(frame, faces)
for face in faces:
# 3. detect face landmarks
try:
landmarks = face_landmarks.detect_marks(frame, landmark_model, face)
except cv2.error:
# Skip this round if failed to detect landmarks.
break
if debug:
face_landmarks.draw_marks(frame, landmarks, color=(0, 255, 0))
frame_points = np.array([
landmarks[30], # Nose tip
landmarks[8], # Chin
landmarks[36], # Left eye left corner
landmarks[45], # Right eye right corne
landmarks[48], # Left Mouth corner
landmarks[54] # Right mouth corner
], dtype='double')
# 4. get rotation & translation vector
success, rotation_vector, translation_vector = cv2.solvePnP(
model_points, frame_points, camera_matrix, dist_coefficient, flags=cv2.SOLVEPNP_UPNP
)
if debug:
print(' ' * 13, 'solvePnP:',
success, rotation_vector.tolist(), translation_vector.tolist(), end='\r')
# TODO: calculate iris, mouth and head's roll pitch yaw
# TODO: create socket client, send calculated data to unity
cv2.imshow('debug_window', frame)
# end time stamp of this frame
time_delta = datetime.now() - time_start
# calculate Frame Per Second
fps = 1e6 / time_delta.microseconds
print(' FPS:', fps, end='\r')
if cv2.waitKey(1) & 0xFF in (27, ord('q')):
break
if debug:
cv2.destroyAllWindows()
camera.release()
def head_pose(img):
faces = find_faces(img, face_model)
size = img.shape
focal_length = size[1]
center = (size[1] / 2, size[0] / 2)
camera_matrix = np.array([[focal_length, 0, center[0]],
[0, focal_length, center[1]], [0, 0, 1]],
dtype="double")
direction = 0
for face in faces:
marks = detect_marks(img, landmark_model, face)
# mark_detector.draw_marks(img, marks, color=(0, 255, 0))
image_points = np.array(
[
marks[30], # Nose tip
marks[8], # Chin
marks[36], # Left eye left corner
marks[45], # Right eye right corne
marks[48], # Left Mouth corner
marks[54] # Right mouth corner
],
dtype="double")
dist_coeffs = np.zeros((4, 1)) # Assuming no lens distortion
(success, rotation_vector,
translation_vector) = cv2.solvePnP(model_points,
image_points,
camera_matrix,
dist_coeffs,
flags=cv2.SOLVEPNP_UPNP)
# Project a 3D point (0, 0, 1000.0) onto the image plane.
# We use this to draw a line sticking out of the nose
(nose_end_point2D,
jacobian) = cv2.projectPoints(np.array([(0.0, 0.0, 1000.0)]),
rotation_vector, translation_vector,
camera_matrix, dist_coeffs)
for p in image_points:
cv2.circle(img, (int(p[0]), int(p[1])), 3, (0, 0, 255), -1)
p1 = (int(image_points[0][0]), int(image_points[0][1]))
p2 = (int(nose_end_point2D[0][0][0]), int(nose_end_point2D[0][0][1]))
x1, x2 = head_pose_points(img, rotation_vector, translation_vector,
camera_matrix)
cv2.line(img, p1, p2, (0, 255, 255), 2)
cv2.line(img, tuple(x1), tuple(x2), (255, 255, 0), 2)
# for (x, y) in marks:
# cv2.circle(img, (x, y), 4, (255, 255, 0), -1)
# cv2.putText(img, str(p1), p1, font, 1, (0, 255, 255), 1)
try:
m = (p2[1] - p1[1]) / (p2[0] - p1[0])
ang1 = int(math.degrees(math.atan(m)))
except:
ang1 = 90
try:
m = (x2[1] - x1[1]) / (x2[0] - x1[0])
ang2 = int(math.degrees(math.atan(-1 / m)))
except:
ang2 = 90
# print('div by zero error')
if ang1 >= 48:
direction = 1
# print('Head down')
# cv2.putText(img, 'Head down', (30, 30), font, 2, (255, 255, 128), 3)
elif ang1 <= -48:
direction = 2
# print('Head up')
# cv2.putText(img, 'Head up', (30, 30), font, 2, (255, 255, 128), 3)
if ang2 >= 48:
direction = 3
# print('Head right')
# cv2.putText(img, 'Head right', (90, 30), font, 2, (255, 255, 128), 3)
elif ang2 <= -48:
direction = 4
# print('Head left')
# cv2.putText(img, 'Head left', (90, 30), font, 2, (255, 255, 128), 3)
return direction
(0.0, -330.0, -65.0), # Chin
(-225.0, 170.0, -135.0), # Left eye left corner
(225.0, 170.0, -135.0), # Right eye right corne
(-150.0, -150.0, -125.0), # Left Mouth corner
(150.0, -150.0, -125.0) # Right mouth corner
])
# Camera internals
focal_length = size[1]
center = (size[1] / 2, size[0] / 2)
camera_matrix = np.array(
[[focal_length, 0, center[0]], [0, focal_length, center[1]], [0, 0, 1]],
dtype="double")
faces = find_faces(imagereal, face_model)
for face in faces:
marks = detect_marks(imagereal, landmark_model, face)
for i in range(100):
for i, (p1, p2) in enumerate(outer_points):
d_outer[i] += marks[p2][1] - marks[p1][1]
for i, (p1, p2) in enumerate(inner_points):
d_inner[i] += marks[p2][1] - marks[p1][1]
d_outer[:] = [x / 100 for x in d_outer]
d_inner[:] = [x / 100 for x in d_inner]
tc = 0 #facespoofcount
mouth_open = 0
speeking_count = 0
speech_checking = 0
BUFFER_SIZE = 2048
CHANNELS = 1
FORMAT = pyaudio.paFloat32
METHOD = "default"
def get_head_position(File_name):
face_model = get_face_detector()
landmark_model = get_landmark_model()
cap = cv2.VideoCapture(File_name)
ret, img = cap.read()
size = img.shape
font = cv2.FONT_HERSHEY_SIMPLEX
# 3D model points.
model_points = np.array([
(0.0, 0.0, 0.0), # Nose tip
(0.0, -330.0, -65.0), # Chin
(-225.0, 170.0, -135.0), # Left eye left corner
(225.0, 170.0, -135.0), # Right eye right corne
(-150.0, -150.0, -125.0), # Left Mouth corner
(150.0, -150.0, -125.0) # Right mouth corner
])
# Camera internals
focal_length = size[1]
center = (size[1] / 2, size[0] / 2)
camera_matrix = np.array([[focal_length, 0, center[0]],
[0, focal_length, center[1]], [0, 0, 1]],
dtype="double")
head_directions = [0] * 4 # 아래, 위, 오른쪽, 왼쪽
while True:
ret, img = cap.read()
if ret == True:
faces = find_faces(img, face_model)
for face in faces:
marks = detect_marks(img, landmark_model, face)
# mark_detector.draw_marks(img, marks, color=(0, 255, 0))
image_points = np.array(
[
marks[30], # Nose tip
marks[8], # Chin
marks[36], # Left eye left corner
marks[45], # Right eye right corne
marks[48], # Left Mouth corner
marks[54] # Right mouth corner
],
dtype="double")
dist_coeffs = np.zeros((4, 1)) # Assuming no lens distortion
(success, rotation_vector,
translation_vector) = cv2.solvePnP(model_points,
image_points,
camera_matrix,
dist_coeffs,
flags=cv2.SOLVEPNP_UPNP)
# Project a 3D point (0, 0, 1000.0) onto the image plane.
# We use this to draw a line sticking out of the nose
(nose_end_point2D,
jacobian) = cv2.projectPoints(np.array([
(0.0, 0.0, 1000.0)
]), rotation_vector, translation_vector, camera_matrix,
dist_coeffs)
for p in image_points:
cv2.circle(img, (int(p[0]), int(p[1])), 3, (0, 0, 255), -1)
p1 = (int(image_points[0][0]), int(image_points[0][1]))
p2 = (int(nose_end_point2D[0][0][0]),
int(nose_end_point2D[0][0][1]))
x1, x2 = head_pose_points(img, rotation_vector,
translation_vector, camera_matrix)
cv2.line(img, p1, p2, (0, 255, 255), 2)
cv2.line(img, tuple(x1), tuple(x2), (255, 255, 0), 2)
# for (x, y) in marks:
# cv2.circle(img, (x, y), 4, (255, 255, 0), -1)
# cv2.putText(img, str(p1), p1, font, 1, (0, 255, 255), 1)
try:
m = (p2[1] - p1[1]) / (p2[0] - p1[0])
ang1 = int(math.degrees(math.atan(m)))
except:
ang1 = 90
try:
m = (x2[1] - x1[1]) / (x2[0] - x1[0])
ang2 = int(math.degrees(math.atan(-1 / m)))
except:
ang2 = 90
# print('div by zero error')
if ang1 >= 48:
head_directions[0] += 1
# print('Head down')
# cv2.putText(img, 'Head down', (30, 30), font, 2, (255, 255, 128), 3)
elif ang1 <= -48:
head_directions[1] += 1
# print('Head up')
# cv2.putText(img, 'Head up', (30, 30), font, 2, (255, 255, 128), 3)
if ang2 >= 48:
head_directions[2] += 1
# print('Head right')
# cv2.putText(img, 'Head right', (90, 30), font, 2, (255, 255, 128), 3)
elif ang2 <= -48:
head_directions[3] += 1
# print('Head left')
# cv2.putText(img, 'Head left', (90, 30), font, 2, (255, 255, 128), 3)
# cv2.putText(img, str(ang1), tuple(p1), font, 2, (128, 255, 255), 3)
# cv2.putText(img, str(ang2), tuple(x1), font, 2, (255, 255, 128), 3)
# cv2.imshow('img', img)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
break
cv2.destroyAllWindows()
cap.release()
return head_directions
def get_frame(imgData):
nparr = np.frombuffer(base64.b64decode(imgData), np.uint8)
image = cv2.imdecode(nparr, cv2.COLOR_BGR2GRAY)
ret = True
size = image.shape
font = cv2.FONT_HERSHEY_SIMPLEX
model_points = np.array([
(0.0, 0.0, 0.0), # Nose tip
(0.0, -330.0, -65.0), # Chin
(-225.0, 170.0, -135.0), # Left eye left corner
(225.0, 170.0, -135.0), # Right eye right corne
(-150.0, -150.0, -125.0), # Left Mouth corner
(150.0, -150.0, -125.0) # Right mouth corner
])
focal_length = size[1]
center = (size[1] / 2, size[0] / 2)
camera_matrix = np.array([[focal_length, 0, center[0]],
[0, focal_length, center[1]], [0, 0, 1]],
dtype="double")
img = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (320, 320))
img = img.astype(np.float32)
img = np.expand_dims(img, 0)
img = img / 255
class_names = [c.strip() for c in open("models/classes.TXT").readlines()]
boxes, scores, classes, nums = yolo(img)
count = 0
mob_status = ""
person_status = ""
for i in range(nums[0]):
if int(classes[0][i] == 0):
count += 1
if int(classes[0][i] == 67):
print('Mobile Phone detected')
mob_status = 1
else:
print('Not Mobile Phone detected')
mob_status = 0
print(mob_status)
if count == 0:
print('No person detected')
person_status = 1
elif count > 1:
print('More than one person detected')
person_status = 2
else:
print('Normal')
person_status = 0
image = draw_outputs(image, (boxes, scores, classes, nums), class_names)
user_move1 = ""
user_move2 = ""
if ret == True:
faces = find_faces(image, face_model)
for face in faces:
marks = detect_marks(image, landmark_model, face)
image_points = np.array(
[
marks[30], # Nose tip
marks[8], # Chin
marks[36], # Left eye left corner
marks[45], # Right eye right corne
marks[48], # Left Mouth corner
marks[54] # Right mouth corner
],
dtype="double")
dist_coeffs = np.zeros((4, 1)) # Assuming no lens distortion
(success, rotation_vector,
translation_vector) = cv2.solvePnP(model_points,
image_points,
camera_matrix,
dist_coeffs,
flags=cv2.SOLVEPNP_UPNP)
(nose_end_point2D,
jacobian) = cv2.projectPoints(np.array([(0.0, 0.0, 1000.0)]),
rotation_vector, translation_vector,
camera_matrix, dist_coeffs)
for p in image_points:
cv2.circle(image, (int(p[0]), int(p[1])), 3, (0, 0, 255), -1)
p1 = (int(image_points[0][0]), int(image_points[0][1]))
p2 = (int(nose_end_point2D[0][0][0]),
int(nose_end_point2D[0][0][1]))
x1, x2 = head_pose_points(image, rotation_vector,
translation_vector, camera_matrix)
try:
m = (p2[1] - p1[1]) / (p2[0] - p1[0])
ang1 = int(math.degrees(math.atan(m)))
except:
ang1 = 90
try:
m = (x2[1] - x1[1]) / (x2[0] - x1[0])
ang2 = int(math.degrees(math.atan(-1 / m)))
except:
ang2 = 90
if ang1 >= 48:
user_move1 = 2
print('Head down')
elif ang1 <= -48:
user_move1 = 1
print('Head up')
else:
user_move1 = 0
if ang2 >= 48:
print('Head right')
user_move2 = 4
elif ang2 <= -48:
print('Head left')
user_move2 = 3
else:
user_move2 = 0
ret, jpeg = cv2.imencode('.jpg', image)
jpg_as_text = base64.b64encode(jpeg)
gaze.refresh(image)
frame = gaze.annotated_frame()
eye_movements = ""
if gaze.is_blinking():
eye_movements = 1
print("Blinking")
elif gaze.is_right():
eye_movements = 4
print("Looking right")
elif gaze.is_left():
eye_movements = 3
print("Looking left")
elif gaze.is_center():
eye_movements = 2
print("Looking center")
else:
eye_movements = 0
print("Not found!")
print(eye_movements)
proctorDict = dict()
proctorDict['jpg_as_text'] = jpg_as_text
proctorDict['mob_status'] = mob_status
proctorDict['person_status'] = person_status
proctorDict['user_move1'] = user_move1
proctorDict['user_move2'] = user_move2
proctorDict['eye_movements'] = eye_movements
return proctorDict
def nothing(x):
pass
cv2.createTrackbar('threshold', 'image', 75, 255, nothing)
j = 0
jc = 0
while True:
ret, img2 = cap.read()
if ret == True:
faces = find_faces(img2, face_model)
for face in faces:
marks = detect_marks(img2, landmark_model, face)
image_points = np.array(
[
marks[30], # Nose tip
marks[8], # Chin
marks[36], # Left eye left corner
marks[45], # Right eye right corne
marks[48], # Left Mouth corner
marks[54] # Right mouth corner
],
dtype="double")
dist_coeffs = np.zeros((4, 1))
(success, rotation_vector,
translation_vector) = cv2.solvePnP(model_points,
image_points,
camera_matrix,
