def test_facial_landmarks_detection():
model = Facial_Landmarks_Detection("models/intel/landmarks-regression-retail-0009/FP16-INT8/landmarks-regression-retail-0009.xml")
model.load_model()
image = cv2.imread("media/face1.jpg")
height, width, _ = image.shape
eye_landmarks = model.predict(image)
landmarks_pos = [(int(l[0] * width), int(l[1] * height)) for l in eye_landmarks]
left_eye_coord = landmarks_pos[0]
right_eye_coord = landmarks_pos[1]
x_offset = 50
y_offset = 25
left_eye = image[left_eye_coord[1] - y_offset:left_eye_coord[1] + y_offset, left_eye_coord[0] - x_offset:left_eye_coord[0] + x_offset]
right_eye = image[right_eye_coord[1] - y_offset:right_eye_coord[1] + y_offset, right_eye_coord[0] - x_offset:right_eye_coord[0] + x_offset]
cv2.imwrite("media/left_eye.jpg", left_eye)
cv2.imwrite("media/right_eye.jpg", right_eye)
radius = 5
color = (255, 0, 0)
thickness = 5
cv2.circle(image, left_eye_coord, radius, color, thickness)
cv2.circle(image, right_eye_coord, radius, color, thickness)
cv2.imshow("Result", image)
cv2.waitKey()
def init_models(device="CPU"):
# Using global variables, not defining new variables
global face_detection
global facial_landmarks_detection
global head_pose_estimation
global gaze_estimation
log.info("Loading Face Detection model...")
face_detection = Face_Detection(path_face_detection, device)
face_detection.load_model()
log.info("DONE\n")
log.info("Loading Face Landmarks Detection model...")
facial_landmarks_detection = Facial_Landmarks_Detection(
path_facial_landmarks_detection, device)
facial_landmarks_detection.load_model()
log.info("DONE\n")
log.info("Loading Head Pose Estimation model...")
head_pose_estimation = Head_Pose_Estimation(path_head_pose_estimation,
device)
head_pose_estimation.load_model()
log.info("DONE\n")
log.info("Loading Gaze Estimation model...")
gaze_estimation = Gaze_Estimation(path_gaze_estimation, device)
gaze_estimation.load_model()
log.info("DONE\n")
def init_models(device="CPU"):
# Using global variables, not defining new variables
global face_detection
global facial_landmarks_detection
global head_pose_estimation
global gaze_estimation
start = time.time()
face_detection = Face_Detection(path_face_detection, device)
face_detection.load_model()
fd_load_time = (time.time() - start)
start = time.time()
facial_landmarks_detection = Facial_Landmarks_Detection(
path_facial_landmarks_detection, device)
facial_landmarks_detection.load_model()
fld_load_time = (time.time() - start)
start = time.time()
head_pose_estimation = Head_Pose_Estimation(path_head_pose_estimation,
device)
head_pose_estimation.load_model()
hpe_load_time = (time.time() - start)
start = time.time()
gaze_estimation = Gaze_Estimation(path_gaze_estimation, device)
gaze_estimation.load_model()
ge_load_time = (time.time() - start)
return (fd_load_time, fld_load_time, hpe_load_time, ge_load_time)
def __init__(self, args):
# load the objects corresponding to the models
self.face_detection = Face_Detection(args.face_detection_model,
args.device, args.extensions,
args.perf_counts)
self.gaze_estimation = Gaze_Estimation(args.gaze_estimation_model,
args.device, args.extensions,
args.perf_counts)
self.head_pose_estimation = Head_Pose_Estimation(
args.head_pose_estimation_model, args.device, args.extensions,
args.perf_counts)
self.facial_landmarks_detection = Facial_Landmarks_Detection(
args.facial_landmarks_detection_model, args.device,
args.extensions, args.perf_counts)
start_models_load_time = time.time()
self.face_detection.load_model()
self.gaze_estimation.load_model()
self.head_pose_estimation.load_model()
self.facial_landmarks_detection.load_model()
logger = logging.getLogger()
input_T = args.input_type
input_F = args.input_file
if input_T.lower() == 'cam':
# open the video feed
self.feed = InputFeeder(args.input_type, args.input_file)
self.feed.load_data()
else:
if not os.path.isfile(input_F):
logger.error('Unable to find specified video file')
exit(1)
file_extension = input_F.split(".")[-1]
if (file_extension in ['jpg', 'jpeg', 'bmp']):
self.feed = InputFeeder(args.input_type, args.input_file)
self.feed.load_data()
elif (file_extension in ['avi', 'mp4']):
self.feed = InputFeeder(args.input_type, args.input_file)
self.feed.load_data()
else:
logger.error(
"Unsupported file Extension. Allowed ['jpg', 'jpeg', 'bmp', 'avi', 'mp4']"
)
exit(1)
print("Models total loading time :",
time.time() - start_models_load_time)
# init mouse controller
self.mouse_controller = MouseController('low', 'fast')
def main():
args = build_argparser().parse_args()
frame_num = 0
inference_time = 0
counter = 0
# Initialize the Inference Engine
fd = FaceDetection()
fld = Facial_Landmarks_Detection()
ge = Gaze_Estimation()
hp = Head_Pose_Estimation()
# Load Models
fd.load_model(args.face_detection_model, args.device, args.cpu_extension)
fld.load_model(args.facial_landmark_model, args.device, args.cpu_extension)
ge.load_model(args.gaze_estimation_model, args.device, args.cpu_extension)
hp.load_model(args.head_pose_model, args.device, args.cpu_extension)
# Mouse Controller precision and speed
mc = MouseController('medium', 'fast')
# feed input from an image, webcam, or video to model
if args.input == "cam":
feed = InputFeeder("cam")
else:
assert os.path.isfile(args.input), "Specified input file doesn't exist"
feed = InputFeeder("video", args.input)
feed.load_data()
frame_count = 0
for frame in feed.next_batch():
frame_count += 1
inf_start = time.time()
if frame is not None:
try:
key = cv2.waitKey(60)
det_time = time.time() - inf_start
# make predictions
detected_face, face_coords = fd.predict(
frame.copy(), args.prob_threshold)
hp_output = hp.predict(detected_face.copy())
left_eye, right_eye, eye_coords = fld.predict(
detected_face.copy())
new_mouse_coord, gaze_vector = ge.predict(
left_eye, right_eye, hp_output)
stop_inference = time.time()
inference_time = inference_time + stop_inference - inf_start
counter = counter + 1
# Visualization
preview = args.visualization
if preview:
preview_frame = frame.copy()
face_frame = detected_face.copy()
draw_face_bbox(preview_frame, face_coords)
display_hp(preview_frame, hp_output, face_coords)
draw_landmarks(face_frame, eye_coords)
draw_gaze(face_frame, gaze_vector, left_eye.copy(),
right_eye.copy(), eye_coords)
if preview:
img = np.hstack((cv2.resize(preview_frame, (500, 500)),
cv2.resize(face_frame, (500, 500))))
else:
img = cv2.resize(frame, (500, 500))
cv2.imshow('Visualization', img)
# set speed
if frame_count % 5 == 0:
mc.move(new_mouse_coord[0], new_mouse_coord[1])
# INFO
log.info("NUMBER OF FRAMES: {} ".format(frame_num))
log.info("INFERENCE TIME: {}ms".format(det_time * 1000))
frame_num += 1
if key == 27:
break
except:
print(
'Not supported image or video file format. Please send in a supported video format.'
)
exit()
feed.close()
def infer_on_video(args):
draw_flag = args.b
device = args.device
input_path = args.input_path
input_type = args.input_type
output_path = args.output_path
precision = args.accuracy
locations = {}
locations[FACE_DETECTION_MODEL] = os.path.join(
MODEL_PATH, FACE_DETECTION_MODEL, 'INT1',
FACE_DETECTION_MODEL + ".xml")
if precision is not None:
log.info(
"The face-detection-adas-binary-0001 always use INT1 precision")
for model_name in [
FACIAL_LANDMARKS_DETECTION_MODEL, HEAD_POSE_ESTIMATION_MODEL,
GAZE_ESTIMATION_MODEL
]:
locations[model_name] = find_exist_model_file(precision, model_name)
# Initilize feeder
feed = InputFeeder(input_type=input_type, input_file=input_path)
feed.load_data()
# Grab the shape of the input
input_width = feed.getWidth()
input_height = feed.getHeight()
# Create a video writer for the output video
# out = cv2.VideoWriter('../out.mp4', CODEC, 30, (input_width,input_height))
mouse_controller = MouseController(MOUSE_PRECISION, MOUSE_SPEED)
start_model_load_time = time.time()
# model initialization
face_detection = Face_Detection(locations[FACE_DETECTION_MODEL],
device,
extensions=CPU_EXTENSION)
facial_landmarks_detection = Facial_Landmarks_Detection(
locations[FACIAL_LANDMARKS_DETECTION_MODEL],
device,
extensions=CPU_EXTENSION)
head_pose_estimation = Head_Pose_Estimation(
locations[HEAD_POSE_ESTIMATION_MODEL],
device,
extensions=CPU_EXTENSION)
gaze_estimation = Gaze_Estimation(locations[GAZE_ESTIMATION_MODEL],
device,
extensions=CPU_EXTENSION)
total_model_load_time = time.time() - start_model_load_time
counter = 0
start_inference_time = time.time()
# Process frames until the video ends, or process is exited
for ret, batch in feed.next_batch(BATCH_SIZE):
if not ret:
break
counter += 1
gaze_lines = []
out_frame = batch.copy()
key = cv2.waitKey(60)
# Face detection
face_detection_output = face_detection.predict(batch)
# face_detection_output = [ image_id, label, conf, xmin, ymin, xmax, ymax ]
face_xmin = abs(int(face_detection_output[3] * input_width))
face_ymin = abs(int(face_detection_output[4] * input_height))
face_xmax = abs(int(face_detection_output[5] * input_width))
face_ymax = abs(int(face_detection_output[6] * input_height))
if (face_ymax - face_ymin) <= 0 or (face_xmax - face_xmin) <= 0:
continue
# Crop the face image
face = batch[face_ymin:face_ymax, face_xmin:face_xmax]
if draw_flag == True:
cv2.rectangle(out_frame, (face_xmin, face_ymin),
(face_xmax, face_ymax), (255, 255, 0), 2)
# Find facial landmarks (to find eyes)
eyes = facial_landmarks_detection.predict(face)
# Estimate head orientation (yaw=Y, pitch=X, role=Z)
yaw, pitch, roll = head_pose_estimation.predict(face)
eye_images = []
for eye in eyes:
face_height, face_width, _ = face.shape
eye_xmin = int(eye[_X] * face_width - EYE_RADIUS)
eye_ymin = int(eye[_Y] * face_height - EYE_RADIUS)
eye_xmax = int(eye[_X] * face_width + EYE_RADIUS)
eye_ymax = int(eye[_Y] * face_height + EYE_RADIUS)
if (eye_ymax - eye_ymin) <= 0 or (eye_xmax - eye_xmin) <= 0:
continue
# crop and resize
eye_images.append(face[eye_ymin:eye_ymax,
eye_xmin:eye_xmax].copy())
# Draw eye boundary boxes
if draw_flag == True:
cv2.rectangle(out_frame,
(eye_xmin + face_xmin, eye_ymin + face_ymin),
(eye_xmax + face_xmin, eye_ymax + face_ymin),
(0, 255, 0), 2)
# gaze estimation
gaze_vec_norm = gaze_estimation.predict(eye_images, [yaw, pitch, 0])
cos = math.cos(math.radians(roll))
sin = math.sin(math.radians(roll))
tmpx = gaze_vec_norm[0] * cos + gaze_vec_norm[1] * sin
tmpy = -gaze_vec_norm[0] * sin + gaze_vec_norm[1] * cos
gaze_vec_norm = [tmpx, tmpy]
# Store gaze line coordinations
for eye in eyes:
eye[_X] = int(eye[_X] * face_width)
eye[_Y] = int(eye[_Y] * face_height)
gaze_lines.append(
get_gaze_line(eye, face_xmin, face_ymin, gaze_vec_norm))
if draw_flag:
# Drawing gaze lines
for gaze_line in gaze_lines:
start_point = (gaze_line[0][_X], gaze_line[0][_Y])
end_point = (gaze_line[1][_X], gaze_line[1][_Y])
draw_gaze_line(out_frame, start_point, end_point)
# start point of middle gaze line
start_point = ((gaze_lines[0][0][_X] + gaze_lines[1][0][_X]) / 2,
(gaze_lines[0][0][_Y] + gaze_lines[1][0][_Y]) / 2)
# end point of middle gaze line
end_point = ((gaze_lines[0][1][_X] + gaze_lines[1][1][_X]) / 2,
(gaze_lines[0][1][_Y] + gaze_lines[1][1][_Y]) / 2)
gaze_mid_line = [start_point, end_point]
mouse_point = get_mouse_point(gaze_mid_line, input_width, input_height)
log.debug("mouse_point[_X], mouse_point[_Y]: %s, %s", mouse_point[_X],
mouse_point[_Y])
# cv2.circle(out_frame, mouse_point, 10, (255, 255, 255), -1)
mouse_controller.move(mouse_point[_X], mouse_point[_Y])
# write out_frames with batch size
for _ in range(BATCH_SIZE):
cv2.imshow("video", out_frame)
# out.write(out_frame)
if key == 27:
break
total_inference_time = time.time() - start_inference_time
total_inference_time = round(total_inference_time, 1)
fps = counter / total_inference_time
with open(os.path.join(output_path, 'stats.txt'), 'w') as f:
f.write(str(total_inference_time) + '\n')
f.write(str(fps) + '\n')
f.write(str(total_model_load_time) + '\n')
# Release the out writer, capture, and destroy any OpenCV windows
log.info("Input stream ended...")
cv2.destroyAllWindows()
# out.release()
feed.close()
class Computer_Pointer_Controller:
def __init__(self, args):
# load the objects corresponding to the models
self.face_detection = Face_Detection(args.face_detection_model,
args.device, args.extensions,
args.perf_counts)
self.gaze_estimation = Gaze_Estimation(args.gaze_estimation_model,
args.device, args.extensions,
args.perf_counts)
self.head_pose_estimation = Head_Pose_Estimation(
args.head_pose_estimation_model, args.device, args.extensions,
args.perf_counts)
self.facial_landmarks_detection = Facial_Landmarks_Detection(
args.facial_landmarks_detection_model, args.device,
args.extensions, args.perf_counts)
start_models_load_time = time.time()
self.face_detection.load_model()
self.gaze_estimation.load_model()
self.head_pose_estimation.load_model()
self.facial_landmarks_detection.load_model()
logger = logging.getLogger()
input_T = args.input_type
input_F = args.input_file
if input_T.lower() == 'cam':
# open the video feed
self.feed = InputFeeder(args.input_type, args.input_file)
self.feed.load_data()
else:
if not os.path.isfile(input_F):
logger.error('Unable to find specified video file')
exit(1)
file_extension = input_F.split(".")[-1]
if (file_extension in ['jpg', 'jpeg', 'bmp']):
self.feed = InputFeeder(args.input_type, args.input_file)
self.feed.load_data()
elif (file_extension in ['avi', 'mp4']):
self.feed = InputFeeder(args.input_type, args.input_file)
self.feed.load_data()
else:
logger.error(
"Unsupported file Extension. Allowed ['jpg', 'jpeg', 'bmp', 'avi', 'mp4']"
)
exit(1)
print("Models total loading time :",
time.time() - start_models_load_time)
# init mouse controller
self.mouse_controller = MouseController('low', 'fast')
def run(self):
inferences_times = []
face_detections_times = []
for batch in self.feed.next_batch():
if batch is None:
break
# as we want the webcam to act as a mirror, flip the frame
batch = cv2.flip(batch, 1)
inference_time = time.time()
face = self.face_detection.predict(batch)
if face is None:
logger.error('Unable to detect the face.')
continue
else:
face_detections_times.append(time.time() - inference_time)
left_eye_image, right_eye_image = self.facial_landmarks_detection.predict(
face)
if left_eye_image is None or right_eye_image is None:
continue
head_pose_angles = self.head_pose_estimation.predict(face)
if head_pose_angles is None:
continue
vector = self.gaze_estimation.predict(left_eye_image,
right_eye_image,
head_pose_angles)
inferences_times.append(time.time() - inference_time)
if args.show_face == "True":
cv2.imshow("Detected face", face)
cv2.waitKey(1)
self.mouse_controller.move(vector[0], vector[1])
self.feed.close()
cv2.destroyAllWindows()
print("Average face detection inference time:",
sum(face_detections_times) / len(face_detections_times))
print("Average total inferences time:",
sum(inferences_times) / len(inferences_times))
def infer_on_stream(args):
network_fd = Face_Detection(args.face_detection_model, args.device)
network_hp = Head_Pose_Estimation(args.head_pose_model, args.device)
network_fl = Facial_Landmarks_Detection(args.facial_landmarks_model,
args.device)
network_ge = Gaze_Estimation(args.gaze_estimation_model, args.device)
mouse_cont = MouseController(args.mouse_precision, args.mouse_speed)
starting_loading = time.time()
network_fd.load_model()
network_hp.load_model()
network_fl.load_model()
network_ge.load_model()
duration_loading = time.time() - starting_loading
input_type = handle_input(args.input)
feed = InputFeeder(input_type=input_type, input_file=args.input)
feed.load_data()
starting_inference = time.time()
for flag, frame in feed.next_batch():
if not flag:
break
key_pressed = cv2.waitKey(60)
out_frame, face, face_coords = network_fd.predict(
frame, args.prob_threshold, args.display)
if len(face_coords) == 0:
log.error("There is no face in the stream!")
continue
out_frame, head_angle = network_hp.predict(out_frame, face,
face_coords, args.display)
out_frame, eye_left, eye_right, eye_center = network_fl.predict(
out_frame, face, face_coords, args.display)
out_frame, gaze = network_ge.predict(out_frame, eye_left, eye_right,
eye_center, head_angle,
args.display)
mouse_cont.move(gaze[0], gaze[1])
if key_pressed == 27:
break
cv2.imshow('Visualization', cv2.resize(out_frame, (600, 400)))
duration_inference = time.time() - starting_inference
print("Total loading time is: {}\nTotal inference time is: {} ".format(
duration_loading, duration_inference))
feed.close()
cv2.destroyAllWindows
def main():
args = build_argparser().parse_args()
frameNum = 0
inferenceTime = 0
counter = 0
# Initialize the Inference Engine
fd = FaceDetection()
ld = Facial_Landmarks_Detection()
ge = gazeEstimation()
hp = headPose()
modelStart = time.time()
# Load Models
fd.loadModel(args.faceDetectionModel, args.device)
ld.loadModel(args.faceLandmarkModel, args.device)
ge.loadModel(args.gazeEstimationModel, args.device)
hp.loadModel(args.headPoseModel, args.device)
print("Model Load timing:", (time.time() - modelStart) * 1000, "ms")
# Get the input feeder
if args.input == "cam":
feed = InputFeeder("cam")
else:
assert os.path.isfile(args.input), "Specified input file doesn't exist"
feed = InputFeeder("video", args.input)
feed.load_data()
frameCount = 0
# Mouse Controller precision and speed
mc = MouseController('medium', 'fast')
for frame in feed.next_batch():
frameCount += 1
if frame is not None:
key = cv2.waitKey(60)
inferenceStart = time.time()
# make predictions
detected_face, faceCoords = fd.predict(frame.copy(),
args.prob_threshold)
hpOutput = hp.predict(detected_face.copy())
leftEye, rightEye, eyeCoords = ld.predict(detected_face.copy())
new_mouse_coord, gazeVector = ge.predict(leftEye, rightEye,
hpOutput)
inferenceTime = time.time() - inferenceStart
counter = counter + 1
# Visualization
preview = args.visualization
if preview:
preview_frame = frame.copy()
faceFrame = detected_face.copy()
drawFaceBoundingBox(preview_frame, faceCoords)
displayHp(preview_frame, hpOutput, faceCoords)
draw_landmarks(faceFrame, eyeCoords)
draw_gaze(faceFrame, gazeVector, leftEye.copy(),
rightEye.copy(), eyeCoords)
if preview:
img = np.hstack((cv2.resize(preview_frame, (500, 500)),
cv2.resize(faceFrame, (500, 500))))
else:
img = cv2.resize(frame, (500, 500))
cv2.imshow('Visualization', img)
# set speed
if frameCount % 5 == 0:
mc.move(new_mouse_coord[0], new_mouse_coord[1])
print("Frame Number:", frameNum)
print("Inference Time:", inferenceTime * 1000)
frameNum += 1
if key == 27:
break
feed.close()
def model_pipelines(args):
# Parameters which were parsed are assigned
#device = args.dev
#customLayers = args.lay
inputFile = args.inp
visual_flag = args.vf
faceDetectionModel = args.mfd
landmarksDetectionModel = args.mld
headPoseEstimationModel = args.mhp
gazeDetectionModel = args.mgd
start_time = time.time()
# Logging is enabled
log = logging.getLogger(__name__)
log.info('----------THE BEGINNING----------')
log.info('Start Time: {0}'. format(str(start_time)))
# The feed is initialised
single_image = ['jpg','tif','png','jpeg', 'bmp']
if inputFile.split(".")[-1].lower() in single_image:
input_feed = InputFeeder('image', inputFile)
elif args.inp == 'cam':
input_feed = InputFeeder('cam')
else:
input_feed = InputFeeder('video', inputFile)
# Feed data is loaded
log.info('Loading data...')
input_feed.load_data()
log.info('Data Loaded. Beginning inference...')
# The models are initialised and loaded here
face_model_load_start_time = time.time()
landmark_model_load_start_time = time.time()
headpose_model_load_start_time = time.time()
gaze_model_load_start_time = time.time()
ppl_fd = Face_Detection(faceDetectionModel)
ppl_fl = Facial_Landmarks_Detection(landmarksDetectionModel)
ppl_hd = Head_Pose_Estimation(headPoseEstimationModel)
ppl_ge = Gaze_Estimation(gazeDetectionModel)
face_model_load_time = time.time() - face_model_load_start_time
landmark_model_load_time = time.time() - landmark_model_load_start_time
headpose_model_load_time = time.time() - headpose_model_load_start_time
gaze_model_load_time = time.time() - gaze_model_load_start_time
log.info('Face Detection object initialized')
log.info('Facial Landmarks object initialized')
log.info('Head Pose object initialized')
log.info('Gaze object initialized')
log.info('All models loaded and checked')
load_time = [face_model_load_time, landmark_model_load_time, headpose_model_load_time, gaze_model_load_time]
# count the number of frames
frameCount = 0
# collate frames from the feeder and feed into the detection pipelines
for _, frame in input_feed.next_batch():
if not _:
break
frameCount += 1
if frameCount % 5 == 0:
cv2.imshow('video', cv2.resize(frame, (500, 500)))
key = cv2.waitKey(100)
# Get the time for the model inference
face_inference_start_time = time.time()
face_crop = ppl_fd.predict(frame)
face_inference_time = time.time() - face_inference_start_time
if 'mfd' in visual_flag:
cv2.imshow('The cropped face', face_crop)
if type(face_crop) == int:
log.info("No face can be detected")
if key == 27:
break
continue
# Get the time for the model inference
landmark_inference_start_time = time.time()
eye_image_left, eye_image_right, face_landmarked = ppl_fl.predict(face_crop.copy())
landmark_inference_time = time.time() - landmark_inference_start_time
# Get face landmark results
if 'mld' in visual_flag:
cv2.imshow('Face output', face_landmarked)
if eye_image_left.any() == None or eye_image_right.any() == None:
log.info("Landmarks could not be detected, check that the eyes are visible and the image is bright")
continue
# Get the time for the model inference
headpose_inference_start_time = time.time()
head_pose_angles, head_pose_image = ppl_hd.predict(face_crop.copy())
headpose_inference_time = time.time() - headpose_inference_start_time
# Get head pose results
if 'mhp' in visual_flag:
cv2.imshow('Head Pose Angles', head_pose_image)
# Get the time for the model inference
gaze_inference_start_time = time.time()
coord_x, coord_y = ppl_ge.predict(eye_image_left ,eye_image_right, head_pose_angles)
gaze_inference_time = time.time() - gaze_inference_start_time
# Get gaze detection results
if 'mgd' in visual_flag:
cv2.putText(face_landmarked, "Estimated x:{:.2f} | Estimated y:{:.2f}".format(coord_x, coord_y), (10,20), cv2.FONT_HERSHEY_COMPLEX, 0.25, (0,255,0),1)
cv2.imshow('Gaze Estimation', face_landmarked)
mCoord = MouseController('medium','fast')
# Move the mouse based on the coordinates received
if frameCount % 5 == 0:
mCoord.move(coord_x, coord_y)
if key == 27:
break
inference_time = [face_inference_time, landmark_inference_time, headpose_inference_time, gaze_inference_time]
results(args, inference_time, load_time)
if key == ord('x'):
log.warning('KeyboardInterrupt: `X` was pressed')
results(args, inference_time, load_time)
sys.exit()
log.info('End Time: {0}'. format(str(time.time() - start_time)))
log.info('----------THE END----------')
cv2.destroyAllWindows()
input_feed.close()
def main():
try:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[
logging.FileHandler("Computer_Pointer_Controller.log"),
logging.StreamHandler()
])
except:
print("File cannot be created")
args = build_argparser()
video_path = args.i
visualize = args.flags
count = 0
fd_inference_time = 0
fld_inference_time = 0
hp_inference_time = 0
ge_inference_time = 0
MC = MouseController('medium', 'fast')
logging.info("############## Model Load Time #############")
start_time = time.time()
first_model_time = start_time
FD = Face_Detection(device=args.d, threshold=args.prob, extensions=args.l)
FD.load_model(model_path=args.f)
logging.info("Face Detection Model: {:.3f}ms".format(
1000 * (time.time() - first_model_time)))
second_model_time = time.time()
FLD = Facial_Landmarks_Detection(device=args.d, extensions=args.l)
FLD.load_model(model_path=args.fl)
logging.info("Facial Landmarks Detection Model: {:.3f}ms".format(
1000 * (time.time() - second_model_time)))
third_model_time = time.time()
HPE = Head_Pose_Estimation(device=args.d, extensions=args.l)
HPE.load_model(model_path=args.hp)
logging.info("Head Pose Estimation Model: {:.3f}ms".format(
1000 * (time.time() - third_model_time)))
fourth_model_time = time.time()
GE = Gaze_Estimation(device=args.d, extensions=args.l)
GE.load_model(model_path=args.g)
logging.info("Gaze Estimation Model: {:.3f}ms".format(
1000 * (time.time() - fourth_model_time)))
logging.info("############## End ######################### ")
Total_Model_Load_Time = 1000 * (time.time() - start_time)
##### LOADING VIDEO FILE #####
if (video_path == "cam"):
IF = InputFeeder("cam")
else:
IF = InputFeeder("video", video_path)
IF.load_data()
##### MODEL INFERENCE #####
start_inf_time = time.time()
for flag, frame in IF.next_batch():
if not flag:
break
if (count % 5 == 0):
cv2.imshow('frame', cv2.resize(frame, (500, 500)))
key = cv2.waitKey(60)
count = count + 1
start_time_1 = time.time()
face, face_coordinates = FD.predict(frame, args.it)
fd_inference_time += (time.time() - start_time_1)
start_time_2 = time.time()
left_eye_image, right_eye_image, eye_coordinates = FLD.predict(
face, args.it)
fld_inference_time += (time.time() - start_time_2)
start_time_3 = time.time()
head_pose_angles = HPE.predict(face, args.it)
hp_inference_time += (time.time() - start_time_3)
start_time_4 = time.time()
mouse_coordinates, gaze_vector = GE.predict(left_eye_image,
right_eye_image,
head_pose_angles, args.it)
ge_inference_time += (time.time() - start_time_4)
if (len(visualize) != 0):
frame_visualize = frame.copy()
if ("fd" in visualize):
if (len(visualize) == 1):
cv2.rectangle(frame_visualize,
(face_coordinates[0], face_coordinates[1]),
(face_coordinates[2], face_coordinates[3]),
(255, 0, 255), 2)
else:
frame_visualize = face.copy()
if ("fld" in visualize):
if not "fd" in visualize:
frame_visualize = face.copy()
cv2.circle(frame_visualize, (eye_coordinates['left_eye'][0],
eye_coordinates['left_eye'][1]),
25, (0, 0, 255), 2)
cv2.circle(frame_visualize, (eye_coordinates['right_eye'][0],
eye_coordinates['right_eye'][1]),
25, (0, 0, 255), 2)
if ("hp" in visualize):
cv2.putText(
frame_visualize,
"Pose Angles: yaw:{:.2f} | pitch:{:.2f} | roll:{:.2f}".
format(head_pose_angles[0], head_pose_angles[1],
head_pose_angles[2]), (10, 20),
cv2.FONT_HERSHEY_COMPLEX, 0.255, (0, 255, 0), 1)
if ("ge" in visualize):
h = face.shape[0]
arrow = h * 0.7
arrow_X = gaze_vector[0] * arrow
arrow_Y = -gaze_vector[1] * arrow
cv2.arrowedLine(
frame_visualize, (eye_coordinates['left_eye'][0],
eye_coordinates['left_eye'][1]),
(int(eye_coordinates['left_eye'][0] + arrow_X),
int(eye_coordinates['left_eye'][1] + arrow_Y)),
(255, 0, 0), 2)
cv2.arrowedLine(
frame_visualize, (eye_coordinates['right_eye'][0],
eye_coordinates['right_eye'][1]),
(int(eye_coordinates['right_eye'][0] + arrow_X),
int(eye_coordinates['right_eye'][1] + arrow_Y)),
(255, 0, 0), 2)
if (count % 5 == 0):
cv2.imshow('Visualization',
cv2.resize(frame_visualize, (500, 500)))
if (count % 5 == 0):
MC.move(mouse_coordinates[0], mouse_coordinates[1])
if key == 27:
break
Total_Inference_Time = time.time() - start_inf_time
if (count > 0):
logging.info("############## Models Inference time #######")
logging.info("Face Detection:{:.3f}ms".format(
1000 * fd_inference_time / count))
logging.info("Facial Landmarks Detection:{:.3f}ms".format(
1000 * fld_inference_time / count))
logging.info("Headpose Estimation:{:.3f}ms".format(
1000 * hp_inference_time / count))
logging.info("Gaze Estimation:{:.3f}ms".format(
1000 * ge_inference_time / count))
logging.info("############## End #########################")
logging.info("############## Summarized Results ##########")
logging.info(
"Total Model Load Time: {:.3f}ms".format(Total_Model_Load_Time))
logging.info("Total Inference Time: {:.3f}s".format(Total_Inference_Time))
logging.info("FPS:{}".format(count / Total_Inference_Time))
logging.info("############ End ###########################")
cv2.destroyAllWindows()
IF.close()