def run_inference(args):
feed = InputFeeder(input_type='video', input_file=args.input)
feed.load_data()
for batch in feed.next_batch():
cv2.imshow("Output", cv2.resize(batch, (500, 500)))
key = cv2.waitKey(60)
if (key == 27):
break
# getting face
faceDetection = FaceDetection(model_name=args.face_detection_model)
faceDetection.load_model()
face = faceDetection.predict(batch)
# getting eyes
facialLandmarksDetection = FacialLandmarksDetection(
args.facial_landmarks_detection_model)
facialLandmarksDetection.load_model()
left_eye, right_eye = facialLandmarksDetection.predict(face)
# getting head pose angles
headPoseEstimation = HeadPoseEstimation(
args.head_pose_estimation_model)
headPoseEstimation.load_model()
head_pose = headPoseEstimation.predict(face)
print("head pose angles: ", head_pose)
# get mouse points
gazeEstimation = GazeEstimation(args.gaze_estimation_model)
gazeEstimation.load_model()
mouse_coords = gazeEstimation.predict(left_eye, right_eye, head_pose)
print("gaze output: ", mouse_coords)
feed.close()
def pipeline(args):
feed = InputFeeder(args.i)
feed.load_data()
FaceDetectionPipe = FaceDetection(args.m_fd, args.pt, args.d, args.cpu_ext)
load_time = time.time()
FaceDetectionPipe.load_model()
load_time_fd = time.time() - load_time
FacialLandmarksPipe = FacialLandmarks(args.m_ld, args.d, args.cpu_ext)
load_time = time.time()
FacialLandmarksPipe.load_model()
load_time_ld = time.time() - load_time
HeadPoseEstimationPipe = HeadPoseEstimation(args.m_hpe, args.d,
args.cpu_ext)
load_time = time.time()
HeadPoseEstimationPipe.load_model()
load_time_hpe = time.time() - load_time
GazeEstimationPipe = GazeEstimation(args.m_ge, args.d, args.cpu_ext)
load_time = time.time()
GazeEstimationPipe.load_model()
load_time_ge = time.time() - load_time
log.info('Load time for face detection model: ' + str(load_time_fd))
log.info('Load time for landmark detection model: ' + str(load_time_ld))
log.info('Load time for head pose estimation model: ' + str(load_time_hpe))
log.info('Load time for gaze estimation model: ' + str(load_time_ge))
inf_time_fd = inf_time_ld = inf_time_hpe = inf_time_ge = frame_count = 0
for frame in feed.next_batch():
if frame is None:
break
if cv2.waitKey(1) & 0xFF == ord('q'):
break
frame_count += 1
inf_time = time.time()
fd_img_output, fd_coords = FaceDetectionPipe.predict(frame)
inf_time_fd = time.time() - inf_time
if (fd_coords == []):
log.info('No face detected')
else:
inf_time = time.time()
eye_l_image, eye_r_image, ld_coords = FacialLandmarksPipe.predict(
fd_img_output)
inf_time_ld = time.time() - inf_time
inf_time = time.time()
hpe_output = HeadPoseEstimationPipe.predict(fd_img_output)
inf_time_hpe = time.time() - inf_time
yaw, pitch, roll = hpe_output
inf_time = time.time()
ge_output = GazeEstimationPipe.predict(eye_l_image, eye_r_image,
[yaw, pitch, roll])
inf_time_ge = time.time() - inf_time
if frame_count % 5 == 0:
pointer = MouseController('medium', 'fast')
pointer.move(ge_output[0], ge_output[1])
fps_fd = 1 / inf_time_fd
fps_ld = 1 / inf_time_ld
fps_hpe = 1 / inf_time_hpe
fps_ge = 1 / inf_time_ge
if (args.v):
v = Visualizer(frame, fd_img_output, fd_coords, ld_coords,
hpe_output)
v.visualize()
log.info('Average inference time for face detection model: ' +
str(inf_time_fd))
log.info('Average inference time for landmark detection model: ' +
str(inf_time_ld))
log.info(
'Average inference time for head pose estimation model: ' +
str(inf_time_hpe))
log.info('Average inference time for gaze estimation model: ' +
str(inf_time_ge))
log.info('FPS for face detection model: ' + str(fps_fd))
log.info('FPS for landmark detection model: ' + str(fps_ld))
log.info('FPS for head pose estimation model: ' + str(fps_hpe))
log.info('FPS for gaze estimation model: ' + str(fps_ge))
log.info('Frames Count:' + str(frame_count))
mm = ModelMetrics()
log.info('Writing stats to file...')
mm.save_to_file('stats_fd.txt', 'FD/' + model_precision,
inf_time_fd, fps_fd, load_time_fd)
mm.save_to_file('stats_ld.txt', model_precision, inf_time_ld,
fps_ld, load_time_ld)
mm.save_to_file('stats_hpe.txt', model_precision, inf_time_hpe,
fps_hpe, load_time_hpe)
mm.save_to_file('stats_ge.txt', model_precision, inf_time_ge,
fps_ge, load_time_ge)
feed.close()
def main():
"""
Load inference networks, stream video to network,
and output stats and video.
:return: None
"""
# Logger init
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
# Get command line args
args = get_arg()
#Load Preferencies
with open(args.config_file, "r") as yamlfile:
cfg = yaml.load(yamlfile, Loader=yaml.FullLoader)
models = cfg['models']
input_source = args.input
video_path = cfg['video_path']
face_model = FaceDetection(models['face_detection'])
head_pose_model = HeadPoseEstimation(models['head_pose_estimation'])
facial_landmarks_model = FacialLandmarksDetection(models['facial_landmarks_detection'])
gaze_estimation_model = GazeEstimation(models['gaze_estimation'])
# Initialise the MouseController
mouse_contr = MouseController("low","fast")
# Load the models and log timing
start_time = time.time()
face_model.load_model(args.device)
logging.info("Load Face Detection model: {:.1f}ms".format(1000 * (time.time() - start_time)) )
start_time = time.time()
facial_landmarks_model.load_model(args.device)
logging.info("Load Facial Landmarks Detection model: {:.1f}ms".format(1000 * (time.time() - start_time)) )
start_time = time.time()
head_pose_model.load_model(args.device)
logging.info("Load Head Pose Estimation model: {:.1f}ms".format(1000 * (time.time() - start_time)) )
start_time = time.time()
gaze_estimation_model.load_model(args.device)
logging.info("Load Gaze Estimation model: {:.1f}ms".format(1000 * (time.time() - start_time)) )
# Get and open video or camera capture
#input_feed = InputFeeder('video', args.input)
#input_feed.load_data()
input_feed = InputFeeder(input_type=input_source, input_file=video_path)
input_feed.load_data()
if not input_feed.cap.isOpened():
log.critical('Error opening input, check --video_path parameter')
sys.exit(1)
# FPS = input_feed.get_fps()
# Grab the shape of the input
# width = input_feed.get_width()
# height = input_feed.get_height()
# init scene variables
frame_count = 0
### Loop until stream is over ###
facedetect_infer_time = 0
landmark_infer_time = 0
headpose_infer_time = 0
gaze_infer_time = 0
while True:
# Read the next frame
try:
frame = next(input_feed.next_batch())
except StopIteration:
break
if frame is None:
break
key_pressed = cv2.waitKey(60)
frame_count += 1
input_height, input_width, _ = frame.shape
logging.info("frame {count} size {w}, {h}".format(count= frame_count, w = input_width, h =input_height))
# face detection
p_frame = face_model.preprocess_input(frame)
start_time = time.time()
fnoutput = face_model.predict(p_frame)
facedetect_infer_time += time.time() - start_time
out_frame,fboxes = face_model.preprocess_output(fnoutput,frame,args.overlay, args.prob_threshold)
#for each face
for fbox in fboxes:
face = frame[fbox[1]:fbox[3],fbox[0]:fbox[2]]
p_frame = facial_landmarks_model.preprocess_input(face)
start_time = time.time()
lmoutput = facial_landmarks_model.predict(p_frame)
landmark_infer_time += time.time() - start_time
out_frame,left_eye_point,right_eye_point = facial_landmarks_model.preprocess_output(lmoutput, fbox, out_frame,args.overlay, args.prob_threshold)
# get head pose estimation
p_frame = head_pose_model.preprocess_input(face)
start_time = time.time()
hpoutput = head_pose_model.predict(p_frame)
headpose_infer_time += time.time() - start_time
out_frame, headpose_angels = head_pose_model.preprocess_output(hpoutput,out_frame, face,fbox,args.overlay, args.prob_threshold)
# get gaze estimation
out_frame, left_eye, right_eye = gaze_estimation_model.preprocess_input(out_frame,face,left_eye_point,right_eye_point,args.overlay)
start_time = time.time()
geoutput = gaze_estimation_model.predict(left_eye, right_eye, headpose_angels)
gaze_infer_time += time.time() - start_time
out_frame, gazevector = gaze_estimation_model.preprocess_output(geoutput,out_frame,fbox, left_eye_point,right_eye_point,args.overlay, args.prob_threshold)
cv2.imshow('im', out_frame)
if(args.mouse_move):
logging.info("mouse move vector : x ={}, y={}".format(gazevector[0], gazevector[1]))
mouse_contr.move(gazevector[0], gazevector[1])
#use only first detected face in the frame
break
# Break if escape key pressed
if key_pressed == 27:
break
#logging inference times
if(frame_count>0):
logging.info("***** Models Inference time *****")
logging.info("Face Detection:{:.1f}ms".format(1000* facedetect_infer_time/frame_count))
logging.info("Facial Landmarks Detection:{:.1f}ms".format(1000* landmark_infer_time/frame_count))
logging.info("Headpose Estimation:{:.1f}ms".format(1000* headpose_infer_time/frame_count))
logging.info("Gaze Estimation:{:.1f}ms".format(1000* gaze_infer_time/frame_count))
# Release the capture and destroy any OpenCV windows
input_feed.close()
cv2.destroyAllWindows()
def main():
# Grab command line args
args = build_argparser().parse_args()
flags = args.models_outputs_flags
logger = logging.getLogger()
input_file_path = args.input
input_feeder = None
if input_file_path.lower() == "cam":
input_feeder = InputFeeder("cam")
else:
if not os.path.isfile(input_file_path):
logger.error("Unable to find specified video file")
exit(1)
input_feeder = InputFeeder("video", input_file_path)
model_path_dict = {
'FaceDetection': args.face_detection_model,
'FacialLandmarks': args.facial_landmarks_model,
'GazeEstimation': args.gaze_estimation_model,
'HeadPoseEstimation': args.head_pose_estimation_model
}
for file_name_key in model_path_dict.keys():
if not os.path.isfile(model_path_dict[file_name_key]):
logger.error("Unable to find specified " + file_name_key +
" xml file")
exit(1)
fdm = FaceDetection(model_path_dict['FaceDetection'], args.device,
args.cpu_extension)
flm = FacialLandmarks(model_path_dict['FacialLandmarks'], args.device,
args.cpu_extension)
gem = GazeEstimation(model_path_dict['GazeEstimation'], args.device,
args.cpu_extension)
hpem = HeadPoseEstimation(model_path_dict['HeadPoseEstimation'],
args.device, args.cpu_extension)
mc = MouseController('medium', 'fast')
input_feeder.load_data()
fdm.load_model()
flm.load_model()
hpem.load_model()
gem.load_model()
frame_count = 0
for ret, frame in input_feeder.next_batch():
if not ret:
break
frame_count += 1
if frame_count % 5 == 0:
cv2.imshow('video', cv2.resize(frame, (500, 500)))
key = cv2.waitKey(60)
cropped_face, face_coords = fdm.predict(frame, args.prob_threshold)
if type(cropped_face) == int:
logger.error("Unable to detect any face.")
if key == 27:
break
continue
hp_output = hpem.predict(cropped_face)
left_eye_img, right_eye_img, eye_coords = flm.predict(cropped_face)
new_mouse_coord, gaze_vector = gem.predict(left_eye_img, right_eye_img,
hp_output)
if (not len(flags) == 0):
preview_frame = frame
if 'fd' in flags:
preview_frame = cropped_face
if 'fld' in flags:
cv2.rectangle(cropped_face,
(eye_coords[0][0] - 10, eye_coords[0][1] - 10),
(eye_coords[0][2] + 10, eye_coords[0][3] + 10),
(0, 255, 0), 3)
cv2.rectangle(cropped_face,
(eye_coords[1][0] - 10, eye_coords[1][1] - 10),
(eye_coords[1][2] + 10, eye_coords[1][3] + 10),
(0, 255, 0), 3)
if 'hp' in flags:
cv2.putText(
preview_frame,
"Pose Angles: yaw:{:.2f} | pitch:{:.2f} | roll:{:.2f}".
format(hp_output[0], hp_output[1], hp_output[2]), (10, 20),
cv2.FONT_HERSHEY_COMPLEX, 0.25, (0, 255, 0), 1)
if 'ge' in flags:
x, y, w = int(gaze_vector[0] * 12), int(gaze_vector[1] *
12), 160
left_eye = cv2.line(left_eye_img, (x - w, y - w),
(x + w, y + w), (255, 0, 255), 2)
cv2.line(left_eye, (x - w, y + w), (x + w, y - w),
(255, 0, 255), 2)
right_eye = cv2.line(right_eye_img, (x - w, y - w),
(x + w, y + w), (255, 0, 255), 2)
cv2.line(right_eye, (x - w, y + w), (x + w, y - w),
(255, 0, 255), 2)
cropped_face[eye_coords[0][1]:eye_coords[0][3],
eye_coords[0][0]:eye_coords[0][2]] = left_eye
cropped_face[eye_coords[1][1]:eye_coords[1][3],
eye_coords[1][0]:eye_coords[1][2]] = right_eye
cv2.imshow("Visualization", cv2.resize(preview_frame, (500, 500)))
if frame_count % 5 == 0:
mc.move(new_mouse_coord[0], new_mouse_coord[1])
if key == 27:
break
logger.error("VideoStream ended...")
cv2.destroyAllWindows()
input_feeder.close()
def main():
args = get_args().parse_args()
path_filender = args.input
four_flags = args.flags_checker
loger = logging.getLogger()
feeder_in = None
out_path = args.out_path
if path_filender.lower() == "cam":
feeder_in = InputFeeder("cam")
else:
if not os.path.isfile(path_filender):
loger.error("The video was not found")
exit(1)
feeder_in = InputFeeder("video", path_filender)
model_locations = {
'FaceDetection': args.face_detection_model,
'HeadPoseEstimation': args.head_pose_estimation_model,
'FacialLandmarksDetection': args.facial_landmarks_detection_model,
'GazeEstimation': args.gaze_estimation_model
}
for key_name in model_locations.keys():
if not os.path.isfile(model_locations[key_name]):
loger.error("The system cannot find the " + key_name + " xml file")
exit(1)
dt = FaceDetection(model_locations['FaceDetection'], args.device,
args.cpu_extension)
pe = HeadPoseEstimation(model_locations['HeadPoseEstimation'], args.device,
args.cpu_extension)
ld = FacialLandmarksDetection(model_locations['FacialLandmarksDetection'],
args.device, args.cpu_extension)
ge = GazeEstimation(model_locations['GazeEstimation'], args.device,
args.cpu_extension)
cursor = MouseController('medium', 'fast')
feeder_in.load_data()
model_load_time_start = time.time()
dt.load_model()
pe.load_model()
ld.load_model()
ge.load_model()
total_load_time = time.time() - model_load_time_start
frame_counter = 0
inference_time_start = time.time()
for ret, frame in feeder_in.next_batch():
if not ret:
break
frame_counter = frame_counter + 1
if frame_counter % 1 == 0:
cv2.imshow('video', cv2.resize(frame, (600, 600)))
key = cv2.waitKey(60)
face_detected, coords_face = dt.predict(frame, args.p_th)
if type(face_detected) == int:
loger.error("The system cannot detect any face.")
if key == 27:
break
continue
head_pose_output = pe.predict(face_detected)
eye_left_detect, eye_right_detect, eye_coordinates_detect = ld.predict(
face_detected)
coordi_update_pointer, coordi_gaze = ge.predict(
eye_left_detect, eye_right_detect, head_pose_output)
if (not len(four_flags) == 0):
result_app = frame
if 'fad' in four_flags:
result_app = face_detected
if 'hpe' in four_flags:
cv2.putText(
result_app,
"HP Angles: YAW:{:.3f} * PITCH:{:.3f} * ROLL:{:.3f}".
format(head_pose_output[0], head_pose_output[1],
head_pose_output[2]), (5, 40),
cv2.FONT_HERSHEY_COMPLEX, 0.25, (153, 76, 0), 0)
if 'fld' in four_flags:
cv2.rectangle(face_detected,
(eye_coordinates_detect[0][0] - 4,
eye_coordinates_detect[0][1] - 4),
(eye_coordinates_detect[0][2] + 4,
eye_coordinates_detect[0][3] + 4),
(255, 255, 0), 4)
cv2.rectangle(face_detected,
(eye_coordinates_detect[1][0] - 4,
eye_coordinates_detect[1][1] - 4),
(eye_coordinates_detect[1][2] + 4,
eye_coordinates_detect[1][3] + 4),
(255, 255, 0), 4)
if 'gae' in four_flags:
x = int(coordi_gaze[0] * 2)
y = int(coordi_gaze[1] * 2)
w = 150
right_E = cv2.line(eye_right_detect, (x - w, y - w),
(x + w, y + w), (51, 255, 153), 1)
cv2.line(right_E, (x - w, y + w), (x + w, y - w),
(51, 255, 253), 1)
left_E = cv2.line(eye_left_detect, (x - w, y - w),
(x + w, y + w), (51, 255, 153), 1)
cv2.line(left_E, (x - w, y + w), (x + w, y - w),
(51, 255, 253), 1)
face_detected[
eye_coordinates_detect[1][1]:eye_coordinates_detect[1][3],
eye_coordinates_detect[1][0]:eye_coordinates_detect[1]
[2]] = right_E
face_detected[
eye_coordinates_detect[0][1]:eye_coordinates_detect[0][3],
eye_coordinates_detect[0][0]:eye_coordinates_detect[0]
[2]] = left_E
cv2.imshow("Result of the App", cv2.resize(result_app, (600, 600)))
if frame_counter % 5 == 0:
cursor.move(coordi_update_pointer[0], coordi_update_pointer[1])
if key == 27:
break
total_time = time.time() - inference_time_start
total_time_for_inference = round(total_time, 1)
fps = frame_counter / total_time_for_inference
with open(out_path + 'stats.txt', 'w') as f:
f.write('Inference time: ' + str(total_time_for_inference) + '\n')
f.write('FPS: ' + str(fps) + '\n')
f.write('Model load time: ' + str(total_load_time) + '\n')
loger.error("The video stream is over...")
cv2.destroyAllWindows()
feeder_in.close()
def main():
"""
Load the network and parse the output.
:return: None
"""
global INFO
global DELAY
global POSE_CHECKED
#controller = MouseController()
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = args_parser().parse_args()
logger = log.getLogger()
if args.input == 'cam':
input_stream = 0
else:
input_stream = args.input
assert os.path.isfile(args.input), "Specified input file doesn't exist"
cap = cv2.VideoCapture(input_stream)
initial_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
initial_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
video_len = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
fps = int(cap.get(cv2.CAP_PROP_FPS))
out = cv2.VideoWriter(os.path.join(args.output_dir, "shopper.mp4"),
cv2.VideoWriter_fourcc(*"MP4V"), fps,
(initial_w, initial_h), True)
frame_count = 0
job_id = 1 #os.environ['PBS_JOBID']
progress_file_path = os.path.join(args.output_dir,
'i_progress_' + str(job_id) + '.txt')
infer_time_start = time.time()
if input_stream:
cap.open(args.input)
# Adjust DELAY to match the number of FPS of the video file
DELAY = 1000 / cap.get(cv2.CAP_PROP_FPS)
if not cap.isOpened():
logger.error("ERROR! Unable to open video source")
return
# Initialise the class
if args.cpu_extension:
facedet = FaceDetection(args.facemodel,
args.confidence,
extensions=args.cpu_extension)
posest = HeadPoseEstimation(args.posemodel,
args.confidence,
extensions=args.cpu_extension)
landest = FaceLandmarksDetection(args.landmarksmodel,
args.confidence,
extensions=args.cpu_extension)
gazeest = GazeEstimation(args.gazemodel,
args.confidence,
extensions=args.cpu_extension)
else:
facedet = FaceDetection(args.facemodel, args.confidence)
posest = HeadPoseEstimation(args.posemodel, args.confidence)
landest = FaceLandmarksDetection(args.landmarksmodel, args.confidence)
gazeest = GazeEstimation(args.gazemodel, args.confidence)
# infer_network_pose = Network()
# Load the network to IE plugin to get shape of input layer
facedet.load_model()
posest.load_model()
landest.load_model()
gazeest.load_model()
print("loaded models")
ret, frame = cap.read()
while ret:
looking = 0
POSE_CHECKED = False
ret, frame = cap.read()
frame_count += 1
if not ret:
print("checkpoint *BREAKING")
break
if frame is None:
log.error("checkpoint ERROR! blank FRAME grabbed")
break
initial_w = int(cap.get(3))
initial_h = int(cap.get(4))
# Start asynchronous inference for specified request
inf_start_fd = time.time()
# Results of the output layer of the network
coords, frame = facedet.predict(frame)
det_time_fd = time.time() - inf_start_fd
if len(coords) > 0:
[xmin, ymin, xmax,
ymax] = coords[0] # use only the first detected face
head_pose = frame[ymin:ymax, xmin:xmax]
inf_start_hp = time.time()
is_looking, pose_angles = posest.predict(head_pose)
if is_looking:
det_time_hp = time.time() - inf_start_hp
POSE_CHECKED = True
#print(is_looking)
inf_start_lm = time.time()
coords, f = landest.predict(head_pose)
frame[ymin:ymax, xmin:xmax] = f
det_time_lm = time.time() - inf_start_lm
[[xlmin, ylmin, xlmax, ylmax], [xrmin, yrmin, xrmax,
yrmax]] = coords
left_eye_image = frame[ylmin:ylmax, xlmin:xlmax]
right_eye_image = frame[yrmin:yrmax, xrmin:xrmax]
output = gazeest.predict(left_eye_image, right_eye_image,
pose_angles)
# Draw performance stats
inf_time_message = "Face Inference time: {:.3f} ms.".format(
det_time_fd * 1000)
if POSE_CHECKED:
cv2.putText(
frame, "Head pose Inference time: {:.3f} ms.".format(
det_time_hp * 1000), (0, 35), cv2.FONT_HERSHEY_SIMPLEX,
0.5, (255, 255, 255), 1)
cv2.putText(frame, inf_time_message, (0, 15),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 255, 255), 1)
out.write(frame)
print("frame", frame_count)
if frame_count % 10 == 0:
print(time.time() - infer_time_start)
progressUpdate(progress_file_path,
int(time.time() - infer_time_start), frame_count,
video_len)
if args.output_dir:
total_time = time.time() - infer_time_start
with open(os.path.join(args.output_dir, 'stats.txt'), 'w') as f:
f.write(str(round(total_time, 1)) + '\n')
f.write(str(frame_count) + '\n')
facedet.clean()
posest.clean()
landest.clean()
gazeest.clean()
out.release()
cap.release()
cv2.destroyAllWindows()
def inference(args):
time_sheet = {
'face_infr': [],
'landmark_infr': [],
'head_infr': [],
'gaze_infr': [],
'infr_per_frame': []
}
logging.basicConfig(filename='result.log', level=logging.INFO)
logging.info(
"================================================================================="
)
logging.info("Precision(face,landmark,head,gaze): FP32-INT1,FP{0},FP{1},FP{2}".format(\
args.landmark_model.split("FP")[1].split("\\")[0],
args.head_model.split("FP")[1].split("\\")[0],
args.gaze_model.split("FP")[1].split("\\")[0]))
model_load_start = time.time()
face_detection = FaceDetection(args.face_model)
face_detection.load_model()
landmark_regression = LandmarkRegression(args.landmark_model)
landmark_regression.load_model()
head_pose = HeadPose(args.head_model)
head_pose.load_model()
gaze_estimation = GazeEstimation(args.gaze_model)
gaze_estimation.load_model()
logging.info("4 models load time: {0:.4f}sec".format(time.time() -
model_load_start))
mouse_controller = MouseController('high', 'fast')
cv2.namedWindow('preview', cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty('preview', cv2.WND_PROP_FULLSCREEN,
cv2.WINDOW_FULLSCREEN)
input_feeder = InputFeeder(args.input_type, args.input_file)
input_feeder.load_data()
total_infr_start = time.time()
for image in input_feeder.next_batch():
if image is None:
break
face_infr_start = time.time()
face_image = face_detection.predict(image)
time_sheet['face_infr'].append(time.time() - face_infr_start)
landmark_infr_start = time.time()
left_eye_image, right_eye_image = landmark_regression.predict(
np.copy(face_image))
time_sheet['landmark_infr'].append(time.time() - landmark_infr_start)
head_infr_start = time.time()
head_pose_angles = head_pose.predict(np.copy(face_image))
time_sheet['head_infr'].append(time.time() - head_infr_start)
gaze_infr_start = time.time()
x, y, z = gaze_estimation.predict(left_eye_image, right_eye_image,
head_pose_angles)
time_sheet['gaze_infr'].append(time.time() - gaze_infr_start)
time_sheet['infr_per_frame'].append(time.time() - face_infr_start)
cv2.imshow('preview', image)
mouse_controller.move(x, y)
key = cv2.waitKey(20)
if key == 27: # exit on ESC
break
logging.info("Face model avg inference per frame: {0:.4f}sec".format(
np.mean(time_sheet['face_infr'])))
logging.info("Landmark model avg inference per frame: {0:.4f}sec".format(
np.mean(time_sheet['landmark_infr'])))
logging.info("Head model avg inference per frame: {0:.4f}sec".format(
np.mean(time_sheet['head_infr'])))
logging.info("Gaze model avg inference per frame: {0:.4f}sec".format(
np.mean(time_sheet['gaze_infr'])))
logging.info("4 Model avg inference per frame: {0:.4f}sec".format(
np.mean(time_sheet['infr_per_frame'])))
logging.info("Total inference time: {0:.4f}sec".format(time.time() -
total_infr_start))
logging.info(
"====================================END==========================================\n"
)
input_feeder.close()
cv2.destroyAllWindows()
'--device',
default='CPU',
type=str,
choices=['CPU', 'GPU', 'FPGA', 'MYRIAD'],
help='Choose device to run inference from CPU, GPU, MYRIAD, FPGA')
parser.add_argument('--visualize',
default=False,
type=bool,
help='visualize the model output')
parser.add_argument('--output',
default='output',
type=str,
help='output directory to save results')
args = parser.parse_args()
from face_detection import FaceDetection
# Read Image from given input
image = cv2.imread(args.input)
FD = FaceDetection(args.face_detection, args.device)
FD.load_model()
crop_face, coords = FD.predict(image, visualize=args.visualize)
FL = FaceLandmark(args.face_landmark, args.device)
FL.load_model()
image = FL.predict(crop_face, visualize=args.visualize)
def main():
arg_parser = ArgParser()
args = arg_parser.get_args()
input_file = args.input
# If input file defined then use it else use the webcam
if input_file:
if not os.path.isfile(input_file):
log.error("Input file cannot be found")
exit()
input_feeder = InputFeeder("video", input_file)
else:
input_feeder = InputFeeder("cam")
face_detection_model = FaceDetection(args.face_detection_model,
args.device, args.extensions)
face_detection_model.load_model()
facial_landmarks_model = FacialLandmarksDetection(
args.facial_landmark_detection_model, args.device, args.extensions)
facial_landmarks_model.load_model()
gaze_model = GazeEstimation(args.gaze_estimation_model, args.device,
args.extensions)
gaze_model.load_model()
head_pose_model = HeadPoseEstimation(args.head_pose_estimation_model,
args.device, args.extensions)
head_pose_model.load_model()
mouse_controller = MouseController('medium', 'fast')
input_feeder.load_data()
frame_count = 0
total_face_detection_inference_time = 0
total_facial_landmark_inference_time = 0
total_head_pose_inference_time = 0
total_gaze_estimation_inference_time = 0
total_inference_time = 0
for ret, frame in input_feeder.next_batch():
if not ret:
log.error("ret variable not found")
break
frame_count += 1
if frame_count % args.mouse_update_interval == 0:
cv2.imshow('Input', frame)
key_pressed = cv2.waitKey(60)
# Run inference on the face detection model
start_time = time.time()
cropped_face, face_coordinates = face_detection_model.predict(
frame.copy(), args.probability_threshold)
finish_time = time.time()
total_face_detection_inference_time += finish_time - start_time
total_inference_time += finish_time - start_time
# If no face detected get the next frame
if len(face_coordinates) == 0:
continue
# Run inference on the facial landmark detection model
start_time = time.time()
results = facial_landmarks_model.predict(cropped_face.copy())
finish_time = time.time()
left_eye_coordinates = results[0]
right_eye_coordinates = results[1]
left_eye_image = results[2]
right_eye_image = results[3]
left_eye_crop_coordinates = results[4]
right_eye_crop_coordinates = results[5]
total_facial_landmark_inference_time += finish_time - start_time
total_inference_time += finish_time - start_time
# Run inference on the head pose estimation model
start_time = time.time()
head_pose = head_pose_model.predict(cropped_face.copy())
finish_time = time.time()
total_head_pose_inference_time += finish_time - start_time
total_inference_time += finish_time - start_time
# Run inference on the gaze estimation model
start_time = time.time()
new_mouse_x_coordinate, new_mouse_y_coordinate, gaze_vector = gaze_model.predict(
left_eye_image, right_eye_image, head_pose)
finish_time = time.time()
total_gaze_estimation_inference_time += finish_time - start_time
total_inference_time += finish_time - start_time
if frame_count % args.mouse_update_interval == 0:
log.info("Mouse controller new coordinates: x = {}, y = {}".format(
new_mouse_x_coordinate, new_mouse_y_coordinate))
mouse_controller.move(new_mouse_x_coordinate,
new_mouse_y_coordinate)
# Optional visualization configuration:
if args.show_detected_face:
showDetectedFace(frame, face_coordinates)
if args.show_head_pose:
showHeadPose(frame, head_pose)
if args.show_facial_landmarks:
showFacialLandmarks(cropped_face, left_eye_crop_coordinates,
right_eye_crop_coordinates)
if args.show_gaze_estimation:
showGazeEstimation(frame, right_eye_coordinates,
left_eye_coordinates, gaze_vector,
cropped_face, face_coordinates)
# Break if escape key pressed
if key_pressed == 27:
log.warning("Keyboard interrupt triggered")
break
# Release the capture and destroy any OpenCV windows
cv2.destroyAllWindows()
input_feeder.close()
log.info("Average face detection inference time: {} seconds".format(
total_face_detection_inference_time / frame_count))
log.info(
"Average facial landmark detection inference time: {} seconds".format(
total_facial_landmark_inference_time / frame_count))
log.info("Average head pose estimation inference time: {} seconds".format(
total_head_pose_inference_time / frame_count))
log.info("Average gaze estimation inference time: {} seconds".format(
total_gaze_estimation_inference_time / frame_count))
log.info("Average total inference time: {} seconds".format(
total_inference_time / frame_count))
def main(args):
#model=args.model
fd_model = args.face
flmd_model = args.landmarks
hp_model = args.head
ge_model = args.gaze
device = args.device
display_flag = args.display
# Init and load models
fd = FaceDetection(fd_model, device)
logger.info("######## Model loading Time #######")
start = time.time()
fd.load_model()
logger.info("Face Detection Model: {:.1f}ms".format(1000 *
(time.time() - start)))
flmd = FacialLandMarksDetection(flmd_model, device)
start = time.time()
flmd.load_model()
logger.info("Facial Landmarks Detection Model: {:.1f}ms".format(
1000 * (time.time() - start)))
hpe = HeadPoseEstimation(hp_model, device)
start = time.time()
hpe.load_model()
logger.info("HeadPose Estimation Model: {:.1f}ms".format(
1000 * (time.time() - start)))
ge = GazeEstimation(ge_model, device)
start = time.time()
ge.load_model()
logger.info("Gaze Estimation Model: {:.1f}ms".format(
1000 * (time.time() - start)))
# Mouse controller
mc = MouseController("low", "fast")
feed = InputFeeder(input_type=args.input_type, input_file=args.input_file)
feed.load_data()
frame_count = 0
fd_inference_time = 0
lm_inference_time = 0
hp_inference_time = 0
ge_inference_time = 0
move_mouse = False
for batch in feed.next_batch():
frame_count += 1
# Preprocessed output from face detection
face_boxes, image, fd_time = fd.predict(batch, display_flag)
fd_inference_time += fd_time
for face in face_boxes:
cropped_face = batch[face[1]:face[3], face[0]:face[2]]
#print(f"Face boxe = {face}")
# Get preprocessed result from landmarks
image, left_eye, right_eye, lm_time = flmd.predict(
image, cropped_face, face, display_flag)
lm_inference_time += lm_time
# Get preprocessed result from pose estimation
image, headpose_angels, hp_time = hpe.predict(
image, cropped_face, face, display_flag)
hp_inference_time += hp_time
# Get preprocessed result from Gaze estimation model
image, gazevector, ge_time = ge.predict(image, cropped_face, face,
left_eye, right_eye,
headpose_angels,
display_flag)
#cv2.imshow('Face', cropped_face)
ge_inference_time += ge_time
#print(f"Gaze vect {gazevector[0],gazevector[1]}")
cv2.imshow('img', image)
if (not move_mouse):
mc.move(gazevector[0], gazevector[1])
break
if cv2.waitKey(1) & 0xFF == ord("k"):
break
if (frame_count > 0):
logger.info("###### Models Inference time ######")
logger.info(
f"Face Detection inference time = {(fd_inference_time*1000)/frame_count} ms"
)
logger.info(
f"Facial Landmarks Detection inference time = {(lm_inference_time*1000)/frame_count} ms"
)
logger.info(
f"Headpose Estimation inference time = {(hp_inference_time*1000)/frame_count} ms"
)
logger.info(
f"Gaze estimation inference time = {(ge_inference_time*1000)/frame_count} ms"
)
feed.close()
def main(args):
device = args.device
video_file = args.video
input_type = args.input_type
toggle = args.toggle
stats = args.stats
model = args.model
if stats == 'true':
stats = True
else:
stats = False
if toggle == 'true':
toggle = True
else:
toggle = False
# Start Model Loading
start_model_load_time = time.time()
print(f'[INFO] Started Model Loading...........')
face_model = FaceDetection(parse_models_file(
label='face_detection', path=model),
device)
face_model.load_model()
# Load Landmark model
landmark_model = LandMarksDetection(
parse_models_file(label='facial_landmarks_detection', path=model),
device)
landmark_model.load_model()
pose_estimation_model = HeadPoseEstimation(
parse_models_file(label='head_pose_estimation', path=model),
device)
pose_estimation_model.load_model()
gaze_estimation_model = GazeEstimation(
parse_models_file(label='gaze_estimation', path=model), device)
gaze_estimation_model.load_model()
total_model_load_time = time.time() - start_model_load_time
print('[TOTAL] Loaded in {:.3f} ms'.format(total_model_load_time))
# End Model Loading
mouse = MouseController('high', 'fast')
if not toggle:
cv2.namedWindow(MAIN_WINDOW_NAME, cv2.WINDOW_AUTOSIZE)
try:
feed = InputFeeder(input_type=input_type, input_file=video_file)
feed.load_data()
initial_w = int(feed.cap.get(cv2.CAP_PROP_FRAME_WIDTH))
initial_h = int(feed.cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
counter = 0
if not toggle:
cv2.namedWindow(MAIN_WINDOW_NAME, cv2.WINDOW_NORMAL)
for frame, _ in feed.next_batch():
if not _:
break
try:
counter += 1
# Start Inferences
coord = face_model.predict(frame, (initial_w, initial_h))
for i in range(len(coord)):
xmin, ymin, xmax, ymax = coord[i]
cropped_image = frame[ymin:ymax, xmin:xmax]
# Landmark Inference
cropped_left, cropped_right = landmark_model.predict(cropped_image)
if cropped_right.shape[0] < 60 or cropped_left.shape[1] < 60:
break
if cropped_right.shape[1] < 60 or cropped_left.shape[0] < 60:
break
# Pose Estimation Inference
poses = pose_estimation_model.predict(cropped_image)
# Gaze Estimation Inference
gz = gaze_estimation_model.predict(poses, cropped_left, cropped_right)
# Mouse Controller
mouse.move(gz[0][0], gz[0][1])
# If user pass statistics argument to true
if stats:
# Print performance
performance_counts(
face_model.performance_counter(0)
)
performance_counts(
pose_estimation_model.performance_counter(0)
)
performance_counts(
landmark_model.performance_counter(0)
)
performance_counts(
gaze_estimation_model.performance_counter(0)
)
if not toggle:
# Output Camera or Video
#cv2.resizeWindow(MAIN_WINDOW_NAME, 480, 320)
cv2.imshow(MAIN_WINDOW_NAME, frame)
else:
# Print Statistics only no camera or video
performance_counts(
face_model.performance_counter(0)
)
performance_counts(
pose_estimation_model.performance_counter(0)
)
performance_counts(
landmark_model.performance_counter(0)
)
performance_counts(
gaze_estimation_model.performance_counter(0)
)
cv2.waitKey(1)
except Exception as e:
print('Could not run Inference', e)
feed.close()
except Exception as e:
print("Could not run Inference: ", e)
def test_run(args):
logging.getLogger().setLevel(logging.INFO)
feeder = None
activate_frame_count = 10
logging.warning("Running default value activate frame count = 10")
if args.input_type == 'video' or args.input_type == 'image':
feeder = InputFeeder(args.input_type, args.input)
if args.input == '../bin/demo.mp4':
logging.warning("Running default setting and input")
elif args.input_type == 'webcam':
feeder = InputFeeder(args.input_type, args.input)
else:
logging.error("Input not found")
exit(1)
mouse_controller = MouseController(args.precision, args.speed)
feeder.load_data()
start_time = 0
face_model_load_time = 0
start_time = time.time()
face_model = FaceDetection(args.face, args.device, args.cpu_extension)
face_model.load_model()
face_model_load_time = time.time() - start_time
logging.info("Face Detection Model Loaded...")
head_pose_estimation_load_time = 0
start_time = time.time()
head_pose_estimation = HeadPoseEstimation(args.headpose, args.device,
args.cpu_extension)
head_pose_estimation.load_model()
head_pose_estimation_load_time = time.time() - start_time
logging.info("Head Pose Detection Model Loaded...")
facial_landmarks_detection_load_time = 0
start_time = time.time()
facial_landmarks_detection = FacialLandmarksDetection(
args.landmarks, args.device, args.cpu_extension)
facial_landmarks_detection.load_model()
facial_landmarks_detection_load_time = time.time() - start_time
logging.info("Facial Landmark Detection Model Loaded...")
gaze_model_load_time = 0
start_time = time.time()
gaze_model = GazeEstimation(args.gazeestimation, args.device,
args.cpu_extension)
gaze_model.load_model()
gaze_model_load_time = time.time() - start_time
logging.info("Gaze Estimation Model Loaded...")
frame_count = 0
total_face_model_inference_time = 0
total_head_pose_estimation_inference_time = 0
total_facial_landmarks_detection_inference_time = 0
total_gaze_model_inference_time = 0
start_time = 0
for frame in feeder.next_batch():
if frame is None:
break
frame_count += 1
key = cv2.waitKey(60)
start_time = time.time()
first_face_box, first_face = face_model.predict(frame.copy())
total_face_model_inference_time = total_face_model_inference_time + (
time.time() - start_time)
start_time = time.time()
head_pose_output = head_pose_estimation.predict(first_face_box.copy())
total_head_pose_estimation_inference_time = total_head_pose_estimation_inference_time + (
time.time() - start_time)
start_time = time.time()
left_eye, right_eye, eye_coords = facial_landmarks_detection.predict(
first_face_box.copy())
total_facial_landmarks_detection_inference_time = total_facial_landmarks_detection_inference_time + (
time.time() - start_time)
start_time = time.time()
move_to_coors_mouse = gaze_model.predict(left_eye, right_eye,
head_pose_output)
total_gaze_model_inference_time = total_gaze_model_inference_time + (
time.time() - start_time)
if frame_count % activate_frame_count == 0 and (args.flag == "3"
or args.flag == "4"):
mouse_controller.move(move_to_coors_mouse[0],
move_to_coors_mouse[1])
cv2.imshow('video', frame)
key = cv2.waitKey(60)
if key == 27:
break
if args.flag == "1":
cv2.rectangle(frame, (first_face[0], first_face[1]),
(first_face[2], first_face[3]), (255, 0, 0))
cv2.imshow('video', frame)
key = cv2.waitKey(60)
elif args.flag == "2":
cv2.rectangle(facial_landmarks_detection.image,
(eye_coords[0], eye_coords[1]),
(eye_coords[2], eye_coords[3]), (255, 0, 0))
cv2.imshow('video', facial_landmarks_detection.image)
key = cv2.waitKey(60)
elif args.flag == "3":
if frame_count == 1:
logging.info("Printing mouse coors: ")
logging.info(move_to_coors_mouse)
#Print Report
if args.flag == "0":
print('------------- BEGIN REPORT -------------')
avg_inference_face_model = total_face_model_inference_time / frame_count
avg_inference_headpose = total_head_pose_estimation_inference_time / frame_count
avg_inference_facial_landmark = total_facial_landmarks_detection_inference_time / frame_count
avg_inference_gaze_model = total_gaze_model_inference_time / frame_count
print("Face Detection Model Load Time: ", args.face)
print("Loading time: ", face_model_load_time)
print("Inference time: ", avg_inference_face_model)
print("Head Pose Detection Model: ", args.headpose)
print("Loading time: ", head_pose_estimation_load_time)
print("Inference time:", avg_inference_headpose)
print("Facial Landmark Detection Model Load Time: ", args.landmarks)
print("Loading time: ", facial_landmarks_detection_load_time)
print("Inference time:", avg_inference_facial_landmark)
print("Gaze Estimation Model Load Time: ", args.gazeestimation)
print("Loading time: ", gaze_model_load_time)
print("Inference time:", avg_inference_gaze_model)
print('------------- END REPORT -------------')
def main(args):
# set log level
levels = {
'debug': logging.DEBUG,
'info': logging.INFO,
'warning': logging.WARNING,
'error': logging.ERROR
}
log_level = levels.get(args.log_level, logging.ERROR)
logging.basicConfig(level=log_level)
mouse_control = MouseController('high', 'fast')
logging.info("Model Loading Please Wait ..")
face_det = FaceDetection(args.face_detection, args.device)
facial_det = FaceLandmark(args.face_landmark, args.device)
head_pose_est = HeadPoseEstimation(args.head_pose, args.device)
gaze_est = GazeEstimation(args.gaze_estimation, args.device)
logging.info("Model loading successfully")
inp = InputFeeder(input_type='video', input_file=args.input)
inp.load_data()
face_det.load_model()
facial_det.load_model()
head_pose_est.load_model()
gaze_est.load_model()
video_writer = cv2.VideoWriter(args.output_dir + '/demo_output11.mp4',
cv2.VideoWriter_fourcc(*'MPEG'), 15,
(1920, 1080), True)
cv2.namedWindow('gaze')
for frame in inp.next_batch():
try:
frame.shape
except Exception as err:
break
crop_face, crop_coords = face_det.predict(frame,
visualize=args.visualize)
left_eye, right_eye, left_eye_crop, right_eye_crop = facial_det.predict(
crop_face, visualize=args.visualize)
head_pose = head_pose_est.predict(crop_face, visualize=args.visualize)
(new_x, new_y), gaze_vector = gaze_est.predict(left_eye_crop,
right_eye_crop,
head_pose)
left_eye_gaze = int(left_eye[0] +
gaze_vector[0] * 100), int(left_eye[1] -
gaze_vector[1] * 100)
right_eye_gaze = int(right_eye[0] +
gaze_vector[0] * 100), int(right_eye[1] -
gaze_vector[1] * 100)
cv2.arrowedLine(crop_face, left_eye, left_eye_gaze, (0, 0, 255), 2)
cv2.arrowedLine(crop_face, right_eye, right_eye_gaze, (0, 0, 255), 2)
video_writer.write(frame)
mouse_control.move(new_x, new_y)
if args.show_result:
cv2.imshow('gaze', frame)
cv2.waitKey(1)
inp.close()
video_writer.release()
cv2.destroyAllWindows()
def main():
# Grab command line args
logger = logging.getLogger()
args = build_argparser().parse_args()
flags=args.previewflags
inputfile=args.input
inputfeed=None
if inputfile.lower()=='cam':
inputfeed=InputFeeder('cam')
elif inputfile.endswith('.jpg') or inputfile.endswith('.bmp') :
inputfeed=InputFeeder("image",inputfile)
#elif inputfile.endswith('.mp4') :
#inputfeed=InputFeeder("video",inputfile)
else:
if not (os.path.isfile(inputfile)):
print((inputfile))
logger.error("Specified input file doesn't exist")
exit(1)
inputfeed=InputFeeder("video",inputfile)
model_paths={'GazeEstimation':args.gazeestimationnmodel,'FacialLandmarkDetection':args.faciallandmarkmodel,'HeadPoseEstimation':args.headposemodel,'FaceDetection':args.facedetectionmodel}
for file in model_paths.keys():
if not os.path.isfile(model_paths[file]):
logger.error("Unable to find specified "+file+" xml file")
flm=FacialLandmarkDetection(model_paths['FacialLandmarkDetection'],args.device,args.cpu_extension)
gze= GazeEstimation(model_paths['GazeEstimation'],args.device,args.cpu_extension)
hpe=HeadPoseEstimation(model_paths['HeadPoseEstimation'],args.device,args.cpu_extension)
fd=FaceDetection(model_paths['FaceDetection'],args.device,args.cpu_extension)
flm.load_model()
fd.load_model()
gze.load_model()
hpe.load_model()
mc=MouseController('medium','fast')
inputfeed.load_data()
frame_count=0
for ret, frame in inputfeed.next_batch():
if not ret:
break
frame_count+=1
if frame_count%3==0:
cv2.imshow('video',cv2.resize(frame,(300,300)))
cv2.waitKey(1)
facecoords,cropped_image=fd.predict(frame.copy(),args.prob_threshold)
if type(cropped_image)==int:
logger.error('unable to detect face')
head_out=hpe.predict(cropped_image)
left_eye,right_eye,eye=flm.predict(cropped_image)
mouse_coords,gaze_vector=gze.predict(left_eye,right_eye,head_out)
if (len(flags)!=0):
preview_frame=frame.copy()
if 'fd' in flags:
preview_frame=cropped_image
if 'fld' in flags:
cv2.rectangle(cropped_image,(eye[0][0]-15,eye[0][1]-15),(eye[0][2]+15,eye[0][3]+15),(0,0,255))
cv2.rectangle(cropped_image,(eye[1][0]-15,eye[1][1]-15),(eye[1][2]+15,eye[1][3]+15),(0,0,255))
if 'hp' in flags:
cv2.putText(preview_frame,"Pose Angles: roll:{:2f}|pitch:{:2f}|yaw:{:2f}|".format(head_out[2],head_out[1],head_out[0]),(10,20),cv2.FONT_HERSHEY_COMPLEX,0.25,(255,0,0),1)
if 'ge' in flags:
x,y,w=(int(gaze_vector[1]*12),int(gaze_vector[0]*12),130)
left =cv2.line(left_eye.copy(), (x-w, y-w), (x+w, y+w), (255,0,0), 2)
cv2.line(left, (x-w, y+w), (x+w, y-w), (255,0,0), 2)
right = cv2.line(right_eye.copy(), (x-w, y-w), (x+w, y+w), (255,0,0), 2)
cv2.line(right, (x-w, y+w), (x+w, y-w), (255,0,0), 2)
cropped_image[eye[0][1]:eye[0][3],eye[0][0]:eye[0][2]] = left
cropped_image[eye[1][1]:eye[1][3],eye[1][0]:eye[1][2]] = right
cv2.imshow("visualisation_frame",cv2.resize(preview_frame,(300,300)))
if frame_count%3==0:
mc.move(mouse_coords[0],mouse_coords[1])
logger.error("video ended...")
cv2.destroyAllWindows()
inputfeed.close()
def infer_on_stream(args):
models = None
# Check selected precision model
if "FP32" in args.precision:
models = select_precision(args.precision)
if "FP16" in args.precision:
models = select_precision(args.precision)
if "INT8" in args.precision:
models = select_precision(args.precision)
# Get Input
input_feeder = InputFeeder(args.input_type, args.input_file)
input_feeder.load_data()
# Load face detection model
face = FaceDetection(model_name=models[0],
device=args.device,
extensions=args.cpu_extension)
face.load_model()
# Load head pose model
head = HeadPoseEstimation(model_name=models[1],
device=args.device,
extensions=args.cpu_extension)
head.load_model()
# Load facial landmark model
landmark = FacialLandmarkDetection(model_name=models[2],
device=args.device,
extensions=args.cpu_extension)
landmark.load_model()
# Load gaze estimation model
gaze = GazeEstimation(model_name=models[3],
device=args.device,
extensions=args.cpu_extension)
gaze.load_model()
# Initalize mouse controller
mouse = MouseController('high', 'fast')
for frame in input_feeder.next_batch():
# Break if number of next frame less then number of batch
if frame is None:
break
# Estimate face region
output_frame, cropped_face, box_coord = face.predict(frame)
# Estimate head pose position
head_pose = head.predict(cropped_face)
head_pose = np.array(head_pose)
# Estimate eyes landmark coordinates
lr_eyes = landmark.predict(cropped_face)
eyes = []
# Calculate eye image region
for coord in lr_eyes:
x = int(coord[0] + box_coord[0])
y = int(coord[1] + box_coord[1])
cv2.circle(output_frame, (x, y), 5, (255, 0, 0), -1)
eye_box, cropped_eye = eyes_crop(output_frame, x, y, 40)
cv2.rectangle(output_frame, eye_box[0], eye_box[1], (255, 0, 0), 1)
eyes.append(cropped_eye)
# Estimate gaze direction
gaze_coords = gaze.predict(eyes[0], eyes[1], head_pose)
# Move the mouse cursor
mouse.move(gaze_coords[0], gaze_coords[1])
if "True" in args.visualize:
cv2.imshow('Capture', output_frame)
if cv2.waitKey(30) & 0xFF == ord('q'):
break
input_feeder.close()
if "True" in args.visualize:
cv2.destroyAllWindows()
def main():
"""
Load the network and parse the output.
:return: None
"""
global POSE_CHECKED
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = args_parser().parse_args()
logger = log.getLogger()
if args.input == 'cam':
input_stream = 0
else:
input_stream = args.input
assert os.path.isfile(args.input), "Specified input file doesn't exist"
cap = cv2.VideoCapture(input_stream)
initial_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
initial_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
video_len = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
fps = int(cap.get(cv2.CAP_PROP_FPS))
out = cv2.VideoWriter(os.path.join(args.output_dir, "output.mp4"),
cv2.VideoWriter_fourcc(*"MP4V"), fps,
(initial_w, initial_h), True)
if args.write_intermediate == 'yes':
out_fm = cv2.VideoWriter(
os.path.join(args.output_dir, "output_fm.mp4"),
cv2.VideoWriter_fourcc(*"MP4V"), fps, (initial_w, initial_h), True)
out_lm = cv2.VideoWriter(
os.path.join(args.output_dir, "output_lm.mp4"),
cv2.VideoWriter_fourcc(*"MP4V"), fps, (initial_w, initial_h), True)
out_pm = cv2.VideoWriter(
os.path.join(args.output_dir, "output_pm.mp4"),
cv2.VideoWriter_fourcc(*"MP4V"), fps, (initial_w, initial_h), True)
out_gm = cv2.VideoWriter(
os.path.join(args.output_dir, "output_gm.mp4"),
cv2.VideoWriter_fourcc(*"MP4V"), fps, (initial_w, initial_h), True)
frame_count = 0
job_id = 1
infer_time_start = time.time()
if input_stream:
cap.open(args.input)
# Adjust DELAY to match the number of FPS of the video file
if not cap.isOpened():
logger.error("ERROR! Unable to open video source")
return
if args.mode == 'sync':
async_mode = False
else:
async_mode = True
# Initialise the class
if args.cpu_extension:
face_det = FaceDetection(args.facemodel,
args.confidence,
extensions=args.cpu_extension,
async_mode=async_mode)
pose_det = HeadPoseEstimation(args.posemodel,
args.confidence,
extensions=args.cpu_extension,
async_mode=async_mode)
land_det = FaceLandmarksDetection(args.landmarksmodel,
args.confidence,
extensions=args.cpu_extension,
async_mode=async_mode)
gaze_est = GazeEstimation(args.gazemodel,
args.confidence,
extensions=args.cpu_extension,
async_mode=async_mode)
else:
face_det = FaceDetection(args.facemodel,
args.confidence,
async_mode=async_mode)
pose_det = HeadPoseEstimation(args.posemodel,
args.confidence,
async_mode=async_mode)
land_det = FaceLandmarksDetection(args.landmarksmodel,
args.confidence,
async_mode=async_mode)
gaze_est = GazeEstimation(args.gazemodel,
args.confidence,
async_mode=async_mode)
# infer_network_pose = Network()
# Load the network to IE plugin to get shape of input layer
face_det.load_model()
pose_det.load_model()
land_det.load_model()
gaze_est.load_model()
model_load_time = time.time() - infer_time_start
print("All models are loaded successfully")
try:
pass
except Exception as e:
print("Could not run Inference: ", e)
while cap.isOpened():
ret, frame = cap.read()
if not ret:
print("checkpoint *BREAKING")
break
frame_count += 1
looking = 0
POSE_CHECKED = False
if frame is None:
log.error("checkpoint ERROR! blank FRAME grabbed")
break
initial_w = int(cap.get(3))
initial_h = int(cap.get(4))
# Start asynchronous inference for specified request
inf_start_fd = time.time()
# Results of the output layer of the network
coords, frame = face_det.predict(frame)
if args.write_intermediate == 'yes':
out_fm.write(frame)
det_time_fd = time.time() - inf_start_fd
if len(coords) > 0:
[xmin, ymin, xmax,
ymax] = coords[0] # use only the first detected face
head_pose = frame[ymin:ymax, xmin:xmax]
inf_start_hp = time.time()
is_looking, pose_angles = pose_det.predict(head_pose)
if args.write_intermediate == 'yes':
p = "Pose Angles {}, is Looking? {}".format(
pose_angles, is_looking)
cv2.putText(frame, p, (50, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5,
(255, 0, 0), 1)
out_pm.write(frame)
if is_looking:
det_time_hp = time.time() - inf_start_hp
POSE_CHECKED = True
inf_start_lm = time.time()
coords, f = land_det.predict(head_pose)
frame[ymin:ymax, xmin:xmax] = f
if args.write_intermediate == "yes":
out_lm.write(frame)
det_time_lm = time.time() - inf_start_lm
[[xlmin, ylmin, xlmax, ylmax], [xrmin, yrmin, xrmax,
yrmax]] = coords
left_eye_image = f[ylmin:ylmax, xlmin:xlmax]
right_eye_image = f[yrmin:yrmax, xrmin:xrmax]
output, gaze_vector = gaze_est.predict(left_eye_image,
right_eye_image,
pose_angles)
if args.write_intermediate == 'yes':
p = "Gaze Vector {}".format(gaze_vector)
cv2.putText(frame, p, (50, 15), cv2.FONT_HERSHEY_COMPLEX,
0.5, (255, 0, 0), 1)
fl = draw_gaze(left_eye_image, gaze_vector)
fr = draw_gaze(right_eye_image, gaze_vector)
f[ylmin:ylmax, xlmin:xlmax] = fl
f[yrmin:yrmax, xrmin:xrmax] = fr
# cv2.arrowedLine(f, (xlmin, ylmin), (xrmin, yrmin), (0,0,255), 5)
out_gm.write(frame)
# Draw performance stats
inf_time_message = "Face Inference time: {:.3f} ms.".format(
det_time_fd * 1000)
#
if POSE_CHECKED:
cv2.putText(
frame, "Head pose Inference time: {:.3f} ms.".format(
det_time_hp * 1000), (0, 35), cv2.FONT_HERSHEY_SIMPLEX,
0.5, (0, 255, 0), 1)
cv2.putText(frame, inf_time_message, (0, 15),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 0, 0), 1)
out.write(frame)
if frame_count % 10 == 0:
print("Inference time = ", int(time.time() - infer_time_start))
print('Frame count {} and vidoe len {}'.format(
frame_count, video_len))
if args.output_dir:
total_time = time.time() - infer_time_start
with open(os.path.join(args.output_dir, 'stats.txt'), 'w') as f:
f.write(str(round(total_time, 1)) + '\n')
f.write(str(frame_count) + '\n')
if args.output_dir:
with open(os.path.join(args.output_dir, 'stats.txt'), 'a') as f:
f.write(str(round(model_load_time)) + '\n')
# Clean all models
face_det.clean()
pose_det.clean()
land_det.clean()
gaze_est.clean()
# release cv2 cap
cap.release()
cv2.destroyAllWindows()
# release all out writer
out.release()
if args.write_intermediate == 'yes':
out_fm.release()
out_pm.release()
out_lm.release()
out_gm.release()
def main():
#Building the arguments
args = build_parser().parse_args()
previewFlag = args.previewFlags
log = logging.getLogger()
input_path = args.input
inputFeed = None
if input_path.lower() == 'cam':
inputFeed = InputFeeder('cam')
else:
if not os.path.isfile(input_path):
log.error("Unable to find the input file specified.")
exit(1)
inputFeed = InputFeeder('video', input_path)
#Creating Model paths
model_path = {
'FaceDetectionModel': args.facedetectionmodel,
'FacialLandmarksDetectionModel': args.faciallandmarkmodel,
'GazeEstimationModel': args.gazeestimationmodel,
'HeadPoseEstimationModel': args.headposemodel
}
for fnameKey in model_path.keys():
if not os.path.isfile(model_path[fnameKey]):
log.error('Unable to find the specified ' + fnameKey +
'binary file(.xml)')
exit(1)
#Creating Model Instances
fd = FaceDetection(model_path['FaceDetectionModel'], args.device,
args.cpu_extension)
flm = FacialLandmarkDetection(model_path['FacialLandmarksDetectionModel'],
args.device, args.cpu_extension)
gm = GazeEstimation(model_path['GazeEstimationModel'], args.device,
args.cpu_extension)
hpe = Head_Pose_estimation(model_path['HeadPoseEstimationModel'],
args.device, args.cpu_extension)
m_control = MouseController('medium', 'fast')
#Loading data
inputFeed.load_data()
fd.load_model()
flm.load_model()
hpe.load_model()
gm.load_model()
frame_count = 0
for ret, frame in inputFeed.next_batch():
if not ret:
break
frame_count += 1
if frame_count % 10 == 0:
cv2.imshow('Original Video', cv2.resize(frame, (500, 500)))
key = cv2.waitKey(60)
coords, img = fd.predict(frame, args.prob_threshold)
if type(img) == int:
log.error("No face detected")
if key == 27:
break
continue
hpout = hpe.predict(img)
left_eye, right_eye, eye_coord = flm.predict(img)
mouse_coord, gaze_vec = gm.predict(left_eye, right_eye, hpout)
if (not len(previewFlag) == 0):
preview_img = img
if 'fd' in previewFlag:
preview_img = img
if 'fld' in previewFlag:
start_l = (eye_coord[0][0] - 10, eye_coord[0][1] - 10)
end_l = (eye_coord[0][2] + 10, eye_coord[0][3] + 10)
start_r = (eye_coord[1][0] - 10, eye_coord[1][1] - 10)
end_r = (eye_coord[1][2] + 10, eye_coord[1][3] + 10)
cv2.rectangle(img, start_l, end_l, (0, 255, 0), 2)
cv2.rectangle(img, start_r, end_r, (0, 255, 0), 2)
if 'hp' in previewFlag:
cv2.putText(
preview_img,
"Pose Angles: yaw:{:.2f} | pitch:{:.2f} | roll:{:.2f}".
format(hpout[0], hpout[1], hpout[2]), (10, 20),
cv2.FONT_HERSHEY_COMPLEX, 0.25, (255, 255, 255), 1)
if 'ge' in previewFlag:
x, y, w = int(gaze_vec[0] * 12), int(gaze_vec[1] * 12), 160
lefteye = cv2.line(left_eye, (x - w, y - w), (x + w, y + w),
(100, 0, 255), 1)
cv2.line(lefteye, (x - w, y + w), (x + w, y - w),
(100, 0, 255), 1)
righteye = cv2.line(right_eye, (x - w, y - w), (x + w, y + w),
(100, 0, 255), 1)
cv2.line(righteye, (x - w, y + w), (x + w, y - w),
(100, 0, 255), 1)
img[eye_coord[0][1]:eye_coord[0][3],
eye_coord[0][0]:eye_coord[0][2]] = lefteye
img[eye_coord[1][1]:eye_coord[1][3],
eye_coord[1][0]:eye_coord[1][2]] = righteye
cv2.imshow("Detections", cv2.resize(preview_img, (500, 500)))
if frame_count % 10 == 0:
m_control.move(mouse_coord[0], mouse_coord[1])
if key == 27:
break
log.error("Videostream Completed")
cv2.destroyAllWindows()
inputFeed.close()
def main(args):
# enable logging for the function
logger = logging.getLogger(__name__)
# grab the parsed parameters
faceModel = args.m_f
facial_LandmarksModel = args.m_l
headPoseEstimationModel = args.m_h
GazeEstimationModel = args.m_g
device = args.d
inputFile = args.i
output_path = args.o_p
modelArchitecture = args.modelAr
visualization_flag = args.vf
# initialize feed
single_image_format = ['jpg', 'tif', 'png', 'jpeg', 'bmp']
if inputFile.split(".")[-1].lower() in single_image_format:
feed = InputFeeder('image', inputFile)
elif args.i == 'cam':
feed = InputFeeder('cam')
else:
feed = InputFeeder('video', inputFile)
##Load model time face detection
faceStart_model_load_time = time.time()
faceDetection = FaceDetection(faceModel, device)
faceModelView = faceDetection.load_model()
faceDetection.check_model()
total_facemodel_load_time = time.time() - faceStart_model_load_time
##Load model time headpose estimatiom
heaadposeStart_model_load_time = time.time()
headPose = headPoseEstimation(headPoseEstimationModel, device)
headPoseModelView = headPose.load_model()
headPose.check_model()
heaadposeTotal_model_load_time = time.time(
) - heaadposeStart_model_load_time
##Load model time face_landmarks estimation
face_landmarksStart_model_load_time = time.time()
face_landmarks = Face_landmarks(facial_LandmarksModel, device)
faceLandmarksModelView = face_landmarks.load_model()
face_landmarks.check_model()
face_landmarksTotal_model_load_time = time.time(
) - face_landmarksStart_model_load_time
##Load model time face_landmarks estimation
GazeEstimationStart_model_load_time = time.time()
GazeEstimation = Gaze_Estimation(GazeEstimationModel, device)
GazeModelView = GazeEstimation.load_model()
GazeEstimation.check_model()
GazeEstimationTotal_model_load_time = time.time(
) - GazeEstimationStart_model_load_time
if modelArchitecture == 'yes':
print("The model architecture of gaze mode is ", GazeModelView)
print("model architecture for landmarks is", faceLandmarksModelView)
print("model architecture for headpose is", headPoseModelView)
print("model architecture for face is", faceModelView)
# count the number of frames
frameCount = 0
input_feeder = InputFeeder('video', inputFile)
w, h = feed.load_data()
for _, frame in feed.next_batch():
if not _:
break
frameCount += 1
key = cv2.waitKey(60)
start_imageface_inference_time = time.time()
imageface = faceDetection.predict(frame, w, h)
imageface_inference_time = time.time() - start_imageface_inference_time
if 'm_f' in visualization_flag:
cv2.imshow('cropped face', imageface)
if type(imageface) == int:
logger.info("no face detected")
if key == 27:
break
continue
start_imagePose_inference_time = time.time()
imageAngles, imagePose = headPose.predict(imageface)
imagePose_inference_time = time.time() - start_imagePose_inference_time
if 'm_h' in visualization_flag:
cv2.imshow('Head Pose Angles', imagePose)
start_landmarkImage_inference_time = time.time()
leftEye, rightEye, landmarkImage = face_landmarks.predict(imageface)
landmarkImage_inference_time = time.time(
) - start_landmarkImage_inference_time
if leftEye.any() == None or rightEye.any() == None:
logger.info(
"image probably too dark or eyes covered, hence could not detect landmarks"
)
continue
if 'm_l' in visualization_flag:
cv2.imshow('Face output', landmarkImage)
start_GazeEstimation_inference_time = time.time()
x, y = GazeEstimation.predict(leftEye, rightEye, imageAngles)
GazeEstimation_inference_time = time.time(
) - start_GazeEstimation_inference_time
if 'm_g' in visualization_flag:
# cv2.putText(landmarkedFace, "Estimated x:{:.2f} | Estimated y:{:.2f}".format(x,y), (10,20), cv2.FONT_HERSHEY_COMPLEX, 0.25, (0,255,0),1)
cv2.imshow('Gaze Estimation', landmarkImage)
mouseVector = MouseController('medium', 'fast')
if frameCount % 5 == 0:
mouseVector.move(x, y)
if key == 27:
break
if imageface_inference_time != 0 and landmarkImage_inference_time != 0 and imagePose_inference_time != 0 and GazeEstimation_inference_time != 0:
fps_face = 1 / imageface_inference_time
fps_landmark = 1 / landmarkImage_inference_time
fps_headpose = 1 / imagePose_inference_time
fps_gaze = 1 / GazeEstimation_inference_time
with open(
os.path.join(output_path, device, 'face',
'face_stats.txt'), 'w') as f:
f.write(str(imageface_inference_time) + '\n')
f.write(str(fps_face) + '\n')
f.write(str(total_facemodel_load_time) + '\n')
with open(
os.path.join(output_path, device, 'landmark',
'landmark_stats.txt'), 'w') as f:
f.write(str(landmarkImage_inference_time) + '\n')
f.write(str(fps_landmark) + '\n')
f.write(str(face_landmarksTotal_model_load_time) + '\n')
with open(
os.path.join(output_path, device, 'headpose',
'headpose_stats.txt'), 'w') as f:
f.write(str(imagePose_inference_time) + '\n')
f.write(str(fps_headpose) + '\n')
f.write(str(heaadposeTotal_model_load_time) + '\n')
with open(
os.path.join(output_path, device, 'gaze',
'gaze_stats.txt'), 'w') as f:
f.write(str(GazeEstimation_inference_time) + '\n')
f.write(str(fps_gaze) + '\n')
f.write(str(GazeEstimationTotal_model_load_time) + '\n')
logger.info("The End")
VIS = visualize(output_path, device)
VIS.visualize1()
VIS.visualize2()
VIS.visualize3()
cv2.destroyAllWindows()
feed.close()
def main():
args = argparser().parse_args()
device = args.device
input_feed = args.input
log = logging.getLogger()
model_paths = {
'facedet': args.face_detection_model + 'xml',
'faceldmdet': args.landmark_detection_model + 'xml',
'headpose': args.pose_estimation_model + 'xml',
'gaze': args.gaze_estimation_model + 'xml'
}
for mp in model_paths.keys():
if not os.path.isfile(model_paths[mp]):
print(model_paths[mp])
print('Recheck file path and try again')
log.error("Not a file")
raise FileNotFoundError
if input_feed == 'cam':
feed = InputFeeder(input_type='cam')
elif not os.path.isfile(input_feed):
print('Recheck file path and try again')
log.error("Unable to find specified video file")
raise FileNotFoundError
else:
feed = InputFeeder(input_type='video', input_file=input_feed)
facedet = FaceDetection(args.face_detection_model, args.device,
args.extensions, args.async_mode)
faceldmdet = FacialLandmarksDetection(args.landmark_detection_model,
args.device, args.extensions,
args.async_mode)
headpose = HeadPose(args.pose_estimation_model, args.device,
args.extensions, args.async_mode)
gaze = GazeEstimation(args.gaze_estimation_model, args.device,
args.extensions, args.async_mode)
try:
log.info('Loading models...')
facedet.load_model()
faceldmdet.load_model()
headpose.load_model()
gaze.load_model()
feed.load_data()
log.info('Models loaded successfully!')
except:
log.error('One or more of the models failed to load..')
exit(1)
log.info('Initializing mouse controller')
mouse = MouseController(precision='medium', speed='fast')
for batch in feed.next_batch():
face = facedet.predict(batch)
eyes, eye_coords = faceldmdet.predict(face)
pose = headpose.predict(face)
point = gaze.predict(pose, eyes)
#print('Gaze values = ', point[0], point[1])
log.info('All inference complete')
#print('view_inter = ', args.view_intermediate)
if args.input == 'cam':
point[0] = -point[0]
mouse.move(point[0], point[1])
if args.view_intermediate == True:
visualize(pose, face, eye_coords, point)
def infer_on_stream(args):
"""
Initialize the inference network, stream video to network,
and output stats and video.
:param args: Command line arguments parsed by `build_argparser()`
:return: None
"""
try:
logging.basicConfig(level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[
logging.FileHandler("gaze-app.log"),
logging.StreamHandler()
])
# Initialise the class
mc = MouseController("low", "fast")
#mc.move(100,100)
fdnet = FaceDetection(args.fdmodel)
lmnet = FacialLandmarks(args.lmmodel)
hpnet = HeadPoseEstimation(args.hpmodel)
genet = GazeEstimation(args.gemodel)
### Load the model through ###
logging.info("============== Models Load time ===============")
start_time = time.time()
fdnet.load_model()
logging.info("Face Detection Model: {:.1f}ms".format(
1000 * (time.time() - start_time)))
fdnet.check_model()
logging.info("Face Detection estimation layers loaded correctly")
start_time = time.time()
lmnet.load_model()
logging.info("Facial Landmarks Detection Model: {:.1f}ms".format(
1000 * (time.time() - start_time)))
lmnet.check_model()
logging.info("Facial Landmarks estimation layers loaded correctly")
start_time = time.time()
hpnet.load_model()
logging.info("Headpose Estimation Model: {:.1f}ms".format(
1000 * (time.time() - start_time)))
hpnet.check_model()
logging.info("Head pose estimation layers loaded correctly")
start_time = time.time()
genet.load_model()
logging.info("Gaze Estimation Model: {:.1f}ms".format(
1000 * (time.time() - start_time)))
genet.check_model()
logging.info("Gaze estimation layers loaded correctly")
logging.info("============== End =====================")
# Get and open video capture
feeder = InputFeeder('video', args.input)
feeder.load_data()
# FPS = feeder.get_fps()
# Grab the shape of the input
# width = feeder.get_width()
# height = feeder.get_height()
# init scene variables
frame_count = 0
### Loop until stream is over ###
fd_infertime = 0
lm_infertime = 0
hp_infertime = 0
ge_infertime = 0
while True:
# Read the next frame
try:
frame = next(feeder.next_batch())
except StopIteration:
break
key_pressed = cv2.waitKey(60)
frame_count += 1
#print(int((frame_count) % int(FPS)))
# face detection
fd_process_time = time.time()
p_frame = fdnet.preprocess_input(frame)
start_time = time.time()
fnoutput = fdnet.predict(p_frame)
fd_infertime += time.time() - start_time
out_frame, fboxes = fdnet.preprocess_output(
fnoutput, frame, args.print)
logging.info(
"Face Detection Model processing time : {:.1f}ms".format(
1000 * (time.time() - fd_process_time)))
#for each face
for fbox in fboxes:
# fbox = (xmin,ymin,xmax,ymax)
# get face landmarks
# crop face from frame
face = frame[fbox[1]:fbox[3], fbox[0]:fbox[2]]
lm_process_time = time.time()
p_frame = lmnet.preprocess_input(face)
start_time = time.time()
lmoutput = lmnet.predict(p_frame)
lm_infertime += time.time() - start_time
out_frame, left_eye_point, right_eye_point = lmnet.preprocess_output(
lmoutput, fbox, out_frame, args.print)
logging.info(
"Landmarks model processing time : {:.1f}ms".format(
1000 * (time.time() - lm_process_time)))
# get head pose estimation
hp_process_time = time.time()
p_frame = hpnet.preprocess_input(face)
start_time = time.time()
hpoutput = hpnet.predict(p_frame)
hp_infertime += time.time() - start_time
out_frame, headpose_angels = hpnet.preprocess_output(
hpoutput, out_frame, face, fbox, args.print)
logging.info(
"Headpose estimation model processing time : {:.1f}ms".
format(1000 * (time.time() - hp_process_time)))
# get gaze estimation
gaze_process_time = time.time()
out_frame, left_eye, right_eye = genet.preprocess_input(
out_frame, face, left_eye_point, right_eye_point,
args.print)
start_time = time.time()
geoutput = genet.predict(left_eye, right_eye, headpose_angels)
ge_infertime += time.time() - start_time
out_frame, gazevector = genet.preprocess_output(
geoutput, out_frame, fbox, left_eye_point, right_eye_point,
args.print)
logging.info(
"Gaze estimation model processing time : {:.1f}ms".format(
1000 * (time.time() - gaze_process_time)))
if (not args.no_video):
cv2.imshow('im', out_frame)
if (not args.no_move):
mc.move(gazevector[0], gazevector[1])
#consider only first detected face in the frame
break
# Break if escape key pressed
if key_pressed == 27:
break
#logging inference times
if (frame_count > 0):
logging.info(
"============== Models Inference time ===============")
logging.info("Face Detection:{:.1f}ms".format(1000 * fd_infertime /
frame_count))
logging.info("Facial Landmarks Detection:{:.1f}ms".format(
1000 * lm_infertime / frame_count))
logging.info("Headpose Estimation:{:.1f}ms".format(
1000 * hp_infertime / frame_count))
logging.info("Gaze Estimation:{:.1f}ms".format(
1000 * ge_infertime / frame_count))
logging.info("============== End ===============================")
# Release the capture and destroy any OpenCV windows
feeder.close()
cv2.destroyAllWindows()
except Exception as ex:
logging.exception("Error in inference:" + str(ex))
def main():
# command line args
args = build_argparser().parse_args()
input_file_path = args.input
log_object = log.getLogger()
oneneneflags = args.visualization_flag
# Initialise the classes
fd_object = FaceDetection(model_name=args.face_detection_model,
device=args.device,
threshold=args.prob_threshold,
extensions=args.cpu_extension)
fl_object = FacialLandmarkDetection(model_name=args.facial_landmarks_model,
device=args.device,
extensions=args.cpu_extension)
hp_object = HeadPoseEstimation(model_name=args.head_pose_model,
device=args.device,
extensions=args.cpu_extension)
ge_object = GazeEstimation(model_name=args.gaze_estimation_model,
device=args.device,
extensions=args.cpu_extension)
mouse_controller_object = MouseController('low', 'fast')
### Loading the models ###
log_object.error(
"=================== Models Load Time ====================")
start_time = time.time()
fd_object.load_model()
log_object.error("Face detection model loaded in {:.3f} ms".format(
(time.time() - start_time) * 1000))
fl_start = time.time()
fl_object.load_model()
log_object.error(
"Facial landmarks detection model loaded in {:.3f} ms".format(
(time.time() - fl_start) * 1000))
hp_start = time.time()
hp_object.load_model()
log_object.error("Head pose estimation model loaded in {:.3f} ms".format(
(time.time() - hp_start) * 1000))
ge_start = time.time()
ge_object.load_model()
log_object.error("Gaze estimation model loaded in {:.3f} ms".format(
(time.time() - ge_start) * 1000))
total_time = time.time() - start_time
log_object.error(
"=================== Models loaded successfully ===================")
log_object.error("Total loading time is {:.3f} ms".format(total_time *
1000))
counter = 0
infer_start = time.time()
log_object.error(
"=================== Start inferencing on input video ===================="
)
if input_file_path == "CAM":
input_feeder = InputFeeder("cam")
else:
if not os.path.isfile(input_file_path):
exit(1)
input_feeder = InputFeeder("video", input_file_path)
log_object.error("Input feeders are loaded")
input_feeder.load_data()
for frame in input_feeder.next_batch():
# if not flag:
# break
pressed_key = cv2.waitKey(60)
counter += 1
face_coordinates, face_image = fd_object.predict(frame.copy())
if face_coordinates == 0:
continue
hp_output = hp_object.predict(face_image)
left_eye_image, right_eye_image, eye_coord = fl_object.predict(
face_image)
mouse_coordinate, gaze_vector = ge_object.predict(
left_eye_image, right_eye_image, hp_output)
if len(oneneneflags) != 0:
preview_window = frame.copy()
if 'fd' in oneneneflags:
if len(oneneneflags) != 1:
preview_window = face_image
else:
cv2.rectangle(preview_window,
(face_coordinates[0], face_coordinates[1]),
(face_coordinates[2], face_coordinates[3]),
(0, 150, 0), 3)
if 'fl' in oneneneflags:
if not 'fd' in oneneneflags:
preview_window = face_image.copy()
cv2.rectangle(preview_window,
(eye_coord[0][0], eye_coord[0][1]),
(eye_coord[0][2], eye_coord[0][3]),
(150, 0, 150))
cv2.rectangle(preview_window,
(eye_coord[1][0], eye_coord[1][1]),
(eye_coord[1][2], eye_coord[1][3]),
(150, 0, 150))
if 'hp' in oneneneflags:
cv2.putText(
preview_window,
"yaw:{:.1f} | pitch:{:.1f} | roll:{:.1f}".format(
hp_output[0], hp_output[1], hp_output[2]), (20, 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 0), 1)
if 'ge' in oneneneflags:
yaw = hp_output[0]
pitch = hp_output[1]
roll = hp_output[2]
focal_length = 950.0
scale = 50
center_of_face = (face_image.shape[1] / 2,
face_image.shape[0] / 2, 0)
if 'fd' in oneneneflags or 'fl' in oneneneflags:
draw_axes(preview_window, center_of_face, yaw, pitch, roll,
scale, focal_length)
else:
draw_axes(frame, center_of_face, yaw, pitch, roll, scale,
focal_length)
if len(oneneneflags) != 0:
img_hor = np.hstack((cv2.resize(frame, (500, 500)),
cv2.resize(preview_window, (500, 500))))
else:
img_hor = cv2.resize(frame, (500, 500))
cv2.imshow('Visualization', img_hor)
mouse_controller_object.move(mouse_coordinate[0], mouse_coordinate[1])
if pressed_key == 27:
log_object.error("exit key is pressed..")
break
infer_time = round(time.time() - infer_start, 1)
fps = int(counter) / infer_time
log_object.error("counter {} seconds".format(counter))
log_object.error("total inference time {} seconds".format(infer_time))
log_object.error("fps {} frame/second".format(fps))
log_object.error("Video session has ended")
with open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'stats.txt'), 'w') as f:
f.write(str(infer_time) + '\n')
f.write(str(fps) + '\n')
f.write(str(total_time) + '\n')
input_feeder.close()
cv2.destroyAllWindows()
def main():
args = build_argparser().parse_args()
device_name = args.device
prob_threshold = args.prob_threshold
logger_object = log.getLogger()
# Initialize variables with the input arguments
model_path_dict = {
'FaceDetectionModel': args.faceDetectionModel,
'FacialLandmarkModel': args.facialLandmarksModel,
'HeadPoseEstimationModel': args.headPoseEstimationModel,
'GazeEstimationModel': args.gazeEstimationModel
}
# Instantiate model
face_model = FaceDetection(model_path_dict['FaceDetectionModel'], device_name, threshold=prob_threshold)
landmark_model = FacialLandmarksDetection(model_path_dict['FacialLandmarkModel'], device_name,
threshold=prob_threshold)
head_pose_model = HeadPoseEstimation(model_path_dict['HeadPoseEstimationModel'], device_name,
threshold=prob_threshold)
gaze_model = GazeEstimation(model_path_dict['GazeEstimationModel'], device_name, threshold=prob_threshold)
mouse_controller = MouseController('medium', 'fast')
# Load Models and get time
start_time = time.time()
face_model.load_model()
logger_object.error("Face detection model loaded: time: {:.3f} ms".format((time.time() - start_time) * 1000))
first_mark = time.time()
landmark_model.load_model()
logger_object.error(
"Facial landmarks detection model loaded: time: {:.3f} ms".format((time.time() - first_mark) * 1000))
second_mark = time.time()
head_pose_model.load_model()
logger_object.error("Head pose estimation model loaded: time: {:.3f} ms".format((time.time() - second_mark) * 1000))
third_mark = time.time()
gaze_model.load_model()
logger_object.error("Gaze estimation model loaded: time: {:.3f} ms".format((time.time() - third_mark) * 1000))
load_total_time = time.time() - start_time
logger_object.error("Total loading time: time: {:.3f} ms".format(load_total_time * 1000))
logger_object.error("All models are loaded successfully..")
# Check extention of these unsupported layers
face_model.check_model()
landmark_model.check_model()
head_pose_model.check_model()
gaze_model.check_model()
preview_flags = args.previewFlags
input_filename = args.input
output_path = args.output_path
prob_threshold = args.prob_threshold
if input_filename.lower() == 'cam':
input_feeder = InputFeeder(input_type='cam')
else:
if not os.path.isfile(input_filename):
logger_object.error("Unable to find specified video file")
exit(1)
input_feeder = InputFeeder(input_type='video', input_file=input_filename)
for model_path in list(model_path_dict.values()):
if not os.path.isfile(model_path):
logger_object.error("Unable to find specified model file" + str(model_path))
exit(1)
input_feeder.load_data()
width = int(input_feeder.cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(input_feeder.cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps = int(input_feeder.cap.get(cv2.CAP_PROP_FPS))
out_video = cv2.VideoWriter(os.path.join('output_video.mp4'), cv2.VideoWriter_fourcc(*'avc1'), fps,
(width, height), True)
frame_counter = 0
start_inf_time = time.time()
for ret, frame in input_feeder.next_batch():
if not ret:
break
frame_counter += 1
key = cv2.waitKey(60)
try:
cropped_image, face_cords = face_model.predict(frame, prob_threshold)
if type(cropped_image) == int:
print("Unable to detect the face")
if key == 27:
break
continue
left_eye, right_eye, eye_cords = landmark_model.predict(cropped_image)
pose_output = head_pose_model.predict(cropped_image)
x, y, z = gaze_model.predict(left_eye, right_eye, pose_output, cropped_image, eye_cords)
mouse_controller.move(x, y)
except Exception as e:
print(str(e) + " for frame " + str(frame_counter))
continue
image = cv2.resize(frame, (width, height))
if not len(preview_flags) == 0:
preview_frame = frame.copy()
if 'fd' in preview_flags:
if len(preview_flags) != 1:
preview_frame = cropped_image
cv2.rectangle(frame, (face_cords[0], face_cords[1]), (face_cords[2], face_cords[3]), (0, 0, 255), 3)
if 'hp' in preview_flags:
cv2.putText(
frame,
"Pose Angles: yaw= {:.2f} , pitch= {:.2f} , roll= {:.2f}".format(
pose_output[0], pose_output[1], pose_output[2]),
(20, 40),
cv2.FONT_HERSHEY_DUPLEX,
1, (255, 0, 0), 3)
if 'ge' in preview_flags:
cv2.putText(
frame,
"Gaze vector: x= {:.2f} , y= {:.2f} , z= {:.2f}".format(
x, y, z),
(15, 100),
cv2.FONT_HERSHEY_COMPLEX,
1, (0, 255, 0), 3)
image = np.hstack((cv2.resize(frame, (500, 500)), cv2.resize(preview_frame, (500, 500))))
cv2.imshow('preview', image)
out_video.write(image)
if frame_counter % 5 == 0:
mouse_controller.move(x, y)
if key == 27:
break
inference_time = round(time.time() - start_inf_time, 1)
fps = int(frame_counter) / inference_time
logger_object.error("counter {} seconds".format(frame_counter))
logger_object.error("total inference time {} seconds".format(inference_time))
logger_object.error("fps {} frame/second".format(fps))
with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'stats.txt'), 'w') as f:
f.write('inference time : ' + str(inference_time) + '\n')
f.write('fps: ' + str(fps) + '\n')
f.write('Models Loading: '+ str(load_total_time) + '\n')
logger_object.error('Video stream ended')
cv2.destroyAllWindows()
input_feeder.close()
def infer(args, logging_enabled):
"""
run inference on input video, display/save output video
"""
face_detection = FaceDetection(args.face_detection)
facial_landmark_detection = FacialLandmarkDetection(
args.facial_landmark_detection)
gaze_estimation = GazeEstimation(args.gaze_estimation)
head_pose_estimation = HeadPoseEstimation(args.head_pose_estimation)
load_start = now()
face_detection.load_model()
fl_start = now()
facial_landmark_detection.load_model()
ge_start = now()
gaze_estimation.load_model()
hp_start = now()
head_pose_estimation.load_model()
log_model_load_times(logging_enabled, load_start, fl_start, ge_start,
hp_start)
feeder = InputFeeder("video", args.input)
feeder.load_data()
frame_count, fd_time, fl_time, ge_time, hp_time = [0] * 5
while 1:
key = cv2.waitKey(20)
try:
frame = next(feeder.next_batch())
except StopIteration:
break
frame_count += 1
fd_frame = face_detection.preprocess_input(frame)
inf_start = now()
fd_output = face_detection.predict(fd_frame)
fd_time += now() - inf_start
out_frame, faces = face_detection.preprocess_output(
fd_output, frame, args.overlay_inference,
args.probability_threshold)
detected_face = frame[faces[0][1]:faces[0][3], faces[0][0]:faces[0][2]]
fl_frame = facial_landmark_detection.preprocess_input(detected_face)
fl_start = now()
fl_output = facial_landmark_detection.predict(fl_frame)
fl_time += now() - fl_start
out_frame, l_coord, r_coord, = facial_landmark_detection.preprocess_output(
fl_output, faces[0], out_frame, args.overlay_inference)
hp_frame = head_pose_estimation.preprocess_input(detected_face)
hp_start = now()
hp_output = head_pose_estimation.predict(hp_frame)
hp_time += now() - hp_start
out_frame, head_pose = head_pose_estimation.preprocess_output(
hp_output, out_frame, detected_face, faces[0],
args.overlay_inference)
out_frame, l_eye, r_eye = gaze_estimation.preprocess_input(
out_frame, detected_face, l_coord, r_coord, args.overlay_inference)
ge_start = now()
ge_output = gaze_estimation.predict(head_pose, l_eye, r_eye)
ge_time += now() - ge_start
out_frame, g_vec = gaze_estimation.preprocess_output(
ge_output, out_frame, faces[0], l_coord, r_coord,
args.overlay_inference)
if args.video_window:
cv2.imshow(
"Computer-Human Interface Peripheral Signal Manipulation via AI Retina Tracking (CHIPSMART)",
out_frame,
)
if args.mouse_control and frame_count % 6 == 0:
mouse_control.move(g_vec[0], g_vec[1])
# Quit if user presses Esc or Q
if key in (27, 81):
user_quit(logging_enabled)
break
log_inference_times(logging_enabled, frame_count, fd_time, fl_time,
ge_time, hp_time)
feeder.close()
cv2.destroyAllWindows()
quit()
class MoveMouse:
'''
Main Class for the Mouse Controller app.
This is the class where all the models are stitched together to control the mouse pointer
'''
def __init__(self, args):
'''
This method instances variables for the Facial Landmarks Detection Model.
Args:
args = All arguments parsed by the arguments parser function
Return:
None
'''
init_start_time = time.time()
self.output_path = args.output_path
self.show_output = args.show_output
self.total_processing_time = 0
self.count_batch = 0
self.inference_speed = []
self.avg_inference_speed = 0
if args.all_devices != 'CPU':
args.face_device = args.all_devices
args.face_landmark_device = args.all_devices
args.head_pose_device = args.all_devices
args.gaze_device = args.all_devices
model_init_start = time.time()
self.face_model = FaceDetection(args.face_model, args.face_device,
args.face_device_ext,
args.face_prob_threshold)
self.landmarks_model = FacialLandmarksDetection(
args.face_landmark_model, args.face_landmark_device,
args.face_landmark_device_ext, args.face_landmark_prob_threshold)
self.head_pose_model = HeadPoseEstimation(
args.head_pose_model, args.head_pose_device,
args.head_pose_device_ext, args.head_pose_prob_threshold)
self.gaze_model = GazeEstimation(args.gaze_model, args.gaze_device,
args.gaze_device_ext,
args.gaze_prob_threshold)
self.model_init_time = time.time() - model_init_start
log.info('[ Main ] All required models initiallized')
self.mouse_control = MouseController(args.precision, args.speed)
log.info('[ Main ] Mouse controller successfully initialized')
self.input_feeder = InputFeeder(args.batch_size, args.input_type,
args.input_file)
log.info('[ Main ] Initialized input feeder')
model_load_start = time.time()
self.face_model.load_model()
self.landmarks_model.load_model()
self.head_pose_model.load_model()
self.gaze_model.load_model()
self.model_load_time = time.time() - model_load_start
self.app_init_time = time.time() - init_start_time
log.info('[ Main ] All moadels loaded to Inference Engine\n')
return None
def draw_face_box(self, frame, face_coords):
'''
Draws face's bounding box on the input frame
Args:
frame = Input frame from video or camera feed. It could also be an input image
Return:
frame = Frame with bounding box of faces drawn on it
'''
start_point = (face_coords[0][0], face_coords[0][1])
end_point = (face_coords[0][2], face_coords[0][3])
thickness = 5
color = (255, 86, 0)
frame = cv2.rectangle(frame, start_point, end_point, color, thickness)
return frame
def draw_eyes_boxes(self, frame, left_eye_coords, right_eye_coords):
'''
Draws face's bounding box on the input frame
Args:
frame = Input frame from video or camera feed. It could also be an input image
Return:
frame = Frame with bounding box of left and right eyes drawn on it
'''
left_eye_start_point = (left_eye_coords[0], left_eye_coords[1])
left_eye_end_point = (left_eye_coords[2], left_eye_coords[3])
right_eye_start_point = (right_eye_coords[0], right_eye_coords[1])
right_eye_end_point = (right_eye_coords[2], right_eye_coords[3])
thickness = 5
color = (0, 210, 0)
frame = cv2.rectangle(frame, left_eye_start_point, left_eye_end_point,
color, thickness)
frame = cv2.rectangle(frame, right_eye_start_point,
right_eye_end_point, color, thickness)
return frame
def draw_outputs(self, frame):
'''
Draws the inference outputs (bounding boxes of the face and both eyes and
the 3D head pose directions) of the four models onto the frames.
Args:
frame = Input frame from video or camera feed. It could also be an input image
Return:
frame = Frame with all inference outputs drawn on it
'''
frame = self.draw_face_box(frame, self.face_coords)
frame = self.draw_eyes_boxes(frame, self.left_eye_coords,
self.right_eye_coords)
frame_id = f'Batch id = {self.count_batch}'
avg_inference_speed = f'Avg. inference speed = {self.avg_inference_speed:.3f}fps'
total_processing_time = f'Total infer. time = {self.total_processing_time:.3f}s'
cv2.putText(frame, frame_id, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.45,
(255, 86, 0), 1)
cv2.putText(frame, avg_inference_speed, (15, 30),
cv2.FONT_HERSHEY_COMPLEX, 0.45, (255, 86, 0), 1)
cv2.putText(frame, total_processing_time, (15, 45),
cv2.FONT_HERSHEY_COMPLEX, 0.45, (255, 86, 0), 1)
return frame
def run_inference(self, frame):
'''
Performs inference on the input video or image by passing it through all four
models to get the desired coordinates for moving the mouse pointer.
Args:
frame = Input image, frame from video or camera feed
Return:
None
'''
self.input_feeder.load_data()
for frame in self.input_feeder.next_batch():
if self.input_feeder.frame_flag == True:
log.info('[ Main ] Started processing a new batch')
start_inference = time.time()
self.face_coords, self.face_crop = self.face_model.predict(
frame)
if self.face_coords == []:
log.info(
'[ Main ] No face detected.. Waiting for you to stare at the camera'
)
f.write('[ Error ] No face was detected')
else:
self.head_pose_angles = self.head_pose_model.predict(
self.face_crop)
self.left_eye_coords, self.left_eye_image, self.right_eye_coords, self.right_eye_image = self.landmarks_model.predict(
self.face_crop)
self.x, self.y = self.gaze_model.predict(
self.left_eye_image, self.right_eye_image,
self.head_pose_angles)
log.info(
f'[ Main ] Relative pointer coordinates: [{self.x:.2f}, {self.y:.2f}]'
)
batch_process_time = time.time() - start_inference
self.total_processing_time += batch_process_time
self.count_batch += 1
log.info(
f'[ Main ] Finished processing batch. Time taken = {batch_process_time}s\n'
)
self.mouse_control.move(self.x, self.y)
if self.show_output:
self.draw_outputs(frame)
cv2.imshow('Computer Pointer Controller Output', frame)
self.inference_speed.append(self.count_batch /
self.total_processing_time)
self.avg_inference_speed = sum(self.inference_speed) / len(
self.inference_speed)
with open(os.path.join(self.output_path, 'outputs.txt'),
'w+') as f:
f.write('INFERENCE STATS\n')
f.write(
f'Total model initialization time : {self.model_init_time:.2f}s\n'
)
f.write(
f'Total model load time: {self.model_load_time:.2f}s\n'
)
f.write(
f'App initialization time: {self.app_init_time:.2f}s\n'
)
f.write(
f'Total processing time: {self.total_processing_time:.2f}s\n'
)
f.write(
f'Average inference speed: {self.avg_inference_speed:.2f}FPS\n'
)
f.write(f'Batch count: {self.count_batch}\n\n')
f.write('LAST OUTPUTS\n')
f.write(f'Face coordinates: {self.face_coords}\n')
f.write(f'Left eye coordinates: {self.left_eye_coords}\n')
f.write(
f'Right eye coordinates: {self.right_eye_coords}\n')
f.write(f'Head pose angles: {self.head_pose_angles}\n')
f.write(
f'Relative pointer coordinates/ Gaze vector: [{self.x:.2f}, {self.y:.2f}]'
)
else:
self.input_feeder.close()
cv2.destroyAllWindows()
log.info(
f'[ Main ] All input Batches processed in {self.total_processing_time:.2f}s'
)
log.info('[ Main ] Shutting down app...')
log.info('[ Main ] Mouse controller app has been shut down.')
break
return
class FaceRecognition:
def __init__(self):
self.facenet_model = load_model('facenet_keras.h5')
self.face_size = (160,160)
self.face_det = FaceDetection()
# extract a single face from a given Image
def _face_detection(self, filename, required_size):
image = cv2.imread(filename)
image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
if type(image) is not np.ndarray:
image = np.array(image)
results, img = self.face_det.predict(image, 0.4)
if not results:
face_array = asarray(image)
else:
for out in results:
x1, y1, width, height = out[0]
if(x1<0):
x1=x1*-1
try:
face = image[y1:height,x1:width]
face_img = cv2.resize(face,required_size)
face_array = asarray(face_img)
except:
print("No face detected")
return face_array
# load images and extract faces for all images in a directory
def _load_faces(self, per_dir):
faces = list()
for filename in listdir(per_dir):
path = per_dir + filename
face = self._face_detection(path, self.face_size)
faces.append(face)
return faces
# load a dataset that contains one subdir for each class that in turn contains images
def _load_dataset(self, directory):
X, y = list(), list()
# enumerate folders, on per class
for subdir in listdir(directory):
path = directory + subdir + '/'
# skip any files that might be in the dir
if not isdir(path):
continue
# load all faces in the subdirectory
faces = self._load_faces(path)
# create labels
labels = [subdir for _ in range(len(faces))]
# summarize face dataset
print('>loaded %d examples for class: %s' % (len(faces), subdir))
# store face and labels
X.extend(faces)
y.extend(labels)
return asarray(X), asarray(y)
# get the face embedding for one face
def _get_embedding(self, face_data):
face_emb = list()
for face_pixels in face_data:
# print(face_pixels)
try:
# scale pixel values
face_pixels = face_pixels.astype('float32')
# standardize pixel values across channels (global)
mean, std = face_pixels.mean(), face_pixels.std()
face_pixels = (face_pixels - mean) / std
# transform face into one sample
samples = expand_dims(face_pixels, axis=0)
# make prediction to get embedding
yhat = self.facenet_model.predict(samples)
face_emb.append(yhat[0])
except:
print('No Embeddings for the face')
face_emb = asarray(face_emb)
print(face_emb.shape)
return face_emb
# develop a classifier for the Face Dataset
def _face_classification(self, face_emb, names):
# normalize input vectors
print('Training Classification Model')
in_encoder = Normalizer(norm='l2')
face_emb = in_encoder.transform(face_emb)
# label encode targets
out_encoder = LabelEncoder()
out_encoder.fit(names)
names = out_encoder.transform(names)
# fit model
model = SVC(kernel='linear', probability=True)
model.fit(face_emb, names)
return model
# train the model for new faces
def _train_model(self, directory):
faces, names = self._load_dataset(directory)
face_emb = self._get_embedding(faces)
savez_compressed('frndUnk-celebrity-faces-embeddings-new.npz', face_emb, names)
svm_model = self._face_classification(face_emb, names)
filename = 'finalized_model_frndUnk_new.sav'
pickle.dump(svm_model, open(filename, 'wb'))
# get the face embedding for one face test time
def _get_embedding_test_face(self, face_pixels):
try:
# scale pixel values
face_pixels = face_pixels.astype('float32')
# standardize pixel values across channels (global)
mean, std = face_pixels.mean(), face_pixels.std()
face_pixels = (face_pixels - mean) / std
# transform face into one sample
samples = expand_dims(face_pixels, axis=0)
# make prediction to get embedding
yhat = self.facenet_model.predict(samples)
except:
print('No Embeddings for the face')
return yhat[0]
# face recognition
def predict(self, img, min_score):
face_embedding = load('frndUnk-celebrity-faces-embeddings-new.npz')
svm_model = pickle.load(open('finalized_model_frndUnk_new.sav', 'rb'))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
if type(img) is not np.ndarray:
img = np.array(img)
name_list = face_embedding['arr_1']
output = []
try:
face_bbox,image = self.face_det.predict(img, 0.4)
for out in face_bbox:
x1, y1, width, height = out[0]
if(x1<0):
x1=x1*-1
face_img = img[y1:height,x1:width]
face_img = cv2.resize(face_img,self.face_size)
embedding = self._get_embedding_test_face(face_img)
random_face_emb = embedding
samples = expand_dims(random_face_emb, axis=0)
out_encoder = LabelEncoder()
out_encoder.fit(name_list)
yhat_class = svm_model.predict(samples)
yhat_prob = svm_model.predict_proba(samples)
class_index = yhat_class[0]
class_probability = yhat_prob[0,class_index] * 100
predict_names = out_encoder.inverse_transform(yhat_class)
bbox = out[0]
score = class_probability
if score >= min_score:
output.append((bbox,score,predict_names[0]))
else:
output.append((bbox,score,'Unknown'))
except:
print('No face detected')
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = asarray(img)
return img, output
def main():
args = build_argparser().parse_args()
frame_num = 0
inference_time = 0
counter = 0
# Initialize the Inference Engine
fd = FaceDetection()
fld = Facial_Landmarks_Detection()
ge = Gaze_Estimation()
hp = Head_Pose_Estimation()
# Load Models
fd.load_model(args.face_detection_model, args.device, args.cpu_extension)
fld.load_model(args.facial_landmark_model, args.device, args.cpu_extension)
ge.load_model(args.gaze_estimation_model, args.device, args.cpu_extension)
hp.load_model(args.head_pose_model, args.device, args.cpu_extension)
# Mouse Controller precision and speed
mc = MouseController('medium', 'fast')
# feed input from an image, webcam, or video to model
if args.input == "cam":
feed = InputFeeder("cam")
else:
assert os.path.isfile(args.input), "Specified input file doesn't exist"
feed = InputFeeder("video", args.input)
feed.load_data()
frame_count = 0
for frame in feed.next_batch():
frame_count += 1
inf_start = time.time()
if frame is not None:
try:
key = cv2.waitKey(60)
det_time = time.time() - inf_start
# make predictions
detected_face, face_coords = fd.predict(
frame.copy(), args.prob_threshold)
hp_output = hp.predict(detected_face.copy())
left_eye, right_eye, eye_coords = fld.predict(
detected_face.copy())
new_mouse_coord, gaze_vector = ge.predict(
left_eye, right_eye, hp_output)
stop_inference = time.time()
inference_time = inference_time + stop_inference - inf_start
counter = counter + 1
# Visualization
preview = args.visualization
if preview:
preview_frame = frame.copy()
face_frame = detected_face.copy()
draw_face_bbox(preview_frame, face_coords)
display_hp(preview_frame, hp_output, face_coords)
draw_landmarks(face_frame, eye_coords)
draw_gaze(face_frame, gaze_vector, left_eye.copy(),
right_eye.copy(), eye_coords)
if preview:
img = np.hstack((cv2.resize(preview_frame, (500, 500)),
cv2.resize(face_frame, (500, 500))))
else:
img = cv2.resize(frame, (500, 500))
cv2.imshow('Visualization', img)
# set speed
if frame_count % 5 == 0:
mc.move(new_mouse_coord[0], new_mouse_coord[1])
# INFO
log.info("NUMBER OF FRAMES: {} ".format(frame_num))
log.info("INFERENCE TIME: {}ms".format(det_time * 1000))
frame_num += 1
if key == 27:
break
except:
print(
'Not supported image or video file format. Please send in a supported video format.'
)
exit()
feed.close()
def main():
try:
args = build_argparser().parse_args()
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[
logging.FileHandler("computer-pointer-controller.log"),
logging.StreamHandler()
])
print_output_frame = args.print_output_frame
logger = logging.getLogger()
input_file_path = args.input
feeder = None
if input_file_path.lower() == "CAM":
feeder = InputFeeder("cam")
else:
if not os.path.isfile(input_file_path):
logger.error("Unable to find specified video file")
exit(1)
feeder = InputFeeder("video", input_file_path)
mc = MouseController('low', 'fast')
feeder.load_data()
modelPathDict = {
'FaceDetectionModel': args.face,
'FacialLandmarksDetectionModel': args.landmark,
'GazeEstimationModel': args.gazeestimation,
'HeadPoseEstimationModel': args.headpose
}
for fileNameKey in modelPathDict.keys():
if not os.path.isfile(modelPathDict[fileNameKey] + '.xml'):
logger.error("Unable to find specified " + fileNameKey +
" xml file")
exit(1)
logging.info("============== Models Load time ===============")
face_detection = FaceDetection(args.face, args.device,
args.prob_threshold, args.cpu_extension)
start_time = time.time()
face_detection.load_model()
logging.info("Face Detection Model: {:.1f}ms".format(
1000 * (time.time() - start_time)))
landmarks_detection = FacialLandmarksDetection(args.landmark,
args.device,
args.cpu_extension)
start_time = time.time()
landmarks_detection.load_model()
logging.info("Facial Landmarks Detection Model: {:.1f}ms".format(
1000 * (time.time() - start_time)))
gaze_estimation = GazeEstimation(args.gazeestimation, args.device,
args.cpu_extension)
start_time = time.time()
gaze_estimation.load_model()
logging.info("Gaze Estimation Model: {:.1f}ms".format(
1000 * (time.time() - start_time)))
headpose_estimation = HeadPoseEstimation(args.headpose, args.device,
args.cpu_extension)
start_time = time.time()
headpose_estimation.load_model()
logging.info("Headpose Estimation Model: {:.1f}ms".format(
1000 * (time.time() - start_time)))
logging.info("============== End =====================")
frame_count = 0
fd_infertime = 0
lm_infertime = 0
hp_infertime = 0
ge_infertime = 0
for ret, frame in feeder.next_batch():
if not ret:
break
frame_count += 1
key = cv2.waitKey(60)
start_time = time.time()
cropped_face, face_coords = face_detection.predict(frame.copy())
fd_infertime += time.time() - start_time
if len(cropped_face) == 0:
logger.error("Unable to detect the face.")
continue
start_time = time.time()
headpose_out = headpose_estimation.predict(cropped_face.copy())
hp_infertime += time.time() - start_time
start_time = time.time()
left_eye, right_eye, eye_coords = landmarks_detection.predict(
cropped_face.copy())
lm_infertime += time.time() - start_time
start_time = time.time()
new_mouse_coord, gaze_vector = gaze_estimation.predict(
left_eye, right_eye, headpose_out)
ge_infertime += time.time() - start_time
if print_output_frame:
preview_frame = frame.copy()
if 'fd' in print_output_frame:
preview_frame = cropped_face
cv2.rectangle(frame, (face_coords[0], face_coords[1]),
(face_coords[2], face_coords[3]),
(255, 0, 0), 3)
if 'fl' in print_output_frame:
cv2.rectangle(cropped_face,
(eye_coords[0][0], eye_coords[0][1]),
(eye_coords[0][2], eye_coords[0][3]),
(0, 255, 0), 2)
cv2.rectangle(cropped_face,
(eye_coords[1][0], eye_coords[1][1]),
(eye_coords[1][2], eye_coords[1][3]),
(0, 255, 0), 2)
if 'hp' in print_output_frame:
cv2.putText(
cropped_face,
"Pose Angles: yaw:{:.2f} | pitch:{:.2f} | roll:{:.2f}".
format(headpose_out[0], headpose_out[1],
headpose_out[2]), (0, 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.25, (0, 0, 0), 1)
face = frame[face_coords[1]:face_coords[3],
face_coords[0]:face_coords[2]]
xmin, ymin, _, _ = face_coords
face_center = (xmin + face.shape[1] / 2,
ymin + face.shape[0] / 2, 0)
headpose_estimation.draw_axes(frame, face_center,
headpose_out[0],
headpose_out[1],
headpose_out[2])
if 'ge' in print_output_frame:
cropped_h, cropped_w = cropped_face.shape[:2]
arrow_length = 0.3 * cropped_h
gaze_arrow_x = gaze_vector[0] * arrow_length
gaze_arrow_y = -gaze_vector[1] * arrow_length
cv2.arrowedLine(cropped_face,
(eye_coords[0][0], eye_coords[0][1]),
(int(eye_coords[0][2] + gaze_arrow_x),
int(eye_coords[0][3] + gaze_arrow_y)),
(0, 255, 0), 2)
cv2.arrowedLine(cropped_face,
(eye_coords[1][0], eye_coords[1][1]),
(int(eye_coords[1][2] + gaze_arrow_x),
int(eye_coords[1][3] + gaze_arrow_y)),
(0, 255, 0), 2)
#frame[face_coords[1]:face_coords[3], face_coords[0]:face_coords[2]] = cropped_face
if len(preview_frame) != 0:
img_hor = np.hstack((cv2.resize(preview_frame, (800, 800)),
cv2.resize(frame, (800, 800))))
else:
img_hor = cv2.resize(frame, (800, 800))
cv2.imshow("Monitor", img_hor)
if frame_count % 5 == 0:
mc.move(new_mouse_coord[0], new_mouse_coord[1])
if key == 27:
break
#logging inference times
if (frame_count > 0):
logging.info(
"============== Models Inference time ===============")
logging.info("Face Detection:{:.1f}ms".format(1000 * fd_infertime /
frame_count))
logging.info("Facial Landmarks Detection:{:.1f}ms".format(
1000 * lm_infertime / frame_count))
logging.info("Headpose Estimation:{:.1f}ms".format(
1000 * hp_infertime / frame_count))
logging.info("Gaze Estimation:{:.1f}ms".format(
1000 * ge_infertime / frame_count))
logging.info("============== End ===============================")
logger.info("Video stream ended...")
cv2.destroyAllWindows()
feeder.close()
except Exception as ex:
logging.exception("Error in inference")
logging.exception("Exception type:")
logging.exception(type(ex))
logging.exception("Exception args:")
logging.exception(ex.args)
logging.exception("Exception:")
logging.exception(ex)
