def run_inference(args): feed = InputFeeder(input_type='video', input_file=args.input) feed.load_data() for batch in feed.next_batch(): cv2.imshow("Output", cv2.resize(batch, (500, 500))) key = cv2.waitKey(60) if (key == 27): break # getting face faceDetection = FaceDetection(model_name=args.face_detection_model) faceDetection.load_model() face = faceDetection.predict(batch) # getting eyes facialLandmarksDetection = FacialLandmarksDetection( args.facial_landmarks_detection_model) facialLandmarksDetection.load_model() left_eye, right_eye = facialLandmarksDetection.predict(face) # getting head pose angles headPoseEstimation = HeadPoseEstimation( args.head_pose_estimation_model) headPoseEstimation.load_model() head_pose = headPoseEstimation.predict(face) print("head pose angles: ", head_pose) # get mouse points gazeEstimation = GazeEstimation(args.gaze_estimation_model) gazeEstimation.load_model() mouse_coords = gazeEstimation.predict(left_eye, right_eye, head_pose) print("gaze output: ", mouse_coords) feed.close()
def main(): """ Load the network and parse the output. :return: None """ global POSE_CHECKED log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = args_parser().parse_args() logger = log.getLogger() if args.input == 'cam': input_stream = 0 else: input_stream = args.input assert os.path.isfile(args.input), "Specified input file doesn't exist" cap = cv2.VideoCapture(input_stream) initial_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) initial_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) video_len = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) fps = int(cap.get(cv2.CAP_PROP_FPS)) out = cv2.VideoWriter(os.path.join(args.output_dir, "output.mp4"), cv2.VideoWriter_fourcc(*"MP4V"), fps, (initial_w, initial_h), True) if args.write_intermediate == 'yes': out_fm = cv2.VideoWriter( os.path.join(args.output_dir, "output_fm.mp4"), cv2.VideoWriter_fourcc(*"MP4V"), fps, (initial_w, initial_h), True) out_lm = cv2.VideoWriter( os.path.join(args.output_dir, "output_lm.mp4"), cv2.VideoWriter_fourcc(*"MP4V"), fps, (initial_w, initial_h), True) out_pm = cv2.VideoWriter( os.path.join(args.output_dir, "output_pm.mp4"), cv2.VideoWriter_fourcc(*"MP4V"), fps, (initial_w, initial_h), True) out_gm = cv2.VideoWriter( os.path.join(args.output_dir, "output_gm.mp4"), cv2.VideoWriter_fourcc(*"MP4V"), fps, (initial_w, initial_h), True) frame_count = 0 job_id = 1 infer_time_start = time.time() if input_stream: cap.open(args.input) # Adjust DELAY to match the number of FPS of the video file if not cap.isOpened(): logger.error("ERROR! Unable to open video source") return if args.mode == 'sync': async_mode = False else: async_mode = True # Initialise the class if args.cpu_extension: face_det = FaceDetection(args.facemodel, args.confidence, extensions=args.cpu_extension, async_mode=async_mode) pose_det = HeadPoseEstimation(args.posemodel, args.confidence, extensions=args.cpu_extension, async_mode=async_mode) land_det = FaceLandmarksDetection(args.landmarksmodel, args.confidence, extensions=args.cpu_extension, async_mode=async_mode) gaze_est = GazeEstimation(args.gazemodel, args.confidence, extensions=args.cpu_extension, async_mode=async_mode) else: face_det = FaceDetection(args.facemodel, args.confidence, async_mode=async_mode) pose_det = HeadPoseEstimation(args.posemodel, args.confidence, async_mode=async_mode) land_det = FaceLandmarksDetection(args.landmarksmodel, args.confidence, async_mode=async_mode) gaze_est = GazeEstimation(args.gazemodel, args.confidence, async_mode=async_mode) # infer_network_pose = Network() # Load the network to IE plugin to get shape of input layer face_det.load_model() pose_det.load_model() land_det.load_model() gaze_est.load_model() model_load_time = time.time() - infer_time_start print("All models are loaded successfully") try: pass except Exception as e: print("Could not run Inference: ", e) while cap.isOpened(): ret, frame = cap.read() if not ret: print("checkpoint *BREAKING") break frame_count += 1 looking = 0 POSE_CHECKED = False if frame is None: log.error("checkpoint ERROR! blank FRAME grabbed") break initial_w = int(cap.get(3)) initial_h = int(cap.get(4)) # Start asynchronous inference for specified request inf_start_fd = time.time() # Results of the output layer of the network coords, frame = face_det.predict(frame) if args.write_intermediate == 'yes': out_fm.write(frame) det_time_fd = time.time() - inf_start_fd if len(coords) > 0: [xmin, ymin, xmax, ymax] = coords[0] # use only the first detected face head_pose = frame[ymin:ymax, xmin:xmax] inf_start_hp = time.time() is_looking, pose_angles = pose_det.predict(head_pose) if args.write_intermediate == 'yes': p = "Pose Angles {}, is Looking? {}".format( pose_angles, is_looking) cv2.putText(frame, p, (50, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 0, 0), 1) out_pm.write(frame) if is_looking: det_time_hp = time.time() - inf_start_hp POSE_CHECKED = True inf_start_lm = time.time() coords, f = land_det.predict(head_pose) frame[ymin:ymax, xmin:xmax] = f if args.write_intermediate == "yes": out_lm.write(frame) det_time_lm = time.time() - inf_start_lm [[xlmin, ylmin, xlmax, ylmax], [xrmin, yrmin, xrmax, yrmax]] = coords left_eye_image = f[ylmin:ylmax, xlmin:xlmax] right_eye_image = f[yrmin:yrmax, xrmin:xrmax] output, gaze_vector = gaze_est.predict(left_eye_image, right_eye_image, pose_angles) if args.write_intermediate == 'yes': p = "Gaze Vector {}".format(gaze_vector) cv2.putText(frame, p, (50, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 0, 0), 1) fl = draw_gaze(left_eye_image, gaze_vector) fr = draw_gaze(right_eye_image, gaze_vector) f[ylmin:ylmax, xlmin:xlmax] = fl f[yrmin:yrmax, xrmin:xrmax] = fr # cv2.arrowedLine(f, (xlmin, ylmin), (xrmin, yrmin), (0,0,255), 5) out_gm.write(frame) # Draw performance stats inf_time_message = "Face Inference time: {:.3f} ms.".format( det_time_fd * 1000) # if POSE_CHECKED: cv2.putText( frame, "Head pose Inference time: {:.3f} ms.".format( det_time_hp * 1000), (0, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1) cv2.putText(frame, inf_time_message, (0, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 0, 0), 1) out.write(frame) if frame_count % 10 == 0: print("Inference time = ", int(time.time() - infer_time_start)) print('Frame count {} and vidoe len {}'.format( frame_count, video_len)) if args.output_dir: total_time = time.time() - infer_time_start with open(os.path.join(args.output_dir, 'stats.txt'), 'w') as f: f.write(str(round(total_time, 1)) + '\n') f.write(str(frame_count) + '\n') if args.output_dir: with open(os.path.join(args.output_dir, 'stats.txt'), 'a') as f: f.write(str(round(model_load_time)) + '\n') # Clean all models face_det.clean() pose_det.clean() land_det.clean() gaze_est.clean() # release cv2 cap cap.release() cv2.destroyAllWindows() # release all out writer out.release() if args.write_intermediate == 'yes': out_fm.release() out_pm.release() out_lm.release() out_gm.release()
def main(): # Grab command line args logger = logging.getLogger() args = build_argparser().parse_args() flags=args.previewflags inputfile=args.input inputfeed=None if inputfile.lower()=='cam': inputfeed=InputFeeder('cam') elif inputfile.endswith('.jpg') or inputfile.endswith('.bmp') : inputfeed=InputFeeder("image",inputfile) #elif inputfile.endswith('.mp4') : #inputfeed=InputFeeder("video",inputfile) else: if not (os.path.isfile(inputfile)): print((inputfile)) logger.error("Specified input file doesn't exist") exit(1) inputfeed=InputFeeder("video",inputfile) model_paths={'GazeEstimation':args.gazeestimationnmodel,'FacialLandmarkDetection':args.faciallandmarkmodel,'HeadPoseEstimation':args.headposemodel,'FaceDetection':args.facedetectionmodel} for file in model_paths.keys(): if not os.path.isfile(model_paths[file]): logger.error("Unable to find specified "+file+" xml file") flm=FacialLandmarkDetection(model_paths['FacialLandmarkDetection'],args.device,args.cpu_extension) gze= GazeEstimation(model_paths['GazeEstimation'],args.device,args.cpu_extension) hpe=HeadPoseEstimation(model_paths['HeadPoseEstimation'],args.device,args.cpu_extension) fd=FaceDetection(model_paths['FaceDetection'],args.device,args.cpu_extension) flm.load_model() fd.load_model() gze.load_model() hpe.load_model() mc=MouseController('medium','fast') inputfeed.load_data() frame_count=0 for ret, frame in inputfeed.next_batch(): if not ret: break frame_count+=1 if frame_count%3==0: cv2.imshow('video',cv2.resize(frame,(300,300))) cv2.waitKey(1) facecoords,cropped_image=fd.predict(frame.copy(),args.prob_threshold) if type(cropped_image)==int: logger.error('unable to detect face') head_out=hpe.predict(cropped_image) left_eye,right_eye,eye=flm.predict(cropped_image) mouse_coords,gaze_vector=gze.predict(left_eye,right_eye,head_out) if (len(flags)!=0): preview_frame=frame.copy() if 'fd' in flags: preview_frame=cropped_image if 'fld' in flags: cv2.rectangle(cropped_image,(eye[0][0]-15,eye[0][1]-15),(eye[0][2]+15,eye[0][3]+15),(0,0,255)) cv2.rectangle(cropped_image,(eye[1][0]-15,eye[1][1]-15),(eye[1][2]+15,eye[1][3]+15),(0,0,255)) if 'hp' in flags: cv2.putText(preview_frame,"Pose Angles: roll:{:2f}|pitch:{:2f}|yaw:{:2f}|".format(head_out[2],head_out[1],head_out[0]),(10,20),cv2.FONT_HERSHEY_COMPLEX,0.25,(255,0,0),1) if 'ge' in flags: x,y,w=(int(gaze_vector[1]*12),int(gaze_vector[0]*12),130) left =cv2.line(left_eye.copy(), (x-w, y-w), (x+w, y+w), (255,0,0), 2) cv2.line(left, (x-w, y+w), (x+w, y-w), (255,0,0), 2) right = cv2.line(right_eye.copy(), (x-w, y-w), (x+w, y+w), (255,0,0), 2) cv2.line(right, (x-w, y+w), (x+w, y-w), (255,0,0), 2) cropped_image[eye[0][1]:eye[0][3],eye[0][0]:eye[0][2]] = left cropped_image[eye[1][1]:eye[1][3],eye[1][0]:eye[1][2]] = right cv2.imshow("visualisation_frame",cv2.resize(preview_frame,(300,300))) if frame_count%3==0: mc.move(mouse_coords[0],mouse_coords[1]) logger.error("video ended...") cv2.destroyAllWindows() inputfeed.close()
def pipeline(args): feed = InputFeeder(args.i) feed.load_data() FaceDetectionPipe = FaceDetection(args.m_fd, args.pt, args.d, args.cpu_ext) load_time = time.time() FaceDetectionPipe.load_model() load_time_fd = time.time() - load_time FacialLandmarksPipe = FacialLandmarks(args.m_ld, args.d, args.cpu_ext) load_time = time.time() FacialLandmarksPipe.load_model() load_time_ld = time.time() - load_time HeadPoseEstimationPipe = HeadPoseEstimation(args.m_hpe, args.d, args.cpu_ext) load_time = time.time() HeadPoseEstimationPipe.load_model() load_time_hpe = time.time() - load_time GazeEstimationPipe = GazeEstimation(args.m_ge, args.d, args.cpu_ext) load_time = time.time() GazeEstimationPipe.load_model() load_time_ge = time.time() - load_time log.info('Load time for face detection model: ' + str(load_time_fd)) log.info('Load time for landmark detection model: ' + str(load_time_ld)) log.info('Load time for head pose estimation model: ' + str(load_time_hpe)) log.info('Load time for gaze estimation model: ' + str(load_time_ge)) inf_time_fd = inf_time_ld = inf_time_hpe = inf_time_ge = frame_count = 0 for frame in feed.next_batch(): if frame is None: break if cv2.waitKey(1) & 0xFF == ord('q'): break frame_count += 1 inf_time = time.time() fd_img_output, fd_coords = FaceDetectionPipe.predict(frame) inf_time_fd = time.time() - inf_time if (fd_coords == []): log.info('No face detected') else: inf_time = time.time() eye_l_image, eye_r_image, ld_coords = FacialLandmarksPipe.predict( fd_img_output) inf_time_ld = time.time() - inf_time inf_time = time.time() hpe_output = HeadPoseEstimationPipe.predict(fd_img_output) inf_time_hpe = time.time() - inf_time yaw, pitch, roll = hpe_output inf_time = time.time() ge_output = GazeEstimationPipe.predict(eye_l_image, eye_r_image, [yaw, pitch, roll]) inf_time_ge = time.time() - inf_time if frame_count % 5 == 0: pointer = MouseController('medium', 'fast') pointer.move(ge_output[0], ge_output[1]) fps_fd = 1 / inf_time_fd fps_ld = 1 / inf_time_ld fps_hpe = 1 / inf_time_hpe fps_ge = 1 / inf_time_ge if (args.v): v = Visualizer(frame, fd_img_output, fd_coords, ld_coords, hpe_output) v.visualize() log.info('Average inference time for face detection model: ' + str(inf_time_fd)) log.info('Average inference time for landmark detection model: ' + str(inf_time_ld)) log.info( 'Average inference time for head pose estimation model: ' + str(inf_time_hpe)) log.info('Average inference time for gaze estimation model: ' + str(inf_time_ge)) log.info('FPS for face detection model: ' + str(fps_fd)) log.info('FPS for landmark detection model: ' + str(fps_ld)) log.info('FPS for head pose estimation model: ' + str(fps_hpe)) log.info('FPS for gaze estimation model: ' + str(fps_ge)) log.info('Frames Count:' + str(frame_count)) mm = ModelMetrics() log.info('Writing stats to file...') mm.save_to_file('stats_fd.txt', 'FD/' + model_precision, inf_time_fd, fps_fd, load_time_fd) mm.save_to_file('stats_ld.txt', model_precision, inf_time_ld, fps_ld, load_time_ld) mm.save_to_file('stats_hpe.txt', model_precision, inf_time_hpe, fps_hpe, load_time_hpe) mm.save_to_file('stats_ge.txt', model_precision, inf_time_ge, fps_ge, load_time_ge) feed.close()
def infer(args, logging_enabled): """ run inference on input video, display/save output video """ face_detection = FaceDetection(args.face_detection) facial_landmark_detection = FacialLandmarkDetection( args.facial_landmark_detection) gaze_estimation = GazeEstimation(args.gaze_estimation) head_pose_estimation = HeadPoseEstimation(args.head_pose_estimation) load_start = now() face_detection.load_model() fl_start = now() facial_landmark_detection.load_model() ge_start = now() gaze_estimation.load_model() hp_start = now() head_pose_estimation.load_model() log_model_load_times(logging_enabled, load_start, fl_start, ge_start, hp_start) feeder = InputFeeder("video", args.input) feeder.load_data() frame_count, fd_time, fl_time, ge_time, hp_time = [0] * 5 while 1: key = cv2.waitKey(20) try: frame = next(feeder.next_batch()) except StopIteration: break frame_count += 1 fd_frame = face_detection.preprocess_input(frame) inf_start = now() fd_output = face_detection.predict(fd_frame) fd_time += now() - inf_start out_frame, faces = face_detection.preprocess_output( fd_output, frame, args.overlay_inference, args.probability_threshold) detected_face = frame[faces[0][1]:faces[0][3], faces[0][0]:faces[0][2]] fl_frame = facial_landmark_detection.preprocess_input(detected_face) fl_start = now() fl_output = facial_landmark_detection.predict(fl_frame) fl_time += now() - fl_start out_frame, l_coord, r_coord, = facial_landmark_detection.preprocess_output( fl_output, faces[0], out_frame, args.overlay_inference) hp_frame = head_pose_estimation.preprocess_input(detected_face) hp_start = now() hp_output = head_pose_estimation.predict(hp_frame) hp_time += now() - hp_start out_frame, head_pose = head_pose_estimation.preprocess_output( hp_output, out_frame, detected_face, faces[0], args.overlay_inference) out_frame, l_eye, r_eye = gaze_estimation.preprocess_input( out_frame, detected_face, l_coord, r_coord, args.overlay_inference) ge_start = now() ge_output = gaze_estimation.predict(head_pose, l_eye, r_eye) ge_time += now() - ge_start out_frame, g_vec = gaze_estimation.preprocess_output( ge_output, out_frame, faces[0], l_coord, r_coord, args.overlay_inference) if args.video_window: cv2.imshow( "Computer-Human Interface Peripheral Signal Manipulation via AI Retina Tracking (CHIPSMART)", out_frame, ) if args.mouse_control and frame_count % 6 == 0: mouse_control.move(g_vec[0], g_vec[1]) # Quit if user presses Esc or Q if key in (27, 81): user_quit(logging_enabled) break log_inference_times(logging_enabled, frame_count, fd_time, fl_time, ge_time, hp_time) feeder.close() cv2.destroyAllWindows() quit()
def main(): """ Load inference networks, stream video to network, and output stats and video. :return: None """ # Logger init logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s") # Get command line args args = get_arg() #Load Preferencies with open(args.config_file, "r") as yamlfile: cfg = yaml.load(yamlfile, Loader=yaml.FullLoader) models = cfg['models'] input_source = args.input video_path = cfg['video_path'] face_model = FaceDetection(models['face_detection']) head_pose_model = HeadPoseEstimation(models['head_pose_estimation']) facial_landmarks_model = FacialLandmarksDetection(models['facial_landmarks_detection']) gaze_estimation_model = GazeEstimation(models['gaze_estimation']) # Initialise the MouseController mouse_contr = MouseController("low","fast") # Load the models and log timing start_time = time.time() face_model.load_model(args.device) logging.info("Load Face Detection model: {:.1f}ms".format(1000 * (time.time() - start_time)) ) start_time = time.time() facial_landmarks_model.load_model(args.device) logging.info("Load Facial Landmarks Detection model: {:.1f}ms".format(1000 * (time.time() - start_time)) ) start_time = time.time() head_pose_model.load_model(args.device) logging.info("Load Head Pose Estimation model: {:.1f}ms".format(1000 * (time.time() - start_time)) ) start_time = time.time() gaze_estimation_model.load_model(args.device) logging.info("Load Gaze Estimation model: {:.1f}ms".format(1000 * (time.time() - start_time)) ) # Get and open video or camera capture #input_feed = InputFeeder('video', args.input) #input_feed.load_data() input_feed = InputFeeder(input_type=input_source, input_file=video_path) input_feed.load_data() if not input_feed.cap.isOpened(): log.critical('Error opening input, check --video_path parameter') sys.exit(1) # FPS = input_feed.get_fps() # Grab the shape of the input # width = input_feed.get_width() # height = input_feed.get_height() # init scene variables frame_count = 0 ### Loop until stream is over ### facedetect_infer_time = 0 landmark_infer_time = 0 headpose_infer_time = 0 gaze_infer_time = 0 while True: # Read the next frame try: frame = next(input_feed.next_batch()) except StopIteration: break if frame is None: break key_pressed = cv2.waitKey(60) frame_count += 1 input_height, input_width, _ = frame.shape logging.info("frame {count} size {w}, {h}".format(count= frame_count, w = input_width, h =input_height)) # face detection p_frame = face_model.preprocess_input(frame) start_time = time.time() fnoutput = face_model.predict(p_frame) facedetect_infer_time += time.time() - start_time out_frame,fboxes = face_model.preprocess_output(fnoutput,frame,args.overlay, args.prob_threshold) #for each face for fbox in fboxes: face = frame[fbox[1]:fbox[3],fbox[0]:fbox[2]] p_frame = facial_landmarks_model.preprocess_input(face) start_time = time.time() lmoutput = facial_landmarks_model.predict(p_frame) landmark_infer_time += time.time() - start_time out_frame,left_eye_point,right_eye_point = facial_landmarks_model.preprocess_output(lmoutput, fbox, out_frame,args.overlay, args.prob_threshold) # get head pose estimation p_frame = head_pose_model.preprocess_input(face) start_time = time.time() hpoutput = head_pose_model.predict(p_frame) headpose_infer_time += time.time() - start_time out_frame, headpose_angels = head_pose_model.preprocess_output(hpoutput,out_frame, face,fbox,args.overlay, args.prob_threshold) # get gaze estimation out_frame, left_eye, right_eye = gaze_estimation_model.preprocess_input(out_frame,face,left_eye_point,right_eye_point,args.overlay) start_time = time.time() geoutput = gaze_estimation_model.predict(left_eye, right_eye, headpose_angels) gaze_infer_time += time.time() - start_time out_frame, gazevector = gaze_estimation_model.preprocess_output(geoutput,out_frame,fbox, left_eye_point,right_eye_point,args.overlay, args.prob_threshold) cv2.imshow('im', out_frame) if(args.mouse_move): logging.info("mouse move vector : x ={}, y={}".format(gazevector[0], gazevector[1])) mouse_contr.move(gazevector[0], gazevector[1]) #use only first detected face in the frame break # Break if escape key pressed if key_pressed == 27: break #logging inference times if(frame_count>0): logging.info("***** Models Inference time *****") logging.info("Face Detection:{:.1f}ms".format(1000* facedetect_infer_time/frame_count)) logging.info("Facial Landmarks Detection:{:.1f}ms".format(1000* landmark_infer_time/frame_count)) logging.info("Headpose Estimation:{:.1f}ms".format(1000* headpose_infer_time/frame_count)) logging.info("Gaze Estimation:{:.1f}ms".format(1000* gaze_infer_time/frame_count)) # Release the capture and destroy any OpenCV windows input_feed.close() cv2.destroyAllWindows()
class MoveMouse: ''' Main Class for the Mouse Controller app. This is the class where all the models are stitched together to control the mouse pointer ''' def __init__(self, args): ''' This method instances variables for the Facial Landmarks Detection Model. Args: args = All arguments parsed by the arguments parser function Return: None ''' init_start_time = time.time() self.output_path = args.output_path self.show_output = args.show_output self.total_processing_time = 0 self.count_batch = 0 self.inference_speed = [] self.avg_inference_speed = 0 if args.all_devices != 'CPU': args.face_device = args.all_devices args.face_landmark_device = args.all_devices args.head_pose_device = args.all_devices args.gaze_device = args.all_devices model_init_start = time.time() self.face_model = FaceDetection(args.face_model, args.face_device, args.face_device_ext, args.face_prob_threshold) self.landmarks_model = FacialLandmarksDetection( args.face_landmark_model, args.face_landmark_device, args.face_landmark_device_ext, args.face_landmark_prob_threshold) self.head_pose_model = HeadPoseEstimation( args.head_pose_model, args.head_pose_device, args.head_pose_device_ext, args.head_pose_prob_threshold) self.gaze_model = GazeEstimation(args.gaze_model, args.gaze_device, args.gaze_device_ext, args.gaze_prob_threshold) self.model_init_time = time.time() - model_init_start log.info('[ Main ] All required models initiallized') self.mouse_control = MouseController(args.precision, args.speed) log.info('[ Main ] Mouse controller successfully initialized') self.input_feeder = InputFeeder(args.batch_size, args.input_type, args.input_file) log.info('[ Main ] Initialized input feeder') model_load_start = time.time() self.face_model.load_model() self.landmarks_model.load_model() self.head_pose_model.load_model() self.gaze_model.load_model() self.model_load_time = time.time() - model_load_start self.app_init_time = time.time() - init_start_time log.info('[ Main ] All moadels loaded to Inference Engine\n') return None def draw_face_box(self, frame, face_coords): ''' Draws face's bounding box on the input frame Args: frame = Input frame from video or camera feed. It could also be an input image Return: frame = Frame with bounding box of faces drawn on it ''' start_point = (face_coords[0][0], face_coords[0][1]) end_point = (face_coords[0][2], face_coords[0][3]) thickness = 5 color = (255, 86, 0) frame = cv2.rectangle(frame, start_point, end_point, color, thickness) return frame def draw_eyes_boxes(self, frame, left_eye_coords, right_eye_coords): ''' Draws face's bounding box on the input frame Args: frame = Input frame from video or camera feed. It could also be an input image Return: frame = Frame with bounding box of left and right eyes drawn on it ''' left_eye_start_point = (left_eye_coords[0], left_eye_coords[1]) left_eye_end_point = (left_eye_coords[2], left_eye_coords[3]) right_eye_start_point = (right_eye_coords[0], right_eye_coords[1]) right_eye_end_point = (right_eye_coords[2], right_eye_coords[3]) thickness = 5 color = (0, 210, 0) frame = cv2.rectangle(frame, left_eye_start_point, left_eye_end_point, color, thickness) frame = cv2.rectangle(frame, right_eye_start_point, right_eye_end_point, color, thickness) return frame def draw_outputs(self, frame): ''' Draws the inference outputs (bounding boxes of the face and both eyes and the 3D head pose directions) of the four models onto the frames. Args: frame = Input frame from video or camera feed. It could also be an input image Return: frame = Frame with all inference outputs drawn on it ''' frame = self.draw_face_box(frame, self.face_coords) frame = self.draw_eyes_boxes(frame, self.left_eye_coords, self.right_eye_coords) frame_id = f'Batch id = {self.count_batch}' avg_inference_speed = f'Avg. inference speed = {self.avg_inference_speed:.3f}fps' total_processing_time = f'Total infer. time = {self.total_processing_time:.3f}s' cv2.putText(frame, frame_id, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.45, (255, 86, 0), 1) cv2.putText(frame, avg_inference_speed, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.45, (255, 86, 0), 1) cv2.putText(frame, total_processing_time, (15, 45), cv2.FONT_HERSHEY_COMPLEX, 0.45, (255, 86, 0), 1) return frame def run_inference(self, frame): ''' Performs inference on the input video or image by passing it through all four models to get the desired coordinates for moving the mouse pointer. Args: frame = Input image, frame from video or camera feed Return: None ''' self.input_feeder.load_data() for frame in self.input_feeder.next_batch(): if self.input_feeder.frame_flag == True: log.info('[ Main ] Started processing a new batch') start_inference = time.time() self.face_coords, self.face_crop = self.face_model.predict( frame) if self.face_coords == []: log.info( '[ Main ] No face detected.. Waiting for you to stare at the camera' ) f.write('[ Error ] No face was detected') else: self.head_pose_angles = self.head_pose_model.predict( self.face_crop) self.left_eye_coords, self.left_eye_image, self.right_eye_coords, self.right_eye_image = self.landmarks_model.predict( self.face_crop) self.x, self.y = self.gaze_model.predict( self.left_eye_image, self.right_eye_image, self.head_pose_angles) log.info( f'[ Main ] Relative pointer coordinates: [{self.x:.2f}, {self.y:.2f}]' ) batch_process_time = time.time() - start_inference self.total_processing_time += batch_process_time self.count_batch += 1 log.info( f'[ Main ] Finished processing batch. Time taken = {batch_process_time}s\n' ) self.mouse_control.move(self.x, self.y) if self.show_output: self.draw_outputs(frame) cv2.imshow('Computer Pointer Controller Output', frame) self.inference_speed.append(self.count_batch / self.total_processing_time) self.avg_inference_speed = sum(self.inference_speed) / len( self.inference_speed) with open(os.path.join(self.output_path, 'outputs.txt'), 'w+') as f: f.write('INFERENCE STATS\n') f.write( f'Total model initialization time : {self.model_init_time:.2f}s\n' ) f.write( f'Total model load time: {self.model_load_time:.2f}s\n' ) f.write( f'App initialization time: {self.app_init_time:.2f}s\n' ) f.write( f'Total processing time: {self.total_processing_time:.2f}s\n' ) f.write( f'Average inference speed: {self.avg_inference_speed:.2f}FPS\n' ) f.write(f'Batch count: {self.count_batch}\n\n') f.write('LAST OUTPUTS\n') f.write(f'Face coordinates: {self.face_coords}\n') f.write(f'Left eye coordinates: {self.left_eye_coords}\n') f.write( f'Right eye coordinates: {self.right_eye_coords}\n') f.write(f'Head pose angles: {self.head_pose_angles}\n') f.write( f'Relative pointer coordinates/ Gaze vector: [{self.x:.2f}, {self.y:.2f}]' ) else: self.input_feeder.close() cv2.destroyAllWindows() log.info( f'[ Main ] All input Batches processed in {self.total_processing_time:.2f}s' ) log.info('[ Main ] Shutting down app...') log.info('[ Main ] Mouse controller app has been shut down.') break return
def main(): args = build_argparser().parse_args() device_name = args.device prob_threshold = args.prob_threshold logger_object = log.getLogger() # Initialize variables with the input arguments model_path_dict = { 'FaceDetectionModel': args.faceDetectionModel, 'FacialLandmarkModel': args.facialLandmarksModel, 'HeadPoseEstimationModel': args.headPoseEstimationModel, 'GazeEstimationModel': args.gazeEstimationModel } # Instantiate model face_model = FaceDetection(model_path_dict['FaceDetectionModel'], device_name, threshold=prob_threshold) landmark_model = FacialLandmarksDetection(model_path_dict['FacialLandmarkModel'], device_name, threshold=prob_threshold) head_pose_model = HeadPoseEstimation(model_path_dict['HeadPoseEstimationModel'], device_name, threshold=prob_threshold) gaze_model = GazeEstimation(model_path_dict['GazeEstimationModel'], device_name, threshold=prob_threshold) mouse_controller = MouseController('medium', 'fast') # Load Models and get time start_time = time.time() face_model.load_model() logger_object.error("Face detection model loaded: time: {:.3f} ms".format((time.time() - start_time) * 1000)) first_mark = time.time() landmark_model.load_model() logger_object.error( "Facial landmarks detection model loaded: time: {:.3f} ms".format((time.time() - first_mark) * 1000)) second_mark = time.time() head_pose_model.load_model() logger_object.error("Head pose estimation model loaded: time: {:.3f} ms".format((time.time() - second_mark) * 1000)) third_mark = time.time() gaze_model.load_model() logger_object.error("Gaze estimation model loaded: time: {:.3f} ms".format((time.time() - third_mark) * 1000)) load_total_time = time.time() - start_time logger_object.error("Total loading time: time: {:.3f} ms".format(load_total_time * 1000)) logger_object.error("All models are loaded successfully..") # Check extention of these unsupported layers face_model.check_model() landmark_model.check_model() head_pose_model.check_model() gaze_model.check_model() preview_flags = args.previewFlags input_filename = args.input output_path = args.output_path prob_threshold = args.prob_threshold if input_filename.lower() == 'cam': input_feeder = InputFeeder(input_type='cam') else: if not os.path.isfile(input_filename): logger_object.error("Unable to find specified video file") exit(1) input_feeder = InputFeeder(input_type='video', input_file=input_filename) for model_path in list(model_path_dict.values()): if not os.path.isfile(model_path): logger_object.error("Unable to find specified model file" + str(model_path)) exit(1) input_feeder.load_data() width = int(input_feeder.cap.get(cv2.CAP_PROP_FRAME_WIDTH)) height = int(input_feeder.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps = int(input_feeder.cap.get(cv2.CAP_PROP_FPS)) out_video = cv2.VideoWriter(os.path.join('output_video.mp4'), cv2.VideoWriter_fourcc(*'avc1'), fps, (width, height), True) frame_counter = 0 start_inf_time = time.time() for ret, frame in input_feeder.next_batch(): if not ret: break frame_counter += 1 key = cv2.waitKey(60) try: cropped_image, face_cords = face_model.predict(frame, prob_threshold) if type(cropped_image) == int: print("Unable to detect the face") if key == 27: break continue left_eye, right_eye, eye_cords = landmark_model.predict(cropped_image) pose_output = head_pose_model.predict(cropped_image) x, y, z = gaze_model.predict(left_eye, right_eye, pose_output, cropped_image, eye_cords) mouse_controller.move(x, y) except Exception as e: print(str(e) + " for frame " + str(frame_counter)) continue image = cv2.resize(frame, (width, height)) if not len(preview_flags) == 0: preview_frame = frame.copy() if 'fd' in preview_flags: if len(preview_flags) != 1: preview_frame = cropped_image cv2.rectangle(frame, (face_cords[0], face_cords[1]), (face_cords[2], face_cords[3]), (0, 0, 255), 3) if 'hp' in preview_flags: cv2.putText( frame, "Pose Angles: yaw= {:.2f} , pitch= {:.2f} , roll= {:.2f}".format( pose_output[0], pose_output[1], pose_output[2]), (20, 40), cv2.FONT_HERSHEY_DUPLEX, 1, (255, 0, 0), 3) if 'ge' in preview_flags: cv2.putText( frame, "Gaze vector: x= {:.2f} , y= {:.2f} , z= {:.2f}".format( x, y, z), (15, 100), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 3) image = np.hstack((cv2.resize(frame, (500, 500)), cv2.resize(preview_frame, (500, 500)))) cv2.imshow('preview', image) out_video.write(image) if frame_counter % 5 == 0: mouse_controller.move(x, y) if key == 27: break inference_time = round(time.time() - start_inf_time, 1) fps = int(frame_counter) / inference_time logger_object.error("counter {} seconds".format(frame_counter)) logger_object.error("total inference time {} seconds".format(inference_time)) logger_object.error("fps {} frame/second".format(fps)) with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'stats.txt'), 'w') as f: f.write('inference time : ' + str(inference_time) + '\n') f.write('fps: ' + str(fps) + '\n') f.write('Models Loading: '+ str(load_total_time) + '\n') logger_object.error('Video stream ended') cv2.destroyAllWindows() input_feeder.close()
def main(): arg_parser = ArgParser() args = arg_parser.get_args() input_file = args.input # If input file defined then use it else use the webcam if input_file: if not os.path.isfile(input_file): log.error("Input file cannot be found") exit() input_feeder = InputFeeder("video", input_file) else: input_feeder = InputFeeder("cam") face_detection_model = FaceDetection(args.face_detection_model, args.device, args.extensions) face_detection_model.load_model() facial_landmarks_model = FacialLandmarksDetection( args.facial_landmark_detection_model, args.device, args.extensions) facial_landmarks_model.load_model() gaze_model = GazeEstimation(args.gaze_estimation_model, args.device, args.extensions) gaze_model.load_model() head_pose_model = HeadPoseEstimation(args.head_pose_estimation_model, args.device, args.extensions) head_pose_model.load_model() mouse_controller = MouseController('medium', 'fast') input_feeder.load_data() frame_count = 0 total_face_detection_inference_time = 0 total_facial_landmark_inference_time = 0 total_head_pose_inference_time = 0 total_gaze_estimation_inference_time = 0 total_inference_time = 0 for ret, frame in input_feeder.next_batch(): if not ret: log.error("ret variable not found") break frame_count += 1 if frame_count % args.mouse_update_interval == 0: cv2.imshow('Input', frame) key_pressed = cv2.waitKey(60) # Run inference on the face detection model start_time = time.time() cropped_face, face_coordinates = face_detection_model.predict( frame.copy(), args.probability_threshold) finish_time = time.time() total_face_detection_inference_time += finish_time - start_time total_inference_time += finish_time - start_time # If no face detected get the next frame if len(face_coordinates) == 0: continue # Run inference on the facial landmark detection model start_time = time.time() results = facial_landmarks_model.predict(cropped_face.copy()) finish_time = time.time() left_eye_coordinates = results[0] right_eye_coordinates = results[1] left_eye_image = results[2] right_eye_image = results[3] left_eye_crop_coordinates = results[4] right_eye_crop_coordinates = results[5] total_facial_landmark_inference_time += finish_time - start_time total_inference_time += finish_time - start_time # Run inference on the head pose estimation model start_time = time.time() head_pose = head_pose_model.predict(cropped_face.copy()) finish_time = time.time() total_head_pose_inference_time += finish_time - start_time total_inference_time += finish_time - start_time # Run inference on the gaze estimation model start_time = time.time() new_mouse_x_coordinate, new_mouse_y_coordinate, gaze_vector = gaze_model.predict( left_eye_image, right_eye_image, head_pose) finish_time = time.time() total_gaze_estimation_inference_time += finish_time - start_time total_inference_time += finish_time - start_time if frame_count % args.mouse_update_interval == 0: log.info("Mouse controller new coordinates: x = {}, y = {}".format( new_mouse_x_coordinate, new_mouse_y_coordinate)) mouse_controller.move(new_mouse_x_coordinate, new_mouse_y_coordinate) # Optional visualization configuration: if args.show_detected_face: showDetectedFace(frame, face_coordinates) if args.show_head_pose: showHeadPose(frame, head_pose) if args.show_facial_landmarks: showFacialLandmarks(cropped_face, left_eye_crop_coordinates, right_eye_crop_coordinates) if args.show_gaze_estimation: showGazeEstimation(frame, right_eye_coordinates, left_eye_coordinates, gaze_vector, cropped_face, face_coordinates) # Break if escape key pressed if key_pressed == 27: log.warning("Keyboard interrupt triggered") break # Release the capture and destroy any OpenCV windows cv2.destroyAllWindows() input_feeder.close() log.info("Average face detection inference time: {} seconds".format( total_face_detection_inference_time / frame_count)) log.info( "Average facial landmark detection inference time: {} seconds".format( total_facial_landmark_inference_time / frame_count)) log.info("Average head pose estimation inference time: {} seconds".format( total_head_pose_inference_time / frame_count)) log.info("Average gaze estimation inference time: {} seconds".format( total_gaze_estimation_inference_time / frame_count)) log.info("Average total inference time: {} seconds".format( total_inference_time / frame_count))
def main(args): #model=args.model fd_model = args.face flmd_model = args.landmarks hp_model = args.head ge_model = args.gaze device = args.device display_flag = args.display # Init and load models fd = FaceDetection(fd_model, device) logger.info("######## Model loading Time #######") start = time.time() fd.load_model() logger.info("Face Detection Model: {:.1f}ms".format(1000 * (time.time() - start))) flmd = FacialLandMarksDetection(flmd_model, device) start = time.time() flmd.load_model() logger.info("Facial Landmarks Detection Model: {:.1f}ms".format( 1000 * (time.time() - start))) hpe = HeadPoseEstimation(hp_model, device) start = time.time() hpe.load_model() logger.info("HeadPose Estimation Model: {:.1f}ms".format( 1000 * (time.time() - start))) ge = GazeEstimation(ge_model, device) start = time.time() ge.load_model() logger.info("Gaze Estimation Model: {:.1f}ms".format( 1000 * (time.time() - start))) # Mouse controller mc = MouseController("low", "fast") feed = InputFeeder(input_type=args.input_type, input_file=args.input_file) feed.load_data() frame_count = 0 fd_inference_time = 0 lm_inference_time = 0 hp_inference_time = 0 ge_inference_time = 0 move_mouse = False for batch in feed.next_batch(): frame_count += 1 # Preprocessed output from face detection face_boxes, image, fd_time = fd.predict(batch, display_flag) fd_inference_time += fd_time for face in face_boxes: cropped_face = batch[face[1]:face[3], face[0]:face[2]] #print(f"Face boxe = {face}") # Get preprocessed result from landmarks image, left_eye, right_eye, lm_time = flmd.predict( image, cropped_face, face, display_flag) lm_inference_time += lm_time # Get preprocessed result from pose estimation image, headpose_angels, hp_time = hpe.predict( image, cropped_face, face, display_flag) hp_inference_time += hp_time # Get preprocessed result from Gaze estimation model image, gazevector, ge_time = ge.predict(image, cropped_face, face, left_eye, right_eye, headpose_angels, display_flag) #cv2.imshow('Face', cropped_face) ge_inference_time += ge_time #print(f"Gaze vect {gazevector[0],gazevector[1]}") cv2.imshow('img', image) if (not move_mouse): mc.move(gazevector[0], gazevector[1]) break if cv2.waitKey(1) & 0xFF == ord("k"): break if (frame_count > 0): logger.info("###### Models Inference time ######") logger.info( f"Face Detection inference time = {(fd_inference_time*1000)/frame_count} ms" ) logger.info( f"Facial Landmarks Detection inference time = {(lm_inference_time*1000)/frame_count} ms" ) logger.info( f"Headpose Estimation inference time = {(hp_inference_time*1000)/frame_count} ms" ) logger.info( f"Gaze estimation inference time = {(ge_inference_time*1000)/frame_count} ms" ) feed.close()
def main(args): device = args.device video_file = args.video input_type = args.input_type toggle = args.toggle stats = args.stats model = args.model if stats == 'true': stats = True else: stats = False if toggle == 'true': toggle = True else: toggle = False # Start Model Loading start_model_load_time = time.time() print(f'[INFO] Started Model Loading...........') face_model = FaceDetection(parse_models_file( label='face_detection', path=model), device) face_model.load_model() # Load Landmark model landmark_model = LandMarksDetection( parse_models_file(label='facial_landmarks_detection', path=model), device) landmark_model.load_model() pose_estimation_model = HeadPoseEstimation( parse_models_file(label='head_pose_estimation', path=model), device) pose_estimation_model.load_model() gaze_estimation_model = GazeEstimation( parse_models_file(label='gaze_estimation', path=model), device) gaze_estimation_model.load_model() total_model_load_time = time.time() - start_model_load_time print('[TOTAL] Loaded in {:.3f} ms'.format(total_model_load_time)) # End Model Loading mouse = MouseController('high', 'fast') if not toggle: cv2.namedWindow(MAIN_WINDOW_NAME, cv2.WINDOW_AUTOSIZE) try: feed = InputFeeder(input_type=input_type, input_file=video_file) feed.load_data() initial_w = int(feed.cap.get(cv2.CAP_PROP_FRAME_WIDTH)) initial_h = int(feed.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) counter = 0 if not toggle: cv2.namedWindow(MAIN_WINDOW_NAME, cv2.WINDOW_NORMAL) for frame, _ in feed.next_batch(): if not _: break try: counter += 1 # Start Inferences coord = face_model.predict(frame, (initial_w, initial_h)) for i in range(len(coord)): xmin, ymin, xmax, ymax = coord[i] cropped_image = frame[ymin:ymax, xmin:xmax] # Landmark Inference cropped_left, cropped_right = landmark_model.predict(cropped_image) if cropped_right.shape[0] < 60 or cropped_left.shape[1] < 60: break if cropped_right.shape[1] < 60 or cropped_left.shape[0] < 60: break # Pose Estimation Inference poses = pose_estimation_model.predict(cropped_image) # Gaze Estimation Inference gz = gaze_estimation_model.predict(poses, cropped_left, cropped_right) # Mouse Controller mouse.move(gz[0][0], gz[0][1]) # If user pass statistics argument to true if stats: # Print performance performance_counts( face_model.performance_counter(0) ) performance_counts( pose_estimation_model.performance_counter(0) ) performance_counts( landmark_model.performance_counter(0) ) performance_counts( gaze_estimation_model.performance_counter(0) ) if not toggle: # Output Camera or Video #cv2.resizeWindow(MAIN_WINDOW_NAME, 480, 320) cv2.imshow(MAIN_WINDOW_NAME, frame) else: # Print Statistics only no camera or video performance_counts( face_model.performance_counter(0) ) performance_counts( pose_estimation_model.performance_counter(0) ) performance_counts( landmark_model.performance_counter(0) ) performance_counts( gaze_estimation_model.performance_counter(0) ) cv2.waitKey(1) except Exception as e: print('Could not run Inference', e) feed.close() except Exception as e: print("Could not run Inference: ", e)
def test_run(args): logging.getLogger().setLevel(logging.INFO) feeder = None activate_frame_count = 10 logging.warning("Running default value activate frame count = 10") if args.input_type == 'video' or args.input_type == 'image': feeder = InputFeeder(args.input_type, args.input) if args.input == '../bin/demo.mp4': logging.warning("Running default setting and input") elif args.input_type == 'webcam': feeder = InputFeeder(args.input_type, args.input) else: logging.error("Input not found") exit(1) mouse_controller = MouseController(args.precision, args.speed) feeder.load_data() start_time = 0 face_model_load_time = 0 start_time = time.time() face_model = FaceDetection(args.face, args.device, args.cpu_extension) face_model.load_model() face_model_load_time = time.time() - start_time logging.info("Face Detection Model Loaded...") head_pose_estimation_load_time = 0 start_time = time.time() head_pose_estimation = HeadPoseEstimation(args.headpose, args.device, args.cpu_extension) head_pose_estimation.load_model() head_pose_estimation_load_time = time.time() - start_time logging.info("Head Pose Detection Model Loaded...") facial_landmarks_detection_load_time = 0 start_time = time.time() facial_landmarks_detection = FacialLandmarksDetection( args.landmarks, args.device, args.cpu_extension) facial_landmarks_detection.load_model() facial_landmarks_detection_load_time = time.time() - start_time logging.info("Facial Landmark Detection Model Loaded...") gaze_model_load_time = 0 start_time = time.time() gaze_model = GazeEstimation(args.gazeestimation, args.device, args.cpu_extension) gaze_model.load_model() gaze_model_load_time = time.time() - start_time logging.info("Gaze Estimation Model Loaded...") frame_count = 0 total_face_model_inference_time = 0 total_head_pose_estimation_inference_time = 0 total_facial_landmarks_detection_inference_time = 0 total_gaze_model_inference_time = 0 start_time = 0 for frame in feeder.next_batch(): if frame is None: break frame_count += 1 key = cv2.waitKey(60) start_time = time.time() first_face_box, first_face = face_model.predict(frame.copy()) total_face_model_inference_time = total_face_model_inference_time + ( time.time() - start_time) start_time = time.time() head_pose_output = head_pose_estimation.predict(first_face_box.copy()) total_head_pose_estimation_inference_time = total_head_pose_estimation_inference_time + ( time.time() - start_time) start_time = time.time() left_eye, right_eye, eye_coords = facial_landmarks_detection.predict( first_face_box.copy()) total_facial_landmarks_detection_inference_time = total_facial_landmarks_detection_inference_time + ( time.time() - start_time) start_time = time.time() move_to_coors_mouse = gaze_model.predict(left_eye, right_eye, head_pose_output) total_gaze_model_inference_time = total_gaze_model_inference_time + ( time.time() - start_time) if frame_count % activate_frame_count == 0 and (args.flag == "3" or args.flag == "4"): mouse_controller.move(move_to_coors_mouse[0], move_to_coors_mouse[1]) cv2.imshow('video', frame) key = cv2.waitKey(60) if key == 27: break if args.flag == "1": cv2.rectangle(frame, (first_face[0], first_face[1]), (first_face[2], first_face[3]), (255, 0, 0)) cv2.imshow('video', frame) key = cv2.waitKey(60) elif args.flag == "2": cv2.rectangle(facial_landmarks_detection.image, (eye_coords[0], eye_coords[1]), (eye_coords[2], eye_coords[3]), (255, 0, 0)) cv2.imshow('video', facial_landmarks_detection.image) key = cv2.waitKey(60) elif args.flag == "3": if frame_count == 1: logging.info("Printing mouse coors: ") logging.info(move_to_coors_mouse) #Print Report if args.flag == "0": print('------------- BEGIN REPORT -------------') avg_inference_face_model = total_face_model_inference_time / frame_count avg_inference_headpose = total_head_pose_estimation_inference_time / frame_count avg_inference_facial_landmark = total_facial_landmarks_detection_inference_time / frame_count avg_inference_gaze_model = total_gaze_model_inference_time / frame_count print("Face Detection Model Load Time: ", args.face) print("Loading time: ", face_model_load_time) print("Inference time: ", avg_inference_face_model) print("Head Pose Detection Model: ", args.headpose) print("Loading time: ", head_pose_estimation_load_time) print("Inference time:", avg_inference_headpose) print("Facial Landmark Detection Model Load Time: ", args.landmarks) print("Loading time: ", facial_landmarks_detection_load_time) print("Inference time:", avg_inference_facial_landmark) print("Gaze Estimation Model Load Time: ", args.gazeestimation) print("Loading time: ", gaze_model_load_time) print("Inference time:", avg_inference_gaze_model) print('------------- END REPORT -------------')
def main(args): # set log level levels = { 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR } log_level = levels.get(args.log_level, logging.ERROR) logging.basicConfig(level=log_level) mouse_control = MouseController('high', 'fast') logging.info("Model Loading Please Wait ..") face_det = FaceDetection(args.face_detection, args.device) facial_det = FaceLandmark(args.face_landmark, args.device) head_pose_est = HeadPoseEstimation(args.head_pose, args.device) gaze_est = GazeEstimation(args.gaze_estimation, args.device) logging.info("Model loading successfully") inp = InputFeeder(input_type='video', input_file=args.input) inp.load_data() face_det.load_model() facial_det.load_model() head_pose_est.load_model() gaze_est.load_model() video_writer = cv2.VideoWriter(args.output_dir + '/demo_output11.mp4', cv2.VideoWriter_fourcc(*'MPEG'), 15, (1920, 1080), True) cv2.namedWindow('gaze') for frame in inp.next_batch(): try: frame.shape except Exception as err: break crop_face, crop_coords = face_det.predict(frame, visualize=args.visualize) left_eye, right_eye, left_eye_crop, right_eye_crop = facial_det.predict( crop_face, visualize=args.visualize) head_pose = head_pose_est.predict(crop_face, visualize=args.visualize) (new_x, new_y), gaze_vector = gaze_est.predict(left_eye_crop, right_eye_crop, head_pose) left_eye_gaze = int(left_eye[0] + gaze_vector[0] * 100), int(left_eye[1] - gaze_vector[1] * 100) right_eye_gaze = int(right_eye[0] + gaze_vector[0] * 100), int(right_eye[1] - gaze_vector[1] * 100) cv2.arrowedLine(crop_face, left_eye, left_eye_gaze, (0, 0, 255), 2) cv2.arrowedLine(crop_face, right_eye, right_eye_gaze, (0, 0, 255), 2) video_writer.write(frame) mouse_control.move(new_x, new_y) if args.show_result: cv2.imshow('gaze', frame) cv2.waitKey(1) inp.close() video_writer.release() cv2.destroyAllWindows()
def infer_on_stream(args): """ Initialize the inference network, stream video to network, and output stats and video. :param args: Command line arguments parsed by `build_argparser()` :return: None """ try: logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", handlers=[ logging.FileHandler("gaze-app.log"), logging.StreamHandler() ]) # Initialise the class mc = MouseController("low", "fast") #mc.move(100,100) fdnet = FaceDetection(args.fdmodel) lmnet = FacialLandmarks(args.lmmodel) hpnet = HeadPoseEstimation(args.hpmodel) genet = GazeEstimation(args.gemodel) ### Load the model through ### logging.info("============== Models Load time ===============") start_time = time.time() fdnet.load_model() logging.info("Face Detection Model: {:.1f}ms".format( 1000 * (time.time() - start_time))) fdnet.check_model() logging.info("Face Detection estimation layers loaded correctly") start_time = time.time() lmnet.load_model() logging.info("Facial Landmarks Detection Model: {:.1f}ms".format( 1000 * (time.time() - start_time))) lmnet.check_model() logging.info("Facial Landmarks estimation layers loaded correctly") start_time = time.time() hpnet.load_model() logging.info("Headpose Estimation Model: {:.1f}ms".format( 1000 * (time.time() - start_time))) hpnet.check_model() logging.info("Head pose estimation layers loaded correctly") start_time = time.time() genet.load_model() logging.info("Gaze Estimation Model: {:.1f}ms".format( 1000 * (time.time() - start_time))) genet.check_model() logging.info("Gaze estimation layers loaded correctly") logging.info("============== End =====================") # Get and open video capture feeder = InputFeeder('video', args.input) feeder.load_data() # FPS = feeder.get_fps() # Grab the shape of the input # width = feeder.get_width() # height = feeder.get_height() # init scene variables frame_count = 0 ### Loop until stream is over ### fd_infertime = 0 lm_infertime = 0 hp_infertime = 0 ge_infertime = 0 while True: # Read the next frame try: frame = next(feeder.next_batch()) except StopIteration: break key_pressed = cv2.waitKey(60) frame_count += 1 #print(int((frame_count) % int(FPS))) # face detection fd_process_time = time.time() p_frame = fdnet.preprocess_input(frame) start_time = time.time() fnoutput = fdnet.predict(p_frame) fd_infertime += time.time() - start_time out_frame, fboxes = fdnet.preprocess_output( fnoutput, frame, args.print) logging.info( "Face Detection Model processing time : {:.1f}ms".format( 1000 * (time.time() - fd_process_time))) #for each face for fbox in fboxes: # fbox = (xmin,ymin,xmax,ymax) # get face landmarks # crop face from frame face = frame[fbox[1]:fbox[3], fbox[0]:fbox[2]] lm_process_time = time.time() p_frame = lmnet.preprocess_input(face) start_time = time.time() lmoutput = lmnet.predict(p_frame) lm_infertime += time.time() - start_time out_frame, left_eye_point, right_eye_point = lmnet.preprocess_output( lmoutput, fbox, out_frame, args.print) logging.info( "Landmarks model processing time : {:.1f}ms".format( 1000 * (time.time() - lm_process_time))) # get head pose estimation hp_process_time = time.time() p_frame = hpnet.preprocess_input(face) start_time = time.time() hpoutput = hpnet.predict(p_frame) hp_infertime += time.time() - start_time out_frame, headpose_angels = hpnet.preprocess_output( hpoutput, out_frame, face, fbox, args.print) logging.info( "Headpose estimation model processing time : {:.1f}ms". format(1000 * (time.time() - hp_process_time))) # get gaze estimation gaze_process_time = time.time() out_frame, left_eye, right_eye = genet.preprocess_input( out_frame, face, left_eye_point, right_eye_point, args.print) start_time = time.time() geoutput = genet.predict(left_eye, right_eye, headpose_angels) ge_infertime += time.time() - start_time out_frame, gazevector = genet.preprocess_output( geoutput, out_frame, fbox, left_eye_point, right_eye_point, args.print) logging.info( "Gaze estimation model processing time : {:.1f}ms".format( 1000 * (time.time() - gaze_process_time))) if (not args.no_video): cv2.imshow('im', out_frame) if (not args.no_move): mc.move(gazevector[0], gazevector[1]) #consider only first detected face in the frame break # Break if escape key pressed if key_pressed == 27: break #logging inference times if (frame_count > 0): logging.info( "============== Models Inference time ===============") logging.info("Face Detection:{:.1f}ms".format(1000 * fd_infertime / frame_count)) logging.info("Facial Landmarks Detection:{:.1f}ms".format( 1000 * lm_infertime / frame_count)) logging.info("Headpose Estimation:{:.1f}ms".format( 1000 * hp_infertime / frame_count)) logging.info("Gaze Estimation:{:.1f}ms".format( 1000 * ge_infertime / frame_count)) logging.info("============== End ===============================") # Release the capture and destroy any OpenCV windows feeder.close() cv2.destroyAllWindows() except Exception as ex: logging.exception("Error in inference:" + str(ex))
def infer_on_stream(args): models = None # Check selected precision model if "FP32" in args.precision: models = select_precision(args.precision) if "FP16" in args.precision: models = select_precision(args.precision) if "INT8" in args.precision: models = select_precision(args.precision) # Get Input input_feeder = InputFeeder(args.input_type, args.input_file) input_feeder.load_data() # Load face detection model face = FaceDetection(model_name=models[0], device=args.device, extensions=args.cpu_extension) face.load_model() # Load head pose model head = HeadPoseEstimation(model_name=models[1], device=args.device, extensions=args.cpu_extension) head.load_model() # Load facial landmark model landmark = FacialLandmarkDetection(model_name=models[2], device=args.device, extensions=args.cpu_extension) landmark.load_model() # Load gaze estimation model gaze = GazeEstimation(model_name=models[3], device=args.device, extensions=args.cpu_extension) gaze.load_model() # Initalize mouse controller mouse = MouseController('high', 'fast') for frame in input_feeder.next_batch(): # Break if number of next frame less then number of batch if frame is None: break # Estimate face region output_frame, cropped_face, box_coord = face.predict(frame) # Estimate head pose position head_pose = head.predict(cropped_face) head_pose = np.array(head_pose) # Estimate eyes landmark coordinates lr_eyes = landmark.predict(cropped_face) eyes = [] # Calculate eye image region for coord in lr_eyes: x = int(coord[0] + box_coord[0]) y = int(coord[1] + box_coord[1]) cv2.circle(output_frame, (x, y), 5, (255, 0, 0), -1) eye_box, cropped_eye = eyes_crop(output_frame, x, y, 40) cv2.rectangle(output_frame, eye_box[0], eye_box[1], (255, 0, 0), 1) eyes.append(cropped_eye) # Estimate gaze direction gaze_coords = gaze.predict(eyes[0], eyes[1], head_pose) # Move the mouse cursor mouse.move(gaze_coords[0], gaze_coords[1]) if "True" in args.visualize: cv2.imshow('Capture', output_frame) if cv2.waitKey(30) & 0xFF == ord('q'): break input_feeder.close() if "True" in args.visualize: cv2.destroyAllWindows()
def main(): """ Load the network and parse the output. :return: None """ global INFO global DELAY global POSE_CHECKED #controller = MouseController() log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = args_parser().parse_args() logger = log.getLogger() if args.input == 'cam': input_stream = 0 else: input_stream = args.input assert os.path.isfile(args.input), "Specified input file doesn't exist" cap = cv2.VideoCapture(input_stream) initial_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) initial_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) video_len = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) fps = int(cap.get(cv2.CAP_PROP_FPS)) out = cv2.VideoWriter(os.path.join(args.output_dir, "shopper.mp4"), cv2.VideoWriter_fourcc(*"MP4V"), fps, (initial_w, initial_h), True) frame_count = 0 job_id = 1 #os.environ['PBS_JOBID'] progress_file_path = os.path.join(args.output_dir, 'i_progress_' + str(job_id) + '.txt') infer_time_start = time.time() if input_stream: cap.open(args.input) # Adjust DELAY to match the number of FPS of the video file DELAY = 1000 / cap.get(cv2.CAP_PROP_FPS) if not cap.isOpened(): logger.error("ERROR! Unable to open video source") return # Initialise the class if args.cpu_extension: facedet = FaceDetection(args.facemodel, args.confidence, extensions=args.cpu_extension) posest = HeadPoseEstimation(args.posemodel, args.confidence, extensions=args.cpu_extension) landest = FaceLandmarksDetection(args.landmarksmodel, args.confidence, extensions=args.cpu_extension) gazeest = GazeEstimation(args.gazemodel, args.confidence, extensions=args.cpu_extension) else: facedet = FaceDetection(args.facemodel, args.confidence) posest = HeadPoseEstimation(args.posemodel, args.confidence) landest = FaceLandmarksDetection(args.landmarksmodel, args.confidence) gazeest = GazeEstimation(args.gazemodel, args.confidence) # infer_network_pose = Network() # Load the network to IE plugin to get shape of input layer facedet.load_model() posest.load_model() landest.load_model() gazeest.load_model() print("loaded models") ret, frame = cap.read() while ret: looking = 0 POSE_CHECKED = False ret, frame = cap.read() frame_count += 1 if not ret: print("checkpoint *BREAKING") break if frame is None: log.error("checkpoint ERROR! blank FRAME grabbed") break initial_w = int(cap.get(3)) initial_h = int(cap.get(4)) # Start asynchronous inference for specified request inf_start_fd = time.time() # Results of the output layer of the network coords, frame = facedet.predict(frame) det_time_fd = time.time() - inf_start_fd if len(coords) > 0: [xmin, ymin, xmax, ymax] = coords[0] # use only the first detected face head_pose = frame[ymin:ymax, xmin:xmax] inf_start_hp = time.time() is_looking, pose_angles = posest.predict(head_pose) if is_looking: det_time_hp = time.time() - inf_start_hp POSE_CHECKED = True #print(is_looking) inf_start_lm = time.time() coords, f = landest.predict(head_pose) frame[ymin:ymax, xmin:xmax] = f det_time_lm = time.time() - inf_start_lm [[xlmin, ylmin, xlmax, ylmax], [xrmin, yrmin, xrmax, yrmax]] = coords left_eye_image = frame[ylmin:ylmax, xlmin:xlmax] right_eye_image = frame[yrmin:yrmax, xrmin:xrmax] output = gazeest.predict(left_eye_image, right_eye_image, pose_angles) # Draw performance stats inf_time_message = "Face Inference time: {:.3f} ms.".format( det_time_fd * 1000) if POSE_CHECKED: cv2.putText( frame, "Head pose Inference time: {:.3f} ms.".format( det_time_hp * 1000), (0, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) cv2.putText(frame, inf_time_message, (0, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 255, 255), 1) out.write(frame) print("frame", frame_count) if frame_count % 10 == 0: print(time.time() - infer_time_start) progressUpdate(progress_file_path, int(time.time() - infer_time_start), frame_count, video_len) if args.output_dir: total_time = time.time() - infer_time_start with open(os.path.join(args.output_dir, 'stats.txt'), 'w') as f: f.write(str(round(total_time, 1)) + '\n') f.write(str(frame_count) + '\n') facedet.clean() posest.clean() landest.clean() gazeest.clean() out.release() cap.release() cv2.destroyAllWindows()
FD = FaceDetection(args.face_detection, args.device) FD.load_model() crop_face, coords = FD.predict(image, visualize=args.visualize) if coords is 0: logging.warning("face not found from image") exit() FL = FaceLandmark(args.face_landmark, args.device) FL.load_model() left_eye, right_eye, left_eye_crop, right_eye_crop = FL.predict( crop_face, visualize=args.visualize) HE = HeadPoseEstimation(args.head_pose, args.device) HE.load_model() HPE = HE.predict(crop_face, args.visualize) GE = GazeEstimation(args.gaze_estimation, args.device) GE.load_model() (new_x, new_y), gaze_vector = GE.predict(left_eye_crop, right_eye_crop, HPE) left_eye_gaze = int(left_eye[0] + gaze_vector[0] * 100), int(left_eye[1] - gaze_vector[1] * 100) right_eye_gaze = int(right_eye[0] + gaze_vector[0] * 100), int(right_eye[1] - gaze_vector[1] * 100) cv2.arrowedLine(crop_face, left_eye, left_eye_gaze, (0, 0, 255), 2)
def main(): args = get_args().parse_args() path_filender = args.input four_flags = args.flags_checker loger = logging.getLogger() feeder_in = None out_path = args.out_path if path_filender.lower() == "cam": feeder_in = InputFeeder("cam") else: if not os.path.isfile(path_filender): loger.error("The video was not found") exit(1) feeder_in = InputFeeder("video", path_filender) model_locations = { 'FaceDetection': args.face_detection_model, 'HeadPoseEstimation': args.head_pose_estimation_model, 'FacialLandmarksDetection': args.facial_landmarks_detection_model, 'GazeEstimation': args.gaze_estimation_model } for key_name in model_locations.keys(): if not os.path.isfile(model_locations[key_name]): loger.error("The system cannot find the " + key_name + " xml file") exit(1) dt = FaceDetection(model_locations['FaceDetection'], args.device, args.cpu_extension) pe = HeadPoseEstimation(model_locations['HeadPoseEstimation'], args.device, args.cpu_extension) ld = FacialLandmarksDetection(model_locations['FacialLandmarksDetection'], args.device, args.cpu_extension) ge = GazeEstimation(model_locations['GazeEstimation'], args.device, args.cpu_extension) cursor = MouseController('medium', 'fast') feeder_in.load_data() model_load_time_start = time.time() dt.load_model() pe.load_model() ld.load_model() ge.load_model() total_load_time = time.time() - model_load_time_start frame_counter = 0 inference_time_start = time.time() for ret, frame in feeder_in.next_batch(): if not ret: break frame_counter = frame_counter + 1 if frame_counter % 1 == 0: cv2.imshow('video', cv2.resize(frame, (600, 600))) key = cv2.waitKey(60) face_detected, coords_face = dt.predict(frame, args.p_th) if type(face_detected) == int: loger.error("The system cannot detect any face.") if key == 27: break continue head_pose_output = pe.predict(face_detected) eye_left_detect, eye_right_detect, eye_coordinates_detect = ld.predict( face_detected) coordi_update_pointer, coordi_gaze = ge.predict( eye_left_detect, eye_right_detect, head_pose_output) if (not len(four_flags) == 0): result_app = frame if 'fad' in four_flags: result_app = face_detected if 'hpe' in four_flags: cv2.putText( result_app, "HP Angles: YAW:{:.3f} * PITCH:{:.3f} * ROLL:{:.3f}". format(head_pose_output[0], head_pose_output[1], head_pose_output[2]), (5, 40), cv2.FONT_HERSHEY_COMPLEX, 0.25, (153, 76, 0), 0) if 'fld' in four_flags: cv2.rectangle(face_detected, (eye_coordinates_detect[0][0] - 4, eye_coordinates_detect[0][1] - 4), (eye_coordinates_detect[0][2] + 4, eye_coordinates_detect[0][3] + 4), (255, 255, 0), 4) cv2.rectangle(face_detected, (eye_coordinates_detect[1][0] - 4, eye_coordinates_detect[1][1] - 4), (eye_coordinates_detect[1][2] + 4, eye_coordinates_detect[1][3] + 4), (255, 255, 0), 4) if 'gae' in four_flags: x = int(coordi_gaze[0] * 2) y = int(coordi_gaze[1] * 2) w = 150 right_E = cv2.line(eye_right_detect, (x - w, y - w), (x + w, y + w), (51, 255, 153), 1) cv2.line(right_E, (x - w, y + w), (x + w, y - w), (51, 255, 253), 1) left_E = cv2.line(eye_left_detect, (x - w, y - w), (x + w, y + w), (51, 255, 153), 1) cv2.line(left_E, (x - w, y + w), (x + w, y - w), (51, 255, 253), 1) face_detected[ eye_coordinates_detect[1][1]:eye_coordinates_detect[1][3], eye_coordinates_detect[1][0]:eye_coordinates_detect[1] [2]] = right_E face_detected[ eye_coordinates_detect[0][1]:eye_coordinates_detect[0][3], eye_coordinates_detect[0][0]:eye_coordinates_detect[0] [2]] = left_E cv2.imshow("Result of the App", cv2.resize(result_app, (600, 600))) if frame_counter % 5 == 0: cursor.move(coordi_update_pointer[0], coordi_update_pointer[1]) if key == 27: break total_time = time.time() - inference_time_start total_time_for_inference = round(total_time, 1) fps = frame_counter / total_time_for_inference with open(out_path + 'stats.txt', 'w') as f: f.write('Inference time: ' + str(total_time_for_inference) + '\n') f.write('FPS: ' + str(fps) + '\n') f.write('Model load time: ' + str(total_load_time) + '\n') loger.error("The video stream is over...") cv2.destroyAllWindows() feeder_in.close()
def main(): # command line args args = build_argparser().parse_args() input_file_path = args.input log_object = log.getLogger() oneneneflags = args.visualization_flag # Initialise the classes fd_object = FaceDetection(model_name=args.face_detection_model, device=args.device, threshold=args.prob_threshold, extensions=args.cpu_extension) fl_object = FacialLandmarkDetection(model_name=args.facial_landmarks_model, device=args.device, extensions=args.cpu_extension) hp_object = HeadPoseEstimation(model_name=args.head_pose_model, device=args.device, extensions=args.cpu_extension) ge_object = GazeEstimation(model_name=args.gaze_estimation_model, device=args.device, extensions=args.cpu_extension) mouse_controller_object = MouseController('low', 'fast') ### Loading the models ### log_object.error( "=================== Models Load Time ====================") start_time = time.time() fd_object.load_model() log_object.error("Face detection model loaded in {:.3f} ms".format( (time.time() - start_time) * 1000)) fl_start = time.time() fl_object.load_model() log_object.error( "Facial landmarks detection model loaded in {:.3f} ms".format( (time.time() - fl_start) * 1000)) hp_start = time.time() hp_object.load_model() log_object.error("Head pose estimation model loaded in {:.3f} ms".format( (time.time() - hp_start) * 1000)) ge_start = time.time() ge_object.load_model() log_object.error("Gaze estimation model loaded in {:.3f} ms".format( (time.time() - ge_start) * 1000)) total_time = time.time() - start_time log_object.error( "=================== Models loaded successfully ===================") log_object.error("Total loading time is {:.3f} ms".format(total_time * 1000)) counter = 0 infer_start = time.time() log_object.error( "=================== Start inferencing on input video ====================" ) if input_file_path == "CAM": input_feeder = InputFeeder("cam") else: if not os.path.isfile(input_file_path): exit(1) input_feeder = InputFeeder("video", input_file_path) log_object.error("Input feeders are loaded") input_feeder.load_data() for frame in input_feeder.next_batch(): # if not flag: # break pressed_key = cv2.waitKey(60) counter += 1 face_coordinates, face_image = fd_object.predict(frame.copy()) if face_coordinates == 0: continue hp_output = hp_object.predict(face_image) left_eye_image, right_eye_image, eye_coord = fl_object.predict( face_image) mouse_coordinate, gaze_vector = ge_object.predict( left_eye_image, right_eye_image, hp_output) if len(oneneneflags) != 0: preview_window = frame.copy() if 'fd' in oneneneflags: if len(oneneneflags) != 1: preview_window = face_image else: cv2.rectangle(preview_window, (face_coordinates[0], face_coordinates[1]), (face_coordinates[2], face_coordinates[3]), (0, 150, 0), 3) if 'fl' in oneneneflags: if not 'fd' in oneneneflags: preview_window = face_image.copy() cv2.rectangle(preview_window, (eye_coord[0][0], eye_coord[0][1]), (eye_coord[0][2], eye_coord[0][3]), (150, 0, 150)) cv2.rectangle(preview_window, (eye_coord[1][0], eye_coord[1][1]), (eye_coord[1][2], eye_coord[1][3]), (150, 0, 150)) if 'hp' in oneneneflags: cv2.putText( preview_window, "yaw:{:.1f} | pitch:{:.1f} | roll:{:.1f}".format( hp_output[0], hp_output[1], hp_output[2]), (20, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 0), 1) if 'ge' in oneneneflags: yaw = hp_output[0] pitch = hp_output[1] roll = hp_output[2] focal_length = 950.0 scale = 50 center_of_face = (face_image.shape[1] / 2, face_image.shape[0] / 2, 0) if 'fd' in oneneneflags or 'fl' in oneneneflags: draw_axes(preview_window, center_of_face, yaw, pitch, roll, scale, focal_length) else: draw_axes(frame, center_of_face, yaw, pitch, roll, scale, focal_length) if len(oneneneflags) != 0: img_hor = np.hstack((cv2.resize(frame, (500, 500)), cv2.resize(preview_window, (500, 500)))) else: img_hor = cv2.resize(frame, (500, 500)) cv2.imshow('Visualization', img_hor) mouse_controller_object.move(mouse_coordinate[0], mouse_coordinate[1]) if pressed_key == 27: log_object.error("exit key is pressed..") break infer_time = round(time.time() - infer_start, 1) fps = int(counter) / infer_time log_object.error("counter {} seconds".format(counter)) log_object.error("total inference time {} seconds".format(infer_time)) log_object.error("fps {} frame/second".format(fps)) log_object.error("Video session has ended") with open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'stats.txt'), 'w') as f: f.write(str(infer_time) + '\n') f.write(str(fps) + '\n') f.write(str(total_time) + '\n') input_feeder.close() cv2.destroyAllWindows()
def main(): # Grab command line args args = build_argparser().parse_args() flags = args.models_outputs_flags logger = logging.getLogger() input_file_path = args.input input_feeder = None if input_file_path.lower() == "cam": input_feeder = InputFeeder("cam") else: if not os.path.isfile(input_file_path): logger.error("Unable to find specified video file") exit(1) input_feeder = InputFeeder("video", input_file_path) model_path_dict = { 'FaceDetection': args.face_detection_model, 'FacialLandmarks': args.facial_landmarks_model, 'GazeEstimation': args.gaze_estimation_model, 'HeadPoseEstimation': args.head_pose_estimation_model } for file_name_key in model_path_dict.keys(): if not os.path.isfile(model_path_dict[file_name_key]): logger.error("Unable to find specified " + file_name_key + " xml file") exit(1) fdm = FaceDetection(model_path_dict['FaceDetection'], args.device, args.cpu_extension) flm = FacialLandmarks(model_path_dict['FacialLandmarks'], args.device, args.cpu_extension) gem = GazeEstimation(model_path_dict['GazeEstimation'], args.device, args.cpu_extension) hpem = HeadPoseEstimation(model_path_dict['HeadPoseEstimation'], args.device, args.cpu_extension) mc = MouseController('medium', 'fast') input_feeder.load_data() fdm.load_model() flm.load_model() hpem.load_model() gem.load_model() frame_count = 0 for ret, frame in input_feeder.next_batch(): if not ret: break frame_count += 1 if frame_count % 5 == 0: cv2.imshow('video', cv2.resize(frame, (500, 500))) key = cv2.waitKey(60) cropped_face, face_coords = fdm.predict(frame, args.prob_threshold) if type(cropped_face) == int: logger.error("Unable to detect any face.") if key == 27: break continue hp_output = hpem.predict(cropped_face) left_eye_img, right_eye_img, eye_coords = flm.predict(cropped_face) new_mouse_coord, gaze_vector = gem.predict(left_eye_img, right_eye_img, hp_output) if (not len(flags) == 0): preview_frame = frame if 'fd' in flags: preview_frame = cropped_face if 'fld' in flags: cv2.rectangle(cropped_face, (eye_coords[0][0] - 10, eye_coords[0][1] - 10), (eye_coords[0][2] + 10, eye_coords[0][3] + 10), (0, 255, 0), 3) cv2.rectangle(cropped_face, (eye_coords[1][0] - 10, eye_coords[1][1] - 10), (eye_coords[1][2] + 10, eye_coords[1][3] + 10), (0, 255, 0), 3) if 'hp' in flags: cv2.putText( preview_frame, "Pose Angles: yaw:{:.2f} | pitch:{:.2f} | roll:{:.2f}". format(hp_output[0], hp_output[1], hp_output[2]), (10, 20), cv2.FONT_HERSHEY_COMPLEX, 0.25, (0, 255, 0), 1) if 'ge' in flags: x, y, w = int(gaze_vector[0] * 12), int(gaze_vector[1] * 12), 160 left_eye = cv2.line(left_eye_img, (x - w, y - w), (x + w, y + w), (255, 0, 255), 2) cv2.line(left_eye, (x - w, y + w), (x + w, y - w), (255, 0, 255), 2) right_eye = cv2.line(right_eye_img, (x - w, y - w), (x + w, y + w), (255, 0, 255), 2) cv2.line(right_eye, (x - w, y + w), (x + w, y - w), (255, 0, 255), 2) cropped_face[eye_coords[0][1]:eye_coords[0][3], eye_coords[0][0]:eye_coords[0][2]] = left_eye cropped_face[eye_coords[1][1]:eye_coords[1][3], eye_coords[1][0]:eye_coords[1][2]] = right_eye cv2.imshow("Visualization", cv2.resize(preview_frame, (500, 500))) if frame_count % 5 == 0: mc.move(new_mouse_coord[0], new_mouse_coord[1]) if key == 27: break logger.error("VideoStream ended...") cv2.destroyAllWindows() input_feeder.close()
def main(): try: args = build_argparser().parse_args() logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", handlers=[ logging.FileHandler("computer-pointer-controller.log"), logging.StreamHandler() ]) print_output_frame = args.print_output_frame logger = logging.getLogger() input_file_path = args.input feeder = None if input_file_path.lower() == "CAM": feeder = InputFeeder("cam") else: if not os.path.isfile(input_file_path): logger.error("Unable to find specified video file") exit(1) feeder = InputFeeder("video", input_file_path) mc = MouseController('low', 'fast') feeder.load_data() modelPathDict = { 'FaceDetectionModel': args.face, 'FacialLandmarksDetectionModel': args.landmark, 'GazeEstimationModel': args.gazeestimation, 'HeadPoseEstimationModel': args.headpose } for fileNameKey in modelPathDict.keys(): if not os.path.isfile(modelPathDict[fileNameKey] + '.xml'): logger.error("Unable to find specified " + fileNameKey + " xml file") exit(1) logging.info("============== Models Load time ===============") face_detection = FaceDetection(args.face, args.device, args.prob_threshold, args.cpu_extension) start_time = time.time() face_detection.load_model() logging.info("Face Detection Model: {:.1f}ms".format( 1000 * (time.time() - start_time))) landmarks_detection = FacialLandmarksDetection(args.landmark, args.device, args.cpu_extension) start_time = time.time() landmarks_detection.load_model() logging.info("Facial Landmarks Detection Model: {:.1f}ms".format( 1000 * (time.time() - start_time))) gaze_estimation = GazeEstimation(args.gazeestimation, args.device, args.cpu_extension) start_time = time.time() gaze_estimation.load_model() logging.info("Gaze Estimation Model: {:.1f}ms".format( 1000 * (time.time() - start_time))) headpose_estimation = HeadPoseEstimation(args.headpose, args.device, args.cpu_extension) start_time = time.time() headpose_estimation.load_model() logging.info("Headpose Estimation Model: {:.1f}ms".format( 1000 * (time.time() - start_time))) logging.info("============== End =====================") frame_count = 0 fd_infertime = 0 lm_infertime = 0 hp_infertime = 0 ge_infertime = 0 for ret, frame in feeder.next_batch(): if not ret: break frame_count += 1 key = cv2.waitKey(60) start_time = time.time() cropped_face, face_coords = face_detection.predict(frame.copy()) fd_infertime += time.time() - start_time if len(cropped_face) == 0: logger.error("Unable to detect the face.") continue start_time = time.time() headpose_out = headpose_estimation.predict(cropped_face.copy()) hp_infertime += time.time() - start_time start_time = time.time() left_eye, right_eye, eye_coords = landmarks_detection.predict( cropped_face.copy()) lm_infertime += time.time() - start_time start_time = time.time() new_mouse_coord, gaze_vector = gaze_estimation.predict( left_eye, right_eye, headpose_out) ge_infertime += time.time() - start_time if print_output_frame: preview_frame = frame.copy() if 'fd' in print_output_frame: preview_frame = cropped_face cv2.rectangle(frame, (face_coords[0], face_coords[1]), (face_coords[2], face_coords[3]), (255, 0, 0), 3) if 'fl' in print_output_frame: cv2.rectangle(cropped_face, (eye_coords[0][0], eye_coords[0][1]), (eye_coords[0][2], eye_coords[0][3]), (0, 255, 0), 2) cv2.rectangle(cropped_face, (eye_coords[1][0], eye_coords[1][1]), (eye_coords[1][2], eye_coords[1][3]), (0, 255, 0), 2) if 'hp' in print_output_frame: cv2.putText( cropped_face, "Pose Angles: yaw:{:.2f} | pitch:{:.2f} | roll:{:.2f}". format(headpose_out[0], headpose_out[1], headpose_out[2]), (0, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.25, (0, 0, 0), 1) face = frame[face_coords[1]:face_coords[3], face_coords[0]:face_coords[2]] xmin, ymin, _, _ = face_coords face_center = (xmin + face.shape[1] / 2, ymin + face.shape[0] / 2, 0) headpose_estimation.draw_axes(frame, face_center, headpose_out[0], headpose_out[1], headpose_out[2]) if 'ge' in print_output_frame: cropped_h, cropped_w = cropped_face.shape[:2] arrow_length = 0.3 * cropped_h gaze_arrow_x = gaze_vector[0] * arrow_length gaze_arrow_y = -gaze_vector[1] * arrow_length cv2.arrowedLine(cropped_face, (eye_coords[0][0], eye_coords[0][1]), (int(eye_coords[0][2] + gaze_arrow_x), int(eye_coords[0][3] + gaze_arrow_y)), (0, 255, 0), 2) cv2.arrowedLine(cropped_face, (eye_coords[1][0], eye_coords[1][1]), (int(eye_coords[1][2] + gaze_arrow_x), int(eye_coords[1][3] + gaze_arrow_y)), (0, 255, 0), 2) #frame[face_coords[1]:face_coords[3], face_coords[0]:face_coords[2]] = cropped_face if len(preview_frame) != 0: img_hor = np.hstack((cv2.resize(preview_frame, (800, 800)), cv2.resize(frame, (800, 800)))) else: img_hor = cv2.resize(frame, (800, 800)) cv2.imshow("Monitor", img_hor) if frame_count % 5 == 0: mc.move(new_mouse_coord[0], new_mouse_coord[1]) if key == 27: break #logging inference times if (frame_count > 0): logging.info( "============== Models Inference time ===============") logging.info("Face Detection:{:.1f}ms".format(1000 * fd_infertime / frame_count)) logging.info("Facial Landmarks Detection:{:.1f}ms".format( 1000 * lm_infertime / frame_count)) logging.info("Headpose Estimation:{:.1f}ms".format( 1000 * hp_infertime / frame_count)) logging.info("Gaze Estimation:{:.1f}ms".format( 1000 * ge_infertime / frame_count)) logging.info("============== End ===============================") logger.info("Video stream ended...") cv2.destroyAllWindows() feeder.close() except Exception as ex: logging.exception("Error in inference") logging.exception("Exception type:") logging.exception(type(ex)) logging.exception("Exception args:") logging.exception(ex.args) logging.exception("Exception:") logging.exception(ex)