class FaceFrameProcessor(FrameProcessor): # 同時に検出できる数 QUEUE_SIZE = 16 def __init__(self, args): # 推論に使用するデバイス一覧 used_devices = set([args.d_fd, args.d_lm, args.d_hp]) # 基底クラスのコンストラクタ呼び出し super(FaceFrameProcessor, self).__init__(args, used_devices) # モデルのロード log.info("Loading models") if (args.m_fd): log.info(" Face Detect model") face_detector_net = self.load_model(args.m_fd) self.face_detector = FaceDetector(face_detector_net, confidence_threshold=args.t_fd, roi_scale_factor=args.exp_r_fd) self.face_detector.deploy(args.d_fd, self.iecore) else: # 顔検出モデルが指定されていなければエラー log.error("--m-fd option is mandatory") raise RuntimeError("--m-fd option is mandatory") if (args.m_lm): log.info(" Face Landmark model") self.lm_enabe = True landmarks_net = self.load_model(args.m_lm) self.landmarks_detector = LandmarksDetector(landmarks_net) self.landmarks_detector.deploy(args.d_lm, self.iecore, queue_size=self.QUEUE_SIZE) if (args.m_hp): log.info(" Head pose model") self.hp_enabe = True headpose_net = self.load_model(args.m_hp) self.headpose_detector = HeadposeDetector(headpose_net) self.headpose_detector.deploy(args.d_hp, self.iecore, queue_size=self.QUEUE_SIZE) log.info("Models are loaded") # フレーム毎の処理 def process(self, frame): assert len( frame.shape) == 3, "Expected input frame in (H, W, C) format" assert frame.shape[2] in [3, 4], "Expected BGR or BGRA input" # 前処理 # orig_image = frame.copy() frame = frame.transpose((2, 0, 1)) # HWC to CHW frame = np.expand_dims(frame, axis=0) # 検出器初期化 self.face_detector.clear() if self.lm_enabe: self.landmarks_detector.clear() if self.hp_enabe: self.headpose_detector.clear() # log.info("Face Detect") # 認識処理 self.face_detector.start_async(frame) # 結果取得 rois = self.face_detector.get_roi_proposals(frame) # 認識された顔が多すぎる場合は最大値まで縮小する if self.QUEUE_SIZE < len(rois): log.warning("Too many faces for processing." \ " Will be processed only {self.QUEUE_SIZE} of {len(rois)}.") rois = rois[:self.QUEUE_SIZE] # 特徴点検出 if self.lm_enabe: # log.info("Landmarks") # 認識処理 self.landmarks_detector.start_async(frame, rois) # 結果取得 landmarks = self.landmarks_detector.get_landmarks() else: landmarks = [None] * len(rois) # 向き検出 if self.hp_enabe: # log.info("Headpose") # 認識処理 self.headpose_detector.start_async(frame, rois) # 結果取得 headposes = self.headpose_detector.get_headposes() else: headposes = [None] * len(rois) return rois, landmarks, headposes # パフォーマンスステータスの取得 def get_performance_stats(self): stats = {'face_detector': self.face_detector.get_performance_stats()} if self.lm_enabe: stats['landmarks'] = self.landmarks_detector.get_performance_stats( ) if self.hp_enabe: stats['headpose'] = self.headpose_detector.get_performance_stats() return stats
class FrameProcessor: QUEUE_SIZE = 16 def __init__(self, args): used_devices = set([args.d_fd, args.d_lm, args.d_reid]) self.context = InferenceContext(used_devices, args.cpu_lib, args.gpu_lib, args.perf_stats) context = self.context log.info("Loading models") face_detector_net = self.load_model(args.m_fd) assert (args.fd_input_height and args.fd_input_width) or \ (args.fd_input_height==0 and args.fd_input_width==0), \ "Both -fd_iw and -fd_ih parameters should be specified for reshape" if args.fd_input_height and args.fd_input_width: face_detector_net.reshape( {"data": [1, 3, args.fd_input_height, args.fd_input_width]}) landmarks_net = self.load_model(args.m_lm) face_reid_net = self.load_model(args.m_reid) self.face_detector = FaceDetector(face_detector_net, confidence_threshold=args.t_fd, roi_scale_factor=args.exp_r_fd) self.landmarks_detector = LandmarksDetector(landmarks_net) self.face_identifier = FaceIdentifier(face_reid_net, match_threshold=args.t_id, match_algo=args.match_algo) self.face_detector.deploy(args.d_fd, context) self.landmarks_detector.deploy(args.d_lm, context, queue_size=self.QUEUE_SIZE) self.face_identifier.deploy(args.d_reid, context, queue_size=self.QUEUE_SIZE) log.info("Models are loaded") log.info("Building faces database using images from '%s'" % (args.fg)) self.faces_database = FacesDatabase( args.fg, self.face_identifier, self.landmarks_detector, self.face_detector if args.run_detector else None, args.no_show) self.face_identifier.set_faces_database(self.faces_database) log.info("Database is built, registered %s identities" % \ (len(self.faces_database))) self.allow_grow = args.allow_grow and not args.no_show def load_model(self, model_path): model_path = osp.abspath(model_path) model_weights_path = osp.splitext(model_path)[0] + ".bin" log.info("Loading the model from '%s'" % (model_path)) assert osp.isfile(model_path), \ "Model description is not found at '%s'" % (model_path) assert osp.isfile(model_weights_path), \ "Model weights are not found at '%s'" % (model_weights_path) model = self.context.ie_core.read_network(model_path, model_weights_path) log.info("Model is loaded") return model def process(self, frame): assert len(frame.shape) == 3, \ "Expected input frame in (H, W, C) format" assert frame.shape[2] in [3, 4], \ "Expected BGR or BGRA input" orig_image = frame.copy() frame = frame.transpose((2, 0, 1)) # HWC to CHW frame = np.expand_dims(frame, axis=0) self.face_detector.clear() self.landmarks_detector.clear() self.face_identifier.clear() self.face_detector.start_async(frame) rois = self.face_detector.get_roi_proposals(frame) if self.QUEUE_SIZE < len(rois): log.warning("Too many faces for processing." \ " Will be processed only %s of %s." % \ (self.QUEUE_SIZE, len(rois))) rois = rois[:self.QUEUE_SIZE] self.landmarks_detector.start_async(frame, rois) landmarks = self.landmarks_detector.get_landmarks() self.face_identifier.start_async(frame, rois, landmarks) face_identities, unknowns = self.face_identifier.get_matches() if self.allow_grow and len(unknowns) > 0: for i in unknowns: # This check is preventing asking to save half-images in the boundary of images if rois[i].position[0] == 0.0 or rois[i].position[1] == 0.0 or \ (rois[i].position[0] + rois[i].size[0] > orig_image.shape[1]) or \ (rois[i].position[1] + rois[i].size[1] > orig_image.shape[0]): continue crop = orig_image[ int(rois[i].position[1]):int(rois[i].position[1] + rois[i].size[1]), int(rois[i].position[0]):int(rois[i].position[0] + rois[i].size[0])] name = self.faces_database.ask_to_save(crop) if name: id = self.faces_database.dump_faces( crop, face_identities[i].descriptor, name) face_identities[i].id = id outputs = [rois, landmarks, face_identities] return outputs def get_performance_stats(self): stats = { 'face_detector': self.face_detector.get_performance_stats(), 'landmarks': self.landmarks_detector.get_performance_stats(), 'face_identifier': self.face_identifier.get_performance_stats(), } return stats
class FrameProcessor: QUEUE_SIZE = 16 def __init__(self, args): self.gpu_ext = args.gpu_lib self.allow_grow = args.allow_grow and not args.no_show log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) core = Core() if args.cpu_lib and 'CPU' in {args.d_fd, args.d_lm, args.d_reid}: core.add_extension(args.cpu_lib, 'CPU') self.face_detector = FaceDetector(core, args.m_fd, args.fd_input_size, confidence_threshold=args.t_fd, roi_scale_factor=args.exp_r_fd) self.landmarks_detector = LandmarksDetector(core, args.m_lm) self.face_identifier = FaceIdentifier(core, args.m_reid, match_threshold=args.t_id, match_algo=args.match_algo) self.face_detector.deploy(args.d_fd, self.get_config(args.d_fd)) self.landmarks_detector.deploy(args.d_lm, self.get_config(args.d_lm), self.QUEUE_SIZE) self.face_identifier.deploy(args.d_reid, self.get_config(args.d_reid), self.QUEUE_SIZE) log.debug('Building faces database using images from {}'.format( args.fg)) self.faces_database = FacesDatabase( args.fg, self.face_identifier, self.landmarks_detector, self.face_detector if args.run_detector else None, args.no_show) self.face_identifier.set_faces_database(self.faces_database) log.info('Database is built, registered {} identities'.format( len(self.faces_database))) def get_config(self, device): config = {} if device == 'GPU' and self.gpu_ext: config['CONFIG_FILE'] = self.gpu_ext return config def process(self, frame): orig_image = frame.copy() rois = self.face_detector.infer((frame, )) if self.QUEUE_SIZE < len(rois): log.warning( 'Too many faces for processing. Will be processed only {} of {}' .format(self.QUEUE_SIZE, len(rois))) rois = rois[:self.QUEUE_SIZE] landmarks = self.landmarks_detector.infer((frame, rois)) face_identities, unknowns = self.face_identifier.infer( (frame, rois, landmarks)) if self.allow_grow and len(unknowns) > 0: for i in unknowns: # This check is preventing asking to save half-images in the boundary of images if rois[i].position[0] == 0.0 or rois[i].position[1] == 0.0 or \ (rois[i].position[0] + rois[i].size[0] > orig_image.shape[1]) or \ (rois[i].position[1] + rois[i].size[1] > orig_image.shape[0]): continue crop_image = crop(orig_image, rois[i]) name = self.faces_database.ask_to_save(crop_image) if name: id = self.faces_database.dump_faces( crop_image, face_identities[i].descriptor, name) face_identities[i].id = id return [rois, landmarks, face_identities]
class FrameProcessor: QUEUE_SIZE = 16 def __init__(self, args): used_devices = set([args.d_fd, args.d_lm, args.d_hp, args.d_reid]) self.context = InferenceContext() context = self.context context.load_plugins(used_devices, args.cpu_lib, args.gpu_lib) for d in used_devices: context.get_plugin(d).set_config( {"PERF_COUNT": "YES" if args.perf_stats else "NO"}) log.info("Loading models") face_detector_net = self.load_model(args.m_fd) landmarks_net = self.load_model(args.m_lm) head_pose_net = self.load_model(args.m_hp) # face_reid_net = self.load_model(args.m_reid) self.face_detector = FaceDetector(face_detector_net, confidence_threshold=args.t_fd, roi_scale_factor=args.exp_r_fd) self.landmarks_detector = LandmarksDetector(landmarks_net) self.head_pose_detector = HeadPoseDetector(head_pose_net) self.face_detector.deploy(args.d_fd, context) self.landmarks_detector.deploy(args.d_lm, context, queue_size=self.QUEUE_SIZE) self.head_pose_detector.deploy(args.d_hp, context, queue_size=self.QUEUE_SIZE) log.info("Models are loaded") def load_model(self, model_path): model_path = osp.abspath(model_path) model_description_path = model_path model_weights_path = osp.splitext(model_path)[0] + ".bin" log.info("Loading the model from '%s'" % (model_description_path)) assert osp.isfile(model_description_path), \ "Model description is not found at '%s'" % (model_description_path) assert osp.isfile(model_weights_path), \ "Model weights are not found at '%s'" % (model_weights_path) model = IENetwork(model_description_path, model_weights_path) log.info("Model is loaded") return model def process(self, frame): # print("frame.shape--", frame.shape) assert len(frame.shape) == 3, \ "Expected input frame in (H, W, C) format" assert frame.shape[2] in [3, 4], \ "Expected BGR or BGRA input" orig_image = frame.copy() frame = frame.transpose((2, 0, 1)) # HWC to CHW # print("frame.shape--", frame.shape) frame = np.expand_dims(frame, axis=0) # print("frame.shape--", frame.shape) self.face_detector.clear() self.landmarks_detector.clear() self.head_pose_detector.clear() # self.face_identifier.clear() self.face_detector.start_async(frame) rois = self.face_detector.get_roi_proposals(frame) if self.QUEUE_SIZE < len(rois): log.warning("Too many faces for processing." \ " Will be processed only %s of %s." % \ (self.QUEUE_SIZE, len(rois))) rois = rois[:self.QUEUE_SIZE] self.landmarks_detector.start_async(frame, rois) self.head_pose_detector.start_async(frame, rois) landmarks = self.landmarks_detector.get_landmarks() head_pose = self.head_pose_detector.get_head_pose() outputs = [rois, landmarks, head_pose, 'test'] return outputs def get_performance_stats(self): stats = { 'face_detector': self.face_detector.get_performance_stats(), 'landmarks': self.landmarks_detector.get_performance_stats(), 'face_identifier': self.face_identifier.get_performance_stats(), } return stats
class FrameProcessor: QUEUE_SIZE = 16 def __init__(self, args): used_devices = set([args.d_fd, args.d_lm, args.d_reid]) self.context = InferenceContext() context = self.context context.load_plugins(used_devices, args.cpu_lib, args.gpu_lib) for d in used_devices: context.get_plugin(d).set_config( {"PERF_COUNT": "YES" if args.perf_stats else "NO"}) log.info("Loading models") face_detector_net = self.load_model(args.m_fd) landmarks_net = self.load_model(args.m_lm) face_reid_net = self.load_model(args.m_reid) self.face_detector = FaceDetector( face_detector_net, confidence_threshold=args.t_fd, roi_scale_factor=args.exp_r_fd, ) self.landmarks_detector = LandmarksDetector(landmarks_net) self.face_identifier = FaceIdentifier(face_reid_net, match_threshold=args.t_id) self.face_detector.deploy(args.d_fd, context) self.landmarks_detector.deploy(args.d_lm, context, queue_size=self.QUEUE_SIZE) self.face_identifier.deploy(args.d_reid, context, queue_size=self.QUEUE_SIZE) log.info("Models are loaded") if args.fc in "LOAD": self.faces_database = pickle.loads(open(args.fpl, "rb").read()) log.info("Face database loaded from {}.".format(args.fpl)) else: log.info("Building faces database using images from '%s'" % (args.fg)) self.faces_database = FacesDatabase( args.fg, self.face_identifier, self.landmarks_detector, self.face_detector if args.run_detector else None, args.no_show, ) if args.fc in "SAVE": with open(args.fps, "wb") as f: f.write(pickle.dumps(self.faces_database)) f.close() log.info("Face database {} saved".format(args.fps)) self.face_identifier.set_faces_database(self.faces_database) log.info("Database is built, registered %s identities" % (len(self.faces_database))) self.allow_grow = args.allow_grow and not args.no_show def load_model(self, model_path): model_path = osp.abspath(model_path) model_description_path = model_path model_weights_path = osp.splitext(model_path)[0] + ".bin" log.info("Loading the model from '%s'" % (model_description_path)) assert osp.isfile( model_description_path ), "Model description is not found at '%s'" % (model_description_path) assert osp.isfile( model_weights_path), "Model weights are not found at '%s'" % ( model_weights_path) model = IENetwork(model_description_path, model_weights_path) log.info("Model is loaded") return model def process(self, frame): assert len( frame.shape) == 3, "Expected input frame in (H, W, C) format" assert frame.shape[2] in [3, 4], "Expected BGR or BGRA input" orig_image = frame.copy() frame = frame.transpose((2, 0, 1)) # HWC to CHW frame = np.expand_dims(frame, axis=0) self.face_detector.clear() self.landmarks_detector.clear() self.face_identifier.clear() self.face_detector.start_async(frame) rois = self.face_detector.get_roi_proposals(frame) if self.QUEUE_SIZE < len(rois): log.warning("Too many faces for processing." " Will be processed only %s of %s." % (self.QUEUE_SIZE, len(rois))) rois = rois[:self.QUEUE_SIZE] self.landmarks_detector.start_async(frame, rois) landmarks = self.landmarks_detector.get_landmarks() self.face_identifier.start_async(frame, rois, landmarks) face_identities, unknowns = self.face_identifier.get_matches() if self.allow_grow and len(unknowns) > 0: for i in unknowns: # This check is preventing asking to save half-images in the boundary of images if (rois[i].position[0] == 0.0 or rois[i].position[1] == 0.0 or (rois[i].position[0] + rois[i].size[0] > orig_image.shape[1]) or (rois[i].position[1] + rois[i].size[1] > orig_image.shape[0])): continue crop = orig_image[ int(rois[i].position[1]):int(rois[i].position[1] + rois[i].size[1]), int(rois[i].position[0]):int(rois[i].position[0] + rois[i].size[0]), ] name = self.faces_database.ask_to_save(crop) if name: id = self.faces_database.dump_faces( crop, face_identities[i].descriptor, name) face_identities[i].id = id outputs = [rois, landmarks, face_identities] return outputs def get_performance_stats(self): stats = { "face_detector": self.face_detector.get_performance_stats(), "landmarks": self.landmarks_detector.get_performance_stats(), "face_identifier": self.face_identifier.get_performance_stats(), } return stats
class FrameProcessor: # 同時に検出できる数 QUEUE_SIZE = 16 def __init__(self, args): # 追加検出処理フラグ self.lm_enabe = False self.hp_enabe = False # 推論エンジン self.iecore = IECore() # 推論に使用するデバイス一覧 used_devices = set([args.d_fd, args.d_lm, args.d_hp]) # puluginのロード start_time = time.time() log.info(f"Loading plugins for devices: {used_devices}") if 'CPU' in used_devices and not len(args.cpu_lib) == 0: log.info(f"Using CPU extensions library '{args.cpu_lib}'") assert os.path.isfile( cpu_ext), "Failed to open CPU extensions library" self.iecore.add_extension(args.cpu_lib, "CPU") if 'GPU' in used_devices and not len(args.gpu_lib) == 0: log.info(f"Using GPU extensions library '{args.gpu_lib}'") assert os.path.isfile( gpu_ext), "Failed to open GPU definitions file" self.iecore.set_config({"CONFIG_FILE": gpu_ext}, "GPU") log.info( f"Plugins are loaded. loading time : {time.time()- start_time:.4f}sec" ) for d in used_devices: self.iecore.set_config( {"PERF_COUNT": "YES" if args.perf_stats else "NO"}, d) # モデルのロード log.info("Loading models") if (args.m_fd): log.info(" Face Detect model") face_detector_net = self.load_model(args.m_fd) self.face_detector = FaceDetector(face_detector_net, confidence_threshold=args.t_fd, roi_scale_factor=args.exp_r_fd) self.face_detector.deploy(args.d_fd, self.iecore) else: # 顔検出モデルが指定されていなければエラー log.error("--m-fd option is mandatory") raise RuntimeError("--m-fd option is mandatory") if (args.m_lm): log.info(" Face Landmark model") self.lm_enabe = True landmarks_net = self.load_model(args.m_lm) self.landmarks_detector = LandmarksDetector(landmarks_net) self.landmarks_detector.deploy(args.d_lm, self.iecore, queue_size=self.QUEUE_SIZE) if (args.m_hp): log.info(" Head pose model") self.hp_enabe = True headpose_net = self.load_model(args.m_hp) self.headpose_detector = HeadposeDetector(headpose_net) self.headpose_detector.deploy(args.d_hp, self.iecore, queue_size=self.QUEUE_SIZE) log.info("Models are loaded") # IR(Intermediate Representation ;中間表現)ファイル(.xml & .bin) の読み込み def load_model(self, model_path): start_time = time.time() # ロード時間測定用 model_path = os.path.abspath(model_path) model_description_path = model_path model_weights_path = os.path.splitext(model_path)[0] + ".bin" log.info(f" Loading the model from '{model_description_path}'") assert os.path.isfile(model_description_path), \ f"Model description is not found at '{model_description_path}'" assert os.path.isfile(model_weights_path), \ f"Model weights are not found at '{model_weights_path}'" model = self.iecore.read_network(model=model_description_path, weights=model_weights_path) log.info( f" Model is loaded loading time : {time.time()- start_time:.4f}sec" ) return model # フレーム毎の処理 def process(self, frame): assert len( frame.shape) == 3, "Expected input frame in (H, W, C) format" assert frame.shape[2] in [3, 4], "Expected BGR or BGRA input" orig_image = frame.copy() frame = frame.transpose((2, 0, 1)) # HWC to CHW frame = np.expand_dims(frame, axis=0) self.face_detector.clear() if self.lm_enabe: self.landmarks_detector.clear() if self.hp_enabe: self.headpose_detector.clear() # log.info("Face Detect") # 認識処理 self.face_detector.start_async(frame) # 結果取得 rois = self.face_detector.get_roi_proposals(frame) # 認識された顔が多すぎる場合は最大値まで縮小する if self.QUEUE_SIZE < len(rois): log.warning("Too many faces for processing." \ " Will be processed only {self.QUEUE_SIZE} of {len(rois)}.") rois = rois[:self.QUEUE_SIZE] # 特徴点検出 if self.lm_enabe: # log.info("Landmarks") # 認識処理 self.landmarks_detector.start_async(frame, rois) # 結果取得 landmarks = self.landmarks_detector.get_landmarks() else: landmarks = [None] * len(rois) # 向き検出 if self.hp_enabe: # log.info("Headpose") # 認識処理 self.headpose_detector.start_async(frame, rois) # 結果取得 headposes = self.headpose_detector.get_headposes() else: headposes = [None] * len(rois) return rois, landmarks, headposes def get_performance_stats(self): stats = {'face_detector': self.face_detector.get_performance_stats()} if self.lm_enabe: stats['landmarks'] = self.landmarks_detector.get_performance_stats( ) if self.hp_enabe: stats['headpose'] = self.headpose_detector.get_performance_stats() return stats