class FaceFrameProcessor(FrameProcessor):
# 同時に検出できる数
QUEUE_SIZE = 16
def __init__(self, args):
# 推論に使用するデバイス一覧
used_devices = set([args.d_fd, args.d_lm, args.d_hp])
# 基底クラスのコンストラクタ呼び出し
super(FaceFrameProcessor, self).__init__(args, used_devices)
# モデルのロード
log.info("Loading models")
if (args.m_fd):
log.info(" Face Detect model")
face_detector_net = self.load_model(args.m_fd)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.face_detector.deploy(args.d_fd, self.iecore)
else:
# 顔検出モデルが指定されていなければエラー
log.error("--m-fd option is mandatory")
raise RuntimeError("--m-fd option is mandatory")
if (args.m_lm):
log.info(" Face Landmark model")
self.lm_enabe = True
landmarks_net = self.load_model(args.m_lm)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.landmarks_detector.deploy(args.d_lm,
self.iecore,
queue_size=self.QUEUE_SIZE)
if (args.m_hp):
log.info(" Head pose model")
self.hp_enabe = True
headpose_net = self.load_model(args.m_hp)
self.headpose_detector = HeadposeDetector(headpose_net)
self.headpose_detector.deploy(args.d_hp,
self.iecore,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
# フレーム毎の処理
def process(self, frame):
assert len(
frame.shape) == 3, "Expected input frame in (H, W, C) format"
assert frame.shape[2] in [3, 4], "Expected BGR or BGRA input"
# 前処理
# orig_image = frame.copy()
frame = frame.transpose((2, 0, 1)) # HWC to CHW
frame = np.expand_dims(frame, axis=0)
# 検出器初期化
self.face_detector.clear()
if self.lm_enabe:
self.landmarks_detector.clear()
if self.hp_enabe:
self.headpose_detector.clear()
# log.info("Face Detect")
# 認識処理
self.face_detector.start_async(frame)
# 結果取得
rois = self.face_detector.get_roi_proposals(frame)
# 認識された顔が多すぎる場合は最大値まで縮小する
if self.QUEUE_SIZE < len(rois):
log.warning("Too many faces for processing." \
" Will be processed only {self.QUEUE_SIZE} of {len(rois)}.")
rois = rois[:self.QUEUE_SIZE]
# 特徴点検出
if self.lm_enabe:
# log.info("Landmarks")
# 認識処理
self.landmarks_detector.start_async(frame, rois)
# 結果取得
landmarks = self.landmarks_detector.get_landmarks()
else:
landmarks = [None] * len(rois)
# 向き検出
if self.hp_enabe:
# log.info("Headpose")
# 認識処理
self.headpose_detector.start_async(frame, rois)
# 結果取得
headposes = self.headpose_detector.get_headposes()
else:
headposes = [None] * len(rois)
return rois, landmarks, headposes
# パフォーマンスステータスの取得
def get_performance_stats(self):
stats = {'face_detector': self.face_detector.get_performance_stats()}
if self.lm_enabe:
stats['landmarks'] = self.landmarks_detector.get_performance_stats(
)
if self.hp_enabe:
stats['headpose'] = self.headpose_detector.get_performance_stats()
return stats
class FrameProcessor:
QUEUE_SIZE = 16
def __init__(self, args):
used_devices = set([args.d_fd, args.d_lm, args.d_reid])
self.context = InferenceContext(used_devices, args.cpu_lib,
args.gpu_lib, args.perf_stats)
context = self.context
log.info("Loading models")
face_detector_net = self.load_model(args.m_fd)
assert (args.fd_input_height and args.fd_input_width) or \
(args.fd_input_height==0 and args.fd_input_width==0), \
"Both -fd_iw and -fd_ih parameters should be specified for reshape"
if args.fd_input_height and args.fd_input_width:
face_detector_net.reshape(
{"data": [1, 3, args.fd_input_height, args.fd_input_width]})
landmarks_net = self.load_model(args.m_lm)
face_reid_net = self.load_model(args.m_reid)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.face_identifier = FaceIdentifier(face_reid_net,
match_threshold=args.t_id,
match_algo=args.match_algo)
self.face_detector.deploy(args.d_fd, context)
self.landmarks_detector.deploy(args.d_lm,
context,
queue_size=self.QUEUE_SIZE)
self.face_identifier.deploy(args.d_reid,
context,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
log.info("Building faces database using images from '%s'" % (args.fg))
self.faces_database = FacesDatabase(
args.fg, self.face_identifier, self.landmarks_detector,
self.face_detector if args.run_detector else None, args.no_show)
self.face_identifier.set_faces_database(self.faces_database)
log.info("Database is built, registered %s identities" % \
(len(self.faces_database)))
self.allow_grow = args.allow_grow and not args.no_show
def load_model(self, model_path):
model_path = osp.abspath(model_path)
model_weights_path = osp.splitext(model_path)[0] + ".bin"
log.info("Loading the model from '%s'" % (model_path))
assert osp.isfile(model_path), \
"Model description is not found at '%s'" % (model_path)
assert osp.isfile(model_weights_path), \
"Model weights are not found at '%s'" % (model_weights_path)
model = self.context.ie_core.read_network(model_path,
model_weights_path)
log.info("Model is loaded")
return model
def process(self, frame):
assert len(frame.shape) == 3, \
"Expected input frame in (H, W, C) format"
assert frame.shape[2] in [3, 4], \
"Expected BGR or BGRA input"
orig_image = frame.copy()
frame = frame.transpose((2, 0, 1)) # HWC to CHW
frame = np.expand_dims(frame, axis=0)
self.face_detector.clear()
self.landmarks_detector.clear()
self.face_identifier.clear()
self.face_detector.start_async(frame)
rois = self.face_detector.get_roi_proposals(frame)
if self.QUEUE_SIZE < len(rois):
log.warning("Too many faces for processing." \
" Will be processed only %s of %s." % \
(self.QUEUE_SIZE, len(rois)))
rois = rois[:self.QUEUE_SIZE]
self.landmarks_detector.start_async(frame, rois)
landmarks = self.landmarks_detector.get_landmarks()
self.face_identifier.start_async(frame, rois, landmarks)
face_identities, unknowns = self.face_identifier.get_matches()
if self.allow_grow and len(unknowns) > 0:
for i in unknowns:
# This check is preventing asking to save half-images in the boundary of images
if rois[i].position[0] == 0.0 or rois[i].position[1] == 0.0 or \
(rois[i].position[0] + rois[i].size[0] > orig_image.shape[1]) or \
(rois[i].position[1] + rois[i].size[1] > orig_image.shape[0]):
continue
crop = orig_image[
int(rois[i].position[1]):int(rois[i].position[1] +
rois[i].size[1]),
int(rois[i].position[0]):int(rois[i].position[0] +
rois[i].size[0])]
name = self.faces_database.ask_to_save(crop)
if name:
id = self.faces_database.dump_faces(
crop, face_identities[i].descriptor, name)
face_identities[i].id = id
outputs = [rois, landmarks, face_identities]
return outputs
def get_performance_stats(self):
stats = {
'face_detector': self.face_detector.get_performance_stats(),
'landmarks': self.landmarks_detector.get_performance_stats(),
'face_identifier': self.face_identifier.get_performance_stats(),
}
return stats
class FrameProcessor:
QUEUE_SIZE = 16
def __init__(self, args):
used_devices = set([args.d_fd, args.d_lm, args.d_hp, args.d_reid])
self.context = InferenceContext()
context = self.context
context.load_plugins(used_devices, args.cpu_lib, args.gpu_lib)
for d in used_devices:
context.get_plugin(d).set_config(
{"PERF_COUNT": "YES" if args.perf_stats else "NO"})
log.info("Loading models")
face_detector_net = self.load_model(args.m_fd)
landmarks_net = self.load_model(args.m_lm)
head_pose_net = self.load_model(args.m_hp)
# face_reid_net = self.load_model(args.m_reid)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.head_pose_detector = HeadPoseDetector(head_pose_net)
self.face_detector.deploy(args.d_fd, context)
self.landmarks_detector.deploy(args.d_lm,
context,
queue_size=self.QUEUE_SIZE)
self.head_pose_detector.deploy(args.d_hp,
context,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
def load_model(self, model_path):
model_path = osp.abspath(model_path)
model_description_path = model_path
model_weights_path = osp.splitext(model_path)[0] + ".bin"
log.info("Loading the model from '%s'" % (model_description_path))
assert osp.isfile(model_description_path), \
"Model description is not found at '%s'" % (model_description_path)
assert osp.isfile(model_weights_path), \
"Model weights are not found at '%s'" % (model_weights_path)
model = IENetwork(model_description_path, model_weights_path)
log.info("Model is loaded")
return model
def process(self, frame):
# print("frame.shape--", frame.shape)
assert len(frame.shape) == 3, \
"Expected input frame in (H, W, C) format"
assert frame.shape[2] in [3, 4], \
"Expected BGR or BGRA input"
orig_image = frame.copy()
frame = frame.transpose((2, 0, 1)) # HWC to CHW
# print("frame.shape--", frame.shape)
frame = np.expand_dims(frame, axis=0)
# print("frame.shape--", frame.shape)
self.face_detector.clear()
self.landmarks_detector.clear()
self.head_pose_detector.clear()
# self.face_identifier.clear()
self.face_detector.start_async(frame)
rois = self.face_detector.get_roi_proposals(frame)
if self.QUEUE_SIZE < len(rois):
log.warning("Too many faces for processing." \
" Will be processed only %s of %s." % \
(self.QUEUE_SIZE, len(rois)))
rois = rois[:self.QUEUE_SIZE]
self.landmarks_detector.start_async(frame, rois)
self.head_pose_detector.start_async(frame, rois)
landmarks = self.landmarks_detector.get_landmarks()
head_pose = self.head_pose_detector.get_head_pose()
outputs = [rois, landmarks, head_pose, 'test']
return outputs
def get_performance_stats(self):
stats = {
'face_detector': self.face_detector.get_performance_stats(),
'landmarks': self.landmarks_detector.get_performance_stats(),
'face_identifier': self.face_identifier.get_performance_stats(),
}
return stats
class FrameProcessor:
QUEUE_SIZE = 16
def __init__(self, args):
used_devices = set([args.d_fd, args.d_lm, args.d_reid])
self.context = InferenceContext()
context = self.context
context.load_plugins(used_devices, args.cpu_lib, args.gpu_lib)
for d in used_devices:
context.get_plugin(d).set_config(
{"PERF_COUNT": "YES" if args.perf_stats else "NO"})
log.info("Loading models")
face_detector_net = self.load_model(args.m_fd)
landmarks_net = self.load_model(args.m_lm)
face_reid_net = self.load_model(args.m_reid)
self.face_detector = FaceDetector(
face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd,
)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.face_identifier = FaceIdentifier(face_reid_net,
match_threshold=args.t_id)
self.face_detector.deploy(args.d_fd, context)
self.landmarks_detector.deploy(args.d_lm,
context,
queue_size=self.QUEUE_SIZE)
self.face_identifier.deploy(args.d_reid,
context,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
if args.fc in "LOAD":
self.faces_database = pickle.loads(open(args.fpl, "rb").read())
log.info("Face database loaded from {}.".format(args.fpl))
else:
log.info("Building faces database using images from '%s'" %
(args.fg))
self.faces_database = FacesDatabase(
args.fg,
self.face_identifier,
self.landmarks_detector,
self.face_detector if args.run_detector else None,
args.no_show,
)
if args.fc in "SAVE":
with open(args.fps, "wb") as f:
f.write(pickle.dumps(self.faces_database))
f.close()
log.info("Face database {} saved".format(args.fps))
self.face_identifier.set_faces_database(self.faces_database)
log.info("Database is built, registered %s identities" %
(len(self.faces_database)))
self.allow_grow = args.allow_grow and not args.no_show
def load_model(self, model_path):
model_path = osp.abspath(model_path)
model_description_path = model_path
model_weights_path = osp.splitext(model_path)[0] + ".bin"
log.info("Loading the model from '%s'" % (model_description_path))
assert osp.isfile(
model_description_path
), "Model description is not found at '%s'" % (model_description_path)
assert osp.isfile(
model_weights_path), "Model weights are not found at '%s'" % (
model_weights_path)
model = IENetwork(model_description_path, model_weights_path)
log.info("Model is loaded")
return model
def process(self, frame):
assert len(
frame.shape) == 3, "Expected input frame in (H, W, C) format"
assert frame.shape[2] in [3, 4], "Expected BGR or BGRA input"
orig_image = frame.copy()
frame = frame.transpose((2, 0, 1)) # HWC to CHW
frame = np.expand_dims(frame, axis=0)
self.face_detector.clear()
self.landmarks_detector.clear()
self.face_identifier.clear()
self.face_detector.start_async(frame)
rois = self.face_detector.get_roi_proposals(frame)
if self.QUEUE_SIZE < len(rois):
log.warning("Too many faces for processing."
" Will be processed only %s of %s." %
(self.QUEUE_SIZE, len(rois)))
rois = rois[:self.QUEUE_SIZE]
self.landmarks_detector.start_async(frame, rois)
landmarks = self.landmarks_detector.get_landmarks()
self.face_identifier.start_async(frame, rois, landmarks)
face_identities, unknowns = self.face_identifier.get_matches()
if self.allow_grow and len(unknowns) > 0:
for i in unknowns:
# This check is preventing asking to save half-images in the boundary of images
if (rois[i].position[0] == 0.0 or rois[i].position[1] == 0.0
or (rois[i].position[0] + rois[i].size[0] >
orig_image.shape[1])
or (rois[i].position[1] + rois[i].size[1] >
orig_image.shape[0])):
continue
crop = orig_image[
int(rois[i].position[1]):int(rois[i].position[1] +
rois[i].size[1]),
int(rois[i].position[0]):int(rois[i].position[0] +
rois[i].size[0]), ]
name = self.faces_database.ask_to_save(crop)
if name:
id = self.faces_database.dump_faces(
crop, face_identities[i].descriptor, name)
face_identities[i].id = id
outputs = [rois, landmarks, face_identities]
return outputs
def get_performance_stats(self):
stats = {
"face_detector": self.face_detector.get_performance_stats(),
"landmarks": self.landmarks_detector.get_performance_stats(),
"face_identifier": self.face_identifier.get_performance_stats(),
}
return stats
class FrameProcessor:
# 同時に検出できる数
QUEUE_SIZE = 16
def __init__(self, args):
# 追加検出処理フラグ
self.lm_enabe = False
self.hp_enabe = False
# 推論エンジン
self.iecore = IECore()
# 推論に使用するデバイス一覧
used_devices = set([args.d_fd, args.d_lm, args.d_hp])
# puluginのロード
start_time = time.time()
log.info(f"Loading plugins for devices: {used_devices}")
if 'CPU' in used_devices and not len(args.cpu_lib) == 0:
log.info(f"Using CPU extensions library '{args.cpu_lib}'")
assert os.path.isfile(
cpu_ext), "Failed to open CPU extensions library"
self.iecore.add_extension(args.cpu_lib, "CPU")
if 'GPU' in used_devices and not len(args.gpu_lib) == 0:
log.info(f"Using GPU extensions library '{args.gpu_lib}'")
assert os.path.isfile(
gpu_ext), "Failed to open GPU definitions file"
self.iecore.set_config({"CONFIG_FILE": gpu_ext}, "GPU")
log.info(
f"Plugins are loaded. loading time : {time.time()- start_time:.4f}sec"
)
for d in used_devices:
self.iecore.set_config(
{"PERF_COUNT": "YES" if args.perf_stats else "NO"}, d)
# モデルのロード
log.info("Loading models")
if (args.m_fd):
log.info(" Face Detect model")
face_detector_net = self.load_model(args.m_fd)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.face_detector.deploy(args.d_fd, self.iecore)
else:
# 顔検出モデルが指定されていなければエラー
log.error("--m-fd option is mandatory")
raise RuntimeError("--m-fd option is mandatory")
if (args.m_lm):
log.info(" Face Landmark model")
self.lm_enabe = True
landmarks_net = self.load_model(args.m_lm)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.landmarks_detector.deploy(args.d_lm,
self.iecore,
queue_size=self.QUEUE_SIZE)
if (args.m_hp):
log.info(" Head pose model")
self.hp_enabe = True
headpose_net = self.load_model(args.m_hp)
self.headpose_detector = HeadposeDetector(headpose_net)
self.headpose_detector.deploy(args.d_hp,
self.iecore,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
# IR(Intermediate Representation ;中間表現)ファイル(.xml & .bin) の読み込み
def load_model(self, model_path):
start_time = time.time() # ロード時間測定用
model_path = os.path.abspath(model_path)
model_description_path = model_path
model_weights_path = os.path.splitext(model_path)[0] + ".bin"
log.info(f" Loading the model from '{model_description_path}'")
assert os.path.isfile(model_description_path), \
f"Model description is not found at '{model_description_path}'"
assert os.path.isfile(model_weights_path), \
f"Model weights are not found at '{model_weights_path}'"
model = self.iecore.read_network(model=model_description_path,
weights=model_weights_path)
log.info(
f" Model is loaded loading time : {time.time()- start_time:.4f}sec"
)
return model
# フレーム毎の処理
def process(self, frame):
assert len(
frame.shape) == 3, "Expected input frame in (H, W, C) format"
assert frame.shape[2] in [3, 4], "Expected BGR or BGRA input"
orig_image = frame.copy()
frame = frame.transpose((2, 0, 1)) # HWC to CHW
frame = np.expand_dims(frame, axis=0)
self.face_detector.clear()
if self.lm_enabe:
self.landmarks_detector.clear()
if self.hp_enabe:
self.headpose_detector.clear()
# log.info("Face Detect")
# 認識処理
self.face_detector.start_async(frame)
# 結果取得
rois = self.face_detector.get_roi_proposals(frame)
# 認識された顔が多すぎる場合は最大値まで縮小する
if self.QUEUE_SIZE < len(rois):
log.warning("Too many faces for processing." \
" Will be processed only {self.QUEUE_SIZE} of {len(rois)}.")
rois = rois[:self.QUEUE_SIZE]
# 特徴点検出
if self.lm_enabe:
# log.info("Landmarks")
# 認識処理
self.landmarks_detector.start_async(frame, rois)
# 結果取得
landmarks = self.landmarks_detector.get_landmarks()
else:
landmarks = [None] * len(rois)
# 向き検出
if self.hp_enabe:
# log.info("Headpose")
# 認識処理
self.headpose_detector.start_async(frame, rois)
# 結果取得
headposes = self.headpose_detector.get_headposes()
else:
headposes = [None] * len(rois)
return rois, landmarks, headposes
def get_performance_stats(self):
stats = {'face_detector': self.face_detector.get_performance_stats()}
if self.lm_enabe:
stats['landmarks'] = self.landmarks_detector.get_performance_stats(
)
if self.hp_enabe:
stats['headpose'] = self.headpose_detector.get_performance_stats()
return stats
class FrameProcessor:
QUEUE_SIZE = 16
def __init__(self, args):
used_devices = set([args.d])
self.context = InferenceContext(used_devices, args.cpu_lib,
args.gpu_lib, args.perf_stats)
context = self.context
log.info("Loading models")
face_detector_net = self.load_model(model_xml)
assert (args.fd_input_height and args.fd_input_width) or (args.fd_input_height==0 and args.fd_input_width==0), \
"Both -fd_iw and -fd_ih parameters should be specified for reshape"
if args.fd_input_height and args.fd_input_width:
face_detector_net.reshape(
{"data": [1, 3, args.fd_input_height, args.fd_input_width]})
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t)
self.face_detector.deploy(args.d_fd, context)
log.info("Models are loaded")
self.allow_grow = args.allow_grow and not args.no_show
def load_model(self, model_path):
model_path = osp.abspath(model_path)
model_weights_path = osp.splitext(model_path)[0] + ".bin"
log.info("Loading the model from '%s'" % (model_path))
assert osp.isfile(
model_path), "Model description is not found at '%s'" % (
model_path)
assert osp.isfile(
model_weights_path), "Model weights are not found at '%s'" % (
model_weights_path)
model = self.context.ie_core.read_network(model_path,
model_weights_path)
log.info("Model is loaded")
return model
def process(self, frame):
assert len(
frame.shape) == 3, "Expected input frame in (H, W, C) format"
assert frame.shape[2] in [3, 4], "Expected BGR or BGRA input"
orig_image = frame.copy()
frame = frame.transpose((2, 0, 1)) # HWC to CHW
frame = np.expand_dims(frame, axis=0)
self.face_detector.clear()
self.face_detector.start_async(frame)
rois = self.face_detector.get_roi_proposals(frame)
if self.QUEUE_SIZE < len(rois):
log.warning("Too many faces for processing."
" Will be processed only %s of %s." %
(self.QUEUE_SIZE, len(rois)))
rois = rois[:self.QUEUE_SIZE]
outputs = [rois]
return outputs
def get_performance_stats(self):
stats = {
'face_detector': self.face_detector.get_performance_stats(),
}
return stats
