def cam_loop(fps, scale_factor, min_neighbors, min_size):
lc = LandmarksDetector("landmarks_detector.h5",
"cascades/haarcascade_frontalface_default.xml")
cv2.namedWindow("real_time_facial_landmarks_capture")
feed = cv2.VideoCapture(0)
while feed.isOpened():
rval, frame = feed.read()
faces, all_landmarks = lc.detect(frame, scale_factor, min_neighbors,
min_size)
for (x, y, w, h), landmarks in zip(faces, all_landmarks):
cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 0, 255), 2)
for (xx, yy) in zip(landmarks[0::2], landmarks[1::2]):
cv2.circle(frame, (xx, yy), 3, (0, 0, 255), -1)
cv2.imshow("real_time_facial_landmarks_capture", frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
feed.release()
cv2.destroyAllWindows()
return
time.sleep(1 / fps)
def __init__(self, args):
self.gpu_ext = args.gpu_lib
self.allow_grow = args.allow_grow and not args.no_show
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
if args.cpu_lib and 'CPU' in {args.d_fd, args.d_lm, args.d_reid}:
core.add_extension(args.cpu_lib, 'CPU')
self.face_detector = FaceDetector(core,
args.m_fd,
args.fd_input_size,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.landmarks_detector = LandmarksDetector(core, args.m_lm)
self.face_identifier = FaceIdentifier(core,
args.m_reid,
match_threshold=args.t_id,
match_algo=args.match_algo)
self.face_detector.deploy(args.d_fd, self.get_config(args.d_fd))
self.landmarks_detector.deploy(args.d_lm, self.get_config(args.d_lm),
self.QUEUE_SIZE)
self.face_identifier.deploy(args.d_reid, self.get_config(args.d_reid),
self.QUEUE_SIZE)
log.debug('Building faces database using images from {}'.format(
args.fg))
self.faces_database = FacesDatabase(
args.fg, self.face_identifier, self.landmarks_detector,
self.face_detector if args.run_detector else None, args.no_show)
self.face_identifier.set_faces_database(self.faces_database)
log.info('Database is built, registered {} identities'.format(
len(self.faces_database)))
def __init__(self, args):
used_devices = set([args.d_fd, args.d_lm, args.d_hp, args.d_reid])
self.context = InferenceContext()
context = self.context
context.load_plugins(used_devices, args.cpu_lib, args.gpu_lib)
for d in used_devices:
context.get_plugin(d).set_config(
{"PERF_COUNT": "YES" if args.perf_stats else "NO"})
log.info("Loading models")
face_detector_net = self.load_model(args.m_fd)
landmarks_net = self.load_model(args.m_lm)
head_pose_net = self.load_model(args.m_hp)
# face_reid_net = self.load_model(args.m_reid)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.head_pose_detector = HeadPoseDetector(head_pose_net)
self.face_detector.deploy(args.d_fd, context)
self.landmarks_detector.deploy(args.d_lm,
context,
queue_size=self.QUEUE_SIZE)
self.head_pose_detector.deploy(args.d_hp,
context,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
def __init__(self, args):
used_devices = set([args.d_fd, args.d_lm, args.d_reid])
self.context = InferenceContext()
context = self.context
context.load_plugins(used_devices, args.cpu_lib, args.gpu_lib)
for d in used_devices:
context.get_plugin(d).set_config(
{"PERF_COUNT": "YES" if args.perf_stats else "NO"})
log.info("Loading models")
face_detector_net = self.load_model(args.m_fd)
landmarks_net = self.load_model(args.m_lm)
face_reid_net = self.load_model(args.m_reid)
self.face_detector = FaceDetector(
face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd,
)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.face_identifier = FaceIdentifier(face_reid_net,
match_threshold=args.t_id)
self.face_detector.deploy(args.d_fd, context)
self.landmarks_detector.deploy(args.d_lm,
context,
queue_size=self.QUEUE_SIZE)
self.face_identifier.deploy(args.d_reid,
context,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
if args.fc in "LOAD":
self.faces_database = pickle.loads(open(args.fpl, "rb").read())
log.info("Face database loaded from {}.".format(args.fpl))
else:
log.info("Building faces database using images from '%s'" %
(args.fg))
self.faces_database = FacesDatabase(
args.fg,
self.face_identifier,
self.landmarks_detector,
self.face_detector if args.run_detector else None,
args.no_show,
)
if args.fc in "SAVE":
with open(args.fps, "wb") as f:
f.write(pickle.dumps(self.faces_database))
f.close()
log.info("Face database {} saved".format(args.fps))
self.face_identifier.set_faces_database(self.faces_database)
log.info("Database is built, registered %s identities" %
(len(self.faces_database)))
self.allow_grow = args.allow_grow and not args.no_show
def __init__(self, varsd):
used_devices = set([varsd["d_fd"], varsd["d_lm"], varsd["d_reid"]])
self.context = InferenceContext(used_devices, varsd["cpu_lib"],
varsd["gpu_lib"], varsd["perf_stats"])
context = self.context
log.info("Loading models")
face_detector_net = self.load_model(varsd["m_fd"])
assert (varsd["fd_input_height"] and varsd["fd_input_width"]) or \
(varsd["fd_input_height"]==0 and varsd["fd_input_width"]==0), \
"Both -fd_iw and -fd_ih parameters should be specified for reshape"
if varsd["fd_input_height"] and varsd["fd_input_width"]:
face_detector_net.reshape({
"data":
[1, 3, varsd["fd_input_height"], varsd["fd_input_width"]]
})
landmarks_net = self.load_model(varsd["m_lm"])
face_reid_net = self.load_model(varsd["m_reid"])
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=varsd["t_fd"],
roi_scale_factor=varsd["exp_r_fd"])
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.face_identifier = FaceIdentifier(face_reid_net,
match_threshold=varsd["t_id"],
match_algo=varsd["match_algo"])
self.face_detector.deploy(varsd["d_fd"], context)
self.landmarks_detector.deploy(varsd["d_lm"],
context,
queue_size=self.QUEUE_SIZE)
self.face_identifier.deploy(varsd["d_reid"],
context,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
log.info("Building faces database using images from '%s'" %
(varsd["fg"]))
self.faces_database = FacesDatabase(
varsd["fg"], self.face_identifier, self.landmarks_detector,
self.face_detector if varsd["run_detector"] else None,
varsd["no_show"])
self.face_identifier.set_faces_database(self.faces_database)
log.info("Database is built, registered %s identities" % \
(len(self.faces_database)))
self.allow_grow = varsd["allow_grow"] and not varsd["no_show"]
def __init__(self, args):
used_devices = set([args.d_fd, args.d_lm, args.d_reid])
self.context = InferenceContext()
context = self.context
context.load_plugins(used_devices, args.cpu_lib, args.gpu_lib)
for d in used_devices:
context.get_plugin(d).set_config(
{"PERF_COUNT": "YES" if args.perf_stats else "NO"})
log.info("Loading models")
face_detector_net = self.load_model(args.m_fd)
assert (args.fd_input_height and args.fd_input_width) or \
(args.fd_input_height==0 and args.fd_input_width==0), \
"Both -fd_iw and -fd_ih parameters should be specified for reshape"
if args.fd_input_height and args.fd_input_width:
face_detector_net.reshape(
{"data": [1, 3, args.fd_input_height, args.fd_input_width]})
landmarks_net = self.load_model(args.m_lm)
face_reid_net = self.load_model(args.m_reid)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.face_identifier = FaceIdentifier(face_reid_net,
match_threshold=args.t_id,
match_algo=args.match_algo)
self.face_detector.deploy(args.d_fd, context)
self.landmarks_detector.deploy(args.d_lm,
context,
queue_size=self.QUEUE_SIZE)
self.face_identifier.deploy(args.d_reid,
context,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
log.info("Building faces database using images from '%s'" % (args.fg))
self.faces_database = FacesDatabase(
args.fg, self.face_identifier, self.landmarks_detector,
self.face_detector if args.run_detector else None, args.no_show)
self.face_identifier.set_faces_database(self.faces_database)
log.info("Database is built, registered %s identities" % \
(len(self.faces_database)))
self.allow_grow = args.allow_grow and not args.no_show
def __init__(self, args):
# 推論に使用するデバイス一覧
used_devices = set([args.d_fd, args.d_lm, args.d_hp])
# 基底クラスのコンストラクタ呼び出し
super(FaceFrameProcessor, self).__init__(args, used_devices)
# モデルのロード
log.info("Loading models")
if (args.m_fd):
log.info(" Face Detect model")
face_detector_net = self.load_model(args.m_fd)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.face_detector.deploy(args.d_fd, self.iecore)
else:
# 顔検出モデルが指定されていなければエラー
log.error("--m-fd option is mandatory")
raise RuntimeError("--m-fd option is mandatory")
if (args.m_lm):
log.info(" Face Landmark model")
self.lm_enabe = True
landmarks_net = self.load_model(args.m_lm)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.landmarks_detector.deploy(args.d_lm,
self.iecore,
queue_size=self.QUEUE_SIZE)
if (args.m_hp):
log.info(" Head pose model")
self.hp_enabe = True
headpose_net = self.load_model(args.m_hp)
self.headpose_detector = HeadposeDetector(headpose_net)
self.headpose_detector.deploy(args.d_hp,
self.iecore,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
def __init__(self, args):
used_devices = set([args.d_fd, args.d_lm, args.d_reid])
print (used_devices)
self.context = InferenceContext()
context = self.context
log.info("Loading models")
# create network
face_detector_net = self.load_model(args.m_fd)
landmarks_net = self.load_model(args.m_lm)
face_reid_net = self.load_model(args.m_reid)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.face_identifier = FaceIdentifier(face_reid_net,
match_threshold=args.t_id,
match_algo = args.match_algo)
#load network onto devices
self.face_detector.deploy(args.d_fd, context, enable_multi = False)
self.landmarks_detector.deploy(args.d_lm, context,
queue_size=self.QUEUE_SIZE,
enable_multi = args.multi)
self.face_identifier.deploy(args.d_reid, context,
queue_size=self.QUEUE_SIZE,
enable_multi = args.multi)
log.info("Models are loaded")
log.info("Building faces database using images from '%s'" % (args.fg))
self.faces_database = FacesDatabase(args.fg, self.face_identifier,
self.landmarks_detector,
self.face_detector if args.run_detector else None, args.no_show)
self.face_identifier.set_faces_database(self.faces_database)
log.info("Database is built, registered %s identities" % \
(len(self.faces_database)))
self.allow_grow = args.allow_grow and not args.no_show
def __init__(self, args):
used_devices = set([args.d_fd, args.d_lm, args.d_reid])
self.context = InferenceContext()
context = self.context
context.load_plugins(used_devices, args.cpu_lib, args.gpu_lib)
for d in used_devices:
context.get_plugin(d).set_config({
"PERF_COUNT": "YES" if args.perf_stats else "NO"})
log.info("Loading models")
face_detector_net = self.load_model(args.m_fd)
landmarks_net = self.load_model(args.m_lm)
face_reid_net = self.load_model(args.m_reid)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.face_identifier = FaceIdentifier(face_reid_net,
match_threshold=args.t_id)
self.face_detector.deploy(args.d_fd, context)
self.landmarks_detector.deploy(args.d_lm, context,
queue_size=self.QUEUE_SIZE)
self.face_identifier.deploy(args.d_reid, context,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
log.info("Building faces database using images from '%s'" % (args.fg))
self.faces_database = FacesDatabase(args.fg, self.face_identifier,
self.landmarks_detector,
self.face_detector if args.run_detector else None, args.no_show)
self.face_identifier.set_faces_database(self.faces_database)
log.info("Database is built, registered %s identities" % \
(len(self.faces_database)))
# [print(s.label) for s in self.faces_database]
self.allow_grow = args.allow_grow and not args.no_show
def __init__(self, args):
self.gpu_ext = args.gpu_lib
self.perf_count = args.perf_stats
self.allow_grow = args.allow_grow and not args.no_show
log.info('Initializing Inference Engine...')
ie = IECore()
if args.cpu_lib and 'CPU' in {args.d_fd, args.d_lm, args.d_reid}:
log.info('Using CPU extensions library "{}"'.format(args.cpu_lib))
ie.add_extension(args.cpu_lib, 'CPU')
log.info('Loading networks...')
self.face_detector = FaceDetector(ie,
args.m_fd,
args.fd_input_size,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.landmarks_detector = LandmarksDetector(ie, args.m_lm)
self.face_identifier = FaceIdentifier(ie,
args.m_reid,
match_threshold=args.t_id,
match_algo=args.match_algo)
self.face_detector.deploy(args.d_fd, self.get_config(args.d_fd))
self.landmarks_detector.deploy(args.d_lm, self.get_config(args.d_lm),
self.QUEUE_SIZE)
self.face_identifier.deploy(args.d_reid, self.get_config(args.d_reid),
self.QUEUE_SIZE)
log.info('Building faces database using images from "{}"'.format(
args.fg))
self.faces_database = FacesDatabase(
args.fg, self.face_identifier, self.landmarks_detector,
self.face_detector if args.run_detector else None, args.no_show)
self.face_identifier.set_faces_database(self.faces_database)
log.info('Database is built, registered {} identities'.format(
len(self.faces_database)))
class FaceFrameProcessor(FrameProcessor):
# 同時に検出できる数
QUEUE_SIZE = 16
def __init__(self, args):
# 推論に使用するデバイス一覧
used_devices = set([args.d_fd, args.d_lm, args.d_hp])
# 基底クラスのコンストラクタ呼び出し
super(FaceFrameProcessor, self).__init__(args, used_devices)
# モデルのロード
log.info("Loading models")
if (args.m_fd):
log.info(" Face Detect model")
face_detector_net = self.load_model(args.m_fd)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.face_detector.deploy(args.d_fd, self.iecore)
else:
# 顔検出モデルが指定されていなければエラー
log.error("--m-fd option is mandatory")
raise RuntimeError("--m-fd option is mandatory")
if (args.m_lm):
log.info(" Face Landmark model")
self.lm_enabe = True
landmarks_net = self.load_model(args.m_lm)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.landmarks_detector.deploy(args.d_lm,
self.iecore,
queue_size=self.QUEUE_SIZE)
if (args.m_hp):
log.info(" Head pose model")
self.hp_enabe = True
headpose_net = self.load_model(args.m_hp)
self.headpose_detector = HeadposeDetector(headpose_net)
self.headpose_detector.deploy(args.d_hp,
self.iecore,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
# フレーム毎の処理
def process(self, frame):
assert len(
frame.shape) == 3, "Expected input frame in (H, W, C) format"
assert frame.shape[2] in [3, 4], "Expected BGR or BGRA input"
# 前処理
# orig_image = frame.copy()
frame = frame.transpose((2, 0, 1)) # HWC to CHW
frame = np.expand_dims(frame, axis=0)
# 検出器初期化
self.face_detector.clear()
if self.lm_enabe:
self.landmarks_detector.clear()
if self.hp_enabe:
self.headpose_detector.clear()
# log.info("Face Detect")
# 認識処理
self.face_detector.start_async(frame)
# 結果取得
rois = self.face_detector.get_roi_proposals(frame)
# 認識された顔が多すぎる場合は最大値まで縮小する
if self.QUEUE_SIZE < len(rois):
log.warning("Too many faces for processing." \
" Will be processed only {self.QUEUE_SIZE} of {len(rois)}.")
rois = rois[:self.QUEUE_SIZE]
# 特徴点検出
if self.lm_enabe:
# log.info("Landmarks")
# 認識処理
self.landmarks_detector.start_async(frame, rois)
# 結果取得
landmarks = self.landmarks_detector.get_landmarks()
else:
landmarks = [None] * len(rois)
# 向き検出
if self.hp_enabe:
# log.info("Headpose")
# 認識処理
self.headpose_detector.start_async(frame, rois)
# 結果取得
headposes = self.headpose_detector.get_headposes()
else:
headposes = [None] * len(rois)
return rois, landmarks, headposes
# パフォーマンスステータスの取得
def get_performance_stats(self):
stats = {'face_detector': self.face_detector.get_performance_stats()}
if self.lm_enabe:
stats['landmarks'] = self.landmarks_detector.get_performance_stats(
)
if self.hp_enabe:
stats['headpose'] = self.headpose_detector.get_performance_stats()
return stats
class FrameProcessor:
QUEUE_SIZE = 16
def __init__(self, args):
used_devices = set([args.d_fd, args.d_lm, args.d_reid])
self.context = InferenceContext(used_devices, args.cpu_lib,
args.gpu_lib, args.perf_stats)
context = self.context
log.info("Loading models")
face_detector_net = self.load_model(args.m_fd)
assert (args.fd_input_height and args.fd_input_width) or \
(args.fd_input_height==0 and args.fd_input_width==0), \
"Both -fd_iw and -fd_ih parameters should be specified for reshape"
if args.fd_input_height and args.fd_input_width:
face_detector_net.reshape(
{"data": [1, 3, args.fd_input_height, args.fd_input_width]})
landmarks_net = self.load_model(args.m_lm)
face_reid_net = self.load_model(args.m_reid)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.face_identifier = FaceIdentifier(face_reid_net,
match_threshold=args.t_id,
match_algo=args.match_algo)
self.face_detector.deploy(args.d_fd, context)
self.landmarks_detector.deploy(args.d_lm,
context,
queue_size=self.QUEUE_SIZE)
self.face_identifier.deploy(args.d_reid,
context,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
log.info("Building faces database using images from '%s'" % (args.fg))
self.faces_database = FacesDatabase(
args.fg, self.face_identifier, self.landmarks_detector,
self.face_detector if args.run_detector else None, args.no_show)
self.face_identifier.set_faces_database(self.faces_database)
log.info("Database is built, registered %s identities" % \
(len(self.faces_database)))
self.allow_grow = args.allow_grow and not args.no_show
def load_model(self, model_path):
model_path = osp.abspath(model_path)
model_weights_path = osp.splitext(model_path)[0] + ".bin"
log.info("Loading the model from '%s'" % (model_path))
assert osp.isfile(model_path), \
"Model description is not found at '%s'" % (model_path)
assert osp.isfile(model_weights_path), \
"Model weights are not found at '%s'" % (model_weights_path)
model = self.context.ie_core.read_network(model_path,
model_weights_path)
log.info("Model is loaded")
return model
def process(self, frame):
assert len(frame.shape) == 3, \
"Expected input frame in (H, W, C) format"
assert frame.shape[2] in [3, 4], \
"Expected BGR or BGRA input"
orig_image = frame.copy()
frame = frame.transpose((2, 0, 1)) # HWC to CHW
frame = np.expand_dims(frame, axis=0)
self.face_detector.clear()
self.landmarks_detector.clear()
self.face_identifier.clear()
self.face_detector.start_async(frame)
rois = self.face_detector.get_roi_proposals(frame)
if self.QUEUE_SIZE < len(rois):
log.warning("Too many faces for processing." \
" Will be processed only %s of %s." % \
(self.QUEUE_SIZE, len(rois)))
rois = rois[:self.QUEUE_SIZE]
self.landmarks_detector.start_async(frame, rois)
landmarks = self.landmarks_detector.get_landmarks()
self.face_identifier.start_async(frame, rois, landmarks)
face_identities, unknowns = self.face_identifier.get_matches()
if self.allow_grow and len(unknowns) > 0:
for i in unknowns:
# This check is preventing asking to save half-images in the boundary of images
if rois[i].position[0] == 0.0 or rois[i].position[1] == 0.0 or \
(rois[i].position[0] + rois[i].size[0] > orig_image.shape[1]) or \
(rois[i].position[1] + rois[i].size[1] > orig_image.shape[0]):
continue
crop = orig_image[
int(rois[i].position[1]):int(rois[i].position[1] +
rois[i].size[1]),
int(rois[i].position[0]):int(rois[i].position[0] +
rois[i].size[0])]
name = self.faces_database.ask_to_save(crop)
if name:
id = self.faces_database.dump_faces(
crop, face_identities[i].descriptor, name)
face_identities[i].id = id
outputs = [rois, landmarks, face_identities]
return outputs
def get_performance_stats(self):
stats = {
'face_detector': self.face_detector.get_performance_stats(),
'landmarks': self.landmarks_detector.get_performance_stats(),
'face_identifier': self.face_identifier.get_performance_stats(),
}
return stats
def __init__(self, args):
# 追加検出処理フラグ
self.lm_enabe = False
self.hp_enabe = False
# 推論エンジン
self.iecore = IECore()
# 推論に使用するデバイス一覧
used_devices = set([args.d_fd, args.d_lm, args.d_hp])
# puluginのロード
start_time = time.time()
log.info(f"Loading plugins for devices: {used_devices}")
if 'CPU' in used_devices and not len(args.cpu_lib) == 0:
log.info(f"Using CPU extensions library '{args.cpu_lib}'")
assert os.path.isfile(
cpu_ext), "Failed to open CPU extensions library"
self.iecore.add_extension(args.cpu_lib, "CPU")
if 'GPU' in used_devices and not len(args.gpu_lib) == 0:
log.info(f"Using GPU extensions library '{args.gpu_lib}'")
assert os.path.isfile(
gpu_ext), "Failed to open GPU definitions file"
self.iecore.set_config({"CONFIG_FILE": gpu_ext}, "GPU")
log.info(
f"Plugins are loaded. loading time : {time.time()- start_time:.4f}sec"
)
for d in used_devices:
self.iecore.set_config(
{"PERF_COUNT": "YES" if args.perf_stats else "NO"}, d)
# モデルのロード
log.info("Loading models")
if (args.m_fd):
log.info(" Face Detect model")
face_detector_net = self.load_model(args.m_fd)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.face_detector.deploy(args.d_fd, self.iecore)
else:
# 顔検出モデルが指定されていなければエラー
log.error("--m-fd option is mandatory")
raise RuntimeError("--m-fd option is mandatory")
if (args.m_lm):
log.info(" Face Landmark model")
self.lm_enabe = True
landmarks_net = self.load_model(args.m_lm)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.landmarks_detector.deploy(args.d_lm,
self.iecore,
queue_size=self.QUEUE_SIZE)
if (args.m_hp):
log.info(" Head pose model")
self.hp_enabe = True
headpose_net = self.load_model(args.m_hp)
self.headpose_detector = HeadposeDetector(headpose_net)
self.headpose_detector.deploy(args.d_hp,
self.iecore,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
class FrameProcessor:
QUEUE_SIZE = 16
def __init__(self, args):
self.gpu_ext = args.gpu_lib
self.allow_grow = args.allow_grow and not args.no_show
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
if args.cpu_lib and 'CPU' in {args.d_fd, args.d_lm, args.d_reid}:
core.add_extension(args.cpu_lib, 'CPU')
self.face_detector = FaceDetector(core,
args.m_fd,
args.fd_input_size,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.landmarks_detector = LandmarksDetector(core, args.m_lm)
self.face_identifier = FaceIdentifier(core,
args.m_reid,
match_threshold=args.t_id,
match_algo=args.match_algo)
self.face_detector.deploy(args.d_fd, self.get_config(args.d_fd))
self.landmarks_detector.deploy(args.d_lm, self.get_config(args.d_lm),
self.QUEUE_SIZE)
self.face_identifier.deploy(args.d_reid, self.get_config(args.d_reid),
self.QUEUE_SIZE)
log.debug('Building faces database using images from {}'.format(
args.fg))
self.faces_database = FacesDatabase(
args.fg, self.face_identifier, self.landmarks_detector,
self.face_detector if args.run_detector else None, args.no_show)
self.face_identifier.set_faces_database(self.faces_database)
log.info('Database is built, registered {} identities'.format(
len(self.faces_database)))
def get_config(self, device):
config = {}
if device == 'GPU' and self.gpu_ext:
config['CONFIG_FILE'] = self.gpu_ext
return config
def process(self, frame):
orig_image = frame.copy()
rois = self.face_detector.infer((frame, ))
if self.QUEUE_SIZE < len(rois):
log.warning(
'Too many faces for processing. Will be processed only {} of {}'
.format(self.QUEUE_SIZE, len(rois)))
rois = rois[:self.QUEUE_SIZE]
landmarks = self.landmarks_detector.infer((frame, rois))
face_identities, unknowns = self.face_identifier.infer(
(frame, rois, landmarks))
if self.allow_grow and len(unknowns) > 0:
for i in unknowns:
# This check is preventing asking to save half-images in the boundary of images
if rois[i].position[0] == 0.0 or rois[i].position[1] == 0.0 or \
(rois[i].position[0] + rois[i].size[0] > orig_image.shape[1]) or \
(rois[i].position[1] + rois[i].size[1] > orig_image.shape[0]):
continue
crop_image = crop(orig_image, rois[i])
name = self.faces_database.ask_to_save(crop_image)
if name:
id = self.faces_database.dump_faces(
crop_image, face_identities[i].descriptor, name)
face_identities[i].id = id
return [rois, landmarks, face_identities]
class FrameProcessor:
QUEUE_SIZE = 16
def __init__(self, args):
used_devices = set([args.d_fd, args.d_lm, args.d_reid])
self.context = InferenceContext()
context = self.context
context.load_plugins(used_devices, args.cpu_lib, args.gpu_lib)
for d in used_devices:
context.get_plugin(d).set_config(
{"PERF_COUNT": "YES" if args.perf_stats else "NO"})
log.info("Loading models")
face_detector_net = self.load_model(args.m_fd)
landmarks_net = self.load_model(args.m_lm)
face_reid_net = self.load_model(args.m_reid)
self.face_detector = FaceDetector(
face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd,
)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.face_identifier = FaceIdentifier(face_reid_net,
match_threshold=args.t_id)
self.face_detector.deploy(args.d_fd, context)
self.landmarks_detector.deploy(args.d_lm,
context,
queue_size=self.QUEUE_SIZE)
self.face_identifier.deploy(args.d_reid,
context,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
if args.fc in "LOAD":
self.faces_database = pickle.loads(open(args.fpl, "rb").read())
log.info("Face database loaded from {}.".format(args.fpl))
else:
log.info("Building faces database using images from '%s'" %
(args.fg))
self.faces_database = FacesDatabase(
args.fg,
self.face_identifier,
self.landmarks_detector,
self.face_detector if args.run_detector else None,
args.no_show,
)
if args.fc in "SAVE":
with open(args.fps, "wb") as f:
f.write(pickle.dumps(self.faces_database))
f.close()
log.info("Face database {} saved".format(args.fps))
self.face_identifier.set_faces_database(self.faces_database)
log.info("Database is built, registered %s identities" %
(len(self.faces_database)))
self.allow_grow = args.allow_grow and not args.no_show
def load_model(self, model_path):
model_path = osp.abspath(model_path)
model_description_path = model_path
model_weights_path = osp.splitext(model_path)[0] + ".bin"
log.info("Loading the model from '%s'" % (model_description_path))
assert osp.isfile(
model_description_path
), "Model description is not found at '%s'" % (model_description_path)
assert osp.isfile(
model_weights_path), "Model weights are not found at '%s'" % (
model_weights_path)
model = IENetwork(model_description_path, model_weights_path)
log.info("Model is loaded")
return model
def process(self, frame):
assert len(
frame.shape) == 3, "Expected input frame in (H, W, C) format"
assert frame.shape[2] in [3, 4], "Expected BGR or BGRA input"
orig_image = frame.copy()
frame = frame.transpose((2, 0, 1)) # HWC to CHW
frame = np.expand_dims(frame, axis=0)
self.face_detector.clear()
self.landmarks_detector.clear()
self.face_identifier.clear()
self.face_detector.start_async(frame)
rois = self.face_detector.get_roi_proposals(frame)
if self.QUEUE_SIZE < len(rois):
log.warning("Too many faces for processing."
" Will be processed only %s of %s." %
(self.QUEUE_SIZE, len(rois)))
rois = rois[:self.QUEUE_SIZE]
self.landmarks_detector.start_async(frame, rois)
landmarks = self.landmarks_detector.get_landmarks()
self.face_identifier.start_async(frame, rois, landmarks)
face_identities, unknowns = self.face_identifier.get_matches()
if self.allow_grow and len(unknowns) > 0:
for i in unknowns:
# This check is preventing asking to save half-images in the boundary of images
if (rois[i].position[0] == 0.0 or rois[i].position[1] == 0.0
or (rois[i].position[0] + rois[i].size[0] >
orig_image.shape[1])
or (rois[i].position[1] + rois[i].size[1] >
orig_image.shape[0])):
continue
crop = orig_image[
int(rois[i].position[1]):int(rois[i].position[1] +
rois[i].size[1]),
int(rois[i].position[0]):int(rois[i].position[0] +
rois[i].size[0]), ]
name = self.faces_database.ask_to_save(crop)
if name:
id = self.faces_database.dump_faces(
crop, face_identities[i].descriptor, name)
face_identities[i].id = id
outputs = [rois, landmarks, face_identities]
return outputs
def get_performance_stats(self):
stats = {
"face_detector": self.face_detector.get_performance_stats(),
"landmarks": self.landmarks_detector.get_performance_stats(),
"face_identifier": self.face_identifier.get_performance_stats(),
}
return stats
def unpack_bz2(src_path):
data = bz2.BZ2File(src_path).read()
dst_path = src_path[:-4]
with open(dst_path, 'wb') as fp:
fp.write(data)
return dst_path
if __name__ == "__main__":
"""
Extracts and aligns all faces from images using DLib and a function from original FFHQ dataset preparation step
python align_images.py /raw_images /aligned_images
"""
landmarks_model_path = unpack_bz2(
tf.keras.utils.get_file('shape_predictor_68_face_landmarks.dat.bz2',
LANDMARKS_MODEL_URL,
cache_subdir='temp'))
RAW_IMAGES_DIR = sys.argv[1]
ALIGNED_IMAGES_DIR = sys.argv[2]
landmarks_detector = LandmarksDetector(landmarks_model_path)
for img_name in os.listdir(RAW_IMAGES_DIR):
raw_img_path = os.path.join(RAW_IMAGES_DIR, img_name)
for i, face_landmarks in enumerate(
landmarks_detector.get_landmarks(raw_img_path), start=1):
face_img_name = '%s_%02d.png' % (os.path.splitext(img_name)[0], i)
aligned_face_path = os.path.join(ALIGNED_IMAGES_DIR, face_img_name)
image_align(raw_img_path, aligned_face_path, face_landmarks)
def align_image(input_path, output_path):
landmarks_model_path = unpack_bz2(get_file('shape_predictor_68_face_landmarks.dat.bz2',
LANDMARKS_MODEL_URL, cache_subdir='temp'))
landmarks_detector = LandmarksDetector(landmarks_model_path)
for i, face_landmarks in enumerate(landmarks_detector.get_landmarks(input_path), start=1):
image_align(input_path, output_path, face_landmarks)
fp.write(data)
return dst_path
def align_image(input_path, output_path):
landmarks_model_path = unpack_bz2(get_file('shape_predictor_68_face_landmarks.dat.bz2',
LANDMARKS_MODEL_URL, cache_subdir='temp'))
landmarks_detector = LandmarksDetector(landmarks_model_path)
for i, face_landmarks in enumerate(landmarks_detector.get_landmarks(input_path), start=1):
image_align(input_path, output_path, face_landmarks)
if __name__ == "__main__":
"""
Extracts and aligns all faces from images using DLib and a function from original FFHQ dataset preparation step
python align_images.py /raw_images /aligned_images
"""
landmarks_model_path = unpack_bz2(get_file('shape_predictor_68_face_landmarks.dat.bz2',
LANDMARKS_MODEL_URL, cache_subdir='temp'))
RAW_IMAGES_DIR = sys.argv[1]
ALIGNED_IMAGES_DIR = sys.argv[2]
landmarks_detector = LandmarksDetector(landmarks_model_path)
for img_name in os.listdir(RAW_IMAGES_DIR):
raw_img_path = os.path.join(RAW_IMAGES_DIR, img_name)
for i, face_landmarks in enumerate(landmarks_detector.get_landmarks(raw_img_path), start=1):
face_img_name = '%s_%02d.png' % (os.path.splitext(img_name)[0], i)
aligned_face_path = os.path.join(ALIGNED_IMAGES_DIR, face_img_name)
image_align(raw_img_path, aligned_face_path, face_landmarks)
class FrameProcessor:
# 同時に検出できる数
QUEUE_SIZE = 16
def __init__(self, args):
# 追加検出処理フラグ
self.lm_enabe = False
self.hp_enabe = False
# 推論エンジン
self.iecore = IECore()
# 推論に使用するデバイス一覧
used_devices = set([args.d_fd, args.d_lm, args.d_hp])
# puluginのロード
start_time = time.time()
log.info(f"Loading plugins for devices: {used_devices}")
if 'CPU' in used_devices and not len(args.cpu_lib) == 0:
log.info(f"Using CPU extensions library '{args.cpu_lib}'")
assert os.path.isfile(
cpu_ext), "Failed to open CPU extensions library"
self.iecore.add_extension(args.cpu_lib, "CPU")
if 'GPU' in used_devices and not len(args.gpu_lib) == 0:
log.info(f"Using GPU extensions library '{args.gpu_lib}'")
assert os.path.isfile(
gpu_ext), "Failed to open GPU definitions file"
self.iecore.set_config({"CONFIG_FILE": gpu_ext}, "GPU")
log.info(
f"Plugins are loaded. loading time : {time.time()- start_time:.4f}sec"
)
for d in used_devices:
self.iecore.set_config(
{"PERF_COUNT": "YES" if args.perf_stats else "NO"}, d)
# モデルのロード
log.info("Loading models")
if (args.m_fd):
log.info(" Face Detect model")
face_detector_net = self.load_model(args.m_fd)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.face_detector.deploy(args.d_fd, self.iecore)
else:
# 顔検出モデルが指定されていなければエラー
log.error("--m-fd option is mandatory")
raise RuntimeError("--m-fd option is mandatory")
if (args.m_lm):
log.info(" Face Landmark model")
self.lm_enabe = True
landmarks_net = self.load_model(args.m_lm)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.landmarks_detector.deploy(args.d_lm,
self.iecore,
queue_size=self.QUEUE_SIZE)
if (args.m_hp):
log.info(" Head pose model")
self.hp_enabe = True
headpose_net = self.load_model(args.m_hp)
self.headpose_detector = HeadposeDetector(headpose_net)
self.headpose_detector.deploy(args.d_hp,
self.iecore,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
# IR(Intermediate Representation ;中間表現)ファイル(.xml & .bin) の読み込み
def load_model(self, model_path):
start_time = time.time() # ロード時間測定用
model_path = os.path.abspath(model_path)
model_description_path = model_path
model_weights_path = os.path.splitext(model_path)[0] + ".bin"
log.info(f" Loading the model from '{model_description_path}'")
assert os.path.isfile(model_description_path), \
f"Model description is not found at '{model_description_path}'"
assert os.path.isfile(model_weights_path), \
f"Model weights are not found at '{model_weights_path}'"
model = self.iecore.read_network(model=model_description_path,
weights=model_weights_path)
log.info(
f" Model is loaded loading time : {time.time()- start_time:.4f}sec"
)
return model
# フレーム毎の処理
def process(self, frame):
assert len(
frame.shape) == 3, "Expected input frame in (H, W, C) format"
assert frame.shape[2] in [3, 4], "Expected BGR or BGRA input"
orig_image = frame.copy()
frame = frame.transpose((2, 0, 1)) # HWC to CHW
frame = np.expand_dims(frame, axis=0)
self.face_detector.clear()
if self.lm_enabe:
self.landmarks_detector.clear()
if self.hp_enabe:
self.headpose_detector.clear()
# log.info("Face Detect")
# 認識処理
self.face_detector.start_async(frame)
# 結果取得
rois = self.face_detector.get_roi_proposals(frame)
# 認識された顔が多すぎる場合は最大値まで縮小する
if self.QUEUE_SIZE < len(rois):
log.warning("Too many faces for processing." \
" Will be processed only {self.QUEUE_SIZE} of {len(rois)}.")
rois = rois[:self.QUEUE_SIZE]
# 特徴点検出
if self.lm_enabe:
# log.info("Landmarks")
# 認識処理
self.landmarks_detector.start_async(frame, rois)
# 結果取得
landmarks = self.landmarks_detector.get_landmarks()
else:
landmarks = [None] * len(rois)
# 向き検出
if self.hp_enabe:
# log.info("Headpose")
# 認識処理
self.headpose_detector.start_async(frame, rois)
# 結果取得
headposes = self.headpose_detector.get_headposes()
else:
headposes = [None] * len(rois)
return rois, landmarks, headposes
def get_performance_stats(self):
stats = {'face_detector': self.face_detector.get_performance_stats()}
if self.lm_enabe:
stats['landmarks'] = self.landmarks_detector.get_performance_stats(
)
if self.hp_enabe:
stats['headpose'] = self.headpose_detector.get_performance_stats()
return stats
def image_align(src_file,
dst_file,
face_landmarks,
output_size=1024,
transform_size=4096,
enable_padding=True):
'''
Function that applies all the data pre-processing steps proposed in StyleGAN-FFHQ
parameters: 'src_file' indicates the path to raw images; 'dst_file' indicates the path to store the aligned images;
'face_landmarks' specify the (x, y) coordinates of the 68-face landmarks; 'output_size' specifies the resolution of the aligned image;
'transform_size' indicates the target up-scaling resolution; 'enable_padding' pads the image if up-scaled
output: Aligned face image at 1024x1024 resolution
'''
# Align function from FFHQ dataset pre-processing step
# https://github.com/NVlabs/ffhq-dataset/blob/master/download_ffhq.py
lm = np.array(face_landmarks)
lm_chin = lm[0:17] # left-right
lm_eyebrow_left = lm[17:22] # left-right
lm_eyebrow_right = lm[22:27] # left-right
lm_nose = lm[27:31] # top-down
lm_nostrils = lm[31:36] # top-down
lm_eye_left = lm[36:42] # left-clockwise
lm_eye_right = lm[42:48] # left-clockwise
lm_mouth_outer = lm[48:60] # left-clockwise
lm_mouth_inner = lm[60:68] # left-clockwise
# Calculate auxiliary vectors.
eye_left = np.mean(lm_eye_left, axis=0)
eye_right = np.mean(lm_eye_right, axis=0)
### Check if the face is tilted and if true, perform de-rotation
# Compute the center of mass for each eye
leftEyeCenter = np.mean(lm_eye_left, axis=0).astype("int")
rightEyeCenter = np.mean(lm_eye_right, axis=0).astype("int")
# Compute the angle between the eye centroids
dY = rightEyeCenter[1] - leftEyeCenter[1]
dX = rightEyeCenter[0] - leftEyeCenter[0]
angle = math.floor(abs(np.degrees(np.arctan2(dY, dX))))
print("\n\t\tThe angle is: ", angle, "\n")
is_rotated = False
if angle > 11:
is_rotated = True
###
eye_avg = (eye_left + eye_right) * 0.5
eye_to_eye = eye_right - eye_left
mouth_left = lm_mouth_outer[0]
mouth_right = lm_mouth_outer[6]
mouth_avg = (mouth_left + mouth_right) * 0.5
eye_to_mouth = mouth_avg - eye_avg
# Choose oriented crop rectangle.
x = eye_to_eye - np.flipud(eye_to_mouth) * [-1, 1]
x /= np.hypot(*x)
x *= max(np.hypot(*eye_to_eye) * 2.0, np.hypot(*eye_to_mouth) * 1.8)
y = np.flipud(x) * [-1, 1]
c = eye_avg + eye_to_mouth * 0.1
quad = np.stack([c - x - y, c - x + y, c + x + y, c + x - y])
qsize = np.hypot(*x) * 2
if is_rotated:
ROTATED_IMAGES_DIR = '../../rotated_images/'
landmarks_model_path = '../../cache/shape_predictor_68_face_landmarks.dat'
os.makedirs(ROTATED_IMAGES_DIR, exist_ok=True)
rot_file = str(os.path.splitext(src_file)[0]).split('/')[-1] + str(
os.path.splitext(src_file)[1])
rot_file_path = os.path.join(ROTATED_IMAGES_DIR, rot_file)
rotate_face(src_file, rot_file_path)
landmarks_detector = LandmarksDetector(landmarks_model_path)
face_landmarks = landmarks_detector.get_landmarks(rot_file_path)
lm = np.array(face_landmarks)
lm_chin = lm[0:17] # left-right
lm_eyebrow_left = lm[17:22] # left-right
lm_eyebrow_right = lm[22:27] # left-right
lm_nose = lm[27:31] # top-down
lm_nostrils = lm[31:36] # top-down
lm_eye_left = lm[36:42] # left-clockwise
lm_eye_right = lm[42:48] # left-clockwise
lm_mouth_outer = lm[48:60] # left-clockwise
lm_mouth_inner = lm[60:68] # left-clockwise
# Calculate auxiliary vectors.
eye_left = np.mean(lm_eye_left, axis=0)
eye_right = np.mean(lm_eye_right, axis=0)
eye_avg = (eye_left + eye_right) * 0.5
eye_to_eye = eye_right - eye_left
mouth_left = lm_mouth_outer[0]
mouth_right = lm_mouth_outer[6]
mouth_avg = (mouth_left + mouth_right) * 0.5
eye_to_mouth = mouth_avg - eye_avg
# Choose oriented crop rectangle.
x = eye_to_eye - np.flipud(eye_to_mouth) * [-1, 1]
x /= np.hypot(*x)
x *= max(np.hypot(*eye_to_eye) * 2.0, np.hypot(*eye_to_mouth) * 1.8)
y = np.flipud(x) * [-1, 1]
c = eye_avg + eye_to_mouth * 0.1
quad = np.stack([c - x - y, c - x + y, c + x + y, c + x - y])
qsize = np.hypot(*x) * 2
img = PIL.Image.open(rot_file_path)
# Shrink.
shrink = int(np.floor(qsize / output_size * 0.5))
if shrink > 1:
rsize = (int(np.rint(float(img.size[0]) / shrink)),
int(np.rint(float(img.size[1]) / shrink)))
img = img.resize(rsize, PIL.Image.ANTIALIAS)
quad /= shrink
qsize /= shrink
# Crop.
border = max(int(np.rint(qsize * 0.1)), 3)
crop = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))),
int(np.ceil(max(quad[:, 0]))), int(np.ceil(max(quad[:, 1]))))
crop = (max(crop[0] - border, 0), max(crop[1] - border, 0),
min(crop[2] + border,
img.size[0]), min(crop[3] + border, img.size[1]))
if crop[2] - crop[0] < img.size[0] or crop[3] - crop[1] < img.size[1]:
img = img.crop(crop)
quad -= crop[0:2]
# Pad.
pad = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))),
int(np.ceil(max(quad[:, 0]))), int(np.ceil(max(quad[:, 1]))))
pad = (max(-pad[0] + border,
0), max(-pad[1] + border,
0), max(pad[2] - img.size[0] + border,
0), max(pad[3] - img.size[1] + border, 0))
if enable_padding and max(pad) > border - 4:
pad = np.maximum(pad, int(np.rint(qsize * 0.3)))
img = np.pad(np.float32(img),
((pad[1], pad[3]), (pad[0], pad[2]), (0, 0)),
'reflect')
h, w, _ = img.shape
y, x, _ = np.ogrid[:h, :w, :1]
mask = np.maximum(
1.0 - np.minimum(
np.float32(x) / pad[0],
np.float32(w - 1 - x) / pad[2]), 1.0 - np.minimum(
np.float32(y) / pad[1],
np.float32(h - 1 - y) / pad[3]))
blur = qsize * 0.02
img += (scipy.ndimage.gaussian_filter(img, [blur, blur, 0]) -
img) * np.clip(mask * 3.0 + 1.0, 0.0, 1.0)
img += (np.median(img, axis=(0, 1)) - img) * np.clip(
mask, 0.0, 1.0)
#img = PIL.Image.fromarray(np.uint8(np.clip(np.rint(img), 0, 255)), 'RGB')
img = np.uint8(np.clip(np.rint(img), 0, 255))
img = PIL.Image.fromarray(img, 'RGB')
quad += pad[:2]
img = img.transform((transform_size, transform_size), PIL.Image.QUAD,
(quad + 0.5).flatten(), PIL.Image.BILINEAR)
if output_size < transform_size:
img = img.resize((output_size, output_size), PIL.Image.ANTIALIAS)
else:
img = PIL.Image.open(src_file)
# Shrink.
shrink = int(np.floor(qsize / output_size * 0.5))
if shrink > 1:
rsize = (int(np.rint(float(img.size[0]) / shrink)),
int(np.rint(float(img.size[1]) / shrink)))
img = img.resize(rsize, PIL.Image.ANTIALIAS)
quad /= shrink
qsize /= shrink
# Crop.
border = max(int(np.rint(qsize * 0.1)), 3)
crop = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))),
int(np.ceil(max(quad[:, 0]))), int(np.ceil(max(quad[:, 1]))))
crop = (max(crop[0] - border, 0), max(crop[1] - border, 0),
min(crop[2] + border,
img.size[0]), min(crop[3] + border, img.size[1]))
if crop[2] - crop[0] < img.size[0] or crop[3] - crop[1] < img.size[1]:
img = img.crop(crop)
quad -= crop[0:2]
# Pad.
pad = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))),
int(np.ceil(max(quad[:, 0]))), int(np.ceil(max(quad[:, 1]))))
pad = (max(-pad[0] + border,
0), max(-pad[1] + border,
0), max(pad[2] - img.size[0] + border,
0), max(pad[3] - img.size[1] + border, 0))
if enable_padding and max(pad) > border - 4:
pad = np.maximum(pad, int(np.rint(qsize * 0.3)))
img = np.pad(np.float32(img),
((pad[1], pad[3]), (pad[0], pad[2]), (0, 0)),
'reflect')
h, w, _ = img.shape
y, x, _ = np.ogrid[:h, :w, :1]
mask = np.maximum(
1.0 - np.minimum(
np.float32(x) / pad[0],
np.float32(w - 1 - x) / pad[2]), 1.0 - np.minimum(
np.float32(y) / pad[1],
np.float32(h - 1 - y) / pad[3]))
blur = qsize * 0.02
img += (scipy.ndimage.gaussian_filter(img, [blur, blur, 0]) -
img) * np.clip(mask * 3.0 + 1.0, 0.0, 1.0)
img += (np.median(img, axis=(0, 1)) - img) * np.clip(
mask, 0.0, 1.0)
#img = PIL.Image.fromarray(np.uint8(np.clip(np.rint(img), 0, 255)), 'RGB')
img = np.uint8(np.clip(np.rint(img), 0, 255))
img = PIL.Image.fromarray(img, 'RGB')
quad += pad[:2]
# Transform.
img = img.transform((transform_size, transform_size), PIL.Image.QUAD,
(quad + 0.5).flatten(), PIL.Image.BILINEAR)
if output_size < transform_size:
img = img.resize((output_size, output_size), PIL.Image.ANTIALIAS)
# Save aligned image.
img.save(dst_file, 'PNG')
class FrameProcessor:
QUEUE_SIZE = 16
def __init__(self, args):
used_devices = set([args.d_fd, args.d_lm, args.d_hp, args.d_reid])
self.context = InferenceContext()
context = self.context
context.load_plugins(used_devices, args.cpu_lib, args.gpu_lib)
for d in used_devices:
context.get_plugin(d).set_config(
{"PERF_COUNT": "YES" if args.perf_stats else "NO"})
log.info("Loading models")
face_detector_net = self.load_model(args.m_fd)
landmarks_net = self.load_model(args.m_lm)
head_pose_net = self.load_model(args.m_hp)
# face_reid_net = self.load_model(args.m_reid)
self.face_detector = FaceDetector(face_detector_net,
confidence_threshold=args.t_fd,
roi_scale_factor=args.exp_r_fd)
self.landmarks_detector = LandmarksDetector(landmarks_net)
self.head_pose_detector = HeadPoseDetector(head_pose_net)
self.face_detector.deploy(args.d_fd, context)
self.landmarks_detector.deploy(args.d_lm,
context,
queue_size=self.QUEUE_SIZE)
self.head_pose_detector.deploy(args.d_hp,
context,
queue_size=self.QUEUE_SIZE)
log.info("Models are loaded")
def load_model(self, model_path):
model_path = osp.abspath(model_path)
model_description_path = model_path
model_weights_path = osp.splitext(model_path)[0] + ".bin"
log.info("Loading the model from '%s'" % (model_description_path))
assert osp.isfile(model_description_path), \
"Model description is not found at '%s'" % (model_description_path)
assert osp.isfile(model_weights_path), \
"Model weights are not found at '%s'" % (model_weights_path)
model = IENetwork(model_description_path, model_weights_path)
log.info("Model is loaded")
return model
def process(self, frame):
# print("frame.shape--", frame.shape)
assert len(frame.shape) == 3, \
"Expected input frame in (H, W, C) format"
assert frame.shape[2] in [3, 4], \
"Expected BGR or BGRA input"
orig_image = frame.copy()
frame = frame.transpose((2, 0, 1)) # HWC to CHW
# print("frame.shape--", frame.shape)
frame = np.expand_dims(frame, axis=0)
# print("frame.shape--", frame.shape)
self.face_detector.clear()
self.landmarks_detector.clear()
self.head_pose_detector.clear()
# self.face_identifier.clear()
self.face_detector.start_async(frame)
rois = self.face_detector.get_roi_proposals(frame)
if self.QUEUE_SIZE < len(rois):
log.warning("Too many faces for processing." \
" Will be processed only %s of %s." % \
(self.QUEUE_SIZE, len(rois)))
rois = rois[:self.QUEUE_SIZE]
self.landmarks_detector.start_async(frame, rois)
self.head_pose_detector.start_async(frame, rois)
landmarks = self.landmarks_detector.get_landmarks()
head_pose = self.head_pose_detector.get_head_pose()
outputs = [rois, landmarks, head_pose, 'test']
return outputs
def get_performance_stats(self):
stats = {
'face_detector': self.face_detector.get_performance_stats(),
'landmarks': self.landmarks_detector.get_performance_stats(),
'face_identifier': self.face_identifier.get_performance_stats(),
}
return stats
