def recognize_files(self, filenames, debug_out_folder,
skip_face_gen=False):
self.__make_debug_out_folder(debug_out_folder)
self.__start_stage(len(filenames))
for f in filenames:
if self.__step_stage():
break
try:
ext = tools.get_low_ext(f)
if ext in tools.IMAGE_EXTS:
encoded_faces, media = self.recognize_image(f)
elif ext in tools.VIDEO_EXTS:
encoded_faces, media = self.recognize_video(f)
else:
log.warning(f'Unknown ext: {ext}')
continue
if media is None:
continue
self.__db.insert(f, encoded_faces, commit=False)
if debug_out_folder:
debug_out_file_name = self.__extract_filename(f)
self.__save_debug_images(
encoded_faces, media,
debug_out_folder, debug_out_file_name,
is_video=ext in tools.VIDEO_EXTS,
skip_face_gen=skip_face_gen)
except Exception as ex:
log.exception(f'Image {f} recognition failed')
self.__end_stage()
def load(self):
try:
data = pickle.loads(open(self.__pickle_file, 'rb').read())
self.__files = data['files']
self.__persons = data['persons']
self.__init_basenames()
except Exception:
log.exception(f"Can't load patterns: {self.__pickle_file}")
def remove_files(self, filenames):
for filename in filenames:
filename = self.__calc_filename(filename)
try:
self.__remove_file(filename)
os.remove(filename)
except (FileNotFoundError, KeyError):
log.exception('Skip file removing')
self.__save()
def __check_basename(self, filename):
basename = os.path.basename(filename)
if basename in self.__basenames:
dup_filename = self.fullpath(self.__basenames[basename])
log.info(f'Duplicate file detected: {basename}')
self.__remove_file(dup_filename)
try:
os.remove(dup_filename)
except FileNotFoundError:
log.exception('Skip file removing')
self.__save()
def do_GET(self):
log.debug('do_GET: ' + self.path)
try:
path, params = self.__path_params()
if path == '/list_cache':
self.__list_cache(params)
return
if path == '/get_names':
self.__get_names()
return
if path == '/get_name_image':
self.__get_name_image(params)
return
if path == '/get_folders':
self.__get_folders()
return
if path == '/get_status':
self.__get_status()
return
if path == '/get_face_src':
self.__get_face_src(params)
return
if path == '/get_face_pattern':
self.__get_face_pattern(params)
return
if path == '/':
path = 'index.html'
if '..' in path:
log.warning('".." in path: ' + path)
self.__not_found_response()
return
ext = tools.get_low_ext(path)
if ext in ('.html', '.js', '.css', '.png', '.jpg'):
self.__file_request(path, params)
return
log.warning('Wrong path: ' + path)
self.__not_found_response()
except Exception as ex:
self.__server_error_response(str(ex))
log.exception(ex)
def run(self):
os.nice(10)
try:
from face_rec_tools import recdb # noqa
from face_rec_tools import config # noqa
from face_rec_tools import cachedb # noqa
from face_rec_tools import patterns # noqa
from face_rec_tools import recognizer # noqa
cfg = config.Config(self.__config)
patterns = patterns.createPatterns(cfg)
patterns.load()
db = recdb.RecDB(cfg.get_path('files', 'db'))
cdb = cachedb.createCacheDB(cfg)
cuda_memory_limit = int(cfg['processing']['cuda_memory_limit'])
log.info(f'Run in process: {self.__method}{self.__args}')
if self.__method == 'recognize_folder':
tools.cuda_init(cuda_memory_limit)
recognizer = recognizer.createRecognizer(
patterns, cfg, cdb, db, self.__status)
recognizer.recognize_folder(*self.__args)
elif self.__method == 'match':
tools.cuda_init(cuda_memory_limit)
recognizer = recognizer.createRecognizer(
patterns, cfg, cdb, db, self.__status)
recognizer.match(*self.__args)
elif self.__method == 'clusterize':
recognizer = recognizer.createRecognizer(
patterns, cfg, cdb, db, self.__status)
recognizer.clusterize(*self.__args)
elif self.__method == 'save_faces':
recognizer = recognizer.createRecognizer(
patterns, cfg, cdb, db, self.__status)
recognizer.save_faces(*self.__args)
elif self.__method == 'get_faces_by_face':
tools.cuda_init(cuda_memory_limit)
recognizer = recognizer.createRecognizer(
patterns, cfg, cdb, db, self.__status)
recognizer.get_faces_by_face(*self.__args)
log.info(f'Process done: {self.__method}')
self.__status['state'] = 'done'
except Exception as ex:
log.exception(ex)
self.__status['state'] = 'error'
self.__status['error'] = str(ex)
def reencode_files(self, files_faces):
self.__start_stage(len(files_faces))
for ff in files_faces:
try:
encoded_faces = ff['faces']
filename = ff['filename']
ext = tools.get_low_ext(filename)
if ext in tools.IMAGE_EXTS:
self.reencode_image(filename, encoded_faces)
encoded_faces = self.__filter_encoded_faces(encoded_faces)
elif ext in tools.VIDEO_EXTS:
encoded_faces, media = self.recognize_video(filename)
else:
log.warning(f'Unknown ext: {ext}')
continue
self.__db.insert(filename, encoded_faces, commit=False)
except Exception as ex:
log.exception(f'{filename} reencoding failed')
self.__end_stage()
def __send_file(self, fname, params={}):
try:
ext = tools.get_low_ext(fname)
cont = ''
if ext == '.html':
cont = 'text/html'
elif ext == '.js':
cont = 'text/javascript'
elif ext == '.css':
cont = 'text/css'
elif ext == '.png':
cont = 'image/png'
elif ext == '.jpg':
cont = 'image/jpeg'
else:
cont = 'text/none'
with open(fname, 'rb') as f:
self.__send_blob(f.read(), cont, params)
except IOError as ex:
self.__not_found_response()
log.exception(ex)
def do_POST(self):
log.debug('do_POST: ' + self.path)
try:
path, params = self.__path_params()
if path == '/add_to_pattern':
self.__add_to_pattern_request(params, self.__data())
return
if path == '/recognize_folder':
self.__recognize_folder_request(params)
return
if path == '/generate_faces':
self.__generate_faces_request(params)
return
if path == '/match':
self.__match_request(params)
return
if path == '/clusterize':
self.__clusterize_request(params)
return
if path == '/get_faces_by_face':
self.__get_faces_by_face_request(params, self.__form_data())
return
if path == '/stop':
self.__stop_request(params)
return
if path == '/clean_cache':
self.__clean_cache()
return
except Exception as ex:
self.__server_error_response(str(ex))
log.exception(ex)
def generate(self, regenerate=False):
log.info(f'Patterns generation: {self.__folder} ({regenerate})')
image_files = {}
for image_file in tools.list_files(self.__folder, tools.IMAGE_EXTS):
if os.path.split(image_file)[1] == FACE_FILENAME:
continue
image_files[image_file] = os.stat(image_file).st_mtime
if not regenerate:
self.load()
filtered = {}
for image_file in image_files:
filename_exsists = self.relpath(image_file) in self.__files
if not filename_exsists or \
self.__files[self.relpath(image_file)][self.FILES_TIME] != \
image_files[image_file]:
filtered[image_file] = image_files[image_file]
if filename_exsists:
self.__remove_file(image_file)
if len(filtered) == 0:
log.info('Nothing changed')
return
image_files = filtered
for (i, image_file) in enumerate(image_files):
splitted = image_file.split(os.path.sep)
name = splitted[-2]
if name == BAD_FOLDERNAME:
name = splitted[-3]
tp = PATTERN_TYPE_BAD
elif name == OTHER_FOLDERNAME:
name = splitted[-3]
tp = PATTERN_TYPE_OTHER
else:
tp = PATTERN_TYPE_GOOD
log.info(f'{i + 1}/{len(image_files)} file: {image_file}')
descr, thumbnail = tools.load_face_description(image_file)
try:
encoding = descr['encoding']
except Exception:
encoding = None
if encoding is None:
import face_recognition
try:
image = tools.read_image(image_file, self.__max_size)
except Exception:
log.exception(f'read_image failed')
continue
boxes = face_recognition.face_locations(image,
model=self.__model)
if len(boxes) != 1:
log.warning(
f'{len(boxes)} faces detected in {image_file}. Skip.')
continue
encodings, landmarks = \
self.__get_encoder().encode(image, boxes)
if not tools.test_landmarks(landmarks[0]):
log.warning(f'bad face detected in {image_file}. Skip.')
continue
encoding = encodings[0]
self.__files[self.relpath(image_file)] = \
[encoding,
name,
image_files[image_file],
tp]
self.__init_basenames()
self.__persons = self.__calc_persons()
self.__save()
