def recognize_files(self, filenames, debug_out_folder, skip_face_gen=False): self.__make_debug_out_folder(debug_out_folder) self.__start_stage(len(filenames)) for f in filenames: if self.__step_stage(): break try: ext = tools.get_low_ext(f) if ext in tools.IMAGE_EXTS: encoded_faces, media = self.recognize_image(f) elif ext in tools.VIDEO_EXTS: encoded_faces, media = self.recognize_video(f) else: log.warning(f'Unknown ext: {ext}') continue if media is None: continue self.__db.insert(f, encoded_faces, commit=False) if debug_out_folder: debug_out_file_name = self.__extract_filename(f) self.__save_debug_images( encoded_faces, media, debug_out_folder, debug_out_file_name, is_video=ext in tools.VIDEO_EXTS, skip_face_gen=skip_face_gen) except Exception as ex: log.exception(f'Image {f} recognition failed') self.__end_stage()
def load(self): try: data = pickle.loads(open(self.__pickle_file, 'rb').read()) self.__files = data['files'] self.__persons = data['persons'] self.__init_basenames() except Exception: log.exception(f"Can't load patterns: {self.__pickle_file}")
def remove_files(self, filenames): for filename in filenames: filename = self.__calc_filename(filename) try: self.__remove_file(filename) os.remove(filename) except (FileNotFoundError, KeyError): log.exception('Skip file removing') self.__save()
def __check_basename(self, filename): basename = os.path.basename(filename) if basename in self.__basenames: dup_filename = self.fullpath(self.__basenames[basename]) log.info(f'Duplicate file detected: {basename}') self.__remove_file(dup_filename) try: os.remove(dup_filename) except FileNotFoundError: log.exception('Skip file removing') self.__save()
def do_GET(self): log.debug('do_GET: ' + self.path) try: path, params = self.__path_params() if path == '/list_cache': self.__list_cache(params) return if path == '/get_names': self.__get_names() return if path == '/get_name_image': self.__get_name_image(params) return if path == '/get_folders': self.__get_folders() return if path == '/get_status': self.__get_status() return if path == '/get_face_src': self.__get_face_src(params) return if path == '/get_face_pattern': self.__get_face_pattern(params) return if path == '/': path = 'index.html' if '..' in path: log.warning('".." in path: ' + path) self.__not_found_response() return ext = tools.get_low_ext(path) if ext in ('.html', '.js', '.css', '.png', '.jpg'): self.__file_request(path, params) return log.warning('Wrong path: ' + path) self.__not_found_response() except Exception as ex: self.__server_error_response(str(ex)) log.exception(ex)
def run(self): os.nice(10) try: from face_rec_tools import recdb # noqa from face_rec_tools import config # noqa from face_rec_tools import cachedb # noqa from face_rec_tools import patterns # noqa from face_rec_tools import recognizer # noqa cfg = config.Config(self.__config) patterns = patterns.createPatterns(cfg) patterns.load() db = recdb.RecDB(cfg.get_path('files', 'db')) cdb = cachedb.createCacheDB(cfg) cuda_memory_limit = int(cfg['processing']['cuda_memory_limit']) log.info(f'Run in process: {self.__method}{self.__args}') if self.__method == 'recognize_folder': tools.cuda_init(cuda_memory_limit) recognizer = recognizer.createRecognizer( patterns, cfg, cdb, db, self.__status) recognizer.recognize_folder(*self.__args) elif self.__method == 'match': tools.cuda_init(cuda_memory_limit) recognizer = recognizer.createRecognizer( patterns, cfg, cdb, db, self.__status) recognizer.match(*self.__args) elif self.__method == 'clusterize': recognizer = recognizer.createRecognizer( patterns, cfg, cdb, db, self.__status) recognizer.clusterize(*self.__args) elif self.__method == 'save_faces': recognizer = recognizer.createRecognizer( patterns, cfg, cdb, db, self.__status) recognizer.save_faces(*self.__args) elif self.__method == 'get_faces_by_face': tools.cuda_init(cuda_memory_limit) recognizer = recognizer.createRecognizer( patterns, cfg, cdb, db, self.__status) recognizer.get_faces_by_face(*self.__args) log.info(f'Process done: {self.__method}') self.__status['state'] = 'done' except Exception as ex: log.exception(ex) self.__status['state'] = 'error' self.__status['error'] = str(ex)
def reencode_files(self, files_faces): self.__start_stage(len(files_faces)) for ff in files_faces: try: encoded_faces = ff['faces'] filename = ff['filename'] ext = tools.get_low_ext(filename) if ext in tools.IMAGE_EXTS: self.reencode_image(filename, encoded_faces) encoded_faces = self.__filter_encoded_faces(encoded_faces) elif ext in tools.VIDEO_EXTS: encoded_faces, media = self.recognize_video(filename) else: log.warning(f'Unknown ext: {ext}') continue self.__db.insert(filename, encoded_faces, commit=False) except Exception as ex: log.exception(f'{filename} reencoding failed') self.__end_stage()
def __send_file(self, fname, params={}): try: ext = tools.get_low_ext(fname) cont = '' if ext == '.html': cont = 'text/html' elif ext == '.js': cont = 'text/javascript' elif ext == '.css': cont = 'text/css' elif ext == '.png': cont = 'image/png' elif ext == '.jpg': cont = 'image/jpeg' else: cont = 'text/none' with open(fname, 'rb') as f: self.__send_blob(f.read(), cont, params) except IOError as ex: self.__not_found_response() log.exception(ex)
def do_POST(self): log.debug('do_POST: ' + self.path) try: path, params = self.__path_params() if path == '/add_to_pattern': self.__add_to_pattern_request(params, self.__data()) return if path == '/recognize_folder': self.__recognize_folder_request(params) return if path == '/generate_faces': self.__generate_faces_request(params) return if path == '/match': self.__match_request(params) return if path == '/clusterize': self.__clusterize_request(params) return if path == '/get_faces_by_face': self.__get_faces_by_face_request(params, self.__form_data()) return if path == '/stop': self.__stop_request(params) return if path == '/clean_cache': self.__clean_cache() return except Exception as ex: self.__server_error_response(str(ex)) log.exception(ex)
def generate(self, regenerate=False): log.info(f'Patterns generation: {self.__folder} ({regenerate})') image_files = {} for image_file in tools.list_files(self.__folder, tools.IMAGE_EXTS): if os.path.split(image_file)[1] == FACE_FILENAME: continue image_files[image_file] = os.stat(image_file).st_mtime if not regenerate: self.load() filtered = {} for image_file in image_files: filename_exsists = self.relpath(image_file) in self.__files if not filename_exsists or \ self.__files[self.relpath(image_file)][self.FILES_TIME] != \ image_files[image_file]: filtered[image_file] = image_files[image_file] if filename_exsists: self.__remove_file(image_file) if len(filtered) == 0: log.info('Nothing changed') return image_files = filtered for (i, image_file) in enumerate(image_files): splitted = image_file.split(os.path.sep) name = splitted[-2] if name == BAD_FOLDERNAME: name = splitted[-3] tp = PATTERN_TYPE_BAD elif name == OTHER_FOLDERNAME: name = splitted[-3] tp = PATTERN_TYPE_OTHER else: tp = PATTERN_TYPE_GOOD log.info(f'{i + 1}/{len(image_files)} file: {image_file}') descr, thumbnail = tools.load_face_description(image_file) try: encoding = descr['encoding'] except Exception: encoding = None if encoding is None: import face_recognition try: image = tools.read_image(image_file, self.__max_size) except Exception: log.exception(f'read_image failed') continue boxes = face_recognition.face_locations(image, model=self.__model) if len(boxes) != 1: log.warning( f'{len(boxes)} faces detected in {image_file}. Skip.') continue encodings, landmarks = \ self.__get_encoder().encode(image, boxes) if not tools.test_landmarks(landmarks[0]): log.warning(f'bad face detected in {image_file}. Skip.') continue encoding = encodings[0] self.__files[self.relpath(image_file)] = \ [encoding, name, image_files[image_file], tp] self.__init_basenames() self.__persons = self.__calc_persons() self.__save()