def extract_vggface2_dataset(input_dir, device_args={} ): multi_gpu = device_args.get('multi_gpu', False) cpu_only = device_args.get('cpu_only', False) input_path = Path(input_dir) if not input_path.exists(): raise ValueError('Input directory not found. Please ensure it exists.') bb_csv = input_path / 'loose_bb_train.csv' if not bb_csv.exists(): raise ValueError('loose_bb_train.csv found. Please ensure it exists.') bb_lines = bb_csv.read_text().split('\n') bb_lines.pop(0) bb_dict = {} for line in bb_lines: name, l, t, w, h = line.split(',') name = name[1:-1] l, t, w, h = [ int(x) for x in (l, t, w, h) ] bb_dict[name] = (l,t,w, h) output_path = input_path.parent / (input_path.name + '_out') dir_names = Path_utils.get_all_dir_names(input_path) if not output_path.exists(): output_path.mkdir(parents=True, exist_ok=True) data = [] for dir_name in io.progress_bar_generator(dir_names, "Collecting"): cur_input_path = input_path / dir_name cur_output_path = output_path / dir_name if not cur_output_path.exists(): cur_output_path.mkdir(parents=True, exist_ok=True) input_path_image_paths = Path_utils.get_image_paths(cur_input_path) for filename in input_path_image_paths: filename_path = Path(filename) name = filename_path.parent.name + '/' + filename_path.stem if name not in bb_dict: continue l,t,w,h = bb_dict[name] if min(w,h) < 128: continue data += [ ExtractSubprocessor.Data(filename=filename,rects=[ (l,t,l+w,t+h) ], landmarks_accurate=False, force_output_path=cur_output_path ) ] face_type = FaceType.fromString('full_face') io.log_info ('Performing 2nd pass...') data = ExtractSubprocessor (data, 'landmarks', 256, face_type, debug_dir=None, multi_gpu=multi_gpu, cpu_only=cpu_only, manual=False).run() io.log_info ('Performing 3rd pass...') ExtractSubprocessor (data, 'final', 256, face_type, debug_dir=None, multi_gpu=multi_gpu, cpu_only=cpu_only, manual=False, final_output_path=None).run()
def dev_test(input_dir): input_path = Path(input_dir) dir_names = Path_utils.get_all_dir_names(input_path) for dir_name in io.progress_bar_generator(dir_names, desc="Processing"): img_paths = Path_utils.get_image_paths (input_path / dir_name) for filename in img_paths: filepath = Path(filename) dflimg = DFLIMG.load (filepath) if dflimg is None: raise ValueError dflimg.embed_and_set(filename, person_name=dir_name)
def dev_test(input_dir): input_path = Path(input_dir) dir_names = Path_utils.get_all_dir_names(input_path) for dir_name in dir_names: img_paths = Path_utils.get_image_paths (input_path / dir_name) for filename in img_paths: filepath = Path(filename) dflimg = DFLIMG.load (filepath) if dflimg is None: raise ValueError import code code.interact(local=dict(globals(), **locals()))
def extract_vggface2_dataset(input_dir, device_args={}): multi_gpu = device_args.get('multi_gpu', False) cpu_only = device_args.get('cpu_only', False) input_path = Path(input_dir) if not input_path.exists(): raise ValueError('Input directory not found. Please ensure it exists.') output_path = input_path.parent / (input_path.name + '_out') dir_names = Path_utils.get_all_dir_names(input_path) if not output_path.exists(): output_path.mkdir(parents=True, exist_ok=True) for dir_name in dir_names: cur_input_path = input_path / dir_name cur_output_path = output_path / dir_name l = len(Path_utils.get_image_paths(cur_input_path)) if l < 250 or l > 350: continue io.log_info(f"Processing: {str(cur_input_path)} ") if not cur_output_path.exists(): cur_output_path.mkdir(parents=True, exist_ok=True) Extractor.main(str(cur_input_path), str(cur_output_path), detector='s3fd', image_size=256, face_type='full_face', max_faces_from_image=1, device_args=device_args) io.log_info(f"Sorting: {str(cur_input_path)} ") Sorter.main(input_path=str(cur_output_path), sort_by_method='hist') try: io.log_info(f"Removing: {str(cur_input_path)} ") shutil.rmtree(cur_input_path) except: io.log_info(f"unable to remove: {str(cur_input_path)} ")
def load(sample_type, samples_path, target_samples_path=None, person_id_mode=False): cache = SampleLoader.cache if str(samples_path) not in cache.keys(): cache[str(samples_path)] = [None] * SampleType.QTY datas = cache[str(samples_path)] if sample_type == SampleType.IMAGE: if datas[sample_type] is None: datas[sample_type] = [ Sample(filename=filename) for filename in io.progress_bar_generator( Path_utils.get_image_paths(samples_path), "Loading") ] elif sample_type == SampleType.FACE: if datas[sample_type] is None: if person_id_mode: dir_names = Path_utils.get_all_dir_names(samples_path) all_samples = [] for i, dir_name in io.progress_bar_generator( [*enumerate(dir_names)], "Loading"): all_samples += SampleLoader.upgradeToFaceSamples( [ Sample(filename=filename, person_id=i) for filename in Path_utils.get_image_paths( samples_path / dir_name) ], silent=True) datas[sample_type] = all_samples else: datas[sample_type] = SampleLoader.upgradeToFaceSamples([ Sample(filename=filename) for filename in Path_utils.get_image_paths(samples_path) ]) elif sample_type == SampleType.FACE_TEMPORAL_SORTED: if datas[sample_type] is None: datas[ sample_type] = SampleLoader.upgradeToFaceTemporalSortedSamples( SampleLoader.load(SampleType.FACE, samples_path)) elif sample_type == SampleType.FACE_YAW_SORTED: if datas[sample_type] is None: datas[ sample_type] = SampleLoader.upgradeToFaceYawSortedSamples( SampleLoader.load(SampleType.FACE, samples_path)) elif sample_type == SampleType.FACE_YAW_SORTED_AS_TARGET: if datas[sample_type] is None: if target_samples_path is None: raise Exception( 'target_samples_path is None for FACE_YAW_SORTED_AS_TARGET' ) datas[ sample_type] = SampleLoader.upgradeToFaceYawSortedAsTargetSamples( SampleLoader.load(SampleType.FACE_YAW_SORTED, samples_path), SampleLoader.load(SampleType.FACE_YAW_SORTED, target_samples_path)) return datas[sample_type]
def get_person_id_max_count(samples_path): return len(Path_utils.get_all_dir_names(samples_path))
def load(sample_type, samples_path, target_samples_path=None, person_id_mode=False, use_caching=False): cache = SampleLoader.cache if str(samples_path) not in cache.keys(): cache[str(samples_path)] = [None]*SampleType.QTY datas = cache[str(samples_path)] if sample_type == SampleType.IMAGE: if datas[sample_type] is None: datas[sample_type] = [ Sample(filename=filename) for filename in io.progress_bar_generator( Path_utils.get_image_paths(samples_path), "Loading") ] elif sample_type == SampleType.FACE: if datas[sample_type] is None: if not use_caching: datas[sample_type] = SampleLoader.upgradeToFaceSamples( [ Sample(filename=filename) for filename in Path_utils.get_image_paths(samples_path) ] ) else: samples_dat = samples_path / 'samples.dat' if samples_dat.exists(): io.log_info (f"Using saved samples info from '{samples_dat}' ") all_samples = pickle.loads(samples_dat.read_bytes()) if person_id_mode: for samples in all_samples: for sample in samples: sample.filename = str( samples_path / Path(sample.filename) ) else: for sample in all_samples: sample.filename = str( samples_path / Path(sample.filename) ) datas[sample_type] = all_samples else: if person_id_mode: dir_names = Path_utils.get_all_dir_names(samples_path) all_samples = [] for i, dir_name in io.progress_bar_generator( [*enumerate(dir_names)] , "Loading"): all_samples += [ SampleLoader.upgradeToFaceSamples( [ Sample(filename=filename, person_id=i) for filename in Path_utils.get_image_paths( samples_path / dir_name ) ], silent=True ) ] datas[sample_type] = all_samples else: datas[sample_type] = all_samples = SampleLoader.upgradeToFaceSamples( [ Sample(filename=filename) for filename in Path_utils.get_image_paths(samples_path) ] ) if person_id_mode: for samples in all_samples: for sample in samples: sample.filename = str(Path(sample.filename).relative_to(samples_path)) else: for sample in all_samples: sample.filename = str(Path(sample.filename).relative_to(samples_path)) samples_dat.write_bytes (pickle.dumps(all_samples)) if person_id_mode: for samples in all_samples: for sample in samples: sample.filename = str( samples_path / Path(sample.filename) ) else: for sample in all_samples: sample.filename = str( samples_path / Path(sample.filename) ) elif sample_type == SampleType.FACE_TEMPORAL_SORTED: if datas[sample_type] is None: datas[sample_type] = SampleLoader.upgradeToFaceTemporalSortedSamples( SampleLoader.load(SampleType.FACE, samples_path) ) elif sample_type == SampleType.FACE_YAW_SORTED: if datas[sample_type] is None: datas[sample_type] = SampleLoader.upgradeToFaceYawSortedSamples( SampleLoader.load(SampleType.FACE, samples_path) ) elif sample_type == SampleType.FACE_YAW_SORTED_AS_TARGET: if datas[sample_type] is None: if target_samples_path is None: raise Exception('target_samples_path is None for FACE_YAW_SORTED_AS_TARGET') datas[sample_type] = SampleLoader.upgradeToFaceYawSortedAsTargetSamples( SampleLoader.load(SampleType.FACE_YAW_SORTED, samples_path), SampleLoader.load(SampleType.FACE_YAW_SORTED, target_samples_path) ) return datas[sample_type]
def pack(samples_path): samples_dat_path = samples_path / packed_faceset_filename if samples_dat_path.exists(): io.log_info(f"{samples_dat_path} : file already exists !") io.input_bool("Press enter to continue and overwrite.", False) as_person_faceset = False dir_names = Path_utils.get_all_dir_names(samples_path) if len(dir_names) != 0: as_person_faceset = io.input_bool( f"{len(dir_names)} subdirectories found, process as person faceset? (y/n) skip:y : ", True) if as_person_faceset: image_paths = [] for dir_name in dir_names: image_paths += Path_utils.get_image_paths(samples_path / dir_name) else: image_paths = Path_utils.get_image_paths(samples_path) samples = samplelib.SampleHost.load_face_samples(image_paths) samples_len = len(samples) samples_configs = [] for sample in io.progress_bar_generator(samples, "Processing"): sample_filepath = Path(sample.filename) sample.filename = sample_filepath.name if as_person_faceset: sample.person_name = sample_filepath.parent.name samples_configs.append(sample.get_config()) samples_bytes = pickle.dumps(samples_configs, 4) of = open(samples_dat_path, "wb") of.write(struct.pack("Q", PackedFaceset.VERSION)) of.write(struct.pack("Q", len(samples_bytes))) of.write(samples_bytes) del samples_bytes #just free mem del samples_configs sample_data_table_offset = of.tell() of.write(bytes(8 * (samples_len + 1))) #sample data offset table data_start_offset = of.tell() offsets = [] for sample in io.progress_bar_generator(samples, "Packing"): try: if sample.person_name is not None: sample_path = samples_path / sample.person_name / sample.filename else: sample_path = samples_path / sample.filename with open(sample_path, "rb") as f: b = f.read() offsets.append(of.tell() - data_start_offset) of.write(b) except: raise Exception(f"error while processing sample {sample_path}") offsets.append(of.tell()) of.seek(sample_data_table_offset, 0) for offset in offsets: of.write(struct.pack("Q", offset)) of.seek(0, 2) of.close() for filename in io.progress_bar_generator(image_paths, "Deleting files"): Path(filename).unlink() if as_person_faceset: for dir_name in io.progress_bar_generator(dir_names, "Deleting dirs"): dir_path = samples_path / dir_name try: shutil.rmtree(dir_path) except: io.log_info(f"unable to remove: {dir_path} ")