def create_file_list( source_dir_root, target_dir_root, output_file, compressions, data_types, samples_per_video, min_sequence_length, ): try: # if file exists, we don't have to create it again file_list = FileList.load(output_file) except FileNotFoundError: file_list = _create_file_list( compressions, data_types, min_sequence_length, output_file, samples_per_video, source_dir_root, ) if target_dir_root: file_list.copy_to(Path(target_dir_root)) file_list.save(output_file) for split in [TRAIN_NAME, VAL_NAME, TEST_NAME]: data_set = FileList.get_dataset_form_file(output_file, split) logger.info(f"{split}-data-set: {data_set}")
f = FileList(str(root_dir), classes=["FAKE", "REAL"], min_sequence_length=1) train_data_numbers = list(range(5, 50)) val_data_numbers = list(range(5)) for train_data_number in train_data_numbers: block = root_dir / f"extracted_images_{train_data_number}" if block.exists(): for label in block.iterdir(): images = list(label.glob("*/*.png")) f.add_data_points(images, label.name, "train", np.arange(0, len(images))) for val_data_number in val_data_numbers: block = root_dir / f"extracted_images_{val_data_number}" if block.exists(): for label in block.iterdir(): images = list(label.glob("*/*.png")) f.add_data_points(images, label.name, "val", np.arange(0, len(images))) f.add_data_points(images, label.name, "test", np.arange(0, len(images))) f.save("/data/ssd1/file_lists/dfdc/5_45_split.json") for split in [TRAIN_NAME, VAL_NAME, TEST_NAME]: data_set = FileList.get_dataset_form_file( "/data/ssd1/file_lists/dfdc/5_45_split.json", split) print(f"{split}-data-set: {data_set}")