def main(): args.dump_root = Path(args.dump_root) args.dump_root.mkdir_p() global data_loader if args.dataset_format == 'kitti': from kitti_raw_loader import KittiRawLoader data_loader = KittiRawLoader(args.dataset_dir, static_frames_file=args.static_frames, img_height=args.height, img_width=args.width, get_depth=args.with_depth, get_pose=args.with_pose, depth_size_ratio=args.depth_size_ratio) if args.dataset_format == 'cityscapes': from cityscapes_loader import cityscapes_loader data_loader = cityscapes_loader(args.dataset_dir, img_height=args.height, img_width=args.width) n_scenes = len(data_loader.scenes) print('Found {} potential scenes'.format(n_scenes)) print('Retrieving frames') if args.num_threads == 1: for scene in tqdm(data_loader.scenes): dump_example(args, scene) else: with ProcessPool(max_workers=args.num_threads) as pool: tasks = pool.map(dump_example, [args]*n_scenes, data_loader.scenes) try: for _ in tqdm(tasks.result(), total=n_scenes): pass except KeyboardInterrupt as e: tasks.cancel() raise e print('Generating train val lists') np.random.seed(8964) # to avoid data snooping, we will make two cameras of the same scene to fall in the same set, train or val subdirs = args.dump_root.dirs() canonic_prefixes = set([subdir.basename()[:-2] for subdir in subdirs]) with open(args.dump_root / 'train.txt', 'w') as tf: with open(args.dump_root / 'val.txt', 'w') as vf: for pr in tqdm(canonic_prefixes): corresponding_dirs = args.dump_root.dirs('{}*'.format(pr)) if np.random.random() < 0.1: for s in corresponding_dirs: vf.write('{}\n'.format(s.name)) else: for s in corresponding_dirs: tf.write('{}\n'.format(s.name)) if args.with_depth and args.no_train_gt: for gt_file in s.files('*.npy'): gt_file.remove_p()
def main(): args.dump_root = Path(args.dump_root) args.dump_root.mkdir_p() global data_loader if args.dataset_format == 'kitti': from kitti_raw_loader import KittiRawLoader data_loader = KittiRawLoader(args.dataset_dir, static_frames_file=args.static_frames, img_height=args.height, img_width=args.width, get_gt=args.with_gt) if args.dataset_format == 'cityscapes': from cityscapes_loader import cityscapes_loader data_loader = cityscapes_loader(args.dataset_dir, img_height=args.height, img_width=args.width) if args.dataset_format == 'sintel': from sintel_loader import sintel_loader data_loader = sintel_loader(args.dataset_dir, img_height=args.height, img_width=args.width) print('Retrieving frames') if args.dataset_format == 'sintel': Parallel(n_jobs=args.num_threads)( delayed(dump_sintel_example)(scene) for scene in tqdm(data_loader.scenes)) else: Parallel(n_jobs=args.num_threads)( delayed(dump_example)(scene) for scene in tqdm(data_loader.scenes)) # Split into train/val print('Generating train val lists') np.random.seed(8964) subfolders = args.dump_root.dirs() with open(args.dump_root / 'train.txt', 'w') as tf: with open(args.dump_root / 'val.txt', 'w') as vf: for s in tqdm(subfolders): if np.random.random() < 0.1: vf.write('{}\n'.format(s.name)) else: tf.write('{}\n'.format(s.name)) # remove useless groundtruth data for training comment if you don't want to erase it for gt_file in s.files('*.npy'): gt_file.remove_p()
def main(): args.dump_root = Path(args.dump_root) args.dump_root.mkdir_p() global data_loader if args.dataset_format == 'kitti': from kitti_raw_loader import KittiRawLoader data_loader = KittiRawLoader(args.dataset_dir, static_frames_file=args.static_frames, img_height=args.height, img_width=args.width, get_depth=args.with_depth, get_pose=args.with_pose, depth_size_ratio=args.depth_size_ratio) if args.dataset_format == 'cityscapes': from cityscapes_loader import cityscapes_loader data_loader = cityscapes_loader(args.dataset_dir, img_height=args.height, img_width=args.width) print('Retrieving frames') if args.num_threads == 1: for scene in tqdm(data_loader.scenes): dump_example(args, scene) else: Parallel(n_jobs=args.num_threads)( delayed(dump_example)(args, scene) for scene in tqdm(data_loader.scenes)) print('Generating train val lists') np.random.seed(8964) # to avoid data snooping, we will make two cameras of the same scene to fall in the same set, train or val subdirs = args.dump_root.dirs() canonic_prefixes = set([subdir.basename()[:-2] for subdir in subdirs]) with open(args.dump_root / 'train.txt', 'w') as tf: with open(args.dump_root / 'val.txt', 'w') as vf: for pr in tqdm(canonic_prefixes): corresponding_dirs = args.dump_root.dirs('{}*'.format(pr)) if np.random.random() < 0.1: for s in corresponding_dirs: vf.write('{}\n'.format(s.name)) else: for s in corresponding_dirs: tf.write('{}\n'.format(s.name)) if args.with_depth and args.no_train_gt: for gt_file in s.files('*.npy'): gt_file.remove_p()
def main(): args.dump_root = Path(args.dump_root) args.dump_root.mkdir_p() global data_loader if args.dataset_format == 'kitti': from kitti_raw_loader import KittiRawLoader data_loader = KittiRawLoader(args.dataset_dir, static_frames_file=args.static_frames, img_height=args.height, img_width=args.width, get_depth=args.with_depth, get_pose=args.with_pose, depth_size_ratio=args.depth_size_ratio) if args.dataset_format == 'cityscapes': from cityscapes_loader import cityscapes_loader data_loader = cityscapes_loader(args.dataset_dir, img_height=args.height, img_width=args.width) print('Retrieving frames') if args.num_threads == 1: for scene in tqdm(data_loader.scenes): dump_example(args, scene) else: Parallel(n_jobs=args.num_threads)( delayed(dump_example)(args, scene) for scene in tqdm(data_loader.scenes)) print('Generating train val lists') np.random.seed(8964) subfolders = args.dump_root.dirs() with open(args.dump_root / 'train.txt', 'w') as tf: with open(args.dump_root / 'val.txt', 'w') as vf: for s in tqdm(subfolders): if np.random.random() < 0.1: vf.write('{}\n'.format(s.name)) else: tf.write('{}\n'.format(s.name)) if args.with_depth and args.no_train_gt: for gt_file in s.files('*.npy'): gt_file.remove_p()