def test_TestSectionLoader_should_load_data_from_path_override_data(): with tempfile.TemporaryDirectory() as data_dir: os.makedirs(os.path.join(data_dir, "volume_name")) os.makedirs(os.path.join(data_dir, "splits")) seimic = np.zeros([IL, XL, D]) generate_npy_files( os.path.join(data_dir, "volume_name", "seismic.npy"), seimic) labels = np.ones([IL, XL, D]) generate_npy_files(os.path.join(data_dir, "volume_name", "labels.npy"), labels) txt_path = os.path.join(data_dir, "splits", "section_volume_name.txt") open(txt_path, "a").close() TestSectionLoader = get_test_loader(config) config.merge_from_list(["DATASET.ROOT", data_dir]) test_set = TestSectionLoader( config, split="volume_name", is_transform=True, augmentations=None, seismic_path=os.path.join(data_dir, "volume_name", "seismic.npy"), label_path=os.path.join(data_dir, "volume_name", "labels.npy"), ) assert_dimensions(test_set)
def get_config( config_paths: Optional[Union[List[str], str]] = None, opts: Optional[list] = None, ) -> CN: r"""Create a unified config with default values overwritten by values from :p:`config_paths` and overwritten by options from :p:`opts`. :param config_paths: List of config paths or string that contains comma separated list of config paths. :param opts: Config options (keys, values) in a list (e.g., passed from command line into the config. For example, :py:`opts = ['FOO.BAR', 0.5]`. Argument can be used for parameter sweeping or quick tests. """ config = _C.clone() if config_paths: if isinstance(config_paths, str): if CONFIG_FILE_SEPARATOR in config_paths: config_paths = config_paths.split(CONFIG_FILE_SEPARATOR) else: config_paths = [config_paths] for config_path in config_paths: config.merge_from_file(config_path) if opts: config.merge_from_list(opts) config.freeze() return config
def test_TrainPatchLoaderWithDepth_should_load_with_one_train_and_label_file( tmpdir): """ Check for successful class instantiation w/ single npy file for train & label """ # Setup os.makedirs(os.path.join(tmpdir, "volume_name")) os.makedirs(os.path.join(tmpdir, "splits")) seimic = np.zeros([IL, XL, D]) generate_npy_files(os.path.join(tmpdir, "volume_name", "seismic.npy"), seimic) labels = np.ones([IL, XL, D]) generate_npy_files(os.path.join(tmpdir, "volume_name", "labels.npy"), labels) txt_dir = os.path.join(tmpdir, "splits") txt_path = os.path.join(txt_dir, "patch_volume_name.txt") open(txt_path, "a").close() config.merge_from_list(["DATASET.ROOT", str(tmpdir)]) # Test train_set = TrainPatchLoaderWithDepth( config, split="volume_name", is_transform=True, augmentations=None, seismic_path=os.path.join(tmpdir, "volume_name", "seismic.npy"), label_path=os.path.join(tmpdir, "volume_name", "labels.npy"), ) assert train_set.labels.shape == (IL, XL, D + 2 * config.TRAIN.PATCH_SIZE) assert train_set.seismic.shape == (IL, XL, D + 2 * config.TRAIN.PATCH_SIZE)
def test_TrainPatchLoaderWithDepth_should_fail_on_missing_label_file(tmpdir): """ Check for exception when training param is empty """ # Setup os.makedirs(os.path.join(tmpdir, "volume_name")) os.makedirs(os.path.join(tmpdir, "splits")) seimic = np.zeros([IL, XL, D]) generate_npy_files(os.path.join(tmpdir, "volume_name", "seismic.npy"), seimic) txt_path = os.path.join(tmpdir, "splits", "patch_volume_name.txt") open(txt_path, "a").close() config.merge_from_list(["DATASET.ROOT", str(tmpdir)]) # Test with pytest.raises(Exception) as excinfo: _ = TrainPatchLoaderWithDepth( config, split="volume_name", is_transform=True, augmentations=None, seismic_path=os.path.join(tmpdir, "volume_name", "seismic.npy"), label_path=os.path.join(tmpdir, "volume_name", "labels.npy"), ) assert "does not exist" in str(excinfo.value)
def get_config( config_paths: Optional[Union[List[str], str]] = None, opts: Optional[list] = None, ) -> CN: r"""Create a unified config with default values overwritten by values from :p:`config_paths` and overwritten by options from :p:`opts`. :param config_paths: List of config paths or string that contains comma separated list of config paths. :param opts: Config options (keys, values) in a list (e.g., passed from command line into the config. For example, :py:`opts = ['FOO.BAR', 0.5]`. Argument can be used for parameter sweeping or quick tests. """ config = _C.clone() if config_paths: if isinstance(config_paths, str): if CONFIG_FILE_SEPARATOR in config_paths: config_paths = config_paths.split(CONFIG_FILE_SEPARATOR) else: config_paths = [config_paths] for config_path in config_paths: config.merge_from_file(config_path) if opts: config.merge_from_list(opts) # multi-task handling # list of tasks to list of config nodes, levaraging `TASK` default values tasks = [] for task in config.MULTI_TASK.TASKS: # get default values t = _C.TASK.clone() task = CN(init_dict=task) # each task can now have a different dataset, if unspecified the global one is used t.DATASET = config.DATASET.clone() # same thing for the episode iterator t.EPISODE_ITERATOR_OPTIONS = ( config.ENVIRONMENT.ITERATOR_OPTIONS.clone() ) t.merge_from_other_cfg(task) tasks.append(t) config.MULTI_TASK.TASKS = tasks config.freeze() return config
def load_config() -> yacs.config.CfgNode: parser = argparse.ArgumentParser() parser.add_argument('--config', type=str) parser.add_argument('options', default=None, nargs=argparse.REMAINDER) args = parser.parse_args() config = get_default_config() if args.config is not None: config.merge_from_file(args.config) config.merge_from_list(args.options) if not torch.cuda.is_available(): config.device = 'cpu' config.train.train_dataloader.pin_memory = False config.train.val_dataloader.pin_memory = False config.test.dataloader.pin_memory = False config.freeze() return config
def test_TestSectionLoader_should_load_data_from_test2_set(): with tempfile.TemporaryDirectory() as data_dir: os.makedirs(os.path.join(data_dir, "test_once")) os.makedirs(os.path.join(data_dir, "splits")) seimic = np.zeros([IL, XL, D]) generate_npy_files( os.path.join(data_dir, "test_once", "test2_seismic.npy"), seimic) A = np.load(os.path.join(data_dir, "test_once", "test2_seismic.npy")) labels = np.ones([IL, XL, D]) generate_npy_files( os.path.join(data_dir, "test_once", "test2_labels.npy"), labels) txt_path = os.path.join(data_dir, "splits", "section_test2.txt") open(txt_path, "a").close() TestSectionLoader = get_test_loader(config) config.merge_from_list(["DATASET.ROOT", data_dir]) test_set = TestSectionLoader(config, split="test2") assert_dimensions(test_set)