def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Return SplitGenerators.""" data_file = join(dl_manager.manual_dir, 'mimic_benchmarking_phenotyping.tar.gz') extracted_path = dl_manager.extract(data_file) train_dir = join(extracted_path, 'train') train_listfile = join(extracted_path, 'train_listfile.csv') val_dir = train_dir val_listfile = join(extracted_path, 'val_listfile.csv') test_dir = join(extracted_path, 'test') test_listfile = join(extracted_path, 'test_listfile.csv') return [ tfds.core.SplitGenerator(name=tfds.Split.TRAIN, gen_kwargs={ 'data_dir': train_dir, 'listfile': train_listfile }), tfds.core.SplitGenerator(name=tfds.Split.VALIDATION, gen_kwargs={ 'data_dir': val_dir, 'listfile': val_listfile }), tfds.core.SplitGenerator(name=tfds.Split.TEST, gen_kwargs={ 'data_dir': test_dir, 'listfile': test_listfile }), ]
def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Download the data and define splits.""" #data_path is a pathlib-like `Path('<manual_dir>/data.zip')` archive_path = dl_manager.manual_dir / 'brazilian_cerrado_dataset.zip' # Extract the manually downloaded `data.zip` extracted_path = dl_manager.extract(archive_path) #extracted_path = dl_manager.download_and_extract("https://homepages.dcc.ufmg.br/~keiller.nogueira/datasets/brazilian_cerrado_dataset.zip") # extracted_path =Path('/home/ami-m-017/Documents/MsComputerScience/research') # dl_manager returns pathlib-like objects with `path.read_text()`, # `path.iterdir()`,... return { 'fold1': self._generate_examples( path=extracted_path / 'Brazilian_Cerrado_Savana_Scenes_Dataset/folds/fold1'), 'fold2': self._generate_examples( path=extracted_path / 'Brazilian_Cerrado_Savana_Scenes_Dataset/folds/fold2'), 'fold3': self._generate_examples( path=extracted_path / 'Brazilian_Cerrado_Savana_Scenes_Dataset/folds/fold3'), 'fold4': self._generate_examples( path=extracted_path / 'Brazilian_Cerrado_Savana_Scenes_Dataset/folds/fold4'), 'fold5': self._generate_examples( path=extracted_path / 'Brazilian_Cerrado_Savana_Scenes_Dataset/folds/fold5'), }
def _split_generators(self, dl_manager: tfds.download.DownloadManager): """ downloads and splits data Returns SplitGenerators. This dataset was created following the guide: https://www.tensorflow.org/datasets/add_dataset """ input_data_path = dl_manager.extract(self.data_source_path) images_path = input_data_path / self.pp_param['output_pp_data_dir_name'] return { 'train': self._generate_examples(images_path, 'train'), 'validation': self._generate_examples(images_path, 'val'), 'test': self._generate_examples(images_path, 'test'), }
def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Returns SplitGenerators.""" # TODO(kappatng): Downloads the data and defines the splits data_path = dl_manager.extract( os.path.join(dl_manager.manual_dir, self.builder_config.file_name)) # TODO(kappatng): Returns the Dict[split names, Iterator[Key, Example]] return [ tfds.core.SplitGenerator( name=tfds.Split.TRAIN, gen_kwargs={ "images_dir_path": os.path.join( data_path, "global/cscratch1/sd/jialiu/kappaTNG/COSMOS/"), }, ), ]
def _split_generators(self, dl_manager: tfds.download.DownloadManager): """ downloads and splits data Returns SplitGenerators. This dataset was created following the guide: https://www.tensorflow.org/datasets/add_dataset """ input_data_path = dl_manager.extract(self.data_source_path) images_path = input_data_path / self.pp_param['output_pp_data_dir_name'] # Create 10 sets where each one have a different degradation level # (form 0% degradation to 90% degradation) return { # 0% degradation 'train_0': self._generate_examples(images_path, 'train', 0), 'val_0': self._generate_examples(images_path, 'val', 0), 'test_0': self._generate_examples(images_path, 'test', 0), # 10% degradation 'train_10': self._generate_examples(images_path, 'train', 0.1), 'val_10': self._generate_examples(images_path, 'val', 0.1), 'test_10': self._generate_examples(images_path, 'test', 0.1), # 20% degradation 'train_20': self._generate_examples(images_path, 'train', 0.2), 'val_20': self._generate_examples(images_path, 'val', 0.2), 'test_20': self._generate_examples(images_path, 'test', 0.2), # 30% degradation 'train_30': self._generate_examples(images_path, 'train', 0.3), 'val_30': self._generate_examples(images_path, 'val', 0.3), 'test_30': self._generate_examples(images_path, 'test', 0.3), # 40% degradation 'train_40': self._generate_examples(images_path, 'train', 0.4), 'val_40': self._generate_examples(images_path, 'val', 0.4), 'test_40': self._generate_examples(images_path, 'test', 0.4), # 50% degradation 'train_50': self._generate_examples(images_path, 'train', 0.5), 'val_50': self._generate_examples(images_path, 'val', 0.5), 'test_50': self._generate_examples(images_path, 'test', 0.5), # 60% degradation 'train_60': self._generate_examples(images_path, 'train', 0.6), 'val_60': self._generate_examples(images_path, 'val', 0.6), 'test_60': self._generate_examples(images_path, 'test', 0.6), # 70% degradation 'train_70': self._generate_examples(images_path, 'train', 0.7), 'val_70': self._generate_examples(images_path, 'val', 0.7), 'test_70': self._generate_examples(images_path, 'test', 0.7), # 80% degradation 'train_80': self._generate_examples(images_path, 'train', 0.8), 'val_80': self._generate_examples(images_path, 'val', 0.8), 'test_80': self._generate_examples(images_path, 'test', 0.8), # 90% degradation 'train_90': self._generate_examples(images_path, 'train', 0.9), 'val_90': self._generate_examples(images_path, 'val', 0.9), 'test_90': self._generate_examples(images_path, 'test', 0.9), }