def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Returns SplitGenerators.""" val_path = os.path.join(dl_manager.manual_dir, 'ILSVRC2012_img_val.tar') if not tf.io.gfile.exists(val_path): raise AssertionError( 'ImageNet requires manual download of the data. Please download ' 'the train and val set and place them into: {}'.format(val_path)) original_labels = _get_original_labels(val_path) (multi_labels, problematic_images, imagenet_m_2022_errors ) = _get_multi_labels_and_problematic_images(dl_manager) imagenet_m_2022 = dict([(k, multi_labels[k]) for k in imagenet_m_2022_errors ]) return { 'validation': self._generate_examples( archive=dl_manager.iter_archive(val_path), original_labels=original_labels, multi_labels=multi_labels, problematic_images=problematic_images), 'imagenet_m': self._generate_examples( archive=dl_manager.iter_archive(val_path), original_labels=original_labels, multi_labels=imagenet_m_2022, problematic_images=problematic_images), }
def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Returns SplitGenerators.""" imnet_path = os.path.join(dl_manager.manual_dir, 'ILSVRC2012_img_train.tar') noisy_images_path = os.path.join(dl_manager.manual_dir, 'noisy_images') noisy_split_path = os.path.join(dl_manager.manual_dir, 'dataset_no_images', 'mini-imagenet', 'split') noisy_annot_path = os.path.join(dl_manager.manual_dir, 'dataset_no_images', 'mini-imagenet-annotations.json') val_path = os.path.join(dl_manager.manual_dir, 'ILSVRC2012_img_val.tar') with tf.io.gfile.GFile(noisy_annot_path) as json_file: data = json.load(json_file) image_data = data['data'] noisy_image_ids = [elem[0]['image/id'] + '.jpg' for elem in image_data] # We first load all mini-ImageNet images in the memory, and # will access them for the other splits paths = dl_manager.download({ 'mini_train': MINI_IMAGENET_TRAIN, 'mini_val': MINI_IMAGENET_VAL, 'mini_test': MINI_IMAGENET_TEST }) train_fnames = self._read_mini_imagenet_csv(paths['mini_train']) val_fnames = self._read_mini_imagenet_csv(paths['mini_val']) test_fnames = self._read_mini_imagenet_csv(paths['mini_test']) mini_imnet_fnames = train_fnames + val_fnames + test_fnames mini_imnet_images = self._get_clean_images( mini_imnet_fnames, dl_manager.iter_archive(imnet_path)) val_split_file = os.path.join(noisy_split_path, 'clean_validation') split_to_generator = {} split_to_generator[ tfds.Split.VALIDATION] = self._generate_val_examples( val_split_file, dl_manager.iter_archive(val_path)) for percent in _PERCENTS: split_name = tfds.Split.TRAIN + '_' + '{:02d}'.format(percent) split_file = os.path.join( noisy_split_path, '{}_noise_nl_{}'.format(self.builder_config.color, str(percent / 100))) split_to_generator[split_name] = self._generate_examples( split_file, noisy_image_ids, noisy_images_path, mini_imnet_images) return split_to_generator
def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Returns SplitGenerators.""" path = dl_manager.download(_IMAGENET_SKETCH_URL) return [ tfds.core.SplitGenerator( name=tfds.Split.TEST, gen_kwargs={ 'archive': dl_manager.iter_archive(path), }, ), ]
def _generate_examples(self, dl_manager: tfds.download.DownloadManager, img_path, latent_path, label): """Yields examples.""" with tf.io.gfile.GFile(latent_path, 'rb') as f: latents = dict(np.load(f)) for key in latents: latents[key] = latents[key].astype(np.float32) for fname, fobj in dl_manager.iter_archive(img_path): idx = int(fname[-9:-4]) % 80000 yield label + '_' + fname[-9:-4], { 'image': fobj, 'label': label, 'illumination': latents['illumination'][idx], 'pose_mat': latents['pose_mat'][idx], 'pose_quat': latents['pose_quat'][idx], }