def __init__( self, root: str, split: str = "train", version: str = "2017", resize: Tuple[int] = (300, 300), augmentations: Callable[[Image, FloatTensor, LongTensor], Tuple[Image, FloatTensor, LongTensor]] = None, keep_crowd: bool = False ): super().__init__(resize, augmentations) self.root = os.path.expanduser(root) verify_str_arg(split, "split", ("train", "val")) self.keep_crowd = keep_crowd self.logger = log_utils.get_master_logger("CocoDetection") # path to image folder, e.g. coco_root/train2017 self.image_folder = os.path.join(self.root, f"{split}{version}") # path to annotation, e.g. coco_root/annotations/instances_train2017 annotation_fp = os.path.join(self.root, "annotations", f"instances_{split}{version}.json") self.logger.info("Parsing COCO %s dataset...", split) self._init_dataset(annotation_fp) self.logger.info("Parsing COCO %s dataset done", split)
def verify_mode_type(self, split, image_mode, image_type): image_mode = self.verify_mode(image_mode) if image_mode == "gtFine": valid_splits = ("train", "test", "val") valid_types = ("_instanceIds.png", "instance", "_labelIds.png", "semantic", "_color.png", "color", "_polygons.json", "polygon") elif image_mode == "gtCoarse": valid_splits = ("train", "train_extra", "val") valid_types = ("_instanceIds.png", "instance", "_labelIds.png", "semantic", "_color.png", "color", "_polygons.json", "polygon") elif image_mode == "leftImg8bit": valid_splits = ("train", "train_extra", "test", "val") valid_types = ( "_leftImg8bit.png", None, ) for i in range(len(valid_types) // 2): if image_type == valid_types[i * 2 + 1]: image_type = valid_types[i * 2] break msg = ("Unknown value '{}' for argument split if image_mode is '{}'. " "Valid values are {{{}}}.") msg = msg.format(split, image_mode, iterable_to_str(valid_splits)) verify_str_arg(split, "split", valid_splits, msg) msg = ( "Unknown value '{}' for argument image_type if image_mode is '{}'. " "Valid values are {{{}}}.") msg = msg.format(image_type, image_mode, iterable_to_str(valid_types)) verify_str_arg(image_type, "image_type", valid_types, msg) return image_mode, image_type
def __init__(self, root, split='train', transform=None, target_transform=None, transforms=None): super(Deepfashion, self).__init__(root, transforms, transform, target_transform) self.images_dir = os.path.join(self.root, 'img') self.targets_dir = os.path.join(self.root, 'lbl') self.split = split self.images = [] self.targets = [] valid_modes = ("train", "test", "val") msg = ("Unknown value '{}' for argument split if mode is '{}'. " "Valid values are {{{}}}.") msg = msg.format(split, mode, iterable_to_str(valid_modes)) verify_str_arg(split, "split", valid_modes, msg) if not os.path.isdir(self.images_dir) or not os.path.isdir(self.targets_dir): image_dir_zip = os.path.join(self.root, '{}'.format('img.zip')) target_dir_zip = os.path.join(self.root, '{}'.format('lbl.zip')) if os.path.isfile(image_dir_zip) and os.path.isfile(target_dir_zip): extract_archive(from_path=image_dir_zip, to_path=self.root) extract_archive(from_path=target_dir_zip, to_path=self.root) else: raise RuntimeError('Dataset not found or incomplete. Please make sure all required folders for the' ' specified "split" and "mode" are inside the "root" directory') data_list = pd.read_csv(os.path.join(self.root, 'list_eval_partition.txt'), sep='\t', skiprows=1) data_list = data_list[data_list['evaluation_status'] == self.split] for image_path in data_list['image_name']: target_path = 'lbl/' + '/'.join(image_path.split('/')[1:]) self.images.append(image_path) self.targets.append(target_path)
def __init__( self, root: str, split: str, image_set: str, view: str, transform: Optional[Callable] = None, target_transform: Optional[Callable] = None, download: bool = False, ): super().__init__(os.path.join(root, self.base_folder), transform=transform, target_transform=target_transform) self.image_set = verify_str_arg(image_set.lower(), "image_set", self.file_dict.keys()) images_dir, self.filename, self.md5 = self.file_dict[self.image_set] self.view = verify_str_arg(view.lower(), "view", ["people", "pairs"]) self.split = verify_str_arg(split.lower(), "split", ["10fold", "train", "test"]) self.labels_file = f"{self.view}{self.annot_file[self.split]}.txt" self.data: List[Any] = [] if download: self.download() if not self._check_integrity(): raise RuntimeError( "Dataset not found or corrupted. You can use download=True to download it" ) self.images_dir = os.path.join(self.root, images_dir)
def __init__(self, root, split='train', mode='fine', target_type='instance', transform=None, target_transform=None, transforms=None): super(Cityscapes, self).__init__(root, transforms, transform, target_transform) self.mode = 'gtFine' if mode == 'fine' else 'gtCoarse' self.images_dir = os.path.join(self.root, 'leftImg8bit', split) self.targets_dir = os.path.join(self.root, self.mode, split) self.target_type = target_type self.split = split self.images = [] self.targets = [] verify_str_arg(mode, "mode", ("fine", "coarse")) if mode == "fine": valid_modes = ("train", "test", "val") else: valid_modes = ("train", "train_extra", "val") msg = ("Unknown value '{}' for argument split if mode is '{}'. " "Valid values are {{{}}}.") msg = msg.format(split, mode, iterable_to_str(valid_modes)) verify_str_arg(split, "split", valid_modes, msg) if not isinstance(target_type, list): self.target_type = [target_type] [verify_str_arg(value, "target_type", ("instance", "semantic", "polygon", "color")) for value in self.target_type] if not os.path.isdir(self.images_dir) or not os.path.isdir(self.targets_dir): if split == 'train_extra': image_dir_zip = os.path.join(self.root, 'leftImg8bit{}'.format('_trainextra.zip')) else: image_dir_zip = os.path.join(self.root, 'leftImg8bit{}'.format('_trainvaltest.zip')) if self.mode == 'gtFine': target_dir_zip = os.path.join(self.root, '{}{}'.format(self.mode, '_trainvaltest.zip')) elif self.mode == 'gtCoarse': target_dir_zip = os.path.join(self.root, '{}{}'.format(self.mode, '.zip')) if os.path.isfile(image_dir_zip) and os.path.isfile(target_dir_zip): extract_archive(from_path=image_dir_zip, to_path=self.root) extract_archive(from_path=target_dir_zip, to_path=self.root) else: raise RuntimeError('Dataset not found or incomplete. Please make sure all required folders for the' ' specified "split" and "mode" are inside the "root" directory') for city in os.listdir(self.images_dir): img_dir = os.path.join(self.images_dir, city) target_dir = os.path.join(self.targets_dir, city) for file_name in os.listdir(img_dir): target_types = [] for t in self.target_type: target_name = file_name target_types.append(os.path.join(target_dir, target_name)) self.images.append(os.path.join(img_dir, file_name)) self.targets.append(target_types)
def verify_mode(self, image_mode): valid_modes = ("gt_color", "color", "gt_labelIds", "semantic", "gt_labelTrainIds", "semantic_train", "RGB", "RGB",) for i in range(len(valid_modes)//2): if image_mode == valid_modes[i*2+1]: image_mode = valid_modes[i*2] break verify_str_arg(image_mode, "image_mode", valid_modes) return image_mode
def _verify_classes(self, classes): categories = ['bedroom', 'bridge', 'church_outdoor', 'classroom', 'conference_room', 'dining_room', 'kitchen', 'living_room', 'restaurant', 'tower'] dset_opts = ['train', 'val', 'test'] try: verify_str_arg(classes, "classes", dset_opts) if classes == 'test': classes = [classes] else: classes = [c + '_' + classes for c in categories] except ValueError: if not isinstance(classes, Iterable): msg = ("Expected type str or Iterable for argument classes, " "but got type {}.") raise ValueError(msg.format(type(classes))) classes = list(classes) msg_fmtstr = ("Expected type str for elements in argument classes, " "but got type {}.") for c in classes: verify_str_arg(c, custom_msg=msg_fmtstr.format(type(c))) c_short = c.split('_') category, dset_opt = '_'.join(c_short[:-1]), c_short[-1] msg_fmtstr = "Unknown value '{}' for {}. Valid values are {{{}}}." msg = msg_fmtstr.format(category, "LSUN class", iterable_to_str(categories)) verify_str_arg(category, valid_values=categories, custom_msg=msg) msg = msg_fmtstr.format(dset_opt, "postfix", iterable_to_str(dset_opts)) verify_str_arg(dset_opt, valid_values=dset_opts, custom_msg=msg) return classes
def verify_mode(self, image_mode): valid_modes = ( "gtFine", "gtFine", "gtCoarse", "gtCoarse", "leftImg8bit", None, ) for i in range(len(valid_modes) // 2): if image_mode == valid_modes[i * 2 + 1]: image_mode = valid_modes[i * 2] break verify_str_arg(image_mode, "image_mode", valid_modes) return image_mode
def __init__(self, root, base_folder, split='train', transform=None, target_transform=None, download=False, color_distortion=False, col=False): super(TinyImageNet, self).__init__(root, transform=transform, target_transform=target_transform) os.makedirs(root, exist_ok=True) self.root = root self.base_folder = base_folder self.color_distortion = color_distortion self.col = col self.dataset_path = os.path.join(root, self.base_folder) self.loader = default_loader self.split = verify_str_arg(split, "split", ( "train", "val", )) _, class_to_idx = find_classes( os.path.join(self.dataset_path, 'wnids.txt')) self.data = make_dataset(self.root, self.base_folder, self.split, class_to_idx)
def __init__(self, root, split='train', download=False, **kwargs): """Use as torchvision.datasets.ImageNet.""" root = self.root = os.path.expanduser(root) self.split = verify_str_arg(split, "split", ("train", "val")) try: wnid_to_classes = load_meta_file(self.root)[0] except RuntimeError: torchvision.datasets.imagenet.META_FILE = os.path.join( os.path.expanduser('~/data/'), 'meta.bin') try: wnid_to_classes = load_meta_file(self.root)[0] except RuntimeError: self.parse_archives() wnid_to_classes = load_meta_file(self.root)[0] torchvision.datasets.ImageFolder.__init__(self, self.split_folder, **kwargs) self.root = root self.wnids = self.classes self.wnid_to_idx = self.class_to_idx self.classes = [wnid_to_classes[wnid] for wnid in self.wnids] self.class_to_idx = { cls: idx for idx, clss in enumerate(self.classes) for cls in clss } """Scrub class names to be a single string.""" scrubbed_names = [] for name in self.classes: if isinstance(name, tuple): scrubbed_names.append(name[0]) else: scrubbed_names.append(name) self.classes = scrubbed_names
def __init__(self, root, split='train', download=None, **kwargs): if download is True: msg = ("The dataset is no longer publicly accessible. You need to " "download the archives externally and place them in the " "root directory.") raise RuntimeError(msg) elif download is False: msg = ("The use of the download flag is deprecated, since the " "dataset is no longer publicly accessible.") warnings.warn(msg, RuntimeWarning) root = self.root = os.path.expanduser(root) self.split = verify_str_arg(split, "split", ("train", "val")) self.parse_archives() wnid_to_classes = load_meta_file(self.root)[0] super(ImageNet, self).__init__(self.split_folder, **kwargs) self.root = root self.wnids = self.classes self.wnid_to_idx = self.class_to_idx self.classes = [wnid_to_classes[wnid] for wnid in self.wnids] self.class_to_idx = { cls: idx for idx, clss in enumerate(self.classes) for cls in clss }
def __init__( self, root, split="train", target_type="attr", transform=None, download=False, ): import pandas super(CelebAHQ, self).__init__(root, transform=transform, target_transform=None) self.split = split if isinstance(target_type, list): self.target_type = target_type else: self.target_type = [target_type] if not self.target_type and self.target_transform is not None: raise RuntimeError( "target_transform is specified but target_type is empty") if download: self.download() if not self._check_integrity(): raise RuntimeError("Dataset not found or corrupted." + " You can use download=True to download it") split_map = { "train": 0, "valid": 1, "test": 2, "all": None, } split = split_map[verify_str_arg(split.lower(), "split", ("train", "valid", "test", "all"))] fn = partial(os.path.join, self.root, self.base_folder) splits = pandas.read_csv( fn("list_eval_partition.txt"), delim_whitespace=True, header=None, index_col=0, ) index = pandas.read_csv( fn("CelebAMask-HQ", "CelebA-HQ-to-CelebA-mapping.txt"), delim_whitespace=True, header=0, usecols=["idx", "orig_idx"], ) splits = index["orig_idx"].apply(lambda i: splits.iloc[i]) index = index["idx"] mask = slice(None) if split is None else (splits[1] == split) self.filename = index[mask].apply(lambda s: str(s) + ".jpg").values
def __init__(self, root, base_folder, split='train', transform=None, target_transform=None, download=False): super(TinyImageNet, self).__init__(root, transform=transform, target_transform=target_transform) os.makedirs(root, exist_ok=True) self.dataset_path = os.path.join(root, self.base_folder) self.loader = default_loader self.split = verify_str_arg(split, "split", ( "train", "val", )) ''' if self._check_integrity(): print('Files already downloaded and verified.') elif download: self._download() else: raise RuntimeError( 'Dataset not found. You can use download=True to download it.') if not os.path.isdir(self.dataset_path): print('Extracting...') extract_archive(os.path.join(root, self.filename)) ''' _, class_to_idx = find_classes( os.path.join(self.dataset_path, 'wnids.txt')) self.data = make_dataset(self.root, self.base_folder, self.split, class_to_idx)
def __init__(self, root='./data/ImageNet', split='val', **kwargs): root = self.root = os.path.expanduser(root) self.split = verify_str_arg(split, "split", ("train", "val")) if not os.path.exists(self.split_folder): os.mkdir(self.split_folder) wnid_to_classes = self.load_label_file() mapped_wnid_to_idx, mapped_idx_to_wnid = self.load_mapping() targets = self.load_ground_truth() target_wnids = [mapped_idx_to_wnid[idx] for idx in targets] self.wnids = list(wnid_to_classes.keys()) self.classes = list(wnid_to_classes.values()) alphbetical_wnid_to_idx = {wnid: i for i, wnid in enumerate(sorted(self.wnids))} imgs = self.parse_image_tar(target_wnids, alphbetical_wnid_to_idx) super(ImageNetLoader, self).__init__(self.split_folder, **kwargs) self.classes = [cls for wnid, clss in wnid_to_classes.items() for cls in clss] self.wnid_to_idx = alphbetical_wnid_to_idx self.class_to_idx = {cls: idx for wnid, idx in alphbetical_wnid_to_idx.items() if wnid in wnid_to_classes for cls in wnid_to_classes[wnid]} self.samples = imgs self.targets = targets self.imgs = imgs
def _verify_str_arg( value: str, arg: Optional[str] = None, valid_values: Optional[Collection[str]] = None, *, custom_msg: Optional[str] = None, ) -> str: return verify_str_arg(value, arg, valid_values, custom_msg=custom_msg)
def __init__(self, root, dataset_name='facades', split='train', download=True, **kwargs): root = os.path.expanduser(root) self.root = os.path.join('.', root) self.dataset_name = verify_str_arg(dataset_name, Pix2Pix_Datasets.keys()) self.split = verify_str_arg(split, ['train', 'val']) if download: self.download() super(Pix2Pix, self).__init__(self.root, **kwargs)
def __init__(self, root, split="train", include_sar=False, transforms=None): super().__init__(pathlib.Path(root), transforms=transforms) verify_str_arg(split, "split", self.splits) if split == "test": self.city_names = self.test_list elif split == "train": self.city_names = self.train_list self.datatypes = ["rgb", "dem", "seg"] if include_sar: self.file_list = self._get_file_list("*sar.tif") self.datatypes.append("sar") else: self.file_list = self._get_file_list("*rgb.jp2")
def __init__(self, root, what=None, compat=True, train=True, **kwargs): if what is None: what = 'train' if train else 'test' self.what = verify_str_arg(what, "what", tuple(self.subsets.keys())) self.compat = compat self.data_file = what + '.pt' self.training_file = self.data_file self.test_file = self.data_file super(QMNIST, self).__init__(root, train, **kwargs)
def __init__(self, root: str, split: str = "trainval", version: str = "2007", resize: Optional[Tuple[int]] = (300, 300), augmentations: Callable[[Image, Dict[str, Any]], Tuple[Image, Dict[str, Any]]] = None, keep_difficult: bool = False, make_partial: List[int] = None): """ Args: root: root to voc path which contains [`VOC2007` `VOC2012`] folders split: split of dataset, e.g. `train`, `val`, `test` and `trainval`. Warning: VOC2012 has no test split version: `2007` or `2012` resize: all images will be resized to given size. If `None`, all images will not be resized make_partial: only keep objects with given classes, w.r.t `self.CLASSES`. If `None`, all objects will be perserved. For multitask learning which one task has 10 classes and the other task has others classes """ super().__init__(resize, augmentations) self.keep_difficult = keep_difficult verify_str_arg(version, "version", ("2007", "2012")) verify_str_arg(split, "split", ("train", "val", "test", "trainval")) self.logger = log_utils.get_master_logger("VOCDetection") self.version = version self.split = split # parse folders self.root = os.path.expanduser(os.path.join(root, f"VOC{version}")) # read split file split_fp = os.path.join(self.root, "ImageSets", "Main", f"{split}.txt") if not os.path.isfile(split_fp): raise FileNotFoundError( f"`{split_fp}` is not found, note that there is no `test.txt` for VOC-2012" ) with open(split_fp, "r") as f: self.file_names = [x.strip() for x in f.readlines()] self.logger.info("Parsing VOC%s %s dataset...", version, split) self._init_dataset(make_partial) self.logger.info("Parsing VOC%s %s dataset done", version, split)
def __init__(self, root, split="train", target_type="attr", transform=None, target_transform=None, download=False ): """ """ self.root = root self.split = split self.target_type = target_type self.transform = transform self.target_transform = target_transform if isinstance(target_type, list): self.target_type = target_type else: self.target_type = [target_type] if not self.target_type and self.target_transform is not None: raise RuntimeError('target_transform is specified but target_type is empty') if download: self.download_from_kaggle() split_map = { "train": 0, "valid": 1, "test": 2, "all": None, } split_ = split_map[verify_str_arg(split.lower(), "split", ("train", "valid", "test", "all"))] fn = partial(os.path.join, self.root) splits = pd.read_csv(fn("list_eval_partition.csv"), delim_whitespace=False, header=0, index_col=0) # This file is not available in Kaggle # identity = pd.read_csv(fn("identity_CelebA.csv"), delim_whitespace=True, header=None, index_col=0) bbox = pd.read_csv(fn("list_bbox_celeba.csv"), delim_whitespace=False, header=0, index_col=0) landmarks_align = pd.read_csv(fn("list_landmarks_align_celeba.csv"), delim_whitespace=False, header=0, index_col=0) attr = pd.read_csv(fn("list_attr_celeba.csv"), delim_whitespace=False, header=0, index_col=0) mask = slice(None) if split_ is None else (splits['partition'] == split_) self.filename = splits[mask].index.values # self.identity = torch.as_tensor(identity[mask].values) self.bbox = torch.as_tensor(bbox[mask].values) self.landmarks_align = torch.as_tensor(landmarks_align[mask].values) self.attr = torch.as_tensor(attr[mask].values) self.attr = (self.attr + 1) // 2 # map from {-1, 1} to {0, 1} self.attr_names = list(attr.columns)
def __init__(self, root, split='train', download=False, **kwargs): self.data_root = os.path.expanduser(root) self.split = verify_str_arg(split, "split", self.splits) if download: self.download() if not self._check_exists(): raise RuntimeError('Dataset not found.' + ' You can use download=True to download it') super().__init__(self.split_folder, **kwargs)
def __init__( self, root: str, split: str = "train", mode: str = "fine", target_type: Union[List[str], str] = "semantic", transform: Optional[Callable] = None, target_transform: Optional[Callable] = None, transforms: Optional[Callable] = None, ) -> None: super(Cityscapes, self).__init__(root, transforms, transform, target_transform) self.mode = 'gtFine' if mode == 'fine' else 'gtCoarse' self.images_dir = os.path.join(self.root, 'leftImg8bit', split) self.targets_dir = os.path.join(self.root, self.mode, split) self.target_type = target_type self.split = split self.images = [] self.targets = [] verify_str_arg(mode, "mode", ("fine", "coarse")) if mode == "fine": valid_modes = ("train", "test", "val") else: valid_modes = ("train", "train_extra", "val") msg = ("Unknown value '{}' for argument split if mode is '{}'. " "Valid values are {{{}}}.") msg = msg.format(split, mode, iterable_to_str(valid_modes)) verify_str_arg(split, "split", valid_modes, msg) if not isinstance(target_type, list): self.target_type = [target_type] [ verify_str_arg(value, "target_type", ("instance", "semantic", "polygon", "color")) for value in self.target_type ] self.images = self._get_files('image', self.split) self.targets = self._get_files('label', self.split)
def __init__(self, root, year='2012', image_set='train', download=False, transform=None, target_transform=None, transforms=None): super(VOCClassification, self).__init__(root, transforms, transform, target_transform) self.year = year self.url = DATASET_YEAR_DICT[year]['url'] self.filename = DATASET_YEAR_DICT[year]['filename'] self.md5 = DATASET_YEAR_DICT[year]['md5'] self.image_set = verify_str_arg(image_set, "image_set", ("train", "trainval", "val")) base_dir = DATASET_YEAR_DICT[year]['base_dir'] voc_root = os.path.join(self.root, base_dir) image_dir = os.path.join(voc_root, 'JPEGImages') annotation_dir = os.path.join(voc_root, 'Annotations') if download: download_extract(self.url, self.root, self.filename, self.md5) if not os.path.isdir(voc_root): raise RuntimeError('Dataset not found or corrupted.' + ' You can use download=True to download it') splits_dir = os.path.join(voc_root, 'ImageSets/Main') split_fs = [ os.path.join(splits_dir, c + '_' + image_set.rstrip('\n') + '.txt') for c in VOC2012_CLASSES ] file_names = [] self.object_nums = [] for split_f in split_fs: with open(os.path.join(split_f), "r") as f: for line in f.readlines(): file_name, object_num = line.strip().split() file_names.append(file_name) self.object_nums.append(int(object_num)) self.images = [os.path.join(image_dir, x + ".jpg") for x in file_names] self.annotations = [ os.path.join(annotation_dir, x + ".xml") for x in file_names ] if image_set == 'val': self.images = self.images[::6] self.annotations = self.annotations[::6] assert (len(self.images) == len(self.annotations))
def __init__(self, root, split='train', folds=None, transform=None, target_transform=None, download=False): super(STL10, self).__init__(root, transform=transform, target_transform=target_transform) self.split = verify_str_arg(split, "split", self.splits) self.folds = self._verify_folds(folds) if download: self.download() if not self._check_integrity(): raise RuntimeError( 'Dataset not found or corrupted. ' 'You can use download=True to download it') # now load the picked numpy arrays if self.split == 'train': self.data, self.labels = self.__loadfile( self.train_list[0][0], self.train_list[1][0]) self.__load_folds(folds) elif self.split == 'train+unlabeled': self.data, self.labels = self.__loadfile( self.train_list[0][0], self.train_list[1][0]) self.__load_folds(folds) unlabeled_data, _ = self.__loadfile(self.train_list[2][0]) self.data = np.concatenate((self.data, unlabeled_data)) self.labels = np.concatenate( (self.labels, np.asarray([-1] * unlabeled_data.shape[0]))) elif self.split == 'train+test+unlabeled': self.data, self.labels = self.__loadfile( self.train_list[0][0], self.train_list[1][0]) self.__load_folds(folds) self.data_test, self.labels_test = self.__loadfile( self.test_list[0][0], self.test_list[1][0]) unlabeled_data, _ = self.__loadfile(self.train_list[2][0]) self.data = np.concatenate((self.data, self.data_test, unlabeled_data)) self.labels = np.concatenate( (self.labels, self.labels_test, np.asarray([-1] * unlabeled_data.shape[0]))) elif self.split == 'unlabeled': self.data, _ = self.__loadfile(self.train_list[2][0]) self.labels = np.asarray([-1] * self.data.shape[0]) else: # self.split == 'test': self.data, self.labels = self.__loadfile( self.test_list[0][0], self.test_list[1][0]) class_file = os.path.join( self.root, self.base_folder, self.class_names_file) if os.path.isfile(class_file): with open(class_file) as f: self.classes = f.read().splitlines()
def __init__(self, root, token='', split='train', download=False, **kwargs): root = self.root = os.path.expanduser(root) self.split = verify_str_arg(split, "split", ("train", "val")) self.archive_dict = { 'train': { 'url': 'http://www.image-net.org/challenges/LSVRC/2012/{}/ILSVRC2012_img_train.tar' .format(token), 'md5': '1d675b47d978889d74fa0da5fadfb00e', }, 'val': { 'url': 'http://www.image-net.org/challenges/LSVRC/2012/{}/ILSVRC2012_img_val.tar' .format(token), 'md5': '29b22e2961454d5413ddabcf34fc5622', }, 'devkit': { 'url': 'http://www.image-net.org/challenges/LSVRC/2012/{}/ILSVRC2012_devkit_t12.tar.gz' .format(token), 'md5': 'fa75699e90414af021442c21a62c3abf', } } if download: if len(token) == 0: raise ValueError( "ImageNet token is empty. Please obtain permission token from the official website." ) self.download() wnid_to_classes = self._load_meta_file()[0] super(ImageNet, self).__init__(self.split_folder, **kwargs) self.root = root self.wnids = self.classes self.wnid_to_idx = self.class_to_idx self.classes = [wnid_to_classes[wnid] for wnid in self.wnids] self.class_to_idx = { cls: idx for idx, clss in enumerate(self.classes) for cls in clss }
def verify_mode(self, image_mode): valid_modes = ( "gtFine", "gtFine", "gtCoarse", "gtCoarse", "leftImg8bit", "clear", "leftImg8bit_foggy", "foggy", "leftImg8bit_foggyDBF", "foggyDBF", "leftImg8bit_transmittance", "transmittance", "leftImg8bit_transmittanceDBF", "transmittanceDBF", ) for i in range(len(valid_modes) // 2): if image_mode == valid_modes[i * 2 + 1]: image_mode = valid_modes[i * 2] break verify_str_arg(image_mode, "image_mode", valid_modes) return image_mode
def verify_mode_split(self, split, image_mode): image_mode = self.verify_mode(image_mode) if image_mode == "gt_color": valid_splits = ("testv1", "testv2") elif image_mode == "gt_labelIds": valid_splits = ("testv1", "testv2") elif image_mode == "gt_labelTrainIds": valid_splits = ("testv1", "testv2") elif image_mode == "RGB": valid_splits = ("light", "medium", "testv1", "testv2") msg = ("Unknown value '{}' for argument split if image_mode is '{}'. " "Valid values are {{{}}}.") msg = msg.format(split, image_mode, iterable_to_str(valid_splits)) verify_str_arg(split, "split", valid_splits, msg) return image_mode # def verify_dataset(self): if not os.path.isdir(os.path.join(self.root, self.images_dir)): if self.image_mode == 'gtFine': image_dir_zip = os.path.join(self.root, '{}_trainvaltest.zip'.format(self.image_mode)) elif self.image_mode == 'gtCoarse': image_dir_zip = os.path.join(self.root, '{}.zip'.format(self.image_mode)) else: if split == 'train_extra': image_dir_zip = os.path.join(self.root, '{}_trainextra.zip'.format(self.image_mode)) else: image_dir_zip = os.path.join(self.root, '{}_trainvaltest.zip'.format(self.image_mode)) if os.path.isfile(image_dir_zip): extract_archive(from_path=image_dir_zip, to_path=self.root) extract_archive(from_path=target_dir_zip, to_path=self.root) else: raise RuntimeError('Dataset not found or incomplete. Please make sure all required folders for the' ' specified "split" and "image_mode" are inside the "root" directory')
def __init__(self, root, target_type="category", transform=None, target_transform=None, download=False): super(Caltech101, self).__init__(os.path.join(root, 'caltech101'), transform=transform, target_transform=target_transform) os.makedirs(self.root, exist_ok=True) if not isinstance(target_type, list): target_type = [target_type] self.target_type = [ verify_str_arg(t, "target_type", ("category", "annotation")) for t in target_type ] if download: self.download() if not self._check_integrity(): raise RuntimeError('Dataset not found or corrupted.' + ' You can use download=True to download it') self.categories = sorted( os.listdir(os.path.join(self.root, "101_ObjectCategories"))) self.categories.remove("BACKGROUND_Google") # this is not a real class # For some reason, the category names in "101_ObjectCategories" and # "Annotations" do not always match. This is a manual map between the # two. Defaults to using same name, since most names are fine. name_map = { "Faces": "Faces_2", "Faces_easy": "Faces_3", "Motorbikes": "Motorbikes_16", "airplanes": "Airplanes_Side_2" } self.annotation_categories = list( map(lambda x: name_map[x] if x in name_map else x, self.categories)) self.index = [] self.y = [] for (i, c) in enumerate(self.categories): n = len( os.listdir(os.path.join(self.root, "101_ObjectCategories", c))) self.index.extend(range(1, n + 1)) self.y.extend(n * [i])
def __init__(self, root, year='2012', image_set='train', download=False, transform=None, target_transform=None, transforms=None): super(VOCDetection, self).__init__(root, transforms, transform, target_transform) self.year = year if year == "2007" and image_set == "test": year = "2007-test" self.url = DATASET_YEAR_DICT[year]['url'] self.filename = DATASET_YEAR_DICT[year]['filename'] self.md5 = DATASET_YEAR_DICT[year]['md5'] valid_sets = ["train", "trainval", "val"] if year == "2007-test": valid_sets.append("test") self.image_set = verify_str_arg(image_set, "image_set", valid_sets) base_dir = DATASET_YEAR_DICT[year]['base_dir'] voc_root = os.path.join(self.root, base_dir) image_dir = os.path.join(voc_root, 'JPEGImages') annotation_dir = os.path.join(voc_root, 'Annotations') if download: download_extract(self.url, self.root, self.filename, self.md5) if not os.path.isdir(voc_root): raise RuntimeError('Dataset not found or corrupted.' + ' You can use download=True to download it') splits_dir = os.path.join(voc_root, 'ImageSets/Main') split_f = os.path.join(splits_dir, image_set.rstrip('\n') + '.txt') with open(os.path.join(split_f), "r") as f: file_names = [x.strip() for x in f.readlines()] self.images = [os.path.join(image_dir, x + ".jpg") for x in file_names] self.annotations = [ os.path.join(annotation_dir, x + ".xml") for x in file_names ] self.images, self.annotations = self.filter_img( self.images, self.annotations) assert (len(self.images) == len(self.annotations))
def __init__( self, root, split="train", transform=None, target_transform=None, download=False, in_memory=False, **kwargs ): super(TinyImageNet, self).__init__(root, transform=transform, target_transform=target_transform) self.root = os.path.expanduser(root) self.root_final = os.path.join(self.root, self.base_folder, "tiny-imagenet-200") self.split = verify_str_arg( split, "split", ("train", "val", "test") ) # training set , validation set or test set self.split_dir = os.path.join(self.root_final, self.split) if download: self._download() if not self._check_integrity(): raise RuntimeError("Dataset not found or corrupted." + " You can use download=True to download it") self.image_paths = sorted(glob.iglob(os.path.join(self.split_dir, "**", "*.%s" % EXTENSION), recursive=True)) self.transform = transform self.target_transform = target_transform self.in_memory = in_memory self.classes = {} # fname - label number mapping self.images = [] # used for in-memory processing # build class label - number mapping with open(os.path.join(self.root_final, CLASS_LIST_FILE), "r") as fp: self.classes_names = sorted([text.strip() for text in fp.readlines()]) self.class_to_idx = {text: i for i, text in enumerate(self.classes_names)} if self.split == "train": for class_text, i in self.class_to_idx.items(): for cnt in range(NUM_IMAGES_PER_CLASS): self.classes["%s_%d.%s" % (class_text, cnt, EXTENSION)] = i elif self.split == "val": with open(os.path.join(self.split_dir, VAL_ANNOTATION_FILE), "r") as fp: for line in fp.readlines(): terms = line.split("\t") file_name, class_text = terms[0], terms[1] self.classes[file_name] = self.class_to_idx[class_text] # read all images into torch tensor in memory to minimize disk IO overhead if self.in_memory: self.images = [self.read_image(path) for path in self.image_paths]