def get_flowers(split='train', batch_size=128, shuffle=True, augm_type='none', size=224, num_workers=8, config_dict=None): augm_config = {} transform = get_imageNet_augmentation(augm_type, out_size=size, config_dict=augm_config) path = get_flowers_path() dataset = Flowers(path, split, transform=transform) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) if config_dict is not None: config_dict['Dataset'] = 'Flowers' config_dict['Batch out_size'] = batch_size config_dict['Augmentation'] = augm_config return loader
def get_ImageNet(train=True, batch_size=None, shuffle=None, augm_type='test', num_workers=8, size=224, config_dict=None): if batch_size == None: if train: batch_size = DEFAULT_TRAIN_BATCHSIZE else: batch_size = DEFAULT_TEST_BATCHSIZE augm_config = {} transform = get_imageNet_augmentation(type=augm_type, out_size=size, config_dict=augm_config) if not train and augm_type != 'none': print('Warning: ImageNet test set with ref_data augmentation') if shuffle is None: shuffle = train path = get_imagenet_path() if train == True: dataset = datasets.ImageNet(path, split='train', transform=transform) else: dataset = datasets.ImageNet(path, split='val', transform=transform) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) if config_dict is not None: config_dict['Dataset'] = 'ImageNet' config_dict['Batch out_size'] = batch_size config_dict['Augmentation'] = augm_config return loader
def get_food_101N(split='train', batch_size=128, shuffle=True, augm_type='none', size=224, num_workers=8): transform = get_imageNet_augmentation(augm_type, out_size=size) path = get_food_101N_path() dataset = Food101N(path, split, transform=transform) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) return loader
def get_LSUN_scenes(split='train', samples_per_class=None, batch_size=None, shuffle=None, augm_type='none', augm_class='imagenet', num_workers=8, size=224, config_dict=None): if batch_size is None: batch_size = DEFAULT_TEST_BATCHSIZE augm_config = {} if augm_class == 'imagenet': transform = get_imageNet_augmentation(type=augm_type, out_size=size, config_dict=augm_config) elif augm_class == 'cifar': raise NotImplementedError() transform = get_cifar10_augmentation(type=augm_type, out_size=size, in_size=224, config_dict=augm_config) else: raise NotImplementedError() path = get_LSUN_scenes_path() dataset = datasets.LSUN(path, classes=split, transform=transform) if samples_per_class is None: loader = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) else: num_classes = len(dataset.dbs) idcs = torch.zeros(num_classes, samples_per_class, dtype=torch.long) start_idx = 0 for i in range(num_classes): idcs[i, :] = torch.arange(start_idx, start_idx + samples_per_class) start_idx = dataset.indices[i] idcs = idcs.view(-1).numpy() sampler = SubsetRandomSampler(idcs) loader = DataLoader(dataset, batch_size=batch_size, sampler=sampler, num_workers=num_workers) return loader
def get_imagenet_o(batch_size=None, shuffle=False, augm_type='none', num_workers=8, size=224): if batch_size == None: batch_size = DEFAULT_TEST_BATCHSIZE transform = get_imageNet_augmentation(type=augm_type, out_size=size) path = get_imagenet_o_path() dataset = datasets.ImageFolder(path, transform=transform) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) return loader
def get_stanford_cars(train=True, batch_size=128, augm_type='none', shuffle=None, size=224, num_workers=8, config_dict=None): if shuffle is None: shuffle = train augm_config = {} transform = get_imageNet_augmentation(augm_type, out_size=size, config_dict=augm_config) path = get_stanford_cars_path() dataset = StanfordCars(path, train, transform=transform) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) if config_dict is not None: config_dict['Dataset'] = 'Stanford Cars' config_dict['Batch out_size'] = batch_size config_dict['Augmentation'] = augm_config return loader
def get_restrictedImageNetValidationTestSplit(val_split=True, batch_size=128, shuffle=False, augm_type='test', num_workers=8, size=224): idxs = np.loadtxt('idxs_rimgnet.txt', dtype=int) if not val_split: all_idcs = np.arange(10150) idxs = np.setdiff1d(all_idcs, idxs) #test idcs if shuffle: sampler = SubsetRandomSampler(idxs) else: sampler = SubsetSampler(idxs) transform = get_imageNet_augmentation(type=augm_type, out_size=size) path = get_imagenet_path() dataset = RestrictedImageNet(path=path, split='val', transform=transform, balanced=False) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=sampler, num_workers=num_workers) return loader
def get_fgvc_aircraft(split='train', class_type='variant', batch_size=128, shuffle=True, augm_type='none', size=224, num_workers=8): transform = get_imageNet_augmentation(type=augm_type, out_size=size) path = get_fgvc_aircraft_path() dataset = FGVCAircraft(path, class_type=class_type, split=split, transform=transform) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) return loader
def get_ImageNetWithout(dataset, train=True, batch_size=None, shuffle=None, augm_type='none', num_workers=8, size=224): if batch_size == None: if train: batch_size = DEFAULT_TRAIN_BATCHSIZE else: batch_size = DEFAULT_TEST_BATCHSIZE augm_config = {} transform = get_imageNet_augmentation(type=augm_type, out_size=size, config_dict=augm_config) if not train and augm_type != 'test' and augm_type != 'none': print('Warning: ImageNet test set with ref_data augmentation') if shuffle is None: shuffle = train path = get_imagenet_path() if dataset == 'pets': dataset_wids = pet_WIDS elif dataset == 'food-101': dataset_wids = food_101_WIDS elif dataset == 'cars': dataset_wids = cars_WIDS elif dataset == 'flowers': dataset_wids = flowers_WIDS else: raise NotImplementedError() od_wids = get_imagenetWID_complement(dataset_wids) if train == True: dataset = ImageNetWIDSubset(path, split='train', wids=od_wids, transform=transform) else: dataset = ImageNetWIDSubset(path, split='val', wids=od_wids, transform=transform) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) return loader
def get_openImages(split='train', batch_size=128, shuffle=None, augm_type='none', num_workers=8, size=224, exclude_dataset=None, config_dict=None): augm_config = {} transform = get_imageNet_augmentation(type=augm_type, out_size=size, config_dict=augm_config) if shuffle is None: shuffle = True if split == 'train' else False path = get_openimages_path() dataset = OpenImages(path, split, transform=transform, exclude_dataset=exclude_dataset) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) if config_dict is not None: config_dict['Dataset'] = 'OpenImages' config_dict['Exclude Dataset'] = exclude_dataset config_dict['Length'] = len(dataset) config_dict['Batch out_size'] = batch_size config_dict['Augmentation'] = augm_config return loader