def get_flowers(split='train',
batch_size=128,
shuffle=True,
augm_type='none',
size=224,
num_workers=8,
config_dict=None):
augm_config = {}
transform = get_imageNet_augmentation(augm_type,
out_size=size,
config_dict=augm_config)
path = get_flowers_path()
dataset = Flowers(path, split, transform=transform)
loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
shuffle=shuffle,
num_workers=num_workers)
if config_dict is not None:
config_dict['Dataset'] = 'Flowers'
config_dict['Batch out_size'] = batch_size
config_dict['Augmentation'] = augm_config
return loader
def get_ImageNet(train=True, batch_size=None, shuffle=None, augm_type='test', num_workers=8, size=224, config_dict=None):
if batch_size == None:
if train:
batch_size = DEFAULT_TRAIN_BATCHSIZE
else:
batch_size = DEFAULT_TEST_BATCHSIZE
augm_config = {}
transform = get_imageNet_augmentation(type=augm_type, out_size=size, config_dict=augm_config)
if not train and augm_type != 'none':
print('Warning: ImageNet test set with ref_data augmentation')
if shuffle is None:
shuffle = train
path = get_imagenet_path()
if train == True:
dataset = datasets.ImageNet(path, split='train', transform=transform)
else:
dataset = datasets.ImageNet(path, split='val', transform=transform)
loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size,
shuffle=shuffle, num_workers=num_workers)
if config_dict is not None:
config_dict['Dataset'] = 'ImageNet'
config_dict['Batch out_size'] = batch_size
config_dict['Augmentation'] = augm_config
return loader
def get_food_101N(split='train',
batch_size=128,
shuffle=True,
augm_type='none',
size=224,
num_workers=8):
transform = get_imageNet_augmentation(augm_type, out_size=size)
path = get_food_101N_path()
dataset = Food101N(path, split, transform=transform)
loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
shuffle=shuffle,
num_workers=num_workers)
return loader
def get_LSUN_scenes(split='train',
samples_per_class=None,
batch_size=None,
shuffle=None,
augm_type='none',
augm_class='imagenet',
num_workers=8,
size=224,
config_dict=None):
if batch_size is None:
batch_size = DEFAULT_TEST_BATCHSIZE
augm_config = {}
if augm_class == 'imagenet':
transform = get_imageNet_augmentation(type=augm_type,
out_size=size,
config_dict=augm_config)
elif augm_class == 'cifar':
raise NotImplementedError()
transform = get_cifar10_augmentation(type=augm_type,
out_size=size,
in_size=224,
config_dict=augm_config)
else:
raise NotImplementedError()
path = get_LSUN_scenes_path()
dataset = datasets.LSUN(path, classes=split, transform=transform)
if samples_per_class is None:
loader = DataLoader(dataset,
batch_size=batch_size,
shuffle=shuffle,
num_workers=num_workers)
else:
num_classes = len(dataset.dbs)
idcs = torch.zeros(num_classes, samples_per_class, dtype=torch.long)
start_idx = 0
for i in range(num_classes):
idcs[i, :] = torch.arange(start_idx, start_idx + samples_per_class)
start_idx = dataset.indices[i]
idcs = idcs.view(-1).numpy()
sampler = SubsetRandomSampler(idcs)
loader = DataLoader(dataset,
batch_size=batch_size,
sampler=sampler,
num_workers=num_workers)
return loader
def get_imagenet_o(batch_size=None,
shuffle=False,
augm_type='none',
num_workers=8,
size=224):
if batch_size == None:
batch_size = DEFAULT_TEST_BATCHSIZE
transform = get_imageNet_augmentation(type=augm_type, out_size=size)
path = get_imagenet_o_path()
dataset = datasets.ImageFolder(path, transform=transform)
loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
shuffle=shuffle,
num_workers=num_workers)
return loader
def get_stanford_cars(train=True, batch_size=128, augm_type='none', shuffle=None, size=224,
num_workers=8, config_dict=None):
if shuffle is None:
shuffle = train
augm_config = {}
transform = get_imageNet_augmentation(augm_type, out_size=size, config_dict=augm_config)
path = get_stanford_cars_path()
dataset = StanfordCars(path, train, transform=transform)
loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size,
shuffle=shuffle, num_workers=num_workers)
if config_dict is not None:
config_dict['Dataset'] = 'Stanford Cars'
config_dict['Batch out_size'] = batch_size
config_dict['Augmentation'] = augm_config
return loader
def get_restrictedImageNetValidationTestSplit(val_split=True, batch_size=128, shuffle=False, augm_type='test', num_workers=8, size=224):
idxs = np.loadtxt('idxs_rimgnet.txt', dtype=int)
if not val_split:
all_idcs = np.arange(10150)
idxs = np.setdiff1d(all_idcs, idxs) #test idcs
if shuffle:
sampler = SubsetRandomSampler(idxs)
else:
sampler = SubsetSampler(idxs)
transform = get_imageNet_augmentation(type=augm_type, out_size=size)
path = get_imagenet_path()
dataset = RestrictedImageNet(path=path, split='val',
transform=transform, balanced=False)
loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=sampler,
num_workers=num_workers)
return loader
def get_fgvc_aircraft(split='train',
class_type='variant',
batch_size=128,
shuffle=True,
augm_type='none',
size=224,
num_workers=8):
transform = get_imageNet_augmentation(type=augm_type, out_size=size)
path = get_fgvc_aircraft_path()
dataset = FGVCAircraft(path,
class_type=class_type,
split=split,
transform=transform)
loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
shuffle=shuffle,
num_workers=num_workers)
return loader
def get_ImageNetWithout(dataset, train=True, batch_size=None, shuffle=None, augm_type='none',
num_workers=8, size=224):
if batch_size == None:
if train:
batch_size = DEFAULT_TRAIN_BATCHSIZE
else:
batch_size = DEFAULT_TEST_BATCHSIZE
augm_config = {}
transform = get_imageNet_augmentation(type=augm_type, out_size=size, config_dict=augm_config)
if not train and augm_type != 'test' and augm_type != 'none':
print('Warning: ImageNet test set with ref_data augmentation')
if shuffle is None:
shuffle = train
path = get_imagenet_path()
if dataset == 'pets':
dataset_wids = pet_WIDS
elif dataset == 'food-101':
dataset_wids = food_101_WIDS
elif dataset == 'cars':
dataset_wids = cars_WIDS
elif dataset == 'flowers':
dataset_wids = flowers_WIDS
else:
raise NotImplementedError()
od_wids = get_imagenetWID_complement(dataset_wids)
if train == True:
dataset = ImageNetWIDSubset(path, split='train', wids=od_wids, transform=transform)
else:
dataset = ImageNetWIDSubset(path, split='val', wids=od_wids, transform=transform)
loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size,
shuffle=shuffle, num_workers=num_workers)
return loader
def get_openImages(split='train', batch_size=128, shuffle=None, augm_type='none', num_workers=8, size=224,
exclude_dataset=None, config_dict=None):
augm_config = {}
transform = get_imageNet_augmentation(type=augm_type, out_size=size, config_dict=augm_config)
if shuffle is None:
shuffle = True if split == 'train' else False
path = get_openimages_path()
dataset = OpenImages(path, split, transform=transform, exclude_dataset=exclude_dataset)
loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size,
shuffle=shuffle, num_workers=num_workers)
if config_dict is not None:
config_dict['Dataset'] = 'OpenImages'
config_dict['Exclude Dataset'] = exclude_dataset
config_dict['Length'] = len(dataset)
config_dict['Batch out_size'] = batch_size
config_dict['Augmentation'] = augm_config
return loader
