def get_cifar100_loader(mode, root_path): ''' 获取cifar10 loader ''' if mode == "train": dst = CIFAR100(root=root_path, train=True, transform=train_transform, download=False) data_loader = DataLoader(dst, batch_size=config.batch_size, shuffle=True, num_workers=config.num_workers) elif mode == "validate": dst = CIFAR100(root=root_path, train=False, transform=validate_transform, download=False) data_loader = DataLoader(dst, batch_size=config.batch_size, shuffle=False, num_workers=config.num_workers) else: raise ValueError("get_loader mode is error") return data_loader
def download(dataset_cfg): train = CIFAR100('/tmp', train=True, transform=None, target_transform=None, download=True) test = CIFAR100('/tmp', train=False, transform=None, target_transform=None, download=True) img_root = dataset_cfg.image_root_folder os.makedirs(img_root, exist_ok=True) os.makedirs(os.path.dirname(dataset_cfg.label_file), exist_ok=True) label_set = set(train.class_to_idx.values()) for label in label_set: os.makedirs(os.path.join(img_root, str(label)), exist_ok=True) d = dict() for i, (img, label) in enumerate(train + test): total_label = np.zeros(len(label_set)) total_label[label] = 1 d[i] = total_label img.save(os.path.join(img_root, str(label), '%s.jpg' % i)) np.save(dataset_cfg.label_file, d)
def load_data(data_root_dir='../data/'): train_transform = transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1), transforms.ToTensor(), transforms.RandomErasing(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ]) # 测试阶段 Ten Crop test test_transform = transforms.Compose([ transforms.Resize(256), transforms.TenCrop(224), transforms.Lambda(lambda crops: torch.stack([transforms.ToTensor()(crop) for crop in crops])), transforms.Lambda(lambda crops: torch.stack( [transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))(crop) for crop in crops])) ]) data_loaders = {} data_sizes = {} for name in ['train', 'test']: if name == 'train': data_set = CIFAR100(data_root_dir, train=True, download=True, transform=train_transform) data_loader = DataLoader(data_set, batch_size=96, shuffle=True, num_workers=8) else: data_set = CIFAR100(data_root_dir, train=False, download=True, transform=test_transform) data_loader = DataLoader(data_set, batch_size=48, shuffle=True, num_workers=8) data_loaders[name] = data_loader data_sizes[name] = len(data_set) return data_loaders, data_sizes
def get_dataset(cls, cutout_length=0): MEAN = [0.5071, 0.4865, 0.4409] STD = [0.1942, 0.1918, 0.1958] cutout = [] if cutout_length > 0: cutout.append(Cutout(cutout_length)) train_transform = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.RandomResizedCrop((32, 32)), transforms.ColorJitter(0.2, 0.2, 0.2, 0.2), transforms.ToTensor(), transforms.Normalize(MEAN, STD) ]) valid_transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize(MEAN, STD)]) if cls == "cifar100": dataset_train = CIFAR100(root="./data", train=True, download=True, transform=train_transform) dataset_valid = CIFAR100(root="./data", train=False, download=True, transform=valid_transform) else: raise NotImplementedError return dataset_train, dataset_valid
def load_cifar_datasets(path='./dataset', n_class=10, train_transform=get_transform()[0], test_transform=get_transform()[1]): if n_class == 10: train_dataset = CIFAR10(path, train=True, download=True, transform=train_transform) test_dataset = CIFAR10(path, train=False, download=True, transform=test_transform) elif n_class == 100: train_dataset = CIFAR100(path, train=True, download=True, transform=train_transform) test_dataset = CIFAR100(path, train=False, download=True, transform=test_transform) else: train_dataset, test_dataset = None, None return train_dataset, test_dataset
def get_dataset(cls, cutout_length=0): MEAN = [0.5070751592371323, 0.48654887331495095, 0.4409178433670343] STD = [0.2673342858792401, 0.2564384629170883, 0.27615047132568404] transf = [ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip() ] normalize = [transforms.ToTensor(), transforms.Normalize(MEAN, STD)] cutout = [] if cutout_length > 0: cutout.append(Cutout(cutout_length)) train_transform = transforms.Compose(transf + normalize + cutout) valid_transform = transforms.Compose(normalize) if cls == "cifar100": dataset_train = CIFAR100(root="./data", train=True, download=True, transform=train_transform) dataset_valid = CIFAR100(root="./data", train=False, download=True, transform=valid_transform) else: raise NotImplementedError return dataset_train, dataset_valid
def load_data(dataset, path, batch_size=64, normalize=False): if normalize: # Wasserstein BiGAN is trained on normalized data. transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) else: # BiGAN is trained on unnormalized data (see Dumoulin et al. ICLR 16). transform = transforms.ToTensor() if dataset == 'svhn': train_set = SVHN(path, split='extra', transform=transform, download=True) val_set = SVHN(path, split='test', transform=transform, download=True) elif dataset == 'cifar10': train_set = CIFAR10(path, train=True, transform=transform, download=True) val_set = CIFAR10(path, train=False, transform=transform, download=True) elif dataset == 'cifar100': train_set = CIFAR100(path, train=True, transform=transform, download=True) val_set = CIFAR100(path, train=False, transform=transform, download=True) train_loader = data.DataLoader( train_set, batch_size, shuffle=True, num_workers=12) val_loader = data.DataLoader( val_set, 1, shuffle=False, num_workers=1, pin_memory=True) return train_loader, val_loader
def get_cifar100(): n_classes = 100 i_channel = 3 i_dim = 32 # transforms taken from https://github.com/kuangliu/pytorch-cifar/blob/master/main.py # transform_train = transforms.Compose([transforms.ToTensor(),\ # transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) transform_train = transforms.Compose([transforms.RandomCrop(32, padding=4), \ transforms.RandomHorizontalFlip(), transforms.ToTensor(), \ transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) # transform_test = transforms.Compose([transforms.ToTensor(),\ # transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) transform_test = transforms.Compose([transforms.ToTensor(), \ transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) train_d = CIFAR100(root=D_PTH, train=True, download=True, transform=transform_train) # self.train_d.data = self.train_d.data.reshape(len(self.train_d),self.i_dim) test_d = CIFAR100(root=D_PTH, train=False, download=True, transform=transform_test) return (n_classes, i_channel, i_dim, train_d, test_d)
def setup(self, stage='fit'): if self.superclass: if stage == 'fit': self.cifar_train = CIFAR100Super( self.data_path, train=True, transform=self.train_transforms) self.cifar_val = CIFAR100Super(self.data_path, train=False, transform=self.test_transforms) elif stage == 'test': self.cifar_test = CIFAR100Super(self.data_path, train=False, transform=self.test_transforms) else: if stage == 'fit': self.cifar_train = CIFAR100(self.data_path, train=True, transform=self.train_transforms) self.cifar_val = CIFAR100(self.data_path, train=False, transform=self.test_transforms) elif stage == 'test': self.cifar_test = CIFAR100(self.data_path, train=False, transform=self.test_transforms)
def get_dataset(data_path, dataset): if dataset == 'cifar10': mean = [x / 255 for x in [125.3, 123.0, 113.9]] std = [x / 255 for x in [63.0, 62.1, 66.7]] elif dataset == 'cifar100': mean = [x / 255 for x in [129.3, 124.1, 112.4]] std = [x / 255 for x in [68.2, 65.4, 70.4]] train_transform = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(), transforms.Normalize(mean, std) ]) test_transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize(mean, std)]) if dataset == 'cifar10': train_data = CIFAR10(data_path, train=True, transform=train_transform, download=True) test_data = CIFAR10(data_path, train=False, transform=test_transform, download=True) elif dataset == 'cifar100': train_data = CIFAR100(data_path, train=True, transform=train_transform, download=True) test_data = CIFAR100(data_path, train=False, transform=test_transform, download=True) return train_data, test_data
def __init__(self, args): # pin_memory = False # if args.gpu is not None: pin_memory = True transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) trainset = CIFAR100(root=args.data_path, train=True, download=True, transform=transform_train) self.trainLoader = DataLoader( trainset, batch_size=args.train_batch_size, shuffle=True, num_workers=2, pin_memory=pin_memory ) testset = CIFAR100(root=args.data_path, train=False, download=False, transform=transform_test) self.testLoader = DataLoader( testset, batch_size=args.eval_batch_size, shuffle=False, num_workers=2, pin_memory=pin_memory)
def get_data_loader(data_path, batch_size): transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)), # ref to EigenDamage code # transforms.Normalize((0.4914, 0.4822, 0.4465), # (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)), # transforms.Normalize((0.4914, 0.4822, 0.4465), # (0.2023, 0.1994, 0.2010)), ]) train_set = CIFAR100(data_path, train=True, download=True, transform=transform_train) test_set = CIFAR100(data_path, train=False, download=True, transform=transform_test) return train_set, test_set
def __init__(self, data_type, model_type, num_groups, num_query, num_epoch, sampling_type='nbs', save_name='0'): self.data_type = data_type self.model_type = model_type self.num_groups = num_groups self.num_query = num_query self.num_epoch = num_epoch self.sampling_type = sampling_type self.save_name = save_name if data_type == 'cifar10': self.dataset = Dataset( CIFAR10(root='.cifar10', train=True, download=True, transform=get_transform(32, 4, 16)['train']) ) self.testset = Dataset( CIFAR10(root='.cifar10', train=False, download=True, transform=get_transform(32, 4, 16)['test']) ) else: self.dataset = Dataset( CIFAR100(root='.cifar100', train=True, download=True, transform=get_transform(32, 4, 8)['train']) ) self.testset = Dataset( CIFAR100(root='.cifar100', train=False, download=True, transform=get_transform(32, 4, 8)['test']) ) self.test_loader = DataLoader(self.testset, batch_size=6144, num_workers=4, pin_memory=True) self.indice = list(range(len(self.dataset))) random.Random(0).shuffle(self.indice)
def get_dataset(dataset_root, dataset, train_transform, test_transform): if dataset == 'cifar10': train = CIFAR10(dataset_root, train=True, download=True, transform=train_transform) unlabeled = CIFAR10(dataset_root, train=True, download=True, transform=train_transform) test = CIFAR10(dataset_root, train=False, download=True, transform=test_transform) elif dataset == 'cifar100': train = CIFAR100(dataset_root, train=True, download=True, transform=train_transform) unlabeled = CIFAR100(dataset_root, train=True, download=True, transform=train_transform) test = CIFAR100(dataset_root, train=False, download=True, transform=test_transform) else: print("Error: No dataset named {}!".format(dataset)) return -1 return train, test, unlabeled
def get_cifar100(): pin_memory = True transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) trainset = CIFAR100(root='./data', train=True, download=True, transform=transform_train) train_loader = DataLoader( trainset, batch_size=128, shuffle=True, num_workers=4, pin_memory=pin_memory ) testset = CIFAR100(root='./data', train=False, download=False, transform=transform_test) test_loader = DataLoader( testset, batch_size=128, shuffle=False, num_workers=4, pin_memory=pin_memory) return train_loader, test_loader
def get_train_val_test_datasets( rnd: np.random.RandomState, root='~/data', validation_ratio=0.05, ) -> tuple: """ Create CIFAR-100 train/val/test data loaders :param rnd: `np.random.RandomState` instance. :param validation_ratio: The ratio of validation data. If this value is `0.`, returned `val_set` is `None`. :param root: Path to save data. :return: Tuple of (train, val, test) or (train, test). """ transform = transforms.Compose([transforms.ToTensor()]) train_set = CIFAR100(root=root, train=True, download=True, transform=transform) # create validation split if validation_ratio > 0.: train_set, val_set = _train_val_split( rnd=rnd, train_dataset=train_set, validation_ratio=validation_ratio) # create a transform to do pre-processing train_loader = DataLoader( train_set, batch_size=len(train_set), shuffle=False, ) data = iter(train_loader).next() dim = [0, 2, 3] mean = data[0].mean(dim=dim).numpy() std = data[0].std(dim=dim).numpy() # end of creating a transform to do pre-processing transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean, std), ]) train_set.transform = transform if validation_ratio > 0.: val_set.transform = transform else: val_set = None test_set = CIFAR100(root=root, train=False, download=True, transform=transform) return train_set, val_set, test_set
def CIFAR100_datagenerator(batch_size, size): transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ]) cifar10_train = CIFAR100(root='./data', train=True, transform=transform_train, download=True) cifar10_test = CIFAR100(root='./data', train=False, transform=transform_test, download=True) train_loader = DataLoader(dataset=cifar10_train, batch_size=batch_size, shuffle=True, drop_last=False) test_loader = DataLoader(dataset=cifar10_test, batch_size=batch_size, shuffle=False, drop_last=False) return train_loader, test_loader
def __init__(self, opt, val=False): super(CustomCIFAR100, self).__init__() dir_dataset = opt.dir_dataset if val: self.dataset = CIFAR100(root=dir_dataset, train=False, download=True) self.transform = Compose([ ToTensor(), Normalize(mean=[0.507, 0.487, 0.441], std=[0.267, 0.256, 0.276]) ]) else: self.dataset = CIFAR100(root=dir_dataset, train=True, download=True) self.transform = Compose([ RandomCrop((32, 32), padding=4, fill=0, padding_mode='constant'), RandomHorizontalFlip(), ToTensor(), Normalize(mean=[0.507, 0.487, 0.441], std=[0.267, 0.256, 0.276]) ])
def get_split_cifar(bsz, num_task): train_transform = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) test_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) trainset = CIFAR100('data/cifar_data', train=True, download=True, transform=train_transform) testset = CIFAR100('data/cifars_data', train=False, download=True, transform=test_transform) workers = list(range(num_task)) partitioner = partition_dataset(trainset, testset, workers, True) train_loader, test_loader, labels = {}, {}, {} for i in workers: train_loader[i], test_loader[i], labels[i] = select_dataset( workers, i, partitioner, bsz) return train_loader, test_loader, labels
def Dataset(dataset): trainset, testset = None, None if dataset == 'cifar10' or 'cifar100': tra_trans = transforms.Compose([ transforms.RandomCrop(32, padding=4), # transforms.Grayscale(num_output_channels=1), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) val_trans = transforms.Compose([ # transforms.Grayscale(num_output_channels=1), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if dataset == 'cifar10': trainset = CIFAR10(root="/home/hyf/data", train=True, download=True, transform=tra_trans) testset = CIFAR10(root="/home/hyf/data", train=False, download=True, transform=val_trans) if dataset == 'cifar100': trainset = CIFAR100(root="/home/hyf/data", train=True, download=True, transform=tra_trans) testset = CIFAR100(root="/home/hyf/data", train=False, download=True, transform=val_trans) if dataset == 'femnist' or 'mnist' or 'fashonmnist': tra_trans = transforms.Compose([ transforms.Pad(2, padding_mode='edge'), # transforms.ToPILImage(), # transforms.Grayscale(num_output_channels=3), transforms.ToTensor(), # transforms.Lambda(lambda x: x.repeat(3, 1, 1)), transforms.Normalize((0.1307,), (0.3081,)), # transforms.Normalize((0.4914, 0.4822, 0.4465), # (0.2023, 0.1994, 0.2010)), ]) val_trans = transforms.Compose([ transforms.Pad(2, padding_mode='edge'), # transforms.ToPILImage(), # transforms.Grayscale(num_output_channels=3), transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)), # transforms.Normalize((0.4914, 0.4822, 0.4465), # (0.2023, 0.1994, 0.2010)), ]) if dataset == 'femnist': trainset = FEMNIST(root='~/data', train=True, download=True, transform=tra_trans) testset = FEMNIST(root='~/data', train=False, download=True, transform=val_trans) if dataset == 'mnist': trainset = MNIST(root='/home/hyf/data', train=True, download=True, transform=tra_trans) testset = MNIST(root='/home/hyf/data', train=False, download=True, transform=val_trans) if dataset == 'fashonmnist': trainset = FashionMNIST(root='~/data', train=True, download=True, transform=tra_trans) testset = FashionMNIST(root='~/data', train=False, download=True, transform=val_trans) return trainset, testset
def get_single_task(dataroot, task): trainset = CIFAR100(dataroot, train=True, transform=transforms.ToTensor()) trainset = filter_by_coarse_label(trainset, task) testset = CIFAR100(dataroot, train=False, transform=transforms.ToTensor()) testset = filter_by_coarse_label(testset, task) return trainset, testset
def _get_cifar100_dataset(dataset_root): if dataset_root is None: dataset_root = default_dataset_location('cifar100') train_set = CIFAR100(dataset_root, train=True, download=True) test_set = CIFAR100(dataset_root, train=False, download=True) return train_set, test_set
def __init__(self, **kwargs): """Construct the Cifar100 class..""" Dataset.__init__(self, **kwargs) CIFAR100.__init__(self, root=self.args.data_path, train=self.train, transform=Compose(self.transforms.__transform__), download=self.args.download)
def load_cifar100(root='data/', batch_size=128, download=True, num_workers=0, drop_last=False): cifar100_train = CIFAR100(root, train=True, transform=cifar100_train_transforms, download=download) cifar100_test = CIFAR100(root, train=False, transform=cifar100_test_transforms, download=download) cifar100_train_dataloader = DataLoader(cifar100_train, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=drop_last) cifar100_test_dataloader = DataLoader(cifar100_test, batch_size=batch_size, shuffle=False, num_workers=num_workers) return cifar100_train_dataloader, cifar100_test_dataloader
def cifar100_altaug(transform_t, transform_v): dataset_train = CIFAR100(root=os.path.expanduser('~/Datasets/cifar100'), train=True, transform=transform_t, download=True) dataset_val = CIFAR100(root=os.path.expanduser('~/Datasets/cifar100'), train=False, transform=transform_v) return dataset_train, dataset_val
def load_cifar100(): a = CIFAR100(root='CIFAR100', train=True) ax, ay = a.train_data, a.train_labels b = CIFAR100(root='CIFAR100', train=False) bx, by = b.test_data, b.test_labels x, y = np.concatenate([ax, bx]), np.concatenate([ay, by]) x = x.transpose([0, 3, 1, 2]).reshape([len(x), -1]) x, y = th.from_numpy(x).float(), th.from_numpy(y) return x, y
def _get_cifar100_dataset(train_transformation, eval_transformation): train_set = CIFAR100(expanduser("~") + "/.avalanche/data/cifar100/", train=True, download=True) test_set = CIFAR100(expanduser("~") + "/.avalanche/data/cifar100/", train=False, download=True) return train_eval_avalanche_datasets( train_set, test_set, train_transformation, eval_transformation)
def _get_cifar100_dataset(): train_set = CIFAR100(expanduser("~") + "/.avalanche/data/cifar100/", train=True, download=True) test_set = CIFAR100(expanduser("~") + "/.avalanche/data/cifar100/", train=False, download=True) return train_set, test_set
def build(self): train_dt = CIFAR100(self.data_dir, transform=self.train_trans, download=True) test_dt = CIFAR100(self.data_dir, train=False, transform=self.test_trans, download=True) return train_dt, test_dt
def cifar100(root): from torchvision.datasets import CIFAR100 transform = transforms.Compose([ transforms.Resize(224), transforms.ToTensor(), transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)), ]) trainset = CIFAR100(root, train=True, transform=transform, download=True) testset = CIFAR100(root, train=False, transform=transform) return trainset, testset