def create_dataset(opt, mode): if opt.dataset == 'CIFAR10': mean = [125.3, 123.0, 113.9] std = [63.0, 62.1, 66.7] elif opt.dataset == 'CIFAR100': mean = [129.3, 124.1, 112.4] std = [68.2, 65.4, 70.4] else: mean = [0, 0, 0] std = [1.0, 1.0, 1.0] convert = tnt.transform.compose([ lambda x: x.astype(np.float32), T.Normalize(mean, std), lambda x: x.transpose(2, 0, 1).astype(np.float32), torch.from_numpy, ]) train_transform = tnt.transform.compose([ T.RandomHorizontalFlip(), T.Pad(opt.randomcrop_pad, cv2.BORDER_REFLECT), T.RandomCrop(32), convert, ]) ds = getattr(datasets, opt.dataset)(opt.dataroot, train=mode, download=True) smode = 'train' if mode else 'test' ds = tnt.dataset.TensorDataset( [getattr(ds, smode + '_data'), getattr(ds, smode + '_labels')]) return ds.transform({0: train_transform if mode else convert})
def create_dataset(opt, mode): convert = tnt.transform.compose([ lambda x: x.astype(np.float32), lambda x: x / 255.0, # cvtransforms.Normalize([125.3, 123.0, 113.9], [63.0, 62.1, 66.7]), lambda x: x.transpose(2, 0, 1).astype(np.float32), torch.from_numpy, ]) train_transform = tnt.transform.compose([ cvtransforms.RandomHorizontalFlip(), cvtransforms.Pad(opt.randomcrop_pad, cv2.BORDER_REFLECT), cvtransforms.RandomCrop(96), convert, ]) smode = 'train' if mode else 'test' ds = getattr(datasets, opt.dataset)('.', split=smode, download=True) #print(len(getattr(ds, 'labels').tolist())) ds = tnt.dataset.TensorDataset([ getattr(ds, 'data').transpose(0, 2, 3, 1), getattr(ds, 'labels').tolist() ]) return ds.transform({0: train_transform if mode else convert})
def create_dataset(opt, mode): convert = tnt.transform.compose([ lambda x: x.astype(np.float32), T.Normalize([125.3, 123.0, 113.9], [63.0, 62.1, 66.7]), lambda x: x.transpose(2, 0, 1), torch.from_numpy, ]) train_transform = tnt.transform.compose([ T.RandomHorizontalFlip(), T.Pad(opt.randomcrop_pad, cv2.BORDER_REFLECT), T.RandomCrop(32), convert, ]) smode = 'train' if mode else 'test' if opt.dataset == "SVHN": ds = getattr(datasets, opt.dataset)(opt.dataroot, split=smode, download=True) ds = tnt.dataset.TensorDataset( [getattr(ds, 'data'), getattr(ds, 'labels')]) return ds else: ds = getattr(datasets, opt.dataset)(opt.dataroot, train=mode, download=True) ds = tnt.dataset.TensorDataset( [getattr(ds, smode + '_data'), getattr(ds, smode + '_labels')]) return ds.transform({0: train_transform if mode else convert})
def create_val_dataset(mode): convert = tnt.transform.compose([ lambda x: x.astype(np.float32), T.Normalize([125.3, 123.0, 113.9], [63.0, 62.1, 66.7]), lambda x: x.transpose(2, 0, 1).astype(np.float32), torch.from_numpy, ]) train_transform = tnt.transform.compose([ T.RandomHorizontalFlip(), T.Pad(randomcrop_pad, cv2.BORDER_REFLECT), T.RandomCrop(32), convert, ]) ds = getattr(datasets, dataset)(data_root, train=mode, download=True) smode = 'train' if mode else 'test' ds = tnt.dataset.TensorDataset( [getattr(ds, smode + '_data'), getattr(ds, smode + '_labels')]) #KK: I think you can do split dataset here since split dataset is also tnt.SplitDtaset(...) partitions = {'trainNotUsed': 0.9, 'validation': 0.1} ds = tnt.dataset.ShuffleDataset(dataset=ds) ds.resample(seed=0) #print(ds[0]) ds = tnt.dataset.SplitDataset(dataset=ds, partitions=partitions) ds.select('validation') print(len(ds)) return ds.transform({0: convert})
def create_dataset(opt, mode): convert = tnt.transform.compose([ lambda x: x.astype(np.float32), lambda x: x / 255.0, # cvtransforms.Normalize([125.3, 123.0, 113.9], [63.0, 62.1, 66.7]), lambda x: x.transpose(2, 0, 1).astype(np.float32), torch.from_numpy, ]) train_transform = tnt.transform.compose([ cvtransforms.RandomHorizontalFlip(), cvtransforms.Pad(opt.randomcrop_pad, cv2.BORDER_REFLECT), cvtransforms.RandomCrop(32), convert, ]) ds = getattr(datasets, opt.dataset)('.', train=mode, download=True) smode = 'train' if mode else 'test' if mode: from numpy.random import RandomState prng = RandomState(opt.seed) assert (opt.sampleSize % 10 == 0) random_permute = prng.permutation(np.arange( 0, 5000))[0:opt.sampleSize / 10] labels = np.array(getattr(ds, 'train_labels')) data = getattr(ds, 'train_data') classes = np.unique(labels) inds_all = np.array([], dtype='int32') for cl in classes: inds = np.where(np.array(labels) == cl)[0][random_permute] inds_all = np.r_[inds, inds_all] ds = tnt.dataset.TensorDataset([ data[inds_all, :].transpose(0, 2, 3, 1), labels[inds_all].tolist() ]) else: ds = tnt.dataset.TensorDataset([ getattr(ds, smode + '_data').transpose(0, 2, 3, 1), getattr(ds, smode + '_labels') ]) return ds.transform({0: train_transform if mode else convert})
def create_dataset(opt, mode, fold=0): convert = tnt.transform.compose([ lambda x: x.astype(np.float32), lambda x: x / 255.0, # cvtransforms.Normalize([125.3, 123.0, 113.9], [63.0, 62.1, 66.7]), lambda x: x.transpose(2, 0, 1).astype(np.float32), torch.from_numpy, ]) train_transform = tnt.transform.compose([ cvtransforms.RandomHorizontalFlip(), cvtransforms.Pad(opt.randomcrop_pad, cv2.BORDER_REFLECT), cvtransforms.RandomCrop(96), convert, ]) smode = 'train' if mode else 'test' ds = getattr(datasets, opt.dataset)('.', split=smode, download=True) if mode: if fold > -1: folds_idx = [ map(int, v.split(' ')[:-1]) for v in [ line.replace('\n', '') for line in open('./stl10_binary/fold_indices.txt') ] ][fold] ds = tnt.dataset.TensorDataset([ getattr(ds, 'data').transpose(0, 2, 3, 1)[folds_idx], getattr(ds, 'labels')[folds_idx].tolist() ]) else: ds = tnt.dataset.TensorDataset([ getattr(ds, 'data').transpose(0, 2, 3, 1), getattr(ds, 'labels').tolist() ]) else: ds = tnt.dataset.TensorDataset([ getattr(ds, 'data').transpose(0, 2, 3, 1), getattr(ds, 'labels').tolist() ]) return ds.transform({0: train_transform if mode else convert})
def create_iterator(opt, mode): if opt.dataset.startswith('CIFAR'): convert = tnt.transform.compose([ lambda x: x.astype(np.float32), T.Normalize([125.3, 123.0, 113.9], [63.0, 62.1, 66.7]), lambda x: x.transpose(2, 0, 1), torch.from_numpy, ]) train_transform = tnt.transform.compose([ T.RandomHorizontalFlip(), T.Pad(opt.randomcrop_pad, cv2.BORDER_REFLECT), T.RandomCrop(32), convert, ]) ds = getattr(datasets, opt.dataset)(opt.dataroot, train=mode, download=True) smode = 'train' if mode else 'test' ds = tnt.dataset.TensorDataset( [getattr(ds, smode + '_data'), getattr(ds, smode + '_labels')]) ds = ds.transform({0: train_transform if mode else convert}) return ds.parallel(batch_size=opt.batchSize, shuffle=mode, num_workers=opt.nthread, pin_memory=True) elif opt.dataset == 'ImageNet': def cvload(path): img = cv2.imread(path, cv2.IMREAD_COLOR) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) return img convert = tnt.transform.compose([ lambda x: x.astype(np.float32) / 255.0, T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), lambda x: x.transpose(2, 0, 1).astype(np.float32), torch.from_numpy, ]) print("| setting up data loader...") if mode: traindir = os.path.join(opt.dataroot, 'train') ds = datasets.ImageFolder(traindir, tnt.transform.compose([ T.RandomSizedCrop(224), T.RandomHorizontalFlip(), convert, ]), loader=cvload) else: valdir = os.path.join(opt.dataroot, 'val') ds = datasets.ImageFolder(valdir, tnt.transform.compose([ T.Scale(256), T.CenterCrop(224), convert, ]), loader=cvload) return torch.utils.data.DataLoader(ds, batch_size=opt.batchSize, shuffle=mode, num_workers=opt.nthread, pin_memory=False) else: raise ValueError('dataset not understood')