def create_dataset(opt, mode):
if opt.dataset == 'CIFAR10':
mean = [125.3, 123.0, 113.9]
std = [63.0, 62.1, 66.7]
elif opt.dataset == 'CIFAR100':
mean = [129.3, 124.1, 112.4]
std = [68.2, 65.4, 70.4]
else:
mean = [0, 0, 0]
std = [1.0, 1.0, 1.0]
convert = tnt.transform.compose([
lambda x: x.astype(np.float32),
T.Normalize(mean, std),
lambda x: x.transpose(2, 0, 1).astype(np.float32),
torch.from_numpy,
])
train_transform = tnt.transform.compose([
T.RandomHorizontalFlip(),
T.Pad(opt.randomcrop_pad, cv2.BORDER_REFLECT),
T.RandomCrop(32),
convert,
])
ds = getattr(datasets, opt.dataset)(opt.dataroot,
train=mode,
download=True)
smode = 'train' if mode else 'test'
ds = tnt.dataset.TensorDataset(
[getattr(ds, smode + '_data'),
getattr(ds, smode + '_labels')])
return ds.transform({0: train_transform if mode else convert})
def create_dataset(opt, mode):
convert = tnt.transform.compose([
lambda x: x.astype(np.float32),
lambda x: x / 255.0,
# cvtransforms.Normalize([125.3, 123.0, 113.9], [63.0, 62.1, 66.7]),
lambda x: x.transpose(2, 0, 1).astype(np.float32),
torch.from_numpy,
])
train_transform = tnt.transform.compose([
cvtransforms.RandomHorizontalFlip(),
cvtransforms.Pad(opt.randomcrop_pad, cv2.BORDER_REFLECT),
cvtransforms.RandomCrop(96),
convert,
])
smode = 'train' if mode else 'test'
ds = getattr(datasets, opt.dataset)('.', split=smode, download=True)
#print(len(getattr(ds, 'labels').tolist()))
ds = tnt.dataset.TensorDataset([
getattr(ds, 'data').transpose(0, 2, 3, 1),
getattr(ds, 'labels').tolist()
])
return ds.transform({0: train_transform if mode else convert})
def create_dataset(opt, mode):
convert = tnt.transform.compose([
lambda x: x.astype(np.float32),
T.Normalize([125.3, 123.0, 113.9], [63.0, 62.1, 66.7]),
lambda x: x.transpose(2, 0, 1),
torch.from_numpy,
])
train_transform = tnt.transform.compose([
T.RandomHorizontalFlip(),
T.Pad(opt.randomcrop_pad, cv2.BORDER_REFLECT),
T.RandomCrop(32),
convert,
])
smode = 'train' if mode else 'test'
if opt.dataset == "SVHN":
ds = getattr(datasets, opt.dataset)(opt.dataroot,
split=smode,
download=True)
ds = tnt.dataset.TensorDataset(
[getattr(ds, 'data'), getattr(ds, 'labels')])
return ds
else:
ds = getattr(datasets, opt.dataset)(opt.dataroot,
train=mode,
download=True)
ds = tnt.dataset.TensorDataset(
[getattr(ds, smode + '_data'),
getattr(ds, smode + '_labels')])
return ds.transform({0: train_transform if mode else convert})
def create_val_dataset(mode):
convert = tnt.transform.compose([
lambda x: x.astype(np.float32),
T.Normalize([125.3, 123.0, 113.9], [63.0, 62.1, 66.7]),
lambda x: x.transpose(2, 0, 1).astype(np.float32),
torch.from_numpy,
])
train_transform = tnt.transform.compose([
T.RandomHorizontalFlip(),
T.Pad(randomcrop_pad, cv2.BORDER_REFLECT),
T.RandomCrop(32),
convert,
])
ds = getattr(datasets, dataset)(data_root, train=mode, download=True)
smode = 'train' if mode else 'test'
ds = tnt.dataset.TensorDataset(
[getattr(ds, smode + '_data'),
getattr(ds, smode + '_labels')])
#KK: I think you can do split dataset here since split dataset is also tnt.SplitDtaset(...)
partitions = {'trainNotUsed': 0.9, 'validation': 0.1}
ds = tnt.dataset.ShuffleDataset(dataset=ds)
ds.resample(seed=0)
#print(ds[0])
ds = tnt.dataset.SplitDataset(dataset=ds, partitions=partitions)
ds.select('validation')
print(len(ds))
return ds.transform({0: convert})
def create_dataset(opt, mode):
convert = tnt.transform.compose([
lambda x: x.astype(np.float32),
lambda x: x / 255.0,
# cvtransforms.Normalize([125.3, 123.0, 113.9], [63.0, 62.1, 66.7]),
lambda x: x.transpose(2, 0, 1).astype(np.float32),
torch.from_numpy,
])
train_transform = tnt.transform.compose([
cvtransforms.RandomHorizontalFlip(),
cvtransforms.Pad(opt.randomcrop_pad, cv2.BORDER_REFLECT),
cvtransforms.RandomCrop(32),
convert,
])
ds = getattr(datasets, opt.dataset)('.', train=mode, download=True)
smode = 'train' if mode else 'test'
if mode:
from numpy.random import RandomState
prng = RandomState(opt.seed)
assert (opt.sampleSize % 10 == 0)
random_permute = prng.permutation(np.arange(
0, 5000))[0:opt.sampleSize / 10]
labels = np.array(getattr(ds, 'train_labels'))
data = getattr(ds, 'train_data')
classes = np.unique(labels)
inds_all = np.array([], dtype='int32')
for cl in classes:
inds = np.where(np.array(labels) == cl)[0][random_permute]
inds_all = np.r_[inds, inds_all]
ds = tnt.dataset.TensorDataset([
data[inds_all, :].transpose(0, 2, 3, 1), labels[inds_all].tolist()
])
else:
ds = tnt.dataset.TensorDataset([
getattr(ds, smode + '_data').transpose(0, 2, 3, 1),
getattr(ds, smode + '_labels')
])
return ds.transform({0: train_transform if mode else convert})
def create_dataset(opt, mode, fold=0):
convert = tnt.transform.compose([
lambda x: x.astype(np.float32),
lambda x: x / 255.0,
# cvtransforms.Normalize([125.3, 123.0, 113.9], [63.0, 62.1, 66.7]),
lambda x: x.transpose(2, 0, 1).astype(np.float32),
torch.from_numpy,
])
train_transform = tnt.transform.compose([
cvtransforms.RandomHorizontalFlip(),
cvtransforms.Pad(opt.randomcrop_pad, cv2.BORDER_REFLECT),
cvtransforms.RandomCrop(96),
convert,
])
smode = 'train' if mode else 'test'
ds = getattr(datasets, opt.dataset)('.', split=smode, download=True)
if mode:
if fold > -1:
folds_idx = [
map(int,
v.split(' ')[:-1]) for v in [
line.replace('\n', '')
for line in open('./stl10_binary/fold_indices.txt')
]
][fold]
ds = tnt.dataset.TensorDataset([
getattr(ds, 'data').transpose(0, 2, 3, 1)[folds_idx],
getattr(ds, 'labels')[folds_idx].tolist()
])
else:
ds = tnt.dataset.TensorDataset([
getattr(ds, 'data').transpose(0, 2, 3, 1),
getattr(ds, 'labels').tolist()
])
else:
ds = tnt.dataset.TensorDataset([
getattr(ds, 'data').transpose(0, 2, 3, 1),
getattr(ds, 'labels').tolist()
])
return ds.transform({0: train_transform if mode else convert})
def get_iterator(opt, mode):
def cvload(path):
img = cv2.imread(path, cv2.IMREAD_COLOR)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
return img
convert = tnt.transform.compose([
lambda x: x.astype(np.float32) / 255.0,
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
lambda x: x.transpose(2, 0, 1).astype(np.float32),
torch.from_numpy,
])
print("| setting up data loader...")
if mode:
traindir = os.path.join(opt.imagenetpath, 'train')
ds = torchvision.datasets.ImageFolder(traindir,
tnt.transform.compose([
T.RandomSizedCrop(224),
T.RandomHorizontalFlip(),
convert,
]),
loader=cvload)
else:
valdir = os.path.join(opt.imagenetpath, 'val')
ds = torchvision.datasets.ImageFolder(valdir,
tnt.transform.compose([
T.Scale(256),
T.CenterCrop(224),
convert,
]),
loader=cvload)
return DataLoader(ds,
batch_size=opt.batchSize,
shuffle=mode,
num_workers=opt.nthread,
pin_memory=False)
def get_iterator(mode, opt):
if (opt.imagenetpath is None):
raise (RuntimeError('Where is imagenet?'))
if (opt.N is None):
raise (RuntimeError('Crop size not provided'))
if (opt.batchSize is None):
raise (RuntimeError('Batch Size not provided '))
if (opt.nthread is None):
raise (RuntimeError('num threads?'))
def cvload(path):
img = cv2.imread(path, cv2.IMREAD_COLOR)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
return img
convert = tnt.transform.compose([
lambda x: x.astype(np.float32) / 255.0,
cvtransforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
lambda x: x.transpose(2, 0, 1).astype(np.float32),
torch.from_numpy,
])
print("| setting up data loader...")
if mode:
traindir = os.path.join(opt.imagenetpath, 'train')
if (opt.max_samples > 0):
ds = datasmall.ImageFolder(traindir,
tnt.transform.compose([
cvtransforms.RandomSizedCrop(opt.N),
cvtransforms.RandomHorizontalFlip(),
convert,
]),
loader=cvload,
maxSamp=opt.max_samples)
else:
ds = torchvision.datasets.ImageFolder(
traindir,
tnt.transform.compose([
cvtransforms.RandomSizedCrop(opt.N),
cvtransforms.RandomHorizontalFlip(),
convert,
]),
loader=cvload)
else:
if opt.N == 224:
crop_scale = 256
else:
crop_scale = 256 * opt.N / 224
valdir = os.path.join(opt.imagenetpath, 'val')
ds = torchvision.datasets.ImageFolder(
valdir,
tnt.transform.compose([
cvtransforms.Scale(crop_scale),
cvtransforms.CenterCrop(opt.N),
convert,
]),
loader=cvload)
return torch.utils.data.DataLoader(ds,
batch_size=opt.batchSize,
shuffle=mode,
num_workers=opt.nthread,
pin_memory=False)
def create_iterator(opt, mode):
if opt.dataset.startswith('CIFAR'):
convert = tnt.transform.compose([
lambda x: x.astype(np.float32),
T.Normalize([125.3, 123.0, 113.9], [63.0, 62.1, 66.7]),
lambda x: x.transpose(2, 0, 1),
torch.from_numpy,
])
train_transform = tnt.transform.compose([
T.RandomHorizontalFlip(),
T.Pad(opt.randomcrop_pad, cv2.BORDER_REFLECT),
T.RandomCrop(32),
convert,
])
ds = getattr(datasets, opt.dataset)(opt.dataroot,
train=mode,
download=True)
smode = 'train' if mode else 'test'
ds = tnt.dataset.TensorDataset(
[getattr(ds, smode + '_data'),
getattr(ds, smode + '_labels')])
ds = ds.transform({0: train_transform if mode else convert})
return ds.parallel(batch_size=opt.batchSize,
shuffle=mode,
num_workers=opt.nthread,
pin_memory=True)
elif opt.dataset == 'ImageNet':
def cvload(path):
img = cv2.imread(path, cv2.IMREAD_COLOR)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
return img
convert = tnt.transform.compose([
lambda x: x.astype(np.float32) / 255.0,
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
lambda x: x.transpose(2, 0, 1).astype(np.float32),
torch.from_numpy,
])
print("| setting up data loader...")
if mode:
traindir = os.path.join(opt.dataroot, 'train')
ds = datasets.ImageFolder(traindir,
tnt.transform.compose([
T.RandomSizedCrop(224),
T.RandomHorizontalFlip(),
convert,
]),
loader=cvload)
else:
valdir = os.path.join(opt.dataroot, 'val')
ds = datasets.ImageFolder(valdir,
tnt.transform.compose([
T.Scale(256),
T.CenterCrop(224),
convert,
]),
loader=cvload)
return torch.utils.data.DataLoader(ds,
batch_size=opt.batchSize,
shuffle=mode,
num_workers=opt.nthread,
pin_memory=False)
else:
raise ValueError('dataset not understood')
