def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = CreateDataset(opt)
#CH: Inclusão de dataset de validação
if self.opt.val:
self.train_set, self.val_set = torch.utils.data.random_split(self.dataset, [int(len(self.dataset)*(1-opt.valsize)), int(len(self.dataset)*opt.valsize)])
self.valdataloader = torch.utils.data.DataLoader(
self.val_set,
batch_size=opt.batchSize,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads))
self.traindataloader = torch.utils.data.DataLoader(
self.train_set,
batch_size=opt.batchSize,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads))
else:
self.traindataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.batchSize,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads))
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = CreateDataset(opt)
#split
num_train = len(self.dataset)
print num_train
indices = list(range(num_train))
split =int(np.floor(self.valid_rate * num_train))
train_idx, valid_idx = indices[split:],indices[:split]
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)
self.train_loader = torch.utils.data.DataLoader(
self.dataset,
sampler=train_sampler,
batch_size=opt.batchSize ,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads))
self.valid_loader = torch.utils.data.DataLoader(
self.dataset,
sampler=valid_sampler,
batch_size=opt.batchSize ,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads))
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
transform = transforms.Compose([
transforms.Scale(opt.loadSize),
transforms.CenterCrop(opt.fineSize),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5),
(0.5, 0.5, 0.5))])
# Dataset A
dataset_A = ImageFolder(root=opt.dataroot + '/' + opt.phase + 'A',
transform=transform, return_paths=True)
data_loader_A = torch.utils.data.DataLoader(
dataset_A,
batch_size=self.opt.batchSize,
shuffle=not self.opt.serial_batches,
num_workers=int(self.opt.nThreads))
# Dataset B
dataset_B = ImageFolder(root=opt.dataroot + '/' + opt.phase + 'B',
transform=transform, return_paths=True)
data_loader_B = torch.utils.data.DataLoader(
dataset_B,
batch_size=self.opt.batchSize,
shuffle=not self.opt.serial_batches,
num_workers=int(self.opt.nThreads))
self.dataset_A = dataset_A
self.dataset_B = dataset_B
self.paired_data = PairedData(data_loader_A, data_loader_B)
def initialize(self, args):
BaseDataLoader.initialize(self, args)
self.dataset = CreateDataset(args)
self.dataloader = torch.utils.data.DataLoader(self.dataset,
batch_size=1,
shuffle=False,
num_workers=1)
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = CreateDataset(opt)
self.dataloader = torch.utils.data.DataLoader(self.dataset,
batch_size=opt.batchSize,
shuffle=True,
num_workers=0)
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.fineSize = opt.fineSize
transformations = [
# TODO: Scale
transforms.Scale(opt.loadSize),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
]
transform = transforms.Compose(transformations)
# Dataset A
dataset = ImageFolder(root=opt.dataroot + '/' + opt.phase,
transform=transform,
return_paths=True)
data_loader = torch.utils.data.DataLoader(
dataset,
batch_size=self.opt.batchSize,
shuffle=not self.opt.serial_batches,
num_workers=int(self.opt.nThreads))
self.dataset = dataset
flip = opt.isTrain and not opt.no_flip
self.paired_data = PairedData(data_loader, opt.fineSize,
opt.max_dataset_size, flip)
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.fineSize = opt.fineSize
# transform the image data to the tensor range in (0, 1)
transform = transforms.Compose([
# this is fake, we will delete it later
# transforms.Scale((opt.loadSize, opt.loadSize), interpolation=Image.BILINEAR), # BICUBIC or ANTIALIAS
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5),
(0.5, 0.5, 0.5))])
# Dataset A
dataset = ImageFolder(root=opt.dataroot + '/' + opt.phase, caption = opt.caption, caption_bucket = opt.bucket_description, vocab = opt.vocab, data_augment = opt.augment_data,
transform=transform, return_paths=True)
data_loader = torch.utils.data.DataLoader(
dataset,
batch_size=self.opt.batchSize,
shuffle=not self.opt.serial_batches,
num_workers=int(self.opt.nThreads),
drop_last=False
)
self.dataset = dataset
self.paired_data = PairedData(data_loader, opt.fineSize)
def initialize(self,
datafolders,
dataroot='./dataset',
dataset_mode='2afc',
load_size=64,
batch_size=1,
serial_batches=True,
nThreads=1,
use_gpu=False):
BaseDataLoader.initialize(self)
if (not isinstance(datafolders, list)):
datafolders = [
datafolders,
]
data_root_folders = [
os.path.join(dataroot, datafolder) for datafolder in datafolders
]
self.dataset = CreateDataset(data_root_folders,
dataset_mode=dataset_mode,
load_size=load_size)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=batch_size,
shuffle=not serial_batches,
num_workers=int(nThreads),
pin_memory=use_gpu)
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = UnalignedDataset(opt)
self.dataloader = data.DataLoader(self.dataset,
batch_size=opt.batchSize,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads))
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.real_train = opt.real_train
if opt.real_train:
self.dataset = CreateDataset(opt)
self.dataset_real = CreateDataset_Real(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.batchSize,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads))
self.dataloader_real = torch.utils.data.DataLoader(
self.dataset_real,
batch_size=opt.real_batchSize,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads))
else:
self.dataset = CreateDataset(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.batchSize,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads))
def initialize(
self,
datafolders,
dataroot="./dataset",
dataset_mode="2afc",
load_size=64,
batch_size=1,
serial_batches=True,
nThreads=1,
):
BaseDataLoader.initialize(self)
if not isinstance(datafolders, list):
datafolders = [datafolders]
data_root_folders = [
os.path.join(dataroot, datafolder) for datafolder in datafolders
]
self.dataset = CreateDataset(data_root_folders,
dataset_mode=dataset_mode,
load_size=load_size)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=batch_size,
shuffle=not serial_batches,
num_workers=int(nThreads),
)
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = SingleDataset()
self.dataset.initialize(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset, batch_size=opt.batch_size,
shuffle=not opt.serial_batches, num_workers=int(opt.nThreads))
def initialize(self, datafolder, dataroot='./bapd',dataset_mode='2afc',load_size=64,batch_size=1,serial_batches=True, nThreads=1):
BaseDataLoader.initialize(self)
self.dataset = CreateDataset(os.path.join(dataroot,datafolder),dataset_mode=dataset_mode,load_size=load_size)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=batch_size,
shuffle=not serial_batches,
num_workers=int(nThreads))
def __init__(self, opt):
BaseDataLoader.__init__(self, opt)
self.dataset = create_dataset(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.batch_size,
shuffle=not opt.serial_batches,
num_workers=int(opt.num_threads))
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = create_dataset(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.batchSize,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads))
def initialize(self, opt):
BaseDataLoader.initialize(self, opt) # initialization parameters
self.dataset = CreateDataset(opt) # create data set
self.dataloader = torch.utils.data.DataLoader( # Load the created data set and customize related parameters
self.dataset,
batch_size=opt.batchSize,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads))
def initialize(self, opt):
BaseDataLoader.initialize(self, opt) #目的是传入参数opt
self.dataset = CreateDataset(opt) #根据opt.model创建一个数据集类“MIMLDataset/KLdataset”，并初始化（包括读取h5文件，放入self.bases,self.labels）
self.dataloader = torch.utils.data.DataLoader( #该接口主要用来将自定义的数据读取接口的输出或者PyTorch已有的数据读取接口的输入按照batch size封装成Tensor，后续只需要再包装成Variable即可作为模型的输入
self.dataset,
batch_size=opt.batchSize,
shuffle=False,
num_workers=int(opt.nThreads))
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = create_dataset(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.batch_size,
shuffle=not opt.serial_batches,
num_workers=0)
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = create_dataset(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.batch_size,
shuffle=not opt.serial_batches,#in training, serial_batches by default is false, shuffle=true
num_workers=int(opt.num_threads))
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = CreateDataset(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.batchSize,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads), drop_last=True)
def initialize(self, args):
BaseDataLoader.initialize(self, args)
self.dataset = CreateSyntheticDataset(args)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=args.batchSize,
shuffle=False,
num_workers=32)
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
transformations = [
#transforms.Scale(opt.loadSize),
MyScale(size=(256, 256), pad=True),
transforms.RandomCrop(opt.fineSize),
transforms.ToTensor(),
# this is wrong! because the fake samples are not normalized like this,
# still they are inferred on the same network,
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
#lambda x: (x - x.min()) / x.max() * 2 - 1, # [-1., 1.]
] # 归一化，会产生负数。
#transformations = [transforms.Scale(opt.loadSize), transforms.RandomCrop(opt.fineSize),
# transforms.ToTensor()]
transform = transforms.Compose(transformations)
# Dataset A, eg.. trainA目录
dataset_A = ImageFolder(root=opt.dataroot + '/' + opt.phase + 'A',
transform=transform,
return_paths=True)
data_loader_A = torch.utils.data.DataLoader(
dataset_A,
batch_size=self.opt.batchSize,
shuffle=not self.opt.serial_batches,
num_workers=int(self.opt.nThreads))
# Dataset B
dataset_B = ImageFolder(root=opt.dataroot + '/' + opt.phase + 'B',
transform=transform,
return_paths=True)
data_loader_B = torch.utils.data.DataLoader(
dataset_B,
batch_size=self.opt.batchSize,
shuffle=not self.opt.serial_batches,
num_workers=int(self.opt.nThreads))
# Dataset C
dataset_C = ImageFolder(root=opt.dataroot + '/' + opt.phase + 'C',
transform=transform,
return_paths=True)
data_loader_C = torch.utils.data.DataLoader(
dataset_C,
batch_size=self.opt.batchSize,
shuffle=not self.opt.serial_batches,
num_workers=int(self.opt.nThreads))
# 如何保证 A、B、C是一一一对应的呢，shuffle=not self.opt.serial_batches 这个参数 serial_batches 为True，代表有序，否则随机
# shuffle 是洗牌，搅乱的意思
self.dataset_A = dataset_A
self.dataset_B = dataset_B
self.dataset_C = dataset_C
flip = opt.isTrain and not opt.no_flip
self.three_paired_data = ThreePairedData(data_loader_A, data_loader_B,
data_loader_C,
self.opt.max_dataset_size,
flip)
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = CreateDataset(opt)
if opt.mode != "test":
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.stepBatchSize,
shuffle=opt.mode == "train",
num_workers=int(opt.nThreads))
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = create_dataset(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.batch_size,
shuffle=not opt.serial_batches, # if true, takes images in order to make batches, otherwise takes them randomly
num_workers=int(opt.num_threads) # default=4
)
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = CreateDataset(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=1,
shuffle=not opt.serial_batches,
num_workers=int(opt.nThreads),
worker_init_fn=lambda _: np.random.seed())
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = CreateDataset(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.batchSize,
shuffle=not opt.noshuffle,
num_workers=int(opt.nThreads),
collate_fn = self.my_collate)
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = CreateDataset(opt)
# Torch Dataloader combines a dataset and sampler, provides settings.
self.dataloader = torch.utils.data.DataLoader(
self.dataset, # dataset class
batch_size=opt.batchSize, # how many samples/batch to load
shuffle=not opt.serial_batches, # reshuffle per epoch
num_workers=int(opt.nThreads))
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = create_dataset(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.batch_size,
shuffle=False,
drop_last=True,
num_workers=int(opt.num_threads))
def initialize(self, config, filename):
BaseDataLoader.initialize(self, config)
self.dataset = CreateDataset(config, filename)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=config['batch_size'],
shuffle=True,
num_workers=int(config['num_workers']),
drop_last=True)
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
self.dataset = create_dataset(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.num_style_samples,
shuffle=not opt.serial_batches,
drop_last=True,
num_workers=int(opt.num_threads))
def initialize(self, opt, dataset=None):
BaseDataLoader.initialize(self, opt)
self.dataset = dataset if dataset else create_dataset(opt)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=opt.batch_size,
shuffle=not opt.serial_batches,
num_workers=int(opt.num_threads))
def initialize(self, datafolders, dataroot='./dataset',dataset_mode='2afc',load_size=64,batch_size=1,serial_batches=True, nThreads=1):
BaseDataLoader.initialize(self)
if(not isinstance(datafolders,list)):
datafolders = [datafolders,]
data_root_folders = [os.path.join(dataroot,datafolder) for datafolder in datafolders]
self.dataset = CreateDataset(data_root_folders,dataset_mode=dataset_mode,load_size=load_size)
self.dataloader = torch.utils.data.DataLoader(
self.dataset,
batch_size=batch_size,
shuffle=not serial_batches,
num_workers=int(nThreads))
def process_console_input(self, entry):
if self._file_preprocessor is None:
file_path, file_name = BaseDataLoader._split_file_to_path_and_name(
self.file_names[0]) # TODO: will be break with multiple filenames
file_name = ConllPreprocessor.VOCABULARY_PREFIX + file_name
self._file_preprocessor = ConllPreprocessor(file_path, file_name, self.field_delim,
self.DEFAULT_VOCABULARY_SIZE, self.DEFAULT_MAX_DATA_LENGTH)
entry = self._file_preprocessor.preprocess_single_entry(entry)
return entry
def initialize(self, opt):
BaseDataLoader.initialize(self, opt)
train_dataset = MNISTEDGE(root='./datasets/mnistedge',
train=True,
transform=transforms.ToTensor(),
target_transform=transforms.ToTensor())
train_dataset_unpaired = MNISTEDGE_unpaired(root='./datasets/mnistedge_unpaired',
train=True,
transform=transforms.ToTensor(),
target_transform=transforms.ToTensor())
test_dataset = MNISTEDGE(root='./datasets/mnistedge',
train=False,
transform=transforms.ToTensor(),
target_transform=transforms.ToTensor())
self.len_paired = len(train_dataset)
self.len_unpaired = len(train_dataset_unpaired)
self.len_test_paired = len(test_dataset)
#paired
data_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=self.opt.batchSize,
shuffle=True)
data_loader_unpaired = torch.utils.data.DataLoader(dataset=train_dataset_unpaired,
batch_size=self.opt.batchSize,
shuffle=True)
data_loader_test = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=self.opt.batchSize,
shuffle=True)
#paired
self.paired_data = data_loader
#unpaired
self.unpaired_data = data_loader_unpaired
#test
self.test_data = data_loader_test
