def loader(config, ctx):
"""
Description : dataloder for omniglot dataset
"""
N = config.N
K = config.K
iterations = config.iterations
batch_size = config.batch_size
download = config.download
train_dataset = OmniglotDataset(mode='train', download=download)
test_dataset = OmniglotDataset(mode='test', download=download)
tr_sampler = BatchSampler(labels=train_dataset.y,\
classes_per_it=N,\
num_samples=K,\
iterations=iterations,\
batch_size=batch_size)
te_sampler = BatchSampler(labels=test_dataset.y,\
classes_per_it=N,\
num_samples=K,\
iterations=iterations,\
batch_size=int(batch_size / len(ctx)))
tr_dataloader = DataLoader(train_dataset, batch_sampler=tr_sampler)
te_dataloader = DataLoader(test_dataset, batch_sampler=te_sampler)
return tr_dataloader, te_dataloader
def init_dataset(opt):
'''
Initialize the datasets, samplers and dataloaders
'''
if opt.dataset == 'omniglot':
train_dataset = OmniglotDataset(mode='train')
val_dataset = OmniglotDataset(mode='val')
trainval_dataset = OmniglotDataset(mode='trainval')
test_dataset = OmniglotDataset(mode='test')
elif opt.dataset == 'mini_imagenet':
train_dataset = MiniImagenetDataset(mode='train')
val_dataset = MiniImagenetDataset(mode='val')
trainval_dataset = MiniImagenetDataset(mode='val')
test_dataset = MiniImagenetDataset(mode='test')
train_bs_class = BatchSampler
eval_bs_class = BatchSampler
if opt.task_shuffling == 'non_overlapping':
train_bs_class = NonOverlappingTasksBatchSampler
elif opt.task_shuffling == 'intratask':
train_bs_class = IntraTaskBatchSampler
# Opt for mini_imagenet:
# Namespace(batch_size=32, cuda=True, dataset='mini_imagenet', epochs=100,
# exp='mini_imagenet_5way_1shot', iterations=10000, lr=0.0001, num_cls=5, num_samples=1)
tr_sampler = train_bs_class(labels=train_dataset.y,
classes_per_it=opt.num_cls,
num_samples=opt.num_samples,
iterations=opt.iterations,
batch_size=opt.batch_size)
val_sampler = eval_bs_class(labels=val_dataset.y,
classes_per_it=opt.num_cls,
num_samples=opt.num_samples,
iterations=opt.iterations,
batch_size=opt.batch_size)
trainval_sampler = eval_bs_class(labels=trainval_dataset.y,
classes_per_it=opt.num_cls,
num_samples=opt.num_samples,
iterations=opt.iterations,
batch_size=opt.batch_size)
test_sampler = eval_bs_class(labels=test_dataset.y,
classes_per_it=opt.num_cls,
num_samples=opt.num_samples,
iterations=opt.iterations,
batch_size=opt.batch_size)
tr_dataloader = torch.utils.data.DataLoader(train_dataset,
batch_sampler=tr_sampler)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_sampler=val_sampler)
trainval_dataloader = torch.utils.data.DataLoader(
trainval_dataset, batch_sampler=trainval_sampler)
test_dataloader = torch.utils.data.DataLoader(test_dataset,
batch_sampler=test_sampler)
return tr_dataloader, val_dataloader, trainval_dataloader, test_dataloader
def init_dataset(opt):
'''
Initialize the datasets, samplers and dataloaders
'''
train_dataset = OmniglotDataset(mode='train',
root=opt.dataset_root)
val_dataset = OmniglotDataset(mode='val',
root=opt.dataset_root)
trainval_dataset = OmniglotDataset(mode='trainval',
root=opt.dataset_root)
test_dataset = OmniglotDataset(mode='test',
root=opt.dataset_root)
tr_sampler = PrototypicalBatchSampler(labels=train_dataset.y,
classes_per_it=opt.classes_per_it_tr,
num_support=opt.num_support_tr,
num_query=opt.num_query_tr,
iterations=opt.iterations)
val_sampler = PrototypicalBatchSampler(labels=val_dataset.y,
classes_per_it=opt.classes_per_it_val,
num_support=opt.num_support_val,
num_query=opt.num_query_val,
iterations=opt.iterations)
trainval_sampler = PrototypicalBatchSampler(labels=trainval_dataset.y,
classes_per_it=opt.classes_per_it_tr,
num_support=opt.num_support_tr,
num_query=opt.num_query_tr,
iterations=opt.iterations)
test_sampler = PrototypicalBatchSampler(labels=test_dataset.y,
classes_per_it=opt.classes_per_it_val,
num_support=opt.num_support_val,
num_query=opt.num_query_val,
iterations=opt.iterations)
tr_dataloader = torch.utils.data.DataLoader(train_dataset,
batch_sampler=tr_sampler)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_sampler=val_sampler)
trainval_dataloader = torch.utils.data.DataLoader(trainval_dataset,
batch_sampler=trainval_sampler)
test_dataloader = torch.utils.data.DataLoader(test_dataset,
batch_sampler=test_sampler)
return tr_dataloader, val_dataloader, trainval_dataloader, test_dataloader
def init_dataset(opt):
'''
Initialize the datasets, samplers and dataloaders
'''
if opt.dataset == 'omniglot':
train_dataset = OmniglotDataset(mode='train')
val_dataset = OmniglotDataset(mode='val')
trainval_dataset = OmniglotDataset(mode='trainval')
test_dataset = OmniglotDataset(mode='test')
elif opt.dataset == 'mini_imagenet':
train_dataset = MiniImagenetDataset(mode='train')
val_dataset = MiniImagenetDataset(mode='val')
trainval_dataset = MiniImagenetDataset(mode='val')
test_dataset = MiniImagenetDataset(mode='test')
tr_sampler = PrototypicalBatchSampler(labels=train_dataset.y,
classes_per_it=opt.classes_per_it_tr,
num_samples=opt.num_support_tr +
opt.num_query_tr,
iterations=opt.iterations)
val_sampler = PrototypicalBatchSampler(
labels=val_dataset.y,
classes_per_it=opt.classes_per_it_val,
num_samples=opt.num_support_val + opt.num_query_val,
iterations=opt.iterations)
trainval_sampler = PrototypicalBatchSampler(
labels=trainval_dataset.y,
classes_per_it=opt.classes_per_it_tr,
num_samples=opt.num_support_tr + opt.num_query_tr,
iterations=opt.iterations)
test_sampler = PrototypicalBatchSampler(
labels=test_dataset.y,
classes_per_it=opt.classes_per_it_val,
num_samples=opt.num_support_val + opt.num_query_val,
iterations=opt.iterations)
tr_dataloader = torch.utils.data.DataLoader(train_dataset,
batch_sampler=tr_sampler)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_sampler=val_sampler)
trainval_dataloader = torch.utils.data.DataLoader(
trainval_dataset, batch_sampler=trainval_sampler)
test_dataloader = torch.utils.data.DataLoader(test_dataset,
batch_sampler=test_sampler)
return tr_dataloader, val_dataloader, trainval_dataloader, test_dataloader
def init_dataset(opt):
'''
Initialize the datasets, samplers and dataloaders
'''
if opt.dataset == 'omniglot':
train_dataset = OmniglotDataset(mode='train')
val_dataset = OmniglotDataset(mode='val')
trainval_dataset = OmniglotDataset(mode='trainval')
test_dataset = OmniglotDataset(mode='test')
elif opt.dataset == 'mini_imagenet':
train_dataset = MiniImagenetDataset(mode='train')
val_dataset = MiniImagenetDataset(mode='val')
trainval_dataset = MiniImagenetDataset(mode='val')
test_dataset = MiniImagenetDataset(mode='test')
tr_sampler = BatchSampler(labels=train_dataset.y,
classes_per_it=opt.num_cls,
num_samples=opt.num_samples,
iterations=opt.iterations,
batch_size=opt.batch_size)
val_sampler = BatchSampler(labels=val_dataset.y,
classes_per_it=opt.num_cls,
num_samples=opt.num_samples,
iterations=opt.iterations,
batch_size=opt.batch_size)
trainval_sampler = BatchSampler(labels=trainval_dataset.y,
classes_per_it=opt.num_cls,
num_samples=opt.num_samples,
iterations=opt.iterations,
batch_size=opt.batch_size)
test_sampler = BatchSampler(labels=test_dataset.y,
classes_per_it=opt.num_cls,
num_samples=opt.num_samples,
iterations=opt.iterations,
batch_size=opt.batch_size)
tr_dataloader = torch.utils.data.DataLoader(train_dataset,
batch_sampler=tr_sampler)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_sampler=val_sampler)
trainval_dataloader = torch.utils.data.DataLoader(
trainval_dataset, batch_sampler=trainval_sampler)
test_dataloader = torch.utils.data.DataLoader(test_dataset,
batch_sampler=test_sampler)
return tr_dataloader, val_dataloader, trainval_dataloader, test_dataloader
def init_dataset(opt, mode):
if opt.dataset == 0:
dataset = OmniglotDataset(mode=mode, root=opt.dataset_root)
n_classes = len(np.unique(dataset.y))
if n_classes < opt.classes_per_it_tr or n_classes < opt.classes_per_it_val:
raise(Exception('There are not enough classes in the dataset in order ' +
'to satisfy the chosen classes_per_it. Decrease the ' +
'classes_per_it_{tr/val} option and try again.'))
elif opt.dataset == 1:
dataset = MiniImageNet(mode)
n_classes = len(dataset.wnids)
if mode == "train" and n_classes < opt.classes_per_it_tr:
raise(Exception('There are not enough classes in the dataset in order ' +
'to satisfy the chosen classes_per_it. Decrease the ' +
'classes_per_it_{tr/val} option and try again.'))
elif mode == "val" and n_classes < opt.classes_per_it_val:
raise(Exception('There are not enough classes in the dataset in order ' +
'to satisfy the chosen classes_per_it. Decrease the ' +
'classes_per_it_{tr/val} option and try again.'))
elif mode == "test" and n_classes < opt.classes_per_it_val:
raise(Exception('There are not enough classes in the dataset in order ' +
'to satisfy the chosen classes_per_it. Decrease the ' +
'classes_per_it_{tr/val} option and try again.'))
else:
raise(Exception("No such dataset!!"))
return dataset
def init_dataset(opt, mode):
dataset = OmniglotDataset(mode=mode, root=opt.dataset_root)
n_classes = len(np.unique(dataset.y))
#if n_classes < opt.classes_per_it_tr or n_classes < opt.classes_per_it_val:
# raise(Exception('There are not enough classes in the dataset in order ' +
# 'to satisfy the chosen classes_per_it. Decrease the ' +
# 'classes_per_it_{tr/val} option and try again.'))
return dataset
def init_dataset(opt, mode):
dataset = OmniglotDataset(mode=mode, root=opt.dataset_root)
# unique返回一个不包含重复数据的List
# dataset.x为tensor格式图片, y为label
n_classes = len(np.unique(dataset.y))
if n_classes < opt.classes_per_it_tr or n_classes < opt.classes_per_it_val:
raise (
Exception('There are not enough classes in the dataset in order ' +
'to satisfy the chosen classes_per_it. Decrease the ' +
'classes_per_it_{tr/val} option and try again.'))
return dataset
def init_dataset(opt):
'''
Initialize the datasets, samplers and dataloaders
'''
train_dataset = OmniglotDataset(mode='train',
root=opt.dataset_root,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
[0.5, 0.5, 0.5], [0.5, 0.5, 0.5]),
]))
val_dataset = OmniglotDataset(mode='val',
root=opt.dataset_root,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
[0.5, 0.5, 0.5], [0.5, 0.5, 0.5]),
]))
tr_sampler = PrototypicalBatchSampler(labels=train_dataset.y,
classes_per_it=opt.classes_per_it_tr,
num_support=opt.num_support_tr,
num_query=opt.num_query_tr,
iterations=opt.iterations)
val_sampler = PrototypicalBatchSampler(labels=val_dataset.y,
classes_per_it=opt.classes_per_it_val,
num_support=opt.num_support_val,
num_query=opt.num_query_val,
iterations=opt.iterations)
tr_dataloader = torch.utils.data.DataLoader(train_dataset,
batch_sampler=tr_sampler)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_sampler=val_sampler)
return tr_dataloader, val_dataloader
def init_dataset(opt):
'''
Initialize the datasets, samplers and dataloaders
'''
if opt.dataset == 'omniglot':
test_dataset = OmniglotDataset(mode='test')
elif opt.dataset == 'mini_imagenet':
test_dataset = MiniImagenetDataset(mode='val')
else:
print('Dataset is not valid')
test_sampler = PrototypicalBatchSampler(labels=test_dataset.y,
classes_per_it=opt.classes_per_it_val,
num_samples=opt.num_support_val + opt.num_query_val,
iterations=opt.iterations)
test_dataloader = torch.utils.data.DataLoader(test_dataset,
batch_sampler=test_sampler)
return test_dataloader
def set_data_loader(self):
if self.arg_settings.data == 'omniglot':
dataset = OmniglotDataset(mode=self.mode,
root=self.arg_settings.dataset_root)
num_classes = len(np.unique(dataset.y))
sampler = self.create_sampler(dataset.y)
# check if number of classes in dataset is sufficient
if num_classes < self.arg_settings.classes_per_it_tr or num_classes < self.arg_settings.classes_per_it_val:
raise (Exception(
'There are not enough classes in the dataset in order ' +
'to satisfy the chosen classes_per_it. Decrease the ' +
'classes_per_it_{tr/val} option and try again.'))
dataloader = torch.utils.data.DataLoader(dataset,
batch_sampler=sampler)
return dataloader