def get_val_dataloader(args, patches=False):
if args.stylized:
names, labels = _dataset_info(
join(dirname(__file__), 'txt_lists', 'Stylized' + args.dataset,
"{}_target".format(args.target), '%s_test.txt' % args.target))
else:
names, labels = _dataset_info(
join(dirname(__file__), 'txt_lists', 'Vanilla' + args.dataset,
'%s_test.txt' % args.target))
img_tr = get_val_transformer(args)
val_dataset = JigsawTestDataset(names,
labels,
patches=patches,
img_transformer=img_tr,
jig_classes=args.jigsaw_n_classes)
if args.limit_target and len(val_dataset) > args.limit_target:
val_dataset = Subset(val_dataset, args.limit_target)
print("Using %d subset of val dataset" % args.limit_target)
dataset = ConcatDataset([val_dataset])
loader = torch.utils.data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4,
pin_memory=True,
drop_last=False)
return loader
def get_train_dataloader(args, patches):
dataset_list = args.source
assert isinstance(dataset_list, list)
datasets = []
val_datasets = []
img_transformer, tile_transformer = get_train_transformers(args)
limit = args.limit_source
for dname in dataset_list:
if dname in new_nex_datasets:
index_root = data_root = '/import/home/share/from_Nexperia_April2021/%s' % dname
else:
index_root = join(dirname(__file__),'correct_txt_lists')
data_root = join(dirname(__file__),'kfold')
name_train, labels_train = _dataset_info(join(index_root, "%s_train.txt" % dname))
name_val, labels_val = _dataset_info(join(index_root, "%s_val.txt" % dname))
train_dataset = JigsawNewDataset(data_root, name_train, labels_train, patches=patches, img_transformer=img_transformer,
tile_transformer=tile_transformer, jig_classes=30, bias_whole_image=args.bias_whole_image)
if limit:
train_dataset = Subset(train_dataset, limit)
datasets.append(train_dataset)
val_datasets.append(JigsawTestNewDataset(data_root,name_val, labels_val, img_transformer=get_val_transformer(args),patches=patches, jig_classes=30))
dataset = ConcatDataset(datasets)
val_dataset = ConcatDataset(val_datasets)
loader = torch.utils.data.DataLoader(dataset, batch_size=args.batch_size, shuffle=True, num_workers=4, pin_memory=True, drop_last=True)
val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=4, pin_memory=True, drop_last=False)
return loader, val_loader
def get_train_dataloader(args, patches):
dataset_list = args.source
assert isinstance(dataset_list, list)
datasets = []
val_datasets = []
img_transformer, tile_transformer = get_train_transformers(args)
limit = args.limit_source
for dname in dataset_list:
# name_train, name_val, labels_train, labels_val = get_split_dataset_info(join(dirname(__file__), 'txt_lists', '%s_train.txt' % dname), args.val_size)
name_train, labels_train = _dataset_info(
join(dirname(__file__), 'correct_txt_lists',
'%s_train_kfold.txt' % dname))
name_val, labels_val = _dataset_info(
join(dirname(__file__), 'correct_txt_lists',
'%s_crossval_kfold.txt' % dname))
train_dataset = JigsawNewDataset(
name_train,
labels_train,
patches=patches,
img_transformer=img_transformer,
tile_transformer=tile_transformer,
jig_classes=args.jigsaw_n_classes,
bias_whole_image=args.bias_whole_image)
if limit:
train_dataset = Subset(train_dataset, limit)
datasets.append(train_dataset)
val_datasets.append(
JigsawTestNewDataset(name_val,
labels_val,
img_transformer=get_val_transformer(args),
patches=patches,
jig_classes=args.jigsaw_n_classes))
dataset = ConcatDataset(datasets)
val_dataset = ConcatDataset(val_datasets)
loader = torch.utils.data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4,
pin_memory=True,
drop_last=True)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4,
pin_memory=True,
drop_last=False)
return loader, val_loader
def get_jigsaw_val_dataloader(args, patches=False):
names, labels = _dataset_info(
join(dirname(__file__), 'txt_lists', '%s_test.txt' % args.target))
img_tr = [transforms.Resize((args.image_size, args.image_size))]
tile_tr = [
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
]
img_transformer = transforms.Compose(img_tr)
tile_transformer = transforms.Compose(tile_tr)
val_dataset = JigsawDataset(names,
labels,
patches=patches,
img_transformer=img_transformer,
tile_transformer=tile_transformer,
jig_classes=args.jigsaw_n_classes,
bias_whole_image=args.bias_whole_image)
if args.limit_target and len(val_dataset) > args.limit_target:
val_dataset = Subset(val_dataset, args.limit_target)
print("Using %d subset of val dataset" % args.limit_target)
dataset = ConcatDataset([val_dataset])
loader = torch.utils.data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4,
pin_memory=True,
drop_last=False)
return loader
def get_val_dataloader(args, patches=False):
dname = args.target
if dname in new_nex_datasets:
index_root = data_root = '/import/home/share/from_Nexperia_April2021/%s' % dname
else:
index_root = join(dirname(__file__), 'correct_txt_lists')
data_root = join(dirname(__file__), 'kfold')
names, labels = _dataset_info(join(index_root, "%s_val.txt" % dname))
img_tr = get_nex_val_transformer(args)
val_dataset = JigsawTestNewDataset(data_root,
names,
labels,
patches=patches,
img_transformer=img_tr,
jig_classes=30)
if args.limit_target and len(val_dataset) > args.limit_target:
val_dataset = Subset(val_dataset, args.limit_target)
print("Using %d subset of val dataset" % args.limit_target)
dataset = ConcatDataset([val_dataset])
loader = torch.utils.data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4,
pin_memory=True,
drop_last=False)
return loader
def get_single_test_dataloader(args, dname, patches=False):
names, labels = _dataset_info(join(dirname(__file__), 'txt_lists', '%s_test.txt' % dname))
img_tr = get_val_transformer(args)
# JigsawTestDataset return [unsorted_image, permutation_order-0, class_label]
test_dataset = JigsawTestDataset(names, labels, patches=patches, img_transformer=img_tr, jig_classes=args.jigsaw_n_classes)
if args.limit_target and len(test_dataset) > args.limit_target:
test_dataset = Subset(test_dataset, args.limit_target)
print("Using %d subset of test dataset" % args.limit_target)
dataset = ConcatDataset([test_dataset])
loader = torch.utils.data.DataLoader(dataset, batch_size=args.batch_size, shuffle=False, num_workers=4, pin_memory=True, drop_last=False)
return loader
def get_tgt_dataloader(args, patches=False):
'''
Load whole domain dataset
'''
img_tr = get_nex_val_transformer(args)
dname = args.target
if dname in new_nex_datasets:
index_root = data_root = '/import/home/share/from_Nexperia_April2021/%s' % dname
else:
index_root = join(dirname(__file__),'correct_txt_lists')
data_root = join(dirname(__file__),'kfold')
if args.downsample_target:
name_train, labels_train = _dataset_info(join(index_root,"%s_train_down.txt" % dname))
name_val, labels_val = _dataset_info(join(index_root, "%s_val_down.txt" % dname))
name_test, labels_test = _dataset_info(join(index_root, "%s_test_down.txt" % dname))
else:
name_train, labels_train = _dataset_info(join(index_root,"%s_train.txt" % dname))
name_val, labels_val = _dataset_info(join(index_root, "%s_val.txt" % dname))
name_test, labels_test = _dataset_info(join(index_root, "%s_test.txt" % dname))
tgt_train_dataset = JigsawTestNewDataset(data_root, name_train, labels_train, patches=patches, img_transformer=img_tr,jig_classes=30)
tgt_val_dataset = JigsawTestNewDataset(data_root, name_val, labels_val, patches=patches, img_transformer=img_tr,
jig_classes=30)
tgt_test_dataset = JigsawTestNewDataset(data_root, name_test, labels_test, patches=patches, img_transformer=img_tr,
jig_classes=30)
tgt_dataset = ConcatDataset([tgt_train_dataset, tgt_val_dataset, tgt_test_dataset])
loader = torch.utils.data.DataLoader(tgt_dataset, batch_size=args.batch_size, shuffle=True, num_workers=4, pin_memory=True, drop_last=False)
return loader
def get_val_dataloader(args):
names, labels = _dataset_info(
join(dirname(__file__), 'txt_lists', '%s_test.txt' % args.target))
img_tr = get_val_transformer(args)
val_dataset = BaselineDataset(names, labels, img_transformer=img_tr)
if args.limit_target and len(val_dataset) > args.limit_target:
val_dataset = Subset(val_dataset, args.limit_target)
print("Using %d subset of val dataset" % args.limit_target)
dataset = ConcatDataset([val_dataset])
loader = torch.utils.data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4,
pin_memory=True,
drop_last=False)
return loader
