def generate_dataloader(args):
# Data loading code
if args.dataset == 'cub200':
traindir = os.path.join(args.data_path, 'splited_image/train')
valdir = os.path.join(args.data_path, 'splited_image/val')
elif args.dataset == 'l-bird':
traindir = os.path.join(args.data_path,
'L-Bird-Subset') #L-Bird-Whole-Condensed
valdir = os.path.join(args.data_path, 'L-Bird-Subset-val')
else:
raise ValueError('the require dataset is not exist', args.dataset)
if not os.path.isdir(traindir):
split_train_test_images(args.data_path)
normalize = transforms.Normalize(
mean=[0.485, 0.456, 0.406], # the mean and std of ImageNet
std=[0.229, 0.224, 0.225])
train_dataset = ImageFolder_new(
traindir,
transforms.Compose([
#transforms.Resize(256),
#transforms.RandomCrop(224),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True,
sampler=None)
val_loader = torch.utils.data.DataLoader(ImageFolder_new(
valdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
return train_loader, val_loader
def generate_dataloader(args):
# Data loading code
if args.auxiliary_dataset == 'imagenet':
traindir = os.path.join(args.data_path, 'Data/CLS-LOC/train')
else:
traindir = args.data_path
# traindir = args.data_path # traindir = os.path.join(args.data_path, 'train')
# valdir = os.path.join(args.data_path, 'val')
if not os.path.isdir(traindir):
print('error directory')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
# train_dataset = datasets.ImageFolder(
# traindir,
# transforms.Compose([
# transforms.Resize(256),
# transforms.CenterCrop(224),
# # transforms.RandomHorizontalFlip(),
# transforms.ToTensor(),
# normalize,
# ])
# )
train_dataset = ImageFolder_new(
traindir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
# transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
# train_loader = torch.utils.data.DataLoader(
# train_dataset, batch_size=args.batch_size, shuffle=False,
# num_workers=args.workers, pin_memory=True, sampler=None
# )
train_loader = DataLoader_new(train_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True,
sampler=None)
# val_loader = torch.utils.data.DataLoader(
# datasets.ImageFolder(valdir, transforms.Compose([
# transforms.Scale(256),
# transforms.CenterCrop(224),
# transforms.ToTensor(),
# normalize,
# ])),
# batch_size=args.batch_size, shuffle=False,
# num_workers=args.workers, pin_memory=True
# )
return train_loader #, val_loader
def generate_dataloader(args):
# Data loading code
# the dataloader for the target dataset.
traindir = os.path.join(args.data_path, 'splited_image/train')
valdir = os.path.join(args.data_path, 'splited_image/val')
if not os.path.isdir(traindir):
split_train_test_images(args.data_path)
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_dataset = ImageFolder_new(
traindir,
transforms.Compose([
# transforms.Resize(256),
# transforms.RandomCrop(224),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])
)
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=args.batch_size, shuffle=True,
num_workers = args.workers, pin_memory=True, sampler=None
)
val_loader = torch.utils.data.DataLoader(
ImageFolder_new(valdir, transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=True
)
# the dataloader for the source dataset.
if args.auxiliary_dataset == 'imagenet':
traindir_source = os.path.join(args.data_path_source, 'Data/CLS-LOC/train')
valdir_source = os.path.join(args.data_path_source, 'Data/CLS-LOC/val')
else:
#traindir_source = args.data_path_source
traindir_source = os.path.join(args.data_path_source, 'L-Bird-Subset') ##L-Bird-Whole-Condensed
valdir_source = os.path.join(args.data_path_source, 'L-Bird-Subset-val')
if len(os.listdir(traindir_source)) != 0:
normalize_source = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_dataset_source = ImageFolder_new(
traindir_source,
transforms.Compose([
#transforms.Resize(256),
#transforms.RandomCrop(224),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize_source,
])
)
train_loader_source = torch.utils.data.DataLoader(
train_dataset_source, batch_size=args.batch_size_source, shuffle=True,
num_workers=args.workers, pin_memory=True, sampler=None
)
val_loader_source = torch.utils.data.DataLoader(
ImageFolder_new(valdir_source, transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize_source,
])),
batch_size=args.batch_size_source, shuffle=False,
num_workers=args.workers, pin_memory=True
)
return train_loader_source, val_loader_source, train_loader, val_loader
else:
return train_loader, val_loader
def generate_dataloader(args):
# Data loading code
traindir_source = os.path.join(args.data_path_source, args.src)
traindir_target = os.path.join(args.data_path_source_t, args.src_t)
valdir = os.path.join(args.data_path_target, args.tar)
if not os.path.isdir(traindir_source):
# split_train_test_images(args.data_path)
raise ValueError('Null path of source train data!!!')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
source_train_dataset = datasets.ImageFolder(
traindir_source,
transforms.Compose([
transforms.Resize(256),
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
source_train_loader = torch.utils.data.DataLoader(
source_train_dataset,
batch_size=args.batch_size_s,
shuffle=True,
drop_last=True,
num_workers=args.workers,
pin_memory=True,
sampler=None)
source_val_dataset = ImageFolder_new(
traindir_source,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
source_val_loader = torch.utils.data.DataLoader(
source_val_dataset,
batch_size=args.batch_size_s,
shuffle=False,
num_workers=args.workers,
pin_memory=True,
sampler=None)
target_train_dataset = datasets.ImageFolder(
traindir_target,
transforms.Compose([
transforms.Resize(256),
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
target_train_loader = torch.utils.data.DataLoader(
target_train_dataset,
batch_size=args.batch_size_t,
shuffle=True,
drop_last=True,
num_workers=args.workers,
pin_memory=True,
sampler=None)
target_val_loader = torch.utils.data.DataLoader(
ImageFolder_new(
valdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size_t,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
return source_train_loader, source_val_loader, target_train_loader, target_val_loader
def generate_dataloader(args):
# Data loading code
traindir = os.path.join(args.data_path_source, args.src)
traindir_t = os.path.join(args.data_path_source_t, args.src_t)
valdir = os.path.join(args.data_path_target, args.tar)
if not os.path.isdir(traindir):
raise ValueError(
'the require data path is not exist, please download the dataset')
normalize = transforms.Normalize(
mean=[0.485, 0.456, 0.406], # the mean and std of ImageNet
std=[0.229, 0.224, 0.225])
train_dataset = ImageFolder_new(
traindir,
transforms.Compose([
transforms.Resize(256),
transforms.RandomCrop(224),
# transforms.RandomResizedCrop(227),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True,
sampler=None,
drop_last=True)
train_dataset_doscore = ImageFolder_new(
traindir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
train_loader_doscore = torch.utils.data.DataLoader(
train_dataset_doscore,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True,
sampler=None,
drop_last=True)
train_t_dataset = ImageFolder_new(
traindir_t,
transforms.Compose([
transforms.Resize(256),
transforms.RandomCrop(224),
# transforms.RandomResizedCrop(227),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
train_t_loader = torch.utils.data.DataLoader(train_t_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True,
sampler=None,
drop_last=True)
val_loader = torch.utils.data.DataLoader(ImageFolder_new(
valdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
return train_loader, train_t_loader, val_loader, train_loader_doscore
def generate_dataloader(args):
# Data loading code
traindir_source = os.path.join(args.data_path_source, args.src)
traindir_target = os.path.join(args.data_path_source_t, args.src_t)
valdir = os.path.join(args.data_path_target, args.tar)
if not os.path.isdir(traindir_source):
# split_train_test_images(args.data_path)
raise ValueError('Null path of source train data!!!')
# normalize_s = transforms.Normalize(mean=[0.9094, 0.9077, 0.9047],
# std=[0.1977, 0.2013, 0.2081])
# normalize_s = transforms.Normalize(mean=[0.459, 0.459, 0.459], ############# should be all the same ...
# std=[0.226, 0.226, 0.226])
normalize_s = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
normalize_gray = transforms.Normalize(mean=[0.459, 0.459, 0.459], ############# should be all the same ...
std=[0.226, 0.226, 0.226])
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
#################### random sampled source and target dataset for cross-entropy training
if args.img_process_s == 'ours':
source_train_dataset = ImageFolder_uniform(
traindir_source,
transforms.Compose([
transforms.Resize(256),
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize_s,
])
)
elif args.img_process_s == 'longs':
####################### below is long's preprocess
source_train_dataset = ImageFolder_uniform(
traindir_source,
transforms.Compose([
transforms.Resize((256, 256)),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize_s,
])
)
elif args.img_process_s == 'simple':
source_train_dataset = ImageFolder_uniform(
traindir_source,
transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
normalize_s,
])
)
source_train_loader_ce = torch.utils.data.DataLoader(
source_train_dataset, batch_size=args.batch_size, shuffle=True,
drop_last=True, num_workers=args.workers, pin_memory=True
)
# uniformbatchsampler = UniformBatchSampler(args.per_category, source_train_dataset.category_index_list,
# source_train_dataset.imgs) ##### select images in the iteration process
# source_train_loader_cas = torch.utils.data.DataLoader(
# source_train_dataset, num_workers=args.workers, pin_memory=True, batch_sampler=uniformbatchsampler
# )
if args.img_process_t == 'ours':
target_train_dataset_ce = ImageFolder_new(
traindir_target,
transforms.Compose([
transforms.Resize(256),
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize_s,
]),
transforms.Compose([
transforms.Resize(256),
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Lambda(lambda x: _random_affine_augmentation(x)),
transforms.Lambda(lambda x: _gaussian_blur(x)),
normalize_s,
])
)
elif args.img_process_t == 'longs':
####################### long's preprocess
target_train_dataset_ce = ImageFolder_new(
traindir_target,
transforms.Compose([
transforms.Resize((256, 256)),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize_s,
]),
transforms.Compose([
transforms.Resize((256, 256)),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Lambda(lambda x: _random_affine_augmentation(x)),
transforms.Lambda(lambda x: _gaussian_blur(x)),
normalize_s,
])
)
elif args.img_process_t == 'simple':
target_train_dataset_ce = ImageFolder_new(
traindir_target,
transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
normalize_s,
]),
transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Lambda(lambda x: _random_affine_augmentation(x)),
transforms.Lambda(lambda x: _gaussian_blur(x)),
normalize_s,
])
)
target_train_loader_ce = torch.utils.data.DataLoader(
target_train_dataset_ce, batch_size=args.batch_size, shuffle=True,
drop_last=True, num_workers=args.workers, pin_memory=True
)
if args.img_process_s == 'ours':
target_train_dataset = ImageFolder_uniform(
traindir_target,
transforms.Compose([
transforms.Resize(256),
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize_s,
])
)
elif args.img_process_s == 'longs':
####################### long's preprocess
target_train_dataset = ImageFolder_uniform(
traindir_target,
transforms.Compose([
transforms.Resize((256, 256)),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize_s,
])
)
elif args.img_process_s == 'simple':
target_train_dataset = ImageFolder_uniform(
traindir_target,
transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
normalize_s,
])
)
if args.img_process_s == 'ours':
source_val_dataset = ImageFolder_uniform(
traindir_source,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize_s,
])
)
elif args.img_process_s == 'longs':
source_val_dataset = ImageFolder_uniform(
traindir_source,
transforms.Compose([
transforms.Resize((256, 256)),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize_s,
])
)
elif args.img_process_s == 'simple':
source_val_dataset = ImageFolder_uniform(
traindir_source,
transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
normalize_s,
])
)
source_val_loader = torch.utils.data.DataLoader(
source_val_dataset, batch_size=500, shuffle=False,
num_workers=args.workers, pin_memory=True, sampler=None
)
if args.img_process_t == 'ours':
target_test_dataset = ImageFolder_uniform(valdir, transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
elif args.img_process_t == 'longs':
target_test_dataset = ImageFolder_uniform(valdir, transforms.Compose([
transforms.Resize(256, 256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
elif args.img_process_t == 'simple':
target_test_dataset = ImageFolder_uniform(valdir, transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
normalize,
]))
target_val_loader = torch.utils.data.DataLoader(
target_test_dataset,
batch_size=500, shuffle=False,
num_workers=args.workers, pin_memory=True
)
return source_train_loader_ce, source_train_dataset, target_train_loader_ce, target_train_dataset, source_val_loader, target_val_loader