def generate_dataloader(args): # Data loading code if args.dataset == 'cub200': traindir = os.path.join(args.data_path, 'splited_image/train') valdir = os.path.join(args.data_path, 'splited_image/val') elif args.dataset == 'l-bird': traindir = os.path.join(args.data_path, 'L-Bird-Subset') #L-Bird-Whole-Condensed valdir = os.path.join(args.data_path, 'L-Bird-Subset-val') else: raise ValueError('the require dataset is not exist', args.dataset) if not os.path.isdir(traindir): split_train_test_images(args.data_path) normalize = transforms.Normalize( mean=[0.485, 0.456, 0.406], # the mean and std of ImageNet std=[0.229, 0.224, 0.225]) train_dataset = ImageFolder_new( traindir, transforms.Compose([ #transforms.Resize(256), #transforms.RandomCrop(224), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True, sampler=None) val_loader = torch.utils.data.DataLoader(ImageFolder_new( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) return train_loader, val_loader
def generate_dataloader(args): # Data loading code if args.auxiliary_dataset == 'imagenet': traindir = os.path.join(args.data_path, 'Data/CLS-LOC/train') else: traindir = args.data_path # traindir = args.data_path # traindir = os.path.join(args.data_path, 'train') # valdir = os.path.join(args.data_path, 'val') if not os.path.isdir(traindir): print('error directory') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) # train_dataset = datasets.ImageFolder( # traindir, # transforms.Compose([ # transforms.Resize(256), # transforms.CenterCrop(224), # # transforms.RandomHorizontalFlip(), # transforms.ToTensor(), # normalize, # ]) # ) train_dataset = ImageFolder_new( traindir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), # transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) # train_loader = torch.utils.data.DataLoader( # train_dataset, batch_size=args.batch_size, shuffle=False, # num_workers=args.workers, pin_memory=True, sampler=None # ) train_loader = DataLoader_new(train_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True, sampler=None) # val_loader = torch.utils.data.DataLoader( # datasets.ImageFolder(valdir, transforms.Compose([ # transforms.Scale(256), # transforms.CenterCrop(224), # transforms.ToTensor(), # normalize, # ])), # batch_size=args.batch_size, shuffle=False, # num_workers=args.workers, pin_memory=True # ) return train_loader #, val_loader
def generate_dataloader(args): # Data loading code # the dataloader for the target dataset. traindir = os.path.join(args.data_path, 'splited_image/train') valdir = os.path.join(args.data_path, 'splited_image/val') if not os.path.isdir(traindir): split_train_test_images(args.data_path) normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = ImageFolder_new( traindir, transforms.Compose([ # transforms.Resize(256), # transforms.RandomCrop(224), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]) ) train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=True, num_workers = args.workers, pin_memory=True, sampler=None ) val_loader = torch.utils.data.DataLoader( ImageFolder_new(valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True ) # the dataloader for the source dataset. if args.auxiliary_dataset == 'imagenet': traindir_source = os.path.join(args.data_path_source, 'Data/CLS-LOC/train') valdir_source = os.path.join(args.data_path_source, 'Data/CLS-LOC/val') else: #traindir_source = args.data_path_source traindir_source = os.path.join(args.data_path_source, 'L-Bird-Subset') ##L-Bird-Whole-Condensed valdir_source = os.path.join(args.data_path_source, 'L-Bird-Subset-val') if len(os.listdir(traindir_source)) != 0: normalize_source = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset_source = ImageFolder_new( traindir_source, transforms.Compose([ #transforms.Resize(256), #transforms.RandomCrop(224), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize_source, ]) ) train_loader_source = torch.utils.data.DataLoader( train_dataset_source, batch_size=args.batch_size_source, shuffle=True, num_workers=args.workers, pin_memory=True, sampler=None ) val_loader_source = torch.utils.data.DataLoader( ImageFolder_new(valdir_source, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize_source, ])), batch_size=args.batch_size_source, shuffle=False, num_workers=args.workers, pin_memory=True ) return train_loader_source, val_loader_source, train_loader, val_loader else: return train_loader, val_loader
def generate_dataloader(args): # Data loading code traindir_source = os.path.join(args.data_path_source, args.src) traindir_target = os.path.join(args.data_path_source_t, args.src_t) valdir = os.path.join(args.data_path_target, args.tar) if not os.path.isdir(traindir_source): # split_train_test_images(args.data_path) raise ValueError('Null path of source train data!!!') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) source_train_dataset = datasets.ImageFolder( traindir_source, transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) source_train_loader = torch.utils.data.DataLoader( source_train_dataset, batch_size=args.batch_size_s, shuffle=True, drop_last=True, num_workers=args.workers, pin_memory=True, sampler=None) source_val_dataset = ImageFolder_new( traindir_source, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])) source_val_loader = torch.utils.data.DataLoader( source_val_dataset, batch_size=args.batch_size_s, shuffle=False, num_workers=args.workers, pin_memory=True, sampler=None) target_train_dataset = datasets.ImageFolder( traindir_target, transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) target_train_loader = torch.utils.data.DataLoader( target_train_dataset, batch_size=args.batch_size_t, shuffle=True, drop_last=True, num_workers=args.workers, pin_memory=True, sampler=None) target_val_loader = torch.utils.data.DataLoader( ImageFolder_new( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size_t, shuffle=False, num_workers=args.workers, pin_memory=True) return source_train_loader, source_val_loader, target_train_loader, target_val_loader
def generate_dataloader(args): # Data loading code traindir = os.path.join(args.data_path_source, args.src) traindir_t = os.path.join(args.data_path_source_t, args.src_t) valdir = os.path.join(args.data_path_target, args.tar) if not os.path.isdir(traindir): raise ValueError( 'the require data path is not exist, please download the dataset') normalize = transforms.Normalize( mean=[0.485, 0.456, 0.406], # the mean and std of ImageNet std=[0.229, 0.224, 0.225]) train_dataset = ImageFolder_new( traindir, transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), # transforms.RandomResizedCrop(227), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True, sampler=None, drop_last=True) train_dataset_doscore = ImageFolder_new( traindir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])) train_loader_doscore = torch.utils.data.DataLoader( train_dataset_doscore, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True, sampler=None, drop_last=True) train_t_dataset = ImageFolder_new( traindir_t, transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), # transforms.RandomResizedCrop(227), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) train_t_loader = torch.utils.data.DataLoader(train_t_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True, sampler=None, drop_last=True) val_loader = torch.utils.data.DataLoader(ImageFolder_new( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) return train_loader, train_t_loader, val_loader, train_loader_doscore
def generate_dataloader(args): # Data loading code traindir_source = os.path.join(args.data_path_source, args.src) traindir_target = os.path.join(args.data_path_source_t, args.src_t) valdir = os.path.join(args.data_path_target, args.tar) if not os.path.isdir(traindir_source): # split_train_test_images(args.data_path) raise ValueError('Null path of source train data!!!') # normalize_s = transforms.Normalize(mean=[0.9094, 0.9077, 0.9047], # std=[0.1977, 0.2013, 0.2081]) # normalize_s = transforms.Normalize(mean=[0.459, 0.459, 0.459], ############# should be all the same ... # std=[0.226, 0.226, 0.226]) normalize_s = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) normalize_gray = transforms.Normalize(mean=[0.459, 0.459, 0.459], ############# should be all the same ... std=[0.226, 0.226, 0.226]) normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) #################### random sampled source and target dataset for cross-entropy training if args.img_process_s == 'ours': source_train_dataset = ImageFolder_uniform( traindir_source, transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize_s, ]) ) elif args.img_process_s == 'longs': ####################### below is long's preprocess source_train_dataset = ImageFolder_uniform( traindir_source, transforms.Compose([ transforms.Resize((256, 256)), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize_s, ]) ) elif args.img_process_s == 'simple': source_train_dataset = ImageFolder_uniform( traindir_source, transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), normalize_s, ]) ) source_train_loader_ce = torch.utils.data.DataLoader( source_train_dataset, batch_size=args.batch_size, shuffle=True, drop_last=True, num_workers=args.workers, pin_memory=True ) # uniformbatchsampler = UniformBatchSampler(args.per_category, source_train_dataset.category_index_list, # source_train_dataset.imgs) ##### select images in the iteration process # source_train_loader_cas = torch.utils.data.DataLoader( # source_train_dataset, num_workers=args.workers, pin_memory=True, batch_sampler=uniformbatchsampler # ) if args.img_process_t == 'ours': target_train_dataset_ce = ImageFolder_new( traindir_target, transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize_s, ]), transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Lambda(lambda x: _random_affine_augmentation(x)), transforms.Lambda(lambda x: _gaussian_blur(x)), normalize_s, ]) ) elif args.img_process_t == 'longs': ####################### long's preprocess target_train_dataset_ce = ImageFolder_new( traindir_target, transforms.Compose([ transforms.Resize((256, 256)), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize_s, ]), transforms.Compose([ transforms.Resize((256, 256)), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Lambda(lambda x: _random_affine_augmentation(x)), transforms.Lambda(lambda x: _gaussian_blur(x)), normalize_s, ]) ) elif args.img_process_t == 'simple': target_train_dataset_ce = ImageFolder_new( traindir_target, transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), normalize_s, ]), transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Lambda(lambda x: _random_affine_augmentation(x)), transforms.Lambda(lambda x: _gaussian_blur(x)), normalize_s, ]) ) target_train_loader_ce = torch.utils.data.DataLoader( target_train_dataset_ce, batch_size=args.batch_size, shuffle=True, drop_last=True, num_workers=args.workers, pin_memory=True ) if args.img_process_s == 'ours': target_train_dataset = ImageFolder_uniform( traindir_target, transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize_s, ]) ) elif args.img_process_s == 'longs': ####################### long's preprocess target_train_dataset = ImageFolder_uniform( traindir_target, transforms.Compose([ transforms.Resize((256, 256)), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize_s, ]) ) elif args.img_process_s == 'simple': target_train_dataset = ImageFolder_uniform( traindir_target, transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), normalize_s, ]) ) if args.img_process_s == 'ours': source_val_dataset = ImageFolder_uniform( traindir_source, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize_s, ]) ) elif args.img_process_s == 'longs': source_val_dataset = ImageFolder_uniform( traindir_source, transforms.Compose([ transforms.Resize((256, 256)), transforms.CenterCrop(224), transforms.ToTensor(), normalize_s, ]) ) elif args.img_process_s == 'simple': source_val_dataset = ImageFolder_uniform( traindir_source, transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), normalize_s, ]) ) source_val_loader = torch.utils.data.DataLoader( source_val_dataset, batch_size=500, shuffle=False, num_workers=args.workers, pin_memory=True, sampler=None ) if args.img_process_t == 'ours': target_test_dataset = ImageFolder_uniform(valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])) elif args.img_process_t == 'longs': target_test_dataset = ImageFolder_uniform(valdir, transforms.Compose([ transforms.Resize(256, 256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])) elif args.img_process_t == 'simple': target_test_dataset = ImageFolder_uniform(valdir, transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), normalize, ])) target_val_loader = torch.utils.data.DataLoader( target_test_dataset, batch_size=500, shuffle=False, num_workers=args.workers, pin_memory=True ) return source_train_loader_ce, source_train_dataset, target_train_loader_ce, target_train_dataset, source_val_loader, target_val_loader