def imagenet_train_transform(ds_metainfo, jitter_param=0.4, lighting_param=0.1): """ Create image transform sequence for training subset. Parameters: ---------- ds_metainfo : DatasetMetaInfo ImageNet-1K dataset metainfo. jitter_param : float How much to jitter values. lighting_param : float How much to noise intensity of the image. Returns ------- Sequential Image transform sequence. """ input_image_size = ds_metainfo.input_image_size if ds_metainfo.aug_type == "aug0": interpolation = ds_metainfo.interpolation transform_list = [] elif ds_metainfo.aug_type == "aug1": interpolation = 10 transform_list = [] elif ds_metainfo.aug_type == "aug2": interpolation = 10 transform_list = [ImgAugTransform()] else: raise RuntimeError("Unknown augmentation type: {}\n".format( ds_metainfo.aug_type)) transform_list += [ transforms.RandomResizedCrop(size=input_image_size, interpolation=interpolation), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param, saturation=jitter_param), transforms.RandomLighting(lighting_param), transforms.ToTensor(), transforms.Normalize(mean=ds_metainfo.mean_rgb, std=ds_metainfo.std_rgb) ] return transforms.Compose(transform_list)
def get_data_loader(data_dir, batch_size, num_workers): normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) jitter_param = 0.4 lighting_param = 0.1 input_size = opt.input_size crop_ratio = opt.crop_ratio if opt.crop_ratio > 0 else 0.875 resize = int(math.ceil(input_size / crop_ratio)) def batch_fn(batch, ctx): data = gluon.utils.split_and_load(batch[0], ctx_list=ctx, batch_axis=0) label = gluon.utils.split_and_load(batch[1], ctx_list=ctx, batch_axis=0) return data, label transform_train = transforms.Compose([ gcv_transforms.RandomCrop(input_size, pad=4), transforms.RandomFlipLeftRight(), # transforms.RandomColorJitter( # brightness=jitter_param, # contrast=jitter_param, # saturation=jitter_param), # transforms.RandomLighting(lighting_param), transforms.ToTensor(), normalize ]) transform_test = transforms.Compose([ # transforms.Resize(resize, keep_ratio=True), # transforms.CenterCrop(input_size), transforms.Resize(input_size), transforms.ToTensor(), normalize ]) train_data = gluon.data.DataLoader(ImageNet32( data_dir, train=True).transform_first(transform_train), batch_size=batch_size, shuffle=True, last_batch='discard', num_workers=num_workers) val_data = gluon.data.DataLoader(ImageNet32( data_dir, train=False).transform_first(transform_test), batch_size=batch_size, shuffle=False, num_workers=num_workers) return train_data, val_data, batch_fn
def generate_transform(train, resize, _is_osx, input_size, jitter_param): if _is_osx: # using PIL to load image (slow) if train: transform = Compose( [ RandomResizedCrop(input_size), RandomHorizontalFlip(), ColorJitter(0.4, 0.4, 0.4), ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ] ) else: transform = Compose( [ Resize(resize), CenterCrop(input_size), ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ] ) else: if train: transform = transforms.Compose( [ transforms.RandomResizedCrop(input_size), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter( brightness=jitter_param, contrast=jitter_param, saturation=jitter_param ), transforms.RandomLighting(0.1), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ] ) else: transform = transforms.Compose( [ transforms.Resize(resize), transforms.CenterCrop(input_size), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ] ) return transform
def __init__(self, width=224, height=224, is_train=False): if is_train: self.T = transforms.Compose([ Random2DTranslation(width, height), transforms.RandomFlipLeftRight(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) else: # for eval self.T = transforms.Compose([ transforms.Resize((width, height)), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ])
def imagenet_train_transform(ds_metainfo, mean_rgb=(0.485, 0.456, 0.406), std_rgb=(0.229, 0.224, 0.225), jitter_param=0.4, lighting_param=0.1): input_image_size = ds_metainfo.input_image_size return transforms.Compose([ transforms.RandomResizedCrop(input_image_size), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param, saturation=jitter_param), transforms.RandomLighting(lighting_param), transforms.ToTensor(), transforms.Normalize(mean=mean_rgb, std=std_rgb) ])
def get_data_loader(logger): jitter_param = 0.4 lighting_param = 0.1 transform_train = transforms.Compose([ transforms.Resize(256), # ImageNetPolicy(), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param, saturation=jitter_param), transforms.RandomLighting(lighting_param), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) transform_test = transforms.Compose([ transforms.Resize(256), # transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) synsets, all_items = list_images(root=config.DATA_PATH) train_items, test_items = train_test_split( all_items, test_size=config.TEST_SIZE, random_state=config.RANDOM_STATE) # 训练集和测试集 train_dataset = ImageFolderDataset(synsets=synsets, list_images=train_items, flag=1) test_dataset = ImageFolderDataset(synsets=synsets, list_images=test_items, flag=1) logger.info('训练集的样本数是:%s,测试集的样本数是:%s' % (len(train_dataset), len(test_dataset))) train_dataloader = gluon.data.DataLoader( train_dataset.transform_first(transform_train), batch_size=config.BATCH_SIZE, last_batch='rollover', shuffle=True, num_workers=config.WORKERS) test_dataloader = gluon.data.DataLoader( test_dataset.transform_first(transform_test), batch_size=config.BATCH_SIZE, last_batch='rollover', shuffle=False, num_workers=config.WORKERS) return train_dataloader, test_dataloader
def create_loader(self): """ Create the data loader :return: """ if self.args.mode.upper() == 'TRAIN': tforms = [] tforms.append(transforms.Resize(self.args.resize)) if self.args.flip: tforms.append(transforms.RandomFlipLeftRight()) if self.args.random_crop: tforms.append( transforms.RandomResizedCrop(self.args.im_size, scale=(0.8, 1))) else: tforms.append(transforms.CenterCrop(self.args.im_size)) if self.args.random_jitter: tforms.append(transforms.RandomColorJitter(0.4, 0.4, 0.4, 0.4)) tforms.append(transforms.ToTensor()) tforms.append( transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))) tforms = transforms.Compose(tforms) tr_db = list(self.cfg['train'].values())[0] dataset = ImageRecordDataset(tr_db['rec'], transform=tforms) self.tr_loader = DataLoader(dataset, batch_size=self.args.bs, num_workers=8, pin_memory=True) else: tforms = transforms.Compose([ transforms.Resize(self.args.resize), transforms.CenterCrop(self.args.im_size), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) ]) self.eval_tforms = tforms
def get_data_loader(data_dir, batch_size, num_workers, input_size, crop_ratio, train_dataset=None, val_dataset=None): normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) jitter_param = 0.4 lighting_param = 0.1 crop_ratio = crop_ratio if crop_ratio > 0 else 0.875 resize = int(math.ceil(input_size / crop_ratio)) transform_train = transforms.Compose([ transforms.RandomResizedCrop(input_size), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param, saturation=jitter_param), transforms.RandomLighting(lighting_param), transforms.ToTensor(), normalize ]) transform_test = transforms.Compose([ transforms.Resize(resize, keep_ratio=True), transforms.CenterCrop(input_size), transforms.ToTensor(), normalize ]) if not train_dataset: train_dataset = imagenet.classification.ImageNet(data_dir, train=True) if not val_dataset: val_dataset = imagenet.classification.ImageNet(data_dir, train=False) train_data = gluon.data.DataLoader( train_dataset.transform_first(transform_train), batch_size=batch_size, shuffle=True, last_batch='discard', num_workers=num_workers) val_data = gluon.data.DataLoader( val_dataset.transform_first(transform_test), batch_size=batch_size, shuffle=False, num_workers=num_workers) return train_data, val_data, loader_batch_fn
def get_data_raw(dataset_path, batch_size, num_workers): train_path = os.path.join(dataset_path, 'train') val_path = os.path.join(dataset_path, 'val') test_path = os.path.join(dataset_path, 'test') normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) jitter_param = 0.4 lighting_param = 0.1 input_size = 224 crop_ratio = 0.875 resize = int(math.ceil(input_size / crop_ratio)) transform_train = transforms.Compose([ transforms.RandomResizedCrop(input_size), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param, saturation=jitter_param), transforms.RandomLighting(lighting_param), transforms.ToTensor(), normalize ]) transform_test = transforms.Compose([ transforms.Resize(resize, keep_ratio=True), transforms.CenterCrop(input_size), transforms.ToTensor(), normalize ]) train_data = gluon.data.DataLoader(gluon.data.vision.ImageFolderDataset( train_path).transform_first(transform_train), batch_size=batch_size, shuffle=True, num_workers=num_workers) val_data = gluon.data.DataLoader(gluon.data.vision.ImageFolderDataset( val_path).transform_first(transform_test), batch_size=batch_size, shuffle=False, num_workers=num_workers) test_data = gluon.data.DataLoader(gluon.data.vision.ImageFolderDataset( test_path).transform_first(transform_test), batch_size=batch_size, shuffle=False, num_workers=num_workers) return train_data, val_data, test_data
def get_data_loader(data_dir, batch_size, num_workers, input_image_size=(224, 224)): normalize = transforms.Normalize( mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)) jitter_param = 0.4 lighting_param = 0.1 resize_value = input_image_size[0] + 32 def batch_fn(batch, ctx): data = gluon.utils.split_and_load(batch[0], ctx_list=ctx, batch_axis=0) label = gluon.utils.split_and_load(batch[1], ctx_list=ctx, batch_axis=0) return data, label transform_train = transforms.Compose([ transforms.RandomResizedCrop(input_image_size), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter( brightness=jitter_param, contrast=jitter_param, saturation=jitter_param), transforms.RandomLighting(lighting_param), transforms.ToTensor(), normalize ]) transform_test = transforms.Compose([ transforms.Resize(resize_value, keep_ratio=True), transforms.CenterCrop(input_image_size), transforms.ToTensor(), normalize ]) train_data = gluon.data.DataLoader( ImageNet(data_dir, train=True).transform_first(transform_train), batch_size=batch_size, shuffle=True, last_batch='discard', num_workers=num_workers) val_data = gluon.data.DataLoader( ImageNet(data_dir, train=False).transform_first(transform_test), batch_size=batch_size, shuffle=False, num_workers=num_workers) return train_data, val_data, batch_fn
def get_train_data(rec_train, batch_size, data_nthreads, input_size, crop_ratio, args): def train_batch_fn(batch, ctx): data = batch[0].as_in_context(ctx) label = batch[1].as_in_context(ctx) return data, label jitter_param = 0.4 lighting_param = 0.1 resize = int(math.ceil(input_size / crop_ratio)) train_transforms = [] if args.auto_aug: print('Using AutoAugment') from resnest.gluon.data_utils import AugmentationBlock, autoaug_imagenet_policies train_transforms.append(AugmentationBlock(autoaug_imagenet_policies())) if input_size >= 320: train_transforms.extend([ ERandomCrop(input_size), pth_transforms.Resize((input_size, input_size), interpolation=Image.BICUBIC), pth_transforms.RandomHorizontalFlip(), pth_transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), transforms.RandomLighting(lighting_param), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) else: train_transforms.extend([ transforms.RandomResizedCrop(input_size), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param, saturation=jitter_param), transforms.RandomLighting(lighting_param), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) transform_train = transforms.Compose(train_transforms) train_set = mx.gluon.data.vision.ImageRecordDataset(rec_train).transform_first(transform_train) train_sampler = SplitSampler(len(train_set), num_parts=num_workers, part_index=rank) train_data = gluon.data.DataLoader(train_set, batch_size=batch_size,# shuffle=True, last_batch='discard', num_workers=data_nthreads, sampler=train_sampler) return train_data, train_batch_fn
def get_data_loader(data_dir, batch_size, num_workers): normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) jitter_param = 0.4 lighting_param = 0.1 def batch_fn(batch, ctx): data = gluon.utils.split_and_load(batch[0], ctx_list=ctx, batch_axis=0) label = gluon.utils.split_and_load(batch[1], ctx_list=ctx, batch_axis=0) return data, label transform_train = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param, saturation=jitter_param), transforms.RandomLighting(lighting_param), transforms.ToTensor(), normalize ]) transform_test = transforms.Compose([ transforms.Resize(256, keep_ratio=True), transforms.CenterCrop(224), transforms.ToTensor(), normalize ]) train_data = gluon.data.DataLoader(imagenet.classification.ImageNet( data_dir, train=True).transform_first(transform_train), batch_size=batch_size, shuffle=True, last_batch='discard', num_workers=num_workers) val_data = gluon.data.DataLoader(imagenet.classification.ImageNet( data_dir, train=False).transform_first(transform_test), batch_size=batch_size, shuffle=False, num_workers=num_workers) if 'sync' in opt.kvstore: raise ValueError( "Need to resize iterator for distributed training to not hang at the end" ) return train_data, val_data, batch_fn
def cifar10_train_transform(ds_metainfo, mean_rgb=(0.4914, 0.4822, 0.4465), std_rgb=(0.2023, 0.1994, 0.2010), jitter_param=0.4, lighting_param=0.1): assert (ds_metainfo is not None) assert (ds_metainfo.input_image_size[0] == 32) return transforms.Compose([ RandomCrop(size=32, pad=4), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param, saturation=jitter_param), transforms.RandomLighting(lighting_param), transforms.ToTensor(), transforms.Normalize(mean=mean_rgb, std=std_rgb) ])
def test_transformer(): from mxnet.gluon.data.vision import transforms transform = transforms.Compose([ transforms.Resize(300), transforms.CenterCrop(256), transforms.RandomResizedCrop(224), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(0.1, 0.1, 0.1, 0.1), transforms.RandomBrightness(0.1), transforms.RandomContrast(0.1), transforms.RandomSaturation(0.1), transforms.RandomHue(0.1), transforms.RandomLighting(0.1), transforms.ToTensor(), transforms.Normalize([0, 0, 0], [1, 1, 1])]) transform(mx.nd.ones((245, 480, 3), dtype='uint8')).wait_to_read()
def transform_cifar10_dataset_train(): """ Should create a transformation that performs both random horizontal flip and random crop :return: A gluon transform object :rtype: gluon.Block """ train_data = gluon.data.vision.datasets.CIFAR10(root=M5_IMAGES, train=True) # RandomFlipLeftRight # RandomCrop to size 16 x 16 transform = transforms.Compose( [transforms.RandomFlipLeftRight(), transforms.RandomResizedCrop(16)]) return transform
def get_train_data_loader(data_dir, batch_size, num_workers, input_image_size, mean_rgb, std_rgb, jitter_param, lighting_param): transform_train = transforms.Compose([ transforms.RandomResizedCrop(input_image_size), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param, saturation=jitter_param), transforms.RandomLighting(lighting_param), transforms.ToTensor(), transforms.Normalize(mean=mean_rgb, std=std_rgb) ]) return gluon.data.DataLoader(dataset=ImageNet( root=data_dir, train=True).transform_first(fn=transform_train), batch_size=batch_size, shuffle=True, last_batch='discard', num_workers=num_workers)
def ReadData(batchSize): transform_train = transforms.Compose([ # transforms.CenterCrop(32) # transforms.RandomFlipTopBottom(), # transforms.RandomColorJitter(brightness=0.0, contrast=0.0,saturation=0.0, hue=0.0), # transforms.RandomLighting(0.0), # transforms.Cast('float32'), # transforms.Resize(32), # 随机按照 scale 和 ratio 裁剪,并放缩为 32x32 的正⽅形 transforms.RandomResizedCrop(32, scale=(0.08, 1.0), ratio=(3.0 / 4.0, 4.0 / 3.0)), # 随机左右翻转图⽚ transforms.RandomFlipLeftRight(), # 将图⽚像素值缩⼩到 (0,1) 内,并将数据格式从" ⾼ * 宽 * 通道" 改为" 通道 * ⾼ * 宽" transforms.ToTensor(), # 对图⽚的每个通道做标准化 transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]) transform_test = transforms.Compose([ # transforms.RandomResizedCrop(32, scale=(0.08, 1.0), ratio=(3.0 / 4.0, 4.0 / 3.0)), transforms.ToTensor(), transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]) data_dir = "D:\program\python\lijing\TrainUI" input_dir = "data" input_str = data_dir + '/' + input_dir + '/' train_ds = vision.ImageFolderDataset(input_str + 'train', flag=1) valid_ds = vision.ImageFolderDataset(input_str + 'valid', flag=1) # test_ds = vision.ImageFolderDataset(input_str + 'valid', flag=1) test_ds = vision.ImageFolderDataset(input_str + 'test', flag=1) loader = gluon.data.DataLoader trainData = loader(train_ds.transform_first(transform_train), batchSize, shuffle=True, last_batch='keep') validData = loader(valid_ds.transform_first(transform_test), batchSize, shuffle=True, last_batch='keep') testData = loader(test_ds.transform_first(transform_test), batchSize, shuffle=False, last_batch='keep') return trainData, validData, testData
def get_train_data(batch_size=8): """ process train data, add transforms. :param batch_size: per process num of samples :return: train data """ transform_train = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), transforms.RandomLighting(0.1), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) ]) img_folder, img_file = get_data_path() td = MultilabelDataset(data_folder=img_folder, data_file=img_file) train_data = DataLoader(td.transform_first(transform_train), batch_size=batch_size, shuffle=True) return train_data
def train_loader(path, batch_size=32, num_workers=4): normalize = transforms.Normalize(mean=0.5, std=0.25) train_transforms = transforms.Compose([ transforms.Resize((96,112)),# W x H transforms.RandomFlipLeftRight(), transforms.ToTensor(), normalize, ]) def my_train_transform(img, label): return train_transforms(img), label train_dataset = datasets.ImageFolderDataset(path, transform=my_train_transform) num_train = len(train_dataset) print("number of total examples is %d" % num_train) train_loader = data.DataLoader( train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) print("number of batches for train, valid and test is %d"%(len(train_loader))) return train_loader
def get_data_iters(batch_size): train_set = [(name.split()[0], int(name.split()[1])) for name in open(opt.train_data).readlines()] val_set = [(name.split()[0], int(name.split()[1])) for name in open(opt.val_data).readlines()] normalizer = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) transform_train = transforms.Compose([ transforms.Resize(size=(opt.img_width, opt.img_height), interpolation=1), transforms.RandomFlipLeftRight(), transforms.ToTensor(), normalizer ]) transform_test = transforms.Compose([ transforms.Resize(size=(opt.img_width, opt.img_height), interpolation=1), transforms.ToTensor(), normalizer ]) train_imgs = ImageTxtDataset(opt.data_dir, train_set, transform=transform_train) val_imgs = ImageTxtDataset(opt.data_dir, val_set, transform=transform_test) train_data = gluon.data.DataLoader(train_imgs, batch_size, sampler=RandomIdentitySampler( train_set, 4), last_batch='discard', num_workers=opt.num_workers) val_data = gluon.data.DataLoader(val_imgs, batch_size, sampler=RandomIdentitySampler( val_imgs, 4), last_batch='discard', num_workers=opt.num_workers) return train_data, val_data
def get_transform(jitter_param=0.4, pca_noise=0.2): # Init transformer # See https://mxnet.apache.org/api/python/docs/tutorials/packages/gluon/data/data_augmentation.html transform_train = transforms.Compose([ transforms.Resize(32), transforms.RandomResizedCrop((32, 32), scale=(0.8, 1.0), ratio=(0.9, 1.1)), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param, saturation=jitter_param, hue=jitter_param), transforms.RandomLighting(alpha=pca_noise), transforms.ToTensor(), transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]) transform_test = transforms.Compose([ transforms.Resize(32), transforms.ToTensor(), transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]) return transform_train, transform_test
def get_triplet_train_data(batch_size=8): """ triplet loss :param batch_size: 批次大小 :return: """ transform_train = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), transforms.RandomLighting(0.1), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) ]) img_folder, img_file = get_data_path() img_saved = os.path.join(img_file + ".tp.npz") td = TripletDataset(data_folder=img_folder, data_file=img_file, saved_path=img_saved, transform=transform_train) train_data = DataLoader(td, batch_size=batch_size, shuffle=True) return train_data
def get_data_iters(batch_size): train_set, val_set = LabelList(ratio=opt.ratio, root=opt.dataset_root, name=opt.dataset) normalizer = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) transform_train = transforms.Compose([ transforms.Resize(size=(opt.img_width, opt.img_height), interpolation=1), transforms.RandomFlipLeftRight(), RandomCrop(size=(opt.img_width, opt.img_height), pad=opt.pad), transforms.ToTensor(), normalizer ]) train_imgs = ImageTxtDataset(train_set, transform=transform_train) train_data = gluon.data.DataLoader(train_imgs, batch_size, shuffle=True, last_batch='discard', num_workers=opt.num_workers) if opt.ratio < 1: transform_test = transforms.Compose([ transforms.Resize(size=(opt.img_width, opt.img_height), interpolation=1), transforms.ToTensor(), normalizer ]) val_imgs = ImageTxtDataset(val_set, transform=transform_test) val_data = gluon.data.DataLoader(val_imgs, batch_size, shuffle=True, last_batch='discard', num_workers=opt.num_workers) else: val_data = None return train_data, val_data
def GluonTransformation(data: mx.nd.array): """ data: mx.nd.array h,w,c retrun data: mx.nd.array """ data = mx.nd.array(data) transform = transforms.Compose([ transforms.RandomResizedCrop(200, (0.8, 1.0)), transforms.CenterCrop((300, 300)), transforms.RandomFlipLeftRight(), transforms.RandomFlipTopBottom(), transforms.RandomLighting(0.3), transforms.RandomColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.2), transforms.Resize(384), transforms.ToTensor(), # h,w,c -> c, h, w transforms.Normalize(0, 1) ]) data = transform(data) return data # if __name__=='__main__': # img=cv2.imread('1.jpg') # img_out=ImageRotate(img,30) # # img_out=transformation(img) # cv2.imshow('ori',img) # cv2.imshow('rotate',img_out) # cv2.waitKey(0) # # cv2.imshow('img',mx.nd.clip(img_out,0,255).asnumpy().astype(np.uint8)) # # cv2.imshow('img',img_out.asnumpy().astype(np.uint8)) # # cv2.waitKey(0) # print('done!')
def get_train_val_dataloader(train_dataset, valid_dataset, batch_size): """ :param train_dataset: :param valid_dataset: :return: train_dataloader,valid_dataloader """ # create data augmetation train_transformer = transforms.Compose([ transforms.Resize(256), # Resize the image to 256 transforms.RandomResizedCrop(128, scale=(0.64, 1.0), ratio=(1.0, 1.0)), # Random flip transforms.RandomFlipLeftRight(), transforms.ToTensor(), # Normalize transforms.Normalize(mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010)) ]) valid_transformer = transforms.Compose([ transforms.Resize(128), transforms.ToTensor(), # Normalize transforms.Normalize(mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010)) ]) # get dataloader train_dataloader = DataLoader( train_dataset.transform_first(train_transformer), batch_size=batch_size, shuffle=True, last_batch='keep') valid_dataloader = DataLoader( valid_dataset.transform_first(valid_transformer), batch_size=batch_size, shuffle=True, last_batch='keep') return train_dataloader, valid_dataloader
def generate_transform(train, input_size, jitter_param=0.4, crop_ratio=1): resize = int(math.ceil(input_size / crop_ratio)) if train: transform = transforms.Compose([ transforms.RandomResizedCrop(input_size, scale=(0.6, 1)), transforms.RandomFlipTopBottom(), transforms.RandomFlipLeftRight(), RandomColorDistort(), RandomExpansion(), transforms.RandomLighting(0.2), transforms.Resize(resize), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) else: transform = transforms.Compose([ transforms.Resize(resize), transforms.CenterCrop(input_size), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) return transform
def get_dataloader(train_dataset, val_dataset, batch_size, num_workers): jitter_param = 0.4 lighting_param = 0.1 normalize = transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) transform_train = transforms.Compose([ transforms.Resize(480), transforms.RandomResizedCrop(224), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=jitter_param, contrast=jitter_param, saturation=jitter_param), transforms.RandomLighting(lighting_param), transforms.ToTensor(), normalize ]) transform_test = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize ]) train_data = gluon.data.DataLoader( train_dataset.transform_first(transform_train), batch_size=batch_size, shuffle=True, num_workers=num_workers, last_batch='rollover') val_data = gluon.data.DataLoader( val_dataset.transform_first(transform_test), batch_size=batch_size, shuffle=False, num_workers=num_workers, last_batch='keep') return train_data, val_data
def __init__(self, data_shape, path_imgrec, transform=None): super(FaceDataset, self).__init__() logging.info('loading recordio %s...', path_imgrec) path_imgidx = path_imgrec[0:-4]+".idx" self.imgrec = recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') # pylint: disable=redefined-variable-type s = self.imgrec.read_idx(0) header, _ = recordio.unpack(s) if header.flag>0: print('header0 label', header.label) self.header0 = (int(header.label[0]), int(header.label[1])) #assert(header.flag==1) self.imgidx = list(range(1, int(header.label[0]))) #self.imgidx = [] #self.id2range = {} #self.seq_identity = range(int(header.label[0]), int(header.label[1])) #for identity in self.seq_identity: # s = self.imgrec.read_idx(identity) # header, _ = recordio.unpack(s) # a,b = int(header.label[0]), int(header.label[1]) # count = b-a # if count<images_filter: # continue # self.id2range[identity] = (a,b) # self.imgidx += range(a, b) #print('id2range', len(self.id2range)) else: self.imgidx = list(self.imgrec.keys) self.seq = self.imgidx self.data_shape = data_shape self.transform = transforms.Compose([ #transforms.RandomBrightness(0.3), #transforms.RandomContrast(0.3), #transforms.RandomSaturation(0.3), transforms.RandomFlipLeftRight(), transforms.ToTensor() ])
def train_valid_test_loader(path, train_valid_ratio=(0.8,0.1), batch_size=32, num_workers=4): normalize = transforms.Normalize(mean=0.5, std=0.25) train_transforms = transforms.Compose([ transforms.Resize((96,112)), transforms.RandomFlipLeftRight(), transforms.ToTensor(), normalize, ]) untrain_transforms = transforms.Compose([ transforms.Resize((96,112)), transforms.ToTensor(), normalize, ]) def my_train_transform(img, label): return train_transforms(img), label def my_untrain_transform(img, label): return untrain_transforms(img), label train_dataset = datasets.ImageFolderDataset(path, transform=my_train_transform) untrain_dataset = datasets.ImageFolderDataset(path, transform=my_untrain_transform) num_train = len(train_dataset) print("number of total examples is %d" % num_train) indices = np.random.permutation(num_train) split1 = int(np.floor(train_valid_ratio[0] * num_train)) split2 = int(np.floor(sum(train_valid_ratio) * num_train)) train_sampler = SubsetRandomSampler(indices, 0, split1) valid_sampler = SubsetRandomSampler(indices, split1, split2) test_sampler = SubsetRandomSampler(indices, split2, -1) train_loader = data.DataLoader( train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers) valid_loader = data.DataLoader( untrain_dataset, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers) test_loader = data.DataLoader( untrain_dataset, batch_size=batch_size, sampler=test_sampler, num_workers=num_workers) print("number of batches for train, valid and test is %d, %d, %d"%(len(train_loader), len(valid_loader), len(test_loader))) return train_loader, valid_loader, test_loader
def get_train_data(self, batch_size): """ 获取训练数据,数据扩充 """ transform_train = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomFlipLeftRight(), transforms.RandomColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), transforms.RandomLighting(0.1), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) ]) td = MultilabelDataset(data_folder=self.train_folder, data_file=self.train_file, transform=transform_train) train_data = DataLoader(dataset=td, batch_size=batch_size, shuffle=True) return train_data, len(td)