def get(args): """ Entry point. Call this function to get all Charades dataloaders """ normalize = arraytransforms.Normalize(mean=[0.502], std=[1.0]) train_file = args.train_file val_file = args.val_file train_dataset = Charadesflow(args.data, 'train', train_file, args.cache, transform=transforms.Compose([ arraytransforms.RandomResizedCrop(224), arraytransforms.ToTensor(), normalize, transforms.Lambda(lambda x: torch.cat(x)), ])) val_transforms = transforms.Compose([ arraytransforms.Resize(256), arraytransforms.CenterCrop(224), arraytransforms.ToTensor(), normalize, transforms.Lambda(lambda x: torch.cat(x)), ]) val_dataset = Charadesflow(args.data, 'val', val_file, args.cache, transform=val_transforms) valvideo_dataset = Charadesflow(args.data, 'val_video', val_file, args.cache, transform=val_transforms) return train_dataset, val_dataset, valvideo_dataset
def get_iterator(mode): normalize = transforms.Normalize(mean=[x / 255.0 for x in [125.3, 123.0, 113.9]], std=[x / 255.0 for x in [63.0, 62.1, 66.7]]) kwargs = {'num_workers': 4, 'pin_memory': True} transform_augment = transforms.Compose([ # transforms.RandomResizedCrop(args.size, scale=(0.8, 1.2)), # random scale 0.8-1 of original image area, crop to args.size transforms.RandomResizedCrop(size), transforms.RandomRotation(15), # random rotation -15 to +15 degrees transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]) transform = transforms.Compose([transforms.Resize((size, size)), transforms.ToTensor(), normalize, ]) if mode: dataset = Dataset.MURA(split="train", transform=(transform_augment if augment else transform), type=type) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True, **kwargs) else: dataset = Dataset.MURA(split="test", transform=transform, type=type) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False, **kwargs) return loader
def get_data_loaders(train_batch_size, val_batch_size): normalize = transforms.Normalize(mean=torch.Tensor([0.5]), std=torch.Tensor([0.2])) train_transform = transforms.Compose([ transforms.RandomResizedCrop(256), transforms.RandomHorizontalFlip(), transforms.RandomVerticalFlip(), transforms.ToTensor(), transforms.MultiplicativeGaussianNoise(1, 0.01), normalize ]) val_transform = transforms.Compose( [transforms.Resize((512, 512)), transforms.ToTensor(), normalize]) train_loader = DataLoader(DWTDataset('dataset', split='train', transform=train_transform), batch_size=train_batch_size, shuffle=True) val_loader = DataLoader(DWTDataset('dataset', split='valid', transform=val_transform), batch_size=val_batch_size, shuffle=False) return train_loader, val_loader
def train(traindir, sz, min_scale=0.08, shuffle_seed=0): train_tfms = [ transforms.RandomResizedCrop(sz, scale=(min_scale, 1.0)), transforms.RandomHorizontalFlip() ] train_dataset = datasets.ImageFolder(traindir, transforms.Compose(train_tfms)) return PaddleDataLoader(train_dataset, shuffle_seed=shuffle_seed).reader()
def get_transform(param): """ Transform input into required image shape""" if 'train' == param: transform_train = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomFlipLeftRight(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) return transform_train elif 'test' == param: transform_test = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) return transform_test
def get_transform(is_train): transforms = [] if is_train: transforms.append(T.RandomResizedCrop()) transforms.append(T.ToTensor()) if is_train: transforms.append(T.RandomHorizontalFlip(0.5)) transforms.append(T.RandomVerticalFlip(0.5)) transforms.append(T.RandomRotation()) return T.Compose(transforms)
def get(args): """ Entry point. Call this function to get all Charades dataloaders """ normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_file = args.train_file val_file = args.val_file train_dataset = Charades( args.data, 'train', train_file, args.cache, transform=transforms.Compose([ transforms.RandomResizedCrop(args.inputsize), transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), # missing PCA lighting jitter normalize, ])) val_dataset = Charades(args.data, 'val', val_file, args.cache, transform=transforms.Compose([ transforms.Resize( int(256. / 224 * args.inputsize)), transforms.CenterCrop(args.inputsize), transforms.ToTensor(), normalize, ])) valvideo_dataset = Charades(args.data, 'val_video', val_file, args.cache, transform=transforms.Compose([ transforms.Resize( int(256. / 224 * args.inputsize)), transforms.CenterCrop(args.inputsize), transforms.ToTensor(), normalize, ])) return train_dataset, val_dataset, valvideo_dataset
def imageNet_loader(train_size, valid_size, test_size, crop_size): # http://blog.outcome.io/pytorch-quick-start-classifying-an-image/ normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_loader = torch.utils.data.DataLoader(datasets.ImageFolder( './data/kaggle/train', transforms.Compose([ transforms.RandomResizedCrop(crop_size), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])), batch_size=train_size, shuffle=True, pin_memory=True, drop_last=True) valid_loader = torch.utils.data.DataLoader(datasets.ImageFolder( './data/kaggle/valid', transforms.Compose([ transforms.CenterCrop(crop_size), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])), batch_size=valid_size, shuffle=True, pin_memory=True, drop_last=True) test_loader = torch.utils.data.DataLoader(datasets.ImageFolder( './data/image_net/small_classes/', transforms.Compose([ transforms.CenterCrop(crop_size), transforms.ToTensor(), normalize, ])), batch_size=test_size, shuffle=False) return train_loader, valid_loader, test_loader
def __init__(self, root='./', mode='train', cropsize=224): super(ImageNet, self).__init__() self.samples = [] self.mode = mode self.cropsize = cropsize if mode == 'train': txtpth = osp.join(root, 'train.txt') img_root_pth = osp.join(root, 'train') with open(txtpth, 'r') as fr: lines = fr.read().splitlines() for line in lines: pth, lb = line.split(' ') pth, lb = osp.join(img_root_pth, pth), int(lb) self.samples.append((pth, lb)) elif mode == 'val': txtpth = osp.join(root, 'val.txt') img_root_pth = osp.join(root, 'val') with open(txtpth, 'r') as fr: lines = fr.read().splitlines() for line in lines: pth, lb = line.split(' ') pth, lb = osp.join(img_root_pth, pth), int(lb) self.samples.append((pth, lb)) img_mean, img_std = (0.485, 0.456, 0.406), (0.229, 0.224, 0.225) self.trans_train = T.Compose([ T.RandomResizedCrop(cropsize), T.RandomHorizontalFlip(), RandomAugment(2, 9), T.ToTensor(), T.PCANoise(0.1), T.Normalize(img_mean, img_std) # T.ColorJitter(0.4, 0.4, 0.4), ]) short_size = int(cropsize * 256 / 224) self.trans_val = T.Compose([ T.ResizeCenterCrop(crop_size=cropsize, short_size=short_size), T.ToTensor(), T.Normalize(img_mean, img_std) ])
## DEFINE DEVICE ## device = torch.device("cuda:0" if (torch.cuda.is_available() and args.cuda) else "cpu") if (not torch.cuda.is_available() and args.cuda): print "cuda is not available. " print "Working on {}.".format(device) if torch.cuda.is_available(): print "using GPU number {}".format(gpu_id) ## CREATE DATASETS ## # defining transormations randomVFlip = transforms.RandomVerticalFlip() randomResizedCrop = transforms.RandomResizedCrop(parameters["input"]["matrix_size"], scale=parameters["transforms"]["scale_range"], ratio=parameters["transforms"]["ratio_range"], dtype=parameters['input']['data_type']) randomRotation = transforms.RandomRotation(parameters["transforms"]["max_angle"]) elasticTransform = transforms.ElasticTransform(alpha_range=parameters["transforms"]["alpha_range"], sigma_range=parameters["transforms"]["sigma_range"], p=parameters["transforms"]["elastic_rate"], dtype=parameters['input']['data_type']) channelShift = transforms.ChannelShift(parameters["transforms"]["channel_shift_range"], dtype=parameters['input']['data_type']) centerCrop = transforms.CenterCrop2D(parameters["input"]["matrix_size"]) # creating composed transformation composed = torch_transforms.Compose([randomVFlip,randomRotation,randomResizedCrop, elasticTransform]) # creating datasets training_dataset = MRI2DSegDataset(paths.training_data, matrix_size=parameters["input"]["matrix_size"], orientation=parameters["input"]["orientation"], resolution=parameters["input"]["resolution"], transform = composed) validation_dataset = MRI2DSegDataset(paths.validation_data, matrix_size=parameters["input"]["matrix_size"], orientation=parameters["input"]["orientation"], resolution=parameters["input"]["resolution"]) # creating data loaders training_dataloader = DataLoader(training_dataset, batch_size=parameters["training"]["batch_size"], shuffle=True, drop_last=True, num_workers=1) validation_dataloader = DataLoader(validation_dataset, batch_size=parameters["training"]["batch_size"], shuffle=True, drop_last=False, num_workers=1)
'use_gpu': torch.cuda.is_available(), # train params 'epoch': 100, 'start_lr': 0.01, 'momentum': 0.9, 'gamma': 0.1, 'lr_decay_step_size': 30, 'loss_fun': nn.CrossEntropyLoss(), } data_trans = { 'train': transforms.Compose([ transforms.RandomRotation(degrees=10), transforms.RandomResizedCrop(args['input_size']), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), 'val': transforms.Compose([ transforms.Resize(args['input_size']), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), } def data_load(data_dir=args['data_dir']): image_datasets = {
# get number of input features for the classifier in_features = faster_rcnn.roi_heads.box_predictor.cls_score.in_features # replace the pre-trained head with a new one faster_rcnn.roi_heads.box_predictor = FastRCNNPredictor(in_features, classes) return faster_rcnn # %% # use our dataset and defined transformations transformations = T.Compose( [ T.RandomHorizontalFlip(), T.RandomGrayscale(), T.ColorJitter(), T.RandomResizedCrop(size=(256, 256)), T.ToTensor(), ] ) test_transformations = T.Compose([T.ToTensor(),]) dataset_train = dataset.FRCNNFrameDataset( "data/images", "data/annotations/instances_default.json", transforms=transformations, ) dataset_test = dataset.FRCNNFrameDataset( "data/images", "data/annotations/instances_default.json",
# load the data data_path = args.data_path train_dir = data_path + 'train/' valid_dir = data_path + 'valid/' test_dir = data_path + 'test/' train_push_dir = train_dir train_batch_size = args.batch_size valid_batch_size = args.batch_size test_batch_size = args.batch_size train_push_batch_size = args.batch_size # dataset setup # processing transform_train = transforms.Compose([ transforms.RandomResizedCrop(img_size), transforms.RandomHorizontalFlip(), # transforms.RandomHorizontalFlip(), # transforms.ColorJitter(brightness=.5,contrast=.9,saturation=.5,hue=.1), transforms.ToTensor(), transforms.Normalize(mean, std), ]) transform_push = transforms.Compose([ transforms.Resize(size=(img_size, img_size)), transforms.ToTensor(), ]) transform_test = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(img_size),
def get_transforms(stage='train', means=None, stds=None): transform = [] stage = stage.lower() assert stage in ['train', 'test'], ('arg [stage]' ' should be one of ["train", "test"]') resize = get_valid_size(cfg.resize) size = get_valid_size(cfg.size) if stage == 'train': # size transform scale = tuple_or_list(cfg.scale) if scale: ratio = tuple_or_list(cfg.ratio) logger.debug('Training samples will be random resized and crop ' 'with size %s, scale %s and ratio %s' % (size, scale, ratio)) transform.append(custom_transforms.RandomResizedCrop( size=size, scale=scale, ratio=ratio)) else: logger.debug('Training samples will be resized to %s and then ' 'random cropped into %s' % (resize, size)) transform.append(transforms.Resize(size=resize)) transform.append(transforms.RandomCrop(size)) # color jitter transform colorjitter = tuple_or_list(cfg.colorjitter) if colorjitter is not None: logger.debug('Training samples will use color jitter to enhance ' 'with args: %s' % (colorjitter,)) transform.append(transforms.ColorJitter(*colorjitter)) # gray scale if cfg.random_grayscale > 0: logger.debug('Training samples will be randomly convert to ' 'grayscale with probability %.2f' % cfg.random_grayscale) transform.append(transforms.RandomGrayscale( p=cfg.random_grayscale)) # random horizontal flip if cfg.random_horizontal_flip: logger.debug('Training samples will be random horizontally flip') transform.append(transforms.RandomHorizontalFlip()) else: logger.debug('Testing samples will be resized to %s and then center ' 'crop to %s' % (resize, size)) transform.extend([transforms.Resize(resize), transforms.CenterCrop(size)]) # to tensor transform.append(transforms.ToTensor()) # normalize means = tuple_or_list(means) stds = tuple_or_list(stds) if not (means is None or stds is None): logger.debug('Samples will be normalized with means: %s and stds: %s' % (means, stds)) transform.append(transforms.Normalize(means, stds)) else: logger.debug('Input images will not be normalized') return transforms.Compose(transform)
def __init__(self, args, use_gpu): super(DataManager, self).__init__() self.args = args self.use_gpu = use_gpu print("Initializing dataset {}".format(args.dataset)) dataset = datasets.init_imgfewshot_dataset(name=args.dataset, root=args.root) if args.load: transform_train = T.Compose([ T.RandomCrop(84, padding=8), T.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), T.RandomHorizontalFlip(), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), T.RandomErasing(0.5) ]) transform_test = T.Compose([ T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) elif args.cifar: transform_train = T.Compose([ T.RandomCrop(32, padding=4), T.RandomHorizontalFlip(), T.ToTensor(), T.Normalize(mean=[0.507, 0.487, 0.441], std=[0.267, 0.256, 0.276]), T.RandomErasing(0.5) ]) transform_test = T.Compose([ T.ToTensor(), T.Normalize(mean=[0.507, 0.487, 0.441], std=[0.267, 0.256, 0.276]), ]) else: transform_train = T.Compose([ T.RandomResizedCrop((84, 84)), T.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), T.RandomHorizontalFlip(), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), T.RandomErasing(0.5) ]) transform_test = T.Compose([ T.Resize((92, 92)), T.CenterCrop((84, 84)), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) pin_memory = True if use_gpu else False self.trainloader = DataLoader( dataset_loader.init_loader( name='train_loader', dataset=dataset.train, labels2inds=dataset.train_labels2inds, labelIds=dataset.train_labelIds, nKnovel=args.nKnovel, nExemplars=args.nExemplars, nTestNovel=args.train_nTestNovel, epoch_size=args.train_epoch_size, transform=transform_train, load=args.load, tiered=args.tiered, ), batch_size=args.train_batch, shuffle=True, num_workers=args.workers, pin_memory=pin_memory, drop_last=True, ) self.valloader = DataLoader( dataset_loader.init_loader( name='test_loader', dataset=dataset.val, labels2inds=dataset.val_labels2inds, labelIds=dataset.val_labelIds, nKnovel=args.nKnovel, nExemplars=args.nExemplars, nTestNovel=args.nTestNovel, epoch_size=args.epoch_size, transform=transform_test, load=args.load, tiered=args.tiered, ), batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=pin_memory, drop_last=False, ) self.testloader = DataLoader( dataset_loader.init_loader( name='test_loader', dataset=dataset.test, labels2inds=dataset.test_labels2inds, labelIds=dataset.test_labelIds, nKnovel=args.nKnovel, nExemplars=args.nExemplars, nTestNovel=args.nTestNovel, epoch_size=args.epoch_size, transform=transform_test, load=args.load, tiered=args.tiered, ), batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=pin_memory, drop_last=False, )
help='max learning rate') parser.add_argument('-num_iter', type=int, default=100, help='num of iteration') parser.add_argument('-gpus', nargs='+', type=int, default=0, help='gpu device') args = parser.parse_args() train_transforms = transforms.Compose([ #transforms.ToPILImage(), transforms.ToCVImage(), transforms.RandomResizedCrop(settings.IMAGE_SIZE), transforms.RandomHorizontalFlip(), transforms.ColorJitter(brightness=0.4, saturation=0.4, hue=0.4), #transforms.RandomErasing(), #transforms.CutOut(56), transforms.ToTensor(), transforms.Normalize(settings.TRAIN_MEAN, settings.TRAIN_STD) ]) train_dataloader = get_train_dataloader(settings.DATA_PATH, train_transforms, args.b, args.w) net = get_network(args) net = init_weights(net) if isinstance(args.gpus, int):
from datetime import datetime if __name__ == '__main__': # load configures file_id = open('./cfgs.yaml', 'r', encoding='UTF-8') cfgs = yaml.load(file_id, Loader=yaml.FullLoader) file_id.close() if not os.path.exists('./validation/'): os.makedirs('./validation/') if not os.path.exists('./checkpoint/'): #存放模型的文件夹 os.makedirs('./checkpoint/') if not os.path.exists('./outside/'): # 存放侧边图的文件夹 os.makedirs('./outside/') trans = transforms.Compose([ transforms.RandomResizedCrop(280, scale=(1.0, 1.0)), transforms.ToTensor(), transforms.Normalize(mean=[[0.485, 0.456, 0.406], [0.], [0.31997], [0.41673], [0.24589], [0.10824], [0.13689], [0.39387]], std=[[0.229, 0.224, 0.225], [1.], [0.19775], [0.19969], [0.19713], [0.11726], [0.15983], [0.19695]]) ]) dataset = BSDS_500(root=cfgs['dataset'], VOC=True, transform=trans) # noinspection PyUnresolvedReferences dataloader = torch.utils.data.DataLoader(dataset, batch_size=cfgs['batch_size'], shuffle=True, pin_memory=True,