def GetTrainTestData(dataset, batch_size, dataset_dir, shuffle_train=True): # define transform for train data # mean and std calculated from utils/GetMeanNdStd if dataset == "cifar10": transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) # load datasets trainData = torchvision.datasets.CIFAR10(root=dataset_dir, train=True, download=True, transform=transform_train) # sampler = list(range(64*10)) trainDataGen = DataLoader( trainData, batch_size=batch_size, num_workers=1, shuffle=shuffle_train ) #, sampler=Sampler.SubsetRandomSampler(sampler)) # change transform for test dataset transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) testData = torchvision.datasets.CIFAR10(root=dataset_dir, train=False, download=True, transform=transform_test) testDataGen = DataLoader(testData, batch_size=batch_size, num_workers=1, shuffle=False) return trainDataGen, testDataGen elif dataset == "cifar100": transform = transforms.Compose([ # transforms.Resize(224), transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)) ]) # load datasets trainData = torchvision.datasets.CIFAR100(root=dataset_dir, train=True, download=True, transform=transform) # sampler = list(range(64*10)) trainDataGen = DataLoader( trainData, batch_size=batch_size, num_workers=1, shuffle=False) #, sampler=Sampler.SubsetRandomSampler(sampler)) # change transform for test dataset transform = transforms.Compose([ # transforms.Resize(224), transforms.ToTensor(), transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)) ]) testData = torchvision.datasets.CIFAR100(root=dataset_dir, train=False, download=True, transform=transform) testDataGen = DataLoader(testData, batch_size=batch_size, num_workers=1, shuffle=False) return trainDataGen, testDataGen
def main_worker(gpu, ngpus_per_node, args): global best_acc1 args.gpu = gpu if args.gpu is not None: print("Use GPU: {} for training".format(args.gpu)) # create model print("=> creating model '{}'".format(args.arch)) num_classes = 100 if args.dataset == 'cifar100' else 10 use_norm = True if args.loss_type == 'LDAM' else False model = models.__dict__[args.arch](num_classes=num_classes, use_norm=use_norm) if args.gpu is not None: torch.cuda.set_device(args.gpu) model = model.cuda(args.gpu) else: # DataParallel will divide and allocate batch_size to all available GPUs model = torch.nn.DataParallel(model).cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume, map_location='cuda:0') args.start_epoch = checkpoint['epoch'] best_acc1 = checkpoint['best_acc1'] if args.gpu is not None: # best_acc1 may be from a checkpoint from a different GPU best_acc1 = best_acc1.to(args.gpu) model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True # Data loading code transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_val = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': train_dataset = IMBALANCECIFAR10(root='./data', imb_type=args.imb_type, imb_factor=args.imb_factor, rand_number=args.rand_number, train=True, download=True, transform=transform_train) val_dataset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_val) elif args.dataset == 'cifar100': train_dataset = IMBALANCECIFAR100(root='./data', imb_type=args.imb_type, imb_factor=args.imb_factor, rand_number=args.rand_number, train=True, download=True, transform=transform_train) val_dataset = datasets.CIFAR100(root='./data', train=False, download=True, transform=transform_val) else: warnings.warn('Dataset is not listed') return cls_num_list = train_dataset.get_cls_num_list() print('cls num list:') print(cls_num_list) args.cls_num_list = cls_num_list train_sampler = None train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler) val_loader = torch.utils.data.DataLoader( val_dataset, batch_size=100, shuffle=False, num_workers=args.workers, pin_memory=True) # init log for training log_training = open(os.path.join(args.root_log, args.store_name, 'log_train.csv'), 'w') log_testing = open(os.path.join(args.root_log, args.store_name, 'log_test.csv'), 'w') with open(os.path.join(args.root_log, args.store_name, 'args.txt'), 'w') as f: f.write(str(args)) tf_writer = SummaryWriter(log_dir=os.path.join(args.root_log, args.store_name)) for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch, args) if args.train_rule == 'None': train_sampler = None per_cls_weights = None elif args.train_rule == 'Resample': train_sampler = ImbalancedDatasetSampler(train_dataset) per_cls_weights = None elif args.train_rule == 'Reweight': train_sampler = None beta = 0.9999 effective_num = 1.0 - np.power(beta, cls_num_list) per_cls_weights = (1.0 - beta) / np.array(effective_num) per_cls_weights = per_cls_weights / np.sum(per_cls_weights) * len(cls_num_list) per_cls_weights = torch.FloatTensor(per_cls_weights).cuda(args.gpu) elif args.train_rule == 'DRW': train_sampler = None idx = epoch // 160 betas = [0, 0.9999] effective_num = 1.0 - np.power(betas[idx], cls_num_list) per_cls_weights = (1.0 - betas[idx]) / np.array(effective_num) per_cls_weights = per_cls_weights / np.sum(per_cls_weights) * len(cls_num_list) per_cls_weights = torch.FloatTensor(per_cls_weights).cuda(args.gpu) else: warnings.warn('Sample rule is not listed') if args.loss_type == 'CE': criterion = nn.CrossEntropyLoss(weight=per_cls_weights).cuda(args.gpu) elif args.loss_type == 'LDAM': criterion = LDAMLoss(cls_num_list=cls_num_list, max_m=0.5, s=30, weight=per_cls_weights).cuda(args.gpu) elif args.loss_type == 'Focal': criterion = FocalLoss(weight=per_cls_weights, gamma=1).cuda(args.gpu) else: warnings.warn('Loss type is not listed') return # train for one epoch train(train_loader, model, criterion, optimizer, epoch, args, log_training, tf_writer) # evaluate on validation set acc1 = validate(val_loader, model, criterion, epoch, args, log_testing, tf_writer) # remember best acc@1 and save checkpoint is_best = acc1 > best_acc1 best_acc1 = max(acc1, best_acc1) tf_writer.add_scalar('acc/test_top1_best', best_acc1, epoch) output_best = 'Best Prec@1: %.3f\n' % (best_acc1) print(output_best) log_testing.write(output_best + '\n') log_testing.flush() save_checkpoint(args, { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_acc1': best_acc1, 'optimizer' : optimizer.state_dict(), }, is_best)
parser.add_argument('--compression', default=0.25, type=float) parser.add_argument('--log_after_steps', default=100, type=int) parser.add_argument('--lambda_bce', default=1, type=float) parser.add_argument('--lambda_ortho', default=1, type=float) parser.add_argument('--lambda_quant', default=1, type=float) parser.add_argument('--lambda_l1', default=1, type=float) args = parser.parse_args() start_epoch = 0 device = 'cuda' if torch.cuda.is_available() else 'cpu' # Data print('==> Preparing data..') transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) trainset = torchvision.datasets.CIFAR10(root='/home/shashant/.torch/', train=True, download=True, transform=transform_train) trainloader = torch.utils.data.DataLoader(trainset,
def main(): parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--result_dir', type=str, help='dir to save result txt files', default='results/') parser.add_argument('--noise_rate', type=float, help='corruption rate, should be less than 1', default=0.5) parser.add_argument('--forget_rate', type=float, help='forget rate', default=None) parser.add_argument('--noise_type', type=str, help='[pairflip, symmetric]', default='symmetric') parser.add_argument( '--num_gradual', type=int, default=10, help= 'how many epochs for linear drop rate, can be 5, 10, 15. This parameter is equal to Tk for R(T) in Co-teaching paper.' ) parser.add_argument( '--exponent', type=float, default=1, help= 'exponent of the forget rate, can be 0.5, 1, 2. This parameter is equal to c in Tc for R(T) in Co-teaching paper.' ) parser.add_argument('--top_bn', action='store_true') parser.add_argument('--dataset', type=str, help='mnist, cifar10, or cifar100', default='mnist') parser.add_argument('--n_epoch', type=int, default=300) parser.add_argument('--seed', type=int, default=1) parser.add_argument('--print_freq', type=int, default=50) parser.add_argument('--num_workers', type=int, default=2, help='how many subprocesses to use for data loading') parser.add_argument('--num_iter_per_epoch', type=int, default=400) parser.add_argument('--epoch_decay_start', type=int, default=80) parser.add_argument('--eps', type=float, default=9.9) parser.add_argument('--batch-size', type=int, default=128, metavar='N', help='input batch size for training (default: 256)') parser.add_argument('--test-batch-size', type=int, default=4000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--lr', type=float, default=0.2, metavar='LR', help='learning rate (default: 0.01)') #sm 0.5 lr=0.5 parser.add_argument('--momentum', type=float, default=0.9, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument( '--log-interval', type=int, default=100, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=False, help='For Saving the current Model') parser.add_argument( '--noise-level', type=float, default=80.0, help= 'percentage of noise added to the data (values from 0. to 100.), default: 80.' ) parser.add_argument( '--root-dir', type=str, default='.', help= 'path to CIFAR dir where cifar-10-batches-py/ and cifar-100-python/ are located. If the datasets are not downloaded, they will automatically be and extracted to this path, default: .' ) args = parser.parse_args() torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) batch_size = args.batch_size # if args.dataset=='mnist': # input_channel=1 # num_classes=10 # args.top_bn = False # args.epoch_decay_start = 80 # args.n_epoch = 200 # train_dataset = MNIST(root='./data/', # download=True, # train=True, # transform=transforms.ToTensor(), # noise_type=args.noise_type, # noise_rate=args.noise_rate # ) # # test_dataset = MNIST(root='./data/', # download=True, # train=False, # transform=transforms.ToTensor(), # noise_type=args.noise_type, # noise_rate=args.noise_rate # ) # # if args.dataset=='cifar10': # input_channel=3 # num_classes=10 # args.top_bn = False # args.epoch_decay_start = 80 # args.n_epoch = 200 # train_dataset = CIFAR10(root='./data/', # download=True, # train=True, # transform=transforms.ToTensor(), # noise_type=args.noise_type, # noise_rate=args.noise_rate # ) # # test_dataset = CIFAR10(root='./data/', # download=True, # train=False, # transform=transforms.ToTensor(), # noise_type=args.noise_type, # noise_rate=args.noise_rate # ) # # if args.dataset=='cifar100': # input_channel=3 # num_classes=100 # args.top_bn = False # args.epoch_decay_start = 100 # args.n_epoch = 200 # train_dataset = CIFAR100(root='./data/', # download=True, # train=True, # transform=transforms.ToTensor(), # noise_type=args.noise_type, # noise_rate=args.noise_rate # ) # # test_dataset = CIFAR100(root='./data/', # download=True, # train=False, # transform=transforms.ToTensor(), # noise_type=args.noise_type, # noise_rate=args.noise_rate # ) # if args.forget_rate is None: # forget_rate=args.noise_rate # else: # forget_rate=args.forget_rate # # noise_or_not = train_dataset.noise_or_not # # Data Loader (Input Pipeline) # print('loading dataset...') # train_loader = torch.utils.data.DataLoader(dataset=train_dataset, # batch_size=batch_size, # num_workers=args.num_workers, # drop_last=True, # shuffle=True) # # test_loader = torch.utils.data.DataLoader(dataset=test_dataset, # batch_size=batch_size, # num_workers=args.num_workers, # drop_last=True, # shuffle=False) ''' mean = [0.4914, 0.4822, 0.4465] std = [0.2023, 0.1994, 0.2010] transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean, std), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean, std), ])''' if args.dataset == 'cifar10': mean = [0.4914, 0.4822, 0.4465] std = [0.2023, 0.1994, 0.2010] transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean, std), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean, std), ]) trainset = datasets.CIFAR10(root=args.root_dir, train=True, download=True, transform=transform_train) trainset_track = datasets.CIFAR10(root=args.root_dir, train=True, transform=transform_train) testset = datasets.CIFAR10(root=args.root_dir, train=False, transform=transform_test) num_classes = 10 elif args.dataset == 'cifar100': mean = [0.4914, 0.4822, 0.4465] std = [0.2023, 0.1994, 0.2010] transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean, std), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean, std), ]) trainset = datasets.CIFAR100(root=args.root_dir, train=True, download=True, transform=transform_train) trainset_track = datasets.CIFAR100(root=args.root_dir, train=True, transform=transform_train) testset = datasets.CIFAR100(root=args.root_dir, train=False, transform=transform_test) num_classes = 100 elif args.dataset == 'imagenet_tiny': init_epoch = 100 num_classes = 200 #data_root = '/home/xingyu/Data/phd/data/imagenet-tiny/tiny-imagenet-200' data_root = '/home/iedl/w00536717/coteaching_plus-master/data/imagenet-tiny/tiny-imagenet-200' train_kv = "train_noisy_%s_%s_kv_list.txt" % (args.noise_type, args.noise_rate) test_kv = "val_kv_list.txt" normalize = transforms.Normalize(mean=[0.4802, 0.4481, 0.3975], std=[0.2302, 0.2265, 0.2262]) trainset = ImageFilelist(root=data_root, flist=os.path.join(data_root, train_kv), transform=transforms.Compose([ transforms.RandomResizedCrop(56), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) trainset_track = ImageFilelist(root=data_root, flist=os.path.join(data_root, train_kv), transform=transforms.Compose([ transforms.RandomResizedCrop(56), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) testset = ImageFilelist(root=data_root, flist=os.path.join(data_root, test_kv), transform=transforms.Compose([ transforms.Resize(64), transforms.CenterCrop(56), transforms.ToTensor(), normalize, ])) elif args.dataset == 'clothing1M': init_epoch = 100 num_classes = 14 data_root = '/home/iedl/w00536717/data/cloting1m/' #train_kv = #test_kv = train_transform = transforms.Compose([ transforms.Resize((256, 256)), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) test_transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) ''' train_transform = transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.6959, 0.6537, 0.6371),(0.3113, 0.3192, 0.3214)), ]) test_transform = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize((0.6959, 0.6537, 0.6371),(0.3113, 0.3192, 0.3214)), ]) ''' ''' train_loader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size, shuffle=True, num_workers=4, pin_memory=True) train_loader_track = torch.utils.data.DataLoader(trainset_track, batch_size=args.batch_size, shuffle=False, num_workers=4, pin_memory=True) test_loader = torch.utils.data.DataLoader(testset, batch_size=args.test_batch_size, shuffle=False, num_workers=4, pin_memory=True) ''' train_dataset = Clothing(root=data_root, img_transform=train_transform, train=True, valid=False, test=False) train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=32) train_dataset_track = Clothing(root=data_root, img_transform=train_transform, train=True, valid=False, test=False) train_loader_track = torch.utils.data.DataLoader( dataset=train_dataset_track, batch_size=args.batch_size, shuffle=False, num_workers=32) valid_dataset = Clothing(root=data_root, img_transform=train_transform, train=False, valid=True, test=False) valid_loader = torch.utils.data.DataLoader(dataset=valid_dataset, batch_size=args.batch_size, shuffle=False, num_workers=32) test_dataset = Clothing(root=data_root, img_transform=test_transform, train=False, valid=False, test=True) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=args.batch_size, shuffle=False, num_workers=32) #print(dir(train_loader)) #labels = get_data_cifar_2(train_loader_track) # it should be "clonning" #noisy_labels = add_noise_cifar_wo(train_loader, args.noise_level, # args.noise_type) # it changes the labels in the train loader directly #noisy_labels_track = add_noise_cifar_wo(train_loader_track, args.noise_level, args.noise_type) # Define models os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3,4,5,6,7" use_cuda = torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") #cnn = PreResNet_two.ResNet18(num_classes=num_classes).to(device) cnn = MyResNet_zero.MyCustomResnet(num_classes) #.to(device) cnn = nn.DataParallel(cnn, device_ids=[0, 1, 2, 3, 4, 5, 6, 7]) cnn.to(device) cnn.cuda() #print(model.parameters) #optimizer1 = torch.optim.SGD(cnn1.parameters(), lr=learning_rate) optimizer = torch.optim.SGD(cnn.parameters(), lr=args.lr, weight_decay=1e-3, momentum=args.momentum) #optimizer = torch.optim.Adam(cnn.parameters(), lr=args.lr,weight_decay=1e-4) #optimizer1 = torch.optim.SGD(cnn.parameters(), lr=1e-2,weight_decay=1e-4,momentum=args.momentum) bmm_model = bmm_model_maxLoss = bmm_model_minLoss = 0 acc = [] loss = [] loss_pure = [] loss_corrupt = [] out = [] temp = 1 for epoch in range(1, args.n_epoch + 1): if epoch < 3: #epoch_losses_train, epoch_probs_train, argmaxXentropy_train, bmm_model, bmm_model_maxLoss, bmm_model_minLoss = \ # track_training_loss(args, cnn, device, train_loader_track, epoch, bmm_model, bmm_model_maxLoss, # bmm_model_minLoss) #l1,temp=train(args, cnn, device, train_loader, optimizer, epoch,bmm_model, bmm_model_maxLoss, bmm_model_minLoss,temp) l1 = train_CE(args, cnn, device, train_loader, optimizer, epoch) #adjust_learning_rate(optimizer, 0.1) #l2=train_uncertainty(args, cnn, device, train_loader, optimizer1, epoch,bmm_model, bmm_model_maxLoss, bmm_model_minLoss) #l1=train_true_label(args, cnn, device, train_loader, optimizer, epoch,bmm_model, bmm_model_maxLoss, bmm_model_minLoss) #l2=train_uncertainty(args, cnn, device, train_loader, optimizer1, epoch,bmm_model, bmm_model_maxLoss, bmm_model_minLoss) #l1=train_CE(args, cnn, device, train_loader, optimizer, epoch) #l2=train_uncertainty(args, cnn, device, train_loader, optimizer1, epoch,bmm_model, bmm_model_maxLoss, bmm_model_minLoss) #loss.append(l1) acc.append(test(args, cnn, device, test_loader)) elif epoch < 80: if epoch == 3: adjust_learning_rate(optimizer, args.lr / 10) epoch_losses_train, epoch_probs_train, argmaxXentropy_train, bmm_model, bmm_model_maxLoss, bmm_model_minLoss = \ track_training_loss(args, cnn, device, train_loader_track, epoch, bmm_model, bmm_model_maxLoss, bmm_model_minLoss) l1, temp = train(args, cnn, device, train_loader, optimizer, epoch, bmm_model, bmm_model_maxLoss, bmm_model_minLoss, temp, num_classes) #l1=train_CE(args, cnn, device, train_loader, optimizer, epoch) #l2=train_uncertainty(args, cnn, device, train_loader, optimizer1, epoch,bmm_model, bmm_model_maxLoss, bmm_model_minLoss) acc.append(test(args, cnn, device, test_loader)) elif epoch < 200: if epoch == 10: adjust_learning_rate(optimizer, args.lr / 1000) elif epoch == 100: adjust_learning_rate(optimizer, args.lr / 5000) elif epoch == 160: adjust_learning_rate(optimizer, args.lr / 25000) epoch_losses_train, epoch_probs_train, argmaxXentropy_train, bmm_model, bmm_model_maxLoss, bmm_model_minLoss = \ track_training_loss(args, cnn, device, train_loader_track, epoch, bmm_model, bmm_model_maxLoss, bmm_model_minLoss) l1, temp = train(args, cnn, device, train_loader, optimizer, epoch, bmm_model, bmm_model_maxLoss, bmm_model_minLoss, temp, num_classes) #adjust_learning_rate(optimizer, args.lr/1000) #epoch_losses_train, epoch_probs_train, argmaxXentropy_train, bmm_model, bmm_model_maxLoss, bmm_model_minLoss = \ # track_training_loss(args, cnn, device, train_loader_track, epoch, bmm_model, bmm_model_maxLoss, # bmm_model_minLoss) #l2=train_uncertainty(args, cnn, device, train_loader, optimizer1, epoch,bmm_model, bmm_model_maxLoss, bmm_model_minLoss) #acc.append(test(args, cnn, device, test_loader)) #l1=train_true_label(args, cnn, device, train_loader, optimizer, epoch,bmm_model, bmm_model_maxLoss, bmm_model_minLoss) #l2=train_uncertainty(args, cnn, device, train_loader, optimizer1, epoch,bmm_model, bmm_model_maxLoss, bmm_model_minLoss) #l3=train_DMI(args, cnn, device, train_loader, optimizer1, epoch, bmm_model, bmm_model_maxLoss, bmm_model_minLoss, criterion = DMI_loss) #l1 = train_together(args, cnn, device, train_loader, optimizer, epoch, bmm_model, bmm_model_maxLoss, # bmm_model_minLoss) #loss.append(l1) #out.append(out10) acc.append(test(args, cnn, device, test_loader)) else: #adjust_learning_rate(optimizer, args.lr/10000) epoch_losses_train, epoch_probs_train, argmaxXentropy_train, bmm_model, bmm_model_maxLoss, bmm_model_minLoss = \ track_training_loss(args, cnn, device, train_loader_track, epoch, bmm_model, bmm_model_maxLoss, bmm_model_minLoss) l1, temp = train(args, cnn, device, train_loader, optimizer, epoch, bmm_model, bmm_model_maxLoss, bmm_model_minLoss, temp, num_classes) #l2=train_uncertainty(args, cnn, device, train_loader, optimizer1, epoch,bmm_model, bmm_model_maxLoss, bmm_model_minLoss) #acc.append(test(args, cnn, device, test_loader)) #l1 = train_true_label(args, cnn, device, train_loader, optimizer, epoch, bmm_model, bmm_model_maxLoss,bmm_model_minLoss) #l2 = train_uncertainty(args, cnn, device, train_loader, optimizer1, epoch, bmm_model, bmm_model_maxLoss,bmm_model_minLoss) #l3=train_DMI(args, cnn, device, train_loader, optimizer1, epoch,bmm_model, bmm_model_maxLoss, bmm_model_minLoss, criterion = DMI_loss) #l1 = train_together(args, cnn, device, train_loader, optimizer, epoch, bmm_model, bmm_model_maxLoss, # bmm_model_minLoss) #loss.append(l1) #out.append(out10) acc.append(test(args, cnn, device, test_loader)) print("Evaluate on validset") test(args, cnn, device, valid_loader) temp -= 0.0024 name = str(args.dataset) + " " + str(args.noise_type) + " " + str( args.noise_rate)
# text_encoder = RNN_ENCODER(27297, nhidden=256) state_dict = \ torch.load('../models/text_encoder100.pth', map_location=lambda storage, loc: storage) text_encoder.load_state_dict(state_dict) text_encoder = text_encoder.cpu() text_encoder.eval() state_dict = \ torch.load('../models/coco_AttnGAN2.pth', map_location=lambda storage, loc: storage) netG.load_state_dict(state_dict) imsize = 64 * (2**(3 - 1)) image_transform = transforms.Compose([ transforms.Resize(int(imsize * 76 / 64)), transforms.RandomCrop(imsize), transforms.RandomHorizontalFlip() ]) dataset = TextDataset('../data/coco/', 'test', base_size=64, transform=image_transform) assert dataset dataloader = torch.utils.data.DataLoader(dataset, batch_size=100, drop_last=True, shuffle=True, num_workers=12) wordtoix = dataset.wordtoix
def main(): # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--world_size', type=int, default=1, help='number of GPUs to use') parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.01, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--wd', type=float, default=1e-4, help='weight decay (default: 5e-4)') parser.add_argument('--lr-decay-every', type=int, default=100, help='learning rate decay by 10 every X epochs') parser.add_argument('--lr-decay-scalar', type=float, default=0.1, help='--') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--run_test', default=False, type=str2bool, nargs='?', help='run test only') parser.add_argument( '--limit_training_batches', type=int, default=-1, help='how many batches to do per training, -1 means as many as possible' ) parser.add_argument('--no_grad_clip', default=False, type=str2bool, nargs='?', help='turn off gradient clipping') parser.add_argument('--get_flops', default=False, type=str2bool, nargs='?', help='add hooks to compute flops') parser.add_argument( '--get_inference_time', default=False, type=str2bool, nargs='?', help='runs valid multiple times and reports the result') parser.add_argument('--mgpu', default=False, type=str2bool, nargs='?', help='use data paralization via multiple GPUs') parser.add_argument('--dataset', default="MNIST", type=str, help='dataset for experiment, choice: MNIST, CIFAR10') parser.add_argument('--data', metavar='DIR', default='/imagenet', help='path to imagenet dataset') parser.add_argument( '--model', default="lenet3", type=str, help='model selection, choices: lenet3, vgg, mobilenetv2, resnet18', choices=[ "lenet3", "vgg", "mobilenetv2", "resnet18", "resnet152", "resnet50", "resnet50_noskip", "resnet20", "resnet34", "resnet101", "resnet101_noskip", "densenet201_imagenet", 'densenet121_imagenet', "multprun_gate5_gpu_0316_1", "mult_prun8_gpu", "multnas5_gpu" ]) parser.add_argument('--tensorboard', type=str2bool, nargs='?', help='Log progress to TensorBoard') parser.add_argument( '--save_models', default=True, type=str2bool, nargs='?', help='if True, models will be saved to the local folder') parser.add_argument('--fineturn_model', type=str2bool, nargs='?', help='Log progress to TensorBoard') # ============================PRUNING added parser.add_argument( '--pruning_config', default=None, type=str, help= 'path to pruning configuration file, will overwrite all pruning parameters in arguments' ) parser.add_argument('--group_wd_coeff', type=float, default=0.0, help='group weight decay') parser.add_argument('--name', default='test', type=str, help='experiment name(folder) to store logs') parser.add_argument( '--augment', default=False, type=str2bool, nargs='?', help= 'enable or not augmentation of training dataset, only for CIFAR, def False' ) parser.add_argument('--load_model', default='', type=str, help='path to model weights') parser.add_argument('--pruning', default=False, type=str2bool, nargs='?', help='enable or not pruning, def False') parser.add_argument( '--pruning-threshold', '--pt', default=100.0, type=float, help= 'Max error perc on validation set while pruning (default: 100.0 means always prune)' ) parser.add_argument( '--pruning-momentum', default=0.0, type=float, help= 'Use momentum on criteria between pruning iterations, def 0.0 means no momentum' ) parser.add_argument('--pruning-step', default=15, type=int, help='How often to check loss and do pruning step') parser.add_argument('--prune_per_iteration', default=10, type=int, help='How many neurons to remove at each iteration') parser.add_argument( '--fixed_layer', default=-1, type=int, help='Prune only a given layer with index, use -1 to prune all') parser.add_argument('--start_pruning_after_n_iterations', default=0, type=int, help='from which iteration to start pruning') parser.add_argument('--maximum_pruning_iterations', default=1e8, type=int, help='maximum pruning iterations') parser.add_argument('--starting_neuron', default=0, type=int, help='starting position for oracle pruning') parser.add_argument('--prune_neurons_max', default=-1, type=int, help='prune_neurons_max') parser.add_argument('--pruning-method', default=0, type=int, help='pruning method to be used, see readme.md') parser.add_argument('--pruning_fixed_criteria', default=False, type=str2bool, nargs='?', help='enable or not criteria reevaluation, def False') parser.add_argument('--fixed_network', default=False, type=str2bool, nargs='?', help='fix network for oracle or criteria computation') parser.add_argument( '--zero_lr_for_epochs', default=-1, type=int, help='Learning rate will be set to 0 for given number of updates') parser.add_argument( '--dynamic_network', default=False, type=str2bool, nargs='?', help= 'Creates a new network graph from pruned model, works with ResNet-101 only' ) parser.add_argument('--use_test_as_train', default=False, type=str2bool, nargs='?', help='use testing dataset instead of training') parser.add_argument('--pruning_mask_from', default='', type=str, help='path to mask file precomputed') parser.add_argument( '--compute_flops', default=True, type=str2bool, nargs='?', help= 'if True, will run dummy inference of batch 1 before training to get conv sizes' ) # ============================END pruning added best_prec1 = 0 global global_iteration global group_wd_optimizer global_iteration = 0 args = parser.parse_args() use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) args.distributed = args.world_size > 1 if args.distributed: dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=0) device = torch.device("cuda" if use_cuda else "cpu") # dataset loading section if args.dataset == "MNIST": kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} train_loader = torch.utils.data.DataLoader(datasets.MNIST( '../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=args.batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=args.test_batch_size, shuffle=True, **kwargs) elif args.dataset == "CIFAR10": # Data loading code normalize = transforms.Normalize( mean=[x / 255.0 for x in [125.3, 123.0, 113.9]], std=[x / 255.0 for x in [63.0, 62.1, 66.7]]) if args.augment: transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]) else: transform_train = transforms.Compose([ transforms.ToTensor(), normalize, ]) transform_test = transforms.Compose([transforms.ToTensor(), normalize]) kwargs = {'num_workers': 8, 'pin_memory': True} train_loader = torch.utils.data.DataLoader(datasets.CIFAR10( '../data', train=True, download=True, transform=transform_train), batch_size=args.batch_size, shuffle=True, drop_last=True, **kwargs) test_loader = torch.utils.data.DataLoader( datasets.CIFAR10('../data', train=False, transform=transform_test), batch_size=args.test_batch_size, shuffle=True, **kwargs) elif args.dataset == "Imagenet": traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = datasets.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset) else: train_sampler = None kwargs = {'num_workers': 16} train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), sampler=train_sampler, pin_memory=True, **kwargs) if args.use_test_as_train: train_loader = torch.utils.data.DataLoader( datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=(train_sampler is None), **kwargs) test_loader = torch.utils.data.DataLoader(datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, pin_memory=True, **kwargs) #wm elif args.dataset == "mult_5T": args.data_root = ['/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/CX_20200709',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TK_20200709',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/ZR_20200709',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TX_20200616',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/WM_20200709'] args.data_root_val = ['/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/CX_20200709',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TK_20200709',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/ZR_20200709',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TX_20200616',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/WM_20200709'] args.train_data_list = ['/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/CX_20200709/txt/cx_train.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TK_20200709/txt/tk_train.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/ZR_20200709/txt/zr_train.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TX_20200616/txt/tx_train.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/WM_20200709/txt/wm_train.txt'] args.val_data_list = ['/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/CX_20200709/txt/cx_val.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TK_20200709/txt/tk_val.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/ZR_20200709/txt/zr_val.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TX_20200616/txt/tx_val.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/WM_20200709/txt/wm_val.txt'] num_tasks = len(args.data_root) args.ngpu = 8 args.workers = 8 args.train_batch_size = [40, 40, 40, 40, 40] #36 args.val_batch_size = [100, 100, 100, 100, 100] args.loss_weight = [1.0, 1.0, 1.0, 1.0, 1.0] args.val_num_classes = [[0, 1, 2, 3, 4], [0, 1, 2], [0, 1], [0, 1], [0, 1, 2, 3, 4, 5, 6]] args.mixup_alpha = None #None for i in range(num_tasks): args.train_batch_size[i] *= args.ngpu args.val_batch_size[i] *= args.ngpu pixel_mean = [0.406, 0.456, 0.485] pixel_std = [0.225, 0.224, 0.229] #私人定制: train_dataset = [] for i in range(num_tasks): if i == 1: train_dataset.append( FileListLabeledDataset( args.train_data_list[i], args.data_root[i], Compose([ RandomResizedCrop( 112, scale=(0.94, 1.), ratio=(1. / 4., 4. / 1.) ), #scale=(0.7, 1.2), ratio=(1. / 1., 4. / 1.) RandomHorizontalFlip(), ColorJitter(brightness=[0.5, 1.5], contrast=[0.5, 1.5], saturation=[0.5, 1.5], hue=0), ToTensor(), Lighting(1, [0.2175, 0.0188, 0.0045], [[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203]]), Normalize(pixel_mean, pixel_std), ]))) else: train_dataset.append( FileListLabeledDataset( args.train_data_list[i], args.data_root[i], Compose([ RandomResizedCrop(112, scale=(0.7, 1.2), ratio=(1. / 1., 4. / 1.)), RandomHorizontalFlip(), ColorJitter(brightness=[0.5, 1.5], contrast=[0.5, 1.5], saturation=[0.5, 1.5], hue=0), ToTensor(), Lighting(1, [0.2175, 0.0188, 0.0045], [[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203]]), Normalize(pixel_mean, pixel_std), ]))) #原来的 # train_dataset = [FileListLabeledDataset( # args.train_data_list[i], args.data_root[i], # Compose([ # RandomResizedCrop(112,scale=(0.7, 1.2), ratio=(1. / 1., 4. / 1.)), # RandomHorizontalFlip(), # ColorJitter(brightness=[0.5,1.5], contrast=[0.5,1.5], saturation=[0.5,1.5], hue= 0), # ToTensor(), # Lighting(1, [0.2175, 0.0188, 0.0045], [[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203]]), # Normalize(pixel_mean, pixel_std),]), # memcached=False, # memcached_client="") for i in range(num_tasks)] args.num_classes = [td.num_class for td in train_dataset] train_longest_size = max([ int(np.ceil(len(td) / float(bs))) for td, bs in zip(train_dataset, args.train_batch_size) ]) train_sampler = [ GivenSizeSampler(td, total_size=train_longest_size * bs, rand_seed=0) for td, bs in zip(train_dataset, args.train_batch_size) ] train_loader = [ DataLoader(train_dataset[k], batch_size=args.train_batch_size[k], shuffle=False, num_workers=args.workers, pin_memory=False, sampler=train_sampler[k]) for k in range(num_tasks) ] val_dataset = [ FileListLabeledDataset( args.val_data_list[i], args.data_root_val[i], Compose([ Resize((112, 112)), # CenterCrop(112), ToTensor(), Normalize(pixel_mean, pixel_std), ]), memcached=False, memcached_client="") for i in range(num_tasks) ] val_longest_size = max([ int(np.ceil(len(vd) / float(bs))) for vd, bs in zip(val_dataset, args.val_batch_size) ]) test_loader = [ DataLoader(val_dataset[k], batch_size=args.val_batch_size[k], shuffle=False, num_workers=args.workers, pin_memory=False) for k in range(num_tasks) ] if args.model == "lenet3": model = LeNet(dataset=args.dataset) elif args.model == "vgg": model = vgg11_bn(pretrained=True) elif args.model == "resnet18": model = PreActResNet18() elif (args.model == "resnet50") or (args.model == "resnet50_noskip"): if args.dataset == "CIFAR10": model = PreActResNet50(dataset=args.dataset) else: from models.resnet import resnet50 skip_gate = True if "noskip" in args.model: skip_gate = False if args.pruning_method not in [22, 40]: skip_gate = False model = resnet50(skip_gate=skip_gate) elif args.model == "resnet34": if not (args.dataset == "CIFAR10"): from models.resnet import resnet34 model = resnet34() elif args.model == "multprun_gate5_gpu_0316_1": from models.multitask import MultiTaskWithLoss model = MultiTaskWithLoss(backbone=args.model, num_classes=args.num_classes, feature_dim=2560, spatial_size=112, arc_fc=False, feat_bn=False) print(model) elif args.model == "mult_prun8_gpu": from models.multitask import MultiTaskWithLoss model = MultiTaskWithLoss(backbone=args.model, num_classes=args.num_classes, feature_dim=18, spatial_size=112, arc_fc=False, feat_bn=False) print(model) elif args.model == "multnas5_gpu": #作为修改项 from models.multitask import MultiTaskWithLoss model = MultiTaskWithLoss(backbone=args.model, num_classes=args.num_classes, feature_dim=512, spatial_size=112, arc_fc=False, feat_bn=False) print(model) elif "resnet101" in args.model: if not (args.dataset == "CIFAR10"): from models.resnet import resnet101 if args.dataset == "Imagenet": classes = 1000 if "noskip" in args.model: model = resnet101(num_classes=classes, skip_gate=False) else: model = resnet101(num_classes=classes) elif args.model == "resnet20": if args.dataset == "CIFAR10": NotImplementedError( "resnet20 is not implemented in the current project") # from models.resnet_cifar import resnet20 # model = resnet20() elif args.model == "resnet152": model = PreActResNet152() elif args.model == "densenet201_imagenet": from models.densenet_imagenet import DenseNet201 model = DenseNet201(gate_types=['output_bn'], pretrained=True) elif args.model == "densenet121_imagenet": from models.densenet_imagenet import DenseNet121 model = DenseNet121(gate_types=['output_bn'], pretrained=True) else: print(args.model, "model is not supported") ####end dataset preparation if args.dynamic_network: # attempts to load pruned model and modify it be removing pruned channels # works for resnet101 only if (len(args.load_model) > 0) and (args.dynamic_network): if os.path.isfile(args.load_model): load_model_pytorch(model, args.load_model, args.model) else: print("=> no checkpoint found at '{}'".format(args.load_model)) exit() dynamic_network_change_local(model) # save the model log_save_folder = "%s" % args.name if not os.path.exists(log_save_folder): os.makedirs(log_save_folder) if not os.path.exists("%s/models" % (log_save_folder)): os.makedirs("%s/models" % (log_save_folder)) model_save_path = "%s/models/pruned.weights" % (log_save_folder) model_state_dict = model.state_dict() if args.save_models: save_checkpoint({'state_dict': model_state_dict}, False, filename=model_save_path) print("model is defined") # aux function to get size of feature maps # First it adds hooks for each conv layer # Then runs inference with 1 image output_sizes = get_conv_sizes(args, model) if use_cuda and not args.mgpu: model = model.to(device) elif args.distributed: model.cuda() print( "\n\n WARNING: distributed pruning was not verified and might not work correctly" ) model = torch.nn.parallel.DistributedDataParallel(model) elif args.mgpu: model = torch.nn.DataParallel(model).cuda() else: model = model.to(device) print( "model is set to device: use_cuda {}, args.mgpu {}, agrs.distributed {}" .format(use_cuda, args.mgpu, args.distributed)) weight_decay = args.wd if args.fixed_network: weight_decay = 0.0 # remove updates from gate layers, because we want them to be 0 or 1 constantly if 1: parameters_for_update = [] parameters_for_update_named = [] for name, m in model.named_parameters(): if "gate" not in name: parameters_for_update.append(m) parameters_for_update_named.append((name, m)) else: print("skipping parameter", name, "shape:", m.shape) total_size_params = sum( [np.prod(par.shape) for par in parameters_for_update]) print("Total number of parameters, w/o usage of bn consts: ", total_size_params) optimizer = optim.SGD(parameters_for_update, lr=args.lr, momentum=args.momentum, weight_decay=weight_decay) if 1: # helping optimizer to implement group lasso (with very small weight that doesn't affect training) # will be used to calculate number of remaining flops and parameters in the network group_wd_optimizer = group_lasso_decay( parameters_for_update, group_lasso_weight=args.group_wd_coeff, named_parameters=parameters_for_update_named, output_sizes=output_sizes) cudnn.benchmark = True # define objective criterion = nn.CrossEntropyLoss() ###=======================added for pruning # logging part log_save_folder = "%s" % args.name if not os.path.exists(log_save_folder): os.makedirs(log_save_folder) if not os.path.exists("%s/models" % (log_save_folder)): os.makedirs("%s/models" % (log_save_folder)) train_writer = None if args.tensorboard: try: # tensorboardX v1.6 train_writer = SummaryWriter(log_dir="%s" % (log_save_folder)) except: # tensorboardX v1.7 train_writer = SummaryWriter(logdir="%s" % (log_save_folder)) time_point = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime()) textfile = "%s/log_%s.txt" % (log_save_folder, time_point) stdout = Logger(textfile) sys.stdout = stdout print(" ".join(sys.argv)) # initializing parameters for pruning # we can add weights of different layers or we can add gates (multiplies output with 1, useful only for gradient computation) pruning_engine = None if args.pruning: pruning_settings = dict() if not (args.pruning_config is None): pruning_settings_reader = PruningConfigReader() pruning_settings_reader.read_config(args.pruning_config) pruning_settings = pruning_settings_reader.get_parameters() # overwrite parameters from config file with those from command line # needs manual entry here # user_specified = [key for key in vars(default_args).keys() if not (vars(default_args)[key]==vars(args)[key])] # argv_of_interest = ['pruning_threshold', 'pruning-momentum', 'pruning_step', 'prune_per_iteration', # 'fixed_layer', 'start_pruning_after_n_iterations', 'maximum_pruning_iterations', # 'starting_neuron', 'prune_neurons_max', 'pruning_method'] has_attribute = lambda x: any([x in a for a in sys.argv]) if has_attribute('pruning-momentum'): pruning_settings['pruning_momentum'] = vars( args)['pruning_momentum'] if has_attribute('pruning-method'): pruning_settings['method'] = vars(args)['pruning_method'] pruning_parameters_list = prepare_pruning_list( pruning_settings, model, model_name=args.model, pruning_mask_from=args.pruning_mask_from, name=args.name) print("Total pruning layers:", len(pruning_parameters_list)) folder_to_write = "%s" % log_save_folder + "/" log_folder = folder_to_write pruning_engine = pytorch_pruning(pruning_parameters_list, pruning_settings=pruning_settings, log_folder=log_folder) pruning_engine.connect_tensorboard(train_writer) pruning_engine.dataset = args.dataset pruning_engine.model = args.model pruning_engine.pruning_mask_from = args.pruning_mask_from pruning_engine.load_mask() gates_to_params = connect_gates_with_parameters_for_flops( args.model, parameters_for_update_named) pruning_engine.gates_to_params = gates_to_params ###=======================end for pruning # loading model file if (len(args.load_model) > 0) and (not args.dynamic_network): if os.path.isfile(args.load_model): if args.fineturn_model: checkpoint = torch.load(args.load_model) state_dict = checkpoint['state_dict'] model = load_module_state_dict_checkpoint(model, state_dict) else: load_model_pytorch(model, args.load_model, args.model) else: print("=> no checkpoint found at '{}'".format(args.load_model)) exit() if args.tensorboard and 0: if args.dataset == "CIFAR10": dummy_input = torch.rand(1, 3, 32, 32).to(device) elif args.dataset == "Imagenet": dummy_input = torch.rand(1, 3, 224, 224).to(device) train_writer.add_graph(model, dummy_input) for epoch in range(1, args.epochs + 1): if args.distributed: train_sampler.set_epoch(epoch) adjust_learning_rate(args, optimizer, epoch, args.zero_lr_for_epochs, train_writer) if not args.run_test and not args.get_inference_time: train(args, model, device, train_loader, optimizer, epoch, criterion, train_writer=train_writer, pruning_engine=pruning_engine) if args.pruning: # skip validation error calculation and model saving if pruning_engine.method == 50: continue # evaluate on validation set prec1 = validate(args, test_loader, model, device, criterion, epoch, train_writer=train_writer) # remember best prec@1 and save checkpoint is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) model_save_path = "%s/models/checkpoint.weights" % (log_save_folder) paths = "%s/models" % (log_save_folder) model_state_dict = model.state_dict() if args.save_models: save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model_state_dict, 'best_prec1': best_prec1, }, is_best, filename=model_save_path) states = { 'epoch': epoch + 1, 'state_dict': model_state_dict, } torch.save(states, '{}/{}.pth.tar'.format(paths, epoch + 1))
transforms.ToTensor(), transforms.Normalize(**p['augmentation_kwargs']['normalize']) #THIS IS RESPONSIBLE FOR DISTORDING THE IMAGE :-! ]) ======= transforms.CenterCrop(p['augmentation_kwargs']['crop_size']), # It was random_resized_crop in simClr now changing batsnet_transformation(), #THIS NEW LINE IS ADDED transforms.ToTensor(), transforms.Normalize(**p['augmentation_kwargs']['normalize']) ]) >>>>>>> db23360031c529a04f0a144b63e5f3fe49feb44f elif p['augmentation_strategy'] == 'ours': # Augmentation strategy from our paper return transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.RandomCrop(p['augmentation_kwargs']['crop_size']), Augment(p['augmentation_kwargs']['num_strong_augs']), transforms.ToTensor(), transforms.Normalize(**p['augmentation_kwargs']['normalize']), Cutout( n_holes = p['augmentation_kwargs']['cutout_kwargs']['n_holes'], length = p['augmentation_kwargs']['cutout_kwargs']['length'], random = p['augmentation_kwargs']['cutout_kwargs']['random'])]) else: raise ValueError('Invalid augmentation strategy {}'.format(p['augmentation_strategy'])) def get_val_transformations(p): return transforms.Compose([ transforms.CenterCrop(p['transformation_kwargs']['crop_size']),
# Hyper Parameters BATCH_SIZE = 8 LR = 0.01 # learning rate EPSILON = 0.9 # greedy policy GAMMA = 0.9 # reward discount TARGET_REPLACE_ITER = 5 # target update frequency MEMORY_CAPACITY = 20 env = AF_env(range_size=3, step_size=2, source_folder='./data/tmp') N_ACTIONS = env.n_actions HIDDEN_SIZE = 256 IMG_SIZE = 224 TEST_INTER = 10 TRANSFORM = transforms.Compose([ transforms.RandomCrop(IMG_SIZE), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) ]) class DQN(object): def __init__(self): self.eval_net, self.target_net = RL_net(HIDDEN_SIZE, N_ACTIONS), RL_net( HIDDEN_SIZE, N_ACTIONS) self.learn_step_counter = 0 # for target updating self.memory = memory(MEMORY_CAPACITY, IMG_SIZE) # initialize memory self.optimizer = torch.optim.Adam(self.eval_net.parameters(), lr=LR) self.loss_func = nn.MSELoss()
straight_through= False # straight-through for gumbel softmax. unclear if it is better one way or the other ).cuda() optimizerVAE = torch.optim.Adam(vae.parameters(), lr=learning_rate) """ text = torch.randint(0, NUM_TOKENS, (BATCH_SIZE, TEXTSEQLEN)) images = torch.randn(BATCH_SIZE, 3, IMAGE_SIZE, IMAGE_SIZE) mask = torch.ones_like(text).bool() """ cap = dset.CocoCaptions( root='./coco/images', annFile='./coco/annotations/captions_val2014.json', transform=transforms.Compose([ transforms.RandomCrop((IMAGE_SIZE, IMAGE_SIZE), pad_if_needed=True), #transforms.Grayscale(), #transforms.Resize((IMAGE_SIZE,IMAGE_SIZE),Image.BILINEAR), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ])) tokenDset = token_dataset('./coco/merged-1000.txt') VAEloss = [] for epoch in range(EPOCHS): for i in range(DATASET_SIZE): #print(i,":",tokenDset.getRand(i),img.size()) optimizerVAE.zero_grad() img, _ = cap[i]
import sys sys.path.append('/opt/cocoapi/PythonAPI') from pycocotools.coco import COCO #!pip install nltk import nltk nltk.download('punkt') from data_loader import get_loader from model import EncoderCNN, DecoderRNN # Define a transform to pre-process the training images. transform_train = transforms.Compose([ transforms.Resize(256), # smaller edge of image resized to 256 transforms.RandomCrop(224), # get 224x224 crop from random location transforms.RandomHorizontalFlip(), # horizontally flip image with probability=0.5 transforms.ToTensor(), # convert the PIL Image to a tensor transforms.Normalize((0.485, 0.456, 0.406), # normalize image for pre-trained model (0.229, 0.224, 0.225))]) # Set the minimum word count threshold. vocab_threshold = 5 # Specify the batch size. batch_size = 10 # Specify the dimensionality of the image embedding. embed_size = 256 data_loader = get_loader(transform=transform_train, mode='train', batch_size=batch_size,
def main(): if args.tensorboard: configure("runs/%s"%(args.name)) if args.augment: transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), ]) else: transform_train = transforms.Compose([ transforms.ToTensor(), ]) transform_test = transforms.Compose([ transforms.ToTensor(), ]) kwargs = {'num_workers': 1, 'pin_memory': True} if args.in_dataset == "CIFAR-10": # Data loading code normalizer = transforms.Normalize(mean=[x/255.0 for x in [125.3, 123.0, 113.9]], std=[x/255.0 for x in [63.0, 62.1, 66.7]]) train_loader = torch.utils.data.DataLoader( datasets.CIFAR10('./datasets/cifar10', train=True, download=True, transform=transform_train), batch_size=args.batch_size, shuffle=True, **kwargs) val_loader = torch.utils.data.DataLoader( datasets.CIFAR10('./datasets/cifar10', train=False, transform=transform_test), batch_size=args.batch_size, shuffle=True, **kwargs) lr_schedule=[50, 75, 90] num_classes = 10 elif args.in_dataset == "CIFAR-100": # Data loading code normalizer = transforms.Normalize(mean=[x/255.0 for x in [125.3, 123.0, 113.9]], std=[x/255.0 for x in [63.0, 62.1, 66.7]]) train_loader = torch.utils.data.DataLoader( datasets.CIFAR100('./datasets/cifar100', train=True, download=True, transform=transform_train), batch_size=args.batch_size, shuffle=True, **kwargs) val_loader = torch.utils.data.DataLoader( datasets.CIFAR100('./datasets/cifar100', train=False, transform=transform_test), batch_size=args.batch_size, shuffle=True, **kwargs) lr_schedule=[50, 75, 90] num_classes = 100 elif args.in_dataset == "SVHN": # Data loading code normalizer = None train_loader = torch.utils.data.DataLoader( svhn.SVHN('datasets/svhn/', split='train', transform=transforms.ToTensor(), download=False), batch_size=args.batch_size, shuffle=True, **kwargs) val_loader = torch.utils.data.DataLoader( svhn.SVHN('datasets/svhn/', split='test', transform=transforms.ToTensor(), download=False), batch_size=args.batch_size, shuffle=False, **kwargs) args.epochs = 20 args.save_epoch = 2 lr_schedule=[10, 15, 18] num_classes = 10 # create model if args.model_arch == 'densenet': model = dn.DenseNet3(args.layers, num_classes, args.growth, reduction=args.reduce, bottleneck=args.bottleneck, dropRate=args.droprate, normalizer=normalizer) elif args.model_arch == 'wideresnet': model = wn.WideResNet(args.depth, num_classes, widen_factor=args.width, dropRate=args.droprate, normalizer=normalizer) else: assert False, 'Not supported model arch: {}'.format(args.model_arch) # get the number of model parameters print('Number of model parameters: {}'.format( sum([p.data.nelement() for p in model.parameters()]))) model = model.cuda() cudnn.benchmark = True # define loss function (criterion) and pptimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, nesterov=True, weight_decay=args.weight_decay) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch, lr_schedule) # train for one epoch train(train_loader, model, criterion, optimizer, epoch) # evaluate on validation set prec1 = validate(val_loader, model, criterion, epoch) # remember best prec@1 and save checkpoint if (epoch + 1) % args.save_epoch == 0: save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), }, epoch + 1)
WORK_DIR = '/tmp/imagenet' NUM_EPOCHS = 10 BATCH_SIZE = 128 LEARNING_RATE = 1e-4 NUM_CLASSES = 10 MODEL_PATH = './models' MODEL_NAME = 'Inception.pth' # Create model if not os.path.exists(MODEL_PATH): os.makedirs(MODEL_PATH) transform = transforms.Compose([ transforms.RandomCrop(256, padding=32), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]) ]) # Load data train_dataset = torchvision.datasets.ImageFolder(root=WORK_DIR + '/' + 'train', transform=transform) train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 参数设置,使得我们能够手动输入命令行参数,就是让风格变得和Linux命令行差不多 parser = argparse.ArgumentParser(description='PyTorch CIFAR100 Training') parser.add_argument('--outf', default='./model/', help='folder to output images and model checkpoints') #输出结果保存路径 parser.add_argument('--net', default='./model/Resnet18.pth', help="path to net (to continue training)") #恢复训练时的模型路径 args = parser.parse_args() EPOCH = 40 #遍历数据集次数 pre_epoch = 0 # 定义已经遍历数据集的次数 BATCH_SIZE = 4 #批处理尺寸(batch_size) LR = 0.001 #学习率 inputs = 32 # 准备数据集并预处理 transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), #先四周填充0,在吧图像随机裁剪成32*32 transforms.RandomHorizontalFlip(), #图像一半的概率翻转,一半的概率不翻转 transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), #R,G,B每层的归一化用到的均值和方差 ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) trainset = torchvision.datasets.CIFAR100(root='/home/wang/data100', train=True, download=True, transform=transform_train) #训练数据集 trainloader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2) #生成一个个batch进行批训练,组成batch的时候顺序打乱取 testset = torchvision.datasets.CIFAR100(root='/home/wang/data100', train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=2, shuffle=False, num_workers=2)
def get_transforms(dataset): transform_train = None transform_test = None if dataset == 'cifar10': transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if dataset == 'cifar100': transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761)), ]) if dataset == 'cinic-10': # cinic_directory = '/path/to/cinic/directory' cinic_mean = [0.47889522, 0.47227842, 0.43047404] cinic_std = [0.24205776, 0.23828046, 0.25874835] transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(cinic_mean, cinic_std) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(cinic_mean, cinic_std) ]) if dataset == 'tiny_imagenet': tiny_mean = [ 0.48024578664982126, 0.44807218089384643, 0.3975477478649648 ] tiny_std = [0.2769864069088257, 0.26906448510256, 0.282081906210584] transform_train = transforms.Compose([ transforms.RandomCrop(64, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(tiny_mean, tiny_std) ]) transform_test = transforms.Compose( [transforms.ToTensor(), transforms.Normalize(tiny_mean, tiny_std)]) assert transform_test is not None and transform_train is not None, 'Error, no dataset %s' % dataset return transform_train, transform_test
print('successfully loaded model') except: raise seed = 42 torch.manual_seed(seed) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = model.to(device) model.eval() test_transform = transforms.Compose( [transforms.Scale(256), transforms.RandomCrop(224), transforms.ToTensor()]) test_df = pd.read_csv(args.test_csv, header=None) test_imgs = test_df[0] pbar = tqdm(total=len(test_imgs)) mean, std = 0.0, 0.0 for i, img in enumerate(test_imgs): im = Image.open(os.path.join(args.test_images, str(img) + '.jpg')) im = im.convert('RGB') imt = test_transform(im) imt = imt.unsqueeze(dim=0) imt = imt.to(device) with torch.no_grad(): out = model(imt)
def train_prednet(lr=0.01, cnn=False, model='PredNetTied', circles=2, gpunum=2): use_cuda = torch.cuda.is_available() best_acc = 0 # best test accuracy start_epoch = 0 # start from epoch 0 or last checkpoint epoch batchsize = 256 root = './' lr = lr models = { 'PredictiveTied_LN_Multi': PredictiveTied_LN_Multi, 'PredictiveTied_GN': PredictiveTied_GN, 'PredictiveTiedKL': PredictiveTiedKL, 'PredictiveTiedInstanceNorm': PredictiveTiedInstanceNorm, 'PredictiveTiedLayerNorm': PredictiveTiedLayerNorm, 'PredictiveTied3': PredictiveTied3, 'PredictiveTied': PredictiveTied, 'PredictiveComb': PredictiveComb, 'PredictiveSELU': PredictiveSELU, 'PredictiveNew': PredictiveNew, 'Predictive': Predictive, 'PredictiveFull': PredictiveFull, 'PredNetNew': PredNetNew, 'PredNet': PredNet, 'PredNetLocal': PredNetLocal, 'PredictiveNew2': PredictiveNew2, 'PredictiveNew3': PredictiveNew3, 'PredictiveTied_LN': PredictiveTied_LN } if cnn: modelname = model + '_' else: modelname = model + '_' + str(circles) + 'CLS_' # clearn folder checkpointpath = root + 'checkpoint/' logpath = root + 'log/' if not os.path.isdir(checkpointpath): os.mkdir(checkpointpath) if not os.path.isdir(logpath): os.mkdir(logpath) # Data print('==> Preparing data..') transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) trainset = torchvision.datasets.CIFAR100(root='./data', train=True, download=True, transform=transform_train) trainloader = torch.utils.data.DataLoader(trainset, batch_size=batchsize, shuffle=True, num_workers=2) testset = torchvision.datasets.CIFAR100(root='./data', train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2) # Model print('==> Building model..') if cnn: net = models[model](num_classes=100) else: net = models[model](num_classes=100, cls=circles) # print(1) # Define objective function criterion = nn.CrossEntropyLoss() # print(2) if model == 'PredictiveComb': convparas = [p for p in net.FFconv.parameters()]+\ [p for p in net.FBconv.parameters()]+\ [p for p in net.linear.parameters()]+\ [p for p in net.comb.parameters()] optimizer = optim.SGD([ { 'params': convparas }, ], lr=lr, momentum=0.9, weight_decay=5e-4) elif model == 'PredictiveSELU': convparas = [p for p in net.FFconv.parameters()]+\ [p for p in net.FBconv.parameters()]+\ [p for p in net.linear.parameters()]+\ [p for p in net.selu.parameters()] rateparas = [p for p in net.b0.parameters()] optimizer = optim.SGD([ { 'params': convparas }, { 'params': rateparas, 'weight_decay': 0 }, ], lr=lr, momentum=0.9, weight_decay=5e-4) elif model == 'PredictiveTied3': convparas = [p for p in net.conv.parameters()]+\ [p for p in net.linear.parameters()] rateparas = [p for p in net.a0.parameters()]+\ [p for p in net.b0.parameters()]+\ [p for p in net.c0.parameters()] optimizer = optim.SGD([ { 'params': convparas }, { 'params': rateparas, 'weight_decay': 0 }, ], lr=lr, momentum=0.9, weight_decay=5e-4) elif model == 'PredictiveTiedKL': convparas = [p for p in net.conv.parameters()]+\ [p for p in net.linear.parameters()] rateparas = [p for p in net.b0.parameters()] optimizer = optim.SGD([ { 'params': convparas }, { 'params': rateparas, 'weight_decay': 0 }, ], lr=lr, momentum=0.9, weight_decay=5e-4) elif 'PredictiveTied' in model: convparas = [p for p in net.conv.parameters()]+\ [p for p in net.linear.parameters()] rateparas = [p for p in net.a0.parameters()]+\ [p for p in net.b0.parameters()] optimizer = optim.SGD([ { 'params': convparas }, { 'params': rateparas, 'weight_decay': 0 }, ], lr=lr, momentum=0.9, weight_decay=5e-4) else: convparas = [p for p in net.FFconv.parameters()]+\ [p for p in net.FBconv.parameters()]+\ [p for p in net.linear.parameters()] rateparas = [p for p in net.a0.parameters()]+\ [p for p in net.b0.parameters()] optimizer = optim.SGD([ { 'params': convparas }, { 'params': rateparas, 'weight_decay': 0 }, ], lr=lr, momentum=0.9, weight_decay=5e-4) print(3) # Parallel computing if use_cuda: net.cuda() net = torch.nn.DataParallel(net, device_ids=range(gpunum)) # net = torch.nn.DataParallel(net, device_ids=[3]) cudnn.benchmark = True print(4) # Training def train(epoch): print('\nEpoch: %d' % epoch) net.train() train_loss = 0 correct = 0 total = 0 training_setting = 'batchsize=%d | epoch=%d | lr=%.1e ' % ( batchsize, epoch, optimizer.param_groups[0]['lr']) statfile.write('\nTraining Setting: ' + training_setting + '\n') for batch_idx, (inputs, targets) in enumerate(trainloader): if use_cuda: inputs, targets = inputs.cuda(), targets.cuda() optimizer.zero_grad() inputs, targets = Variable(inputs), Variable(targets) outputs = net(inputs) #print(outputs.size()) #print(targets.size()) if model == 'PredictiveTied_LN_Multi': loss = 0 for i in range(len(outputs)): loss = loss + criterion(outputs[i], targets) loss.backward() optimizer.step() else: loss = criterion(outputs, targets) loss.backward() optimizer.step() # train_loss += loss.data[0] train_loss += loss.item() if model == 'PredictiveTied_LN_Multi': _, predicted = torch.max(outputs[-1].data, 1) else: _, predicted = torch.max(outputs.data, 1) total += targets.size(0) correct += predicted.eq(targets.data).cpu().sum() progress_bar( batch_idx, len(trainloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)' % (train_loss / (batch_idx + 1), 100. * (float)(correct) / (float)(total), correct, total)) statstr = 'Training: Epoch=%d | Loss: %.3f | Acc: %.3f%% (%d/%d) | best acc: %.3f' \ % (epoch, train_loss/(batch_idx+1), 100.*(float)(correct)/(float)(total), correct, total, best_acc) statfile.write(statstr + '\n') return train_loss / (batch_idx + 1) # Testing def test(epoch): nonlocal best_acc net.eval() test_loss = 0 correct = 0 total = 0 for batch_idx, (inputs, targets) in enumerate(testloader): if use_cuda: inputs, targets = inputs.cuda(), targets.cuda() optimizer.zero_grad() inputs, targets = Variable(inputs), Variable(targets) outputs = net(inputs) # print(outputs.size(),targets.size()) if model == 'PredictiveTied_LN_Multi': loss = 0 for i in range(len(outputs)): loss = criterion(outputs[i], targets) test_loss += loss.item() else: loss = criterion(outputs, targets) test_loss += loss.item() if model == 'PredictiveTied_LN_Multi': _, predicted = torch.max(outputs[-1].data, 1) else: _, predicted = torch.max(outputs.data, 1) total += targets.size(0) correct += predicted.eq(targets.data).cpu().sum() progress_bar( batch_idx, len(testloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)' % (test_loss / (batch_idx + 1), 100. * (float)(correct) / (float)(total), correct, total)) statstr = 'Testing: Epoch=%d | Loss: %.3f | Acc: %.3f%% (%d/%d) | best_acc: %.3f' \ % (epoch, test_loss/(batch_idx+1), 100.*(float)(correct)/(float)(total), correct, total, best_acc) statfile.write(statstr + '\n') # Save checkpoint. acc = 100. * correct / total state = { 'state_dict': net.state_dict(), 'acc': acc, 'epoch': epoch, # 'net': net.module if use_cuda else net, } torch.save(state, checkpointpath + modelname + '_last_ckpt.t7') if acc >= best_acc: print('Saving..') torch.save(state, checkpointpath + modelname + '_best_ckpt.t7') best_acc = acc # Set adaptive learning rates def decrease_learning_rate(): """Decay the previous learning rate by 2""" for param_group in optimizer.param_groups: param_group['lr'] /= 2 print('decreasing lr') def decrease_learning_rate_large(): """Decay the previous learning rate by 10""" for param_group in optimizer.param_groups: param_group['lr'] /= 10 print('decreasing lr') LossList = [] for epoch in range(start_epoch, start_epoch + 200): statfile = open( logpath + 'training_stats_' + modelname + '_with' + '.txt', 'a+') if len(LossList) < 5: LossList.append(train(epoch)) else: LossList[0] = LossList[1] LossList[1] = LossList[2] LossList[2] = LossList[3] LossList[3] = LossList[4] LossList[4] = train(epoch) slope = (-2 * (float)(LossList[0]) - (float)(LossList[1]) + (float)(LossList[3]) + 2 * (float)(LossList[4])) / (float)(LossList[4]) print(-slope) if model == 'PridictiveRELU': if -slope < 0.5: decrease_learning_rate() elif LossList[4] < 0.005: break elif epoch % 10 == 0: if -slope > 0.3 and -slope < 0.5: decrease_learning_rate() elif -slope < 0.3: decrease_learning_rate_large() if LossList[4] < 0.01: break test(epoch)
NUM_WORKERS = cfg['NUM_WORKERS'] print("=" * 60) print("Overall Configurations:") print(cfg) print("=" * 60) writer = SummaryWriter( LOG_ROOT) # writer for buffering intermedium results train_transform = transforms.Compose( [ # refer to https://pytorch.org/docs/stable/torchvision/transforms.html for more build-in online data augmentation transforms.Resize([ int(128 * INPUT_SIZE[0] / 112), int(128 * INPUT_SIZE[0] / 112) ]), # smaller side resized transforms.RandomCrop([INPUT_SIZE[0], INPUT_SIZE[1]]), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean=RGB_MEAN, std=RGB_STD), ]) dataset_train = datasets.ImageFolder(os.path.join(DATA_ROOT, 'imgs'), train_transform) # create a weighted random sampler to process imbalanced data weights = make_weights_for_balanced_classes(dataset_train.imgs, len(dataset_train.classes)) weights = torch.DoubleTensor(weights) sampler = torch.utils.data.sampler.WeightedRandomSampler( weights, len(weights))
'val_hard': 'DVQA_val_hard_qa_oracle.json'} val_files['FigureQA'] = {'val1': 'FigureQA_val1_qa.json', 'val2': 'FigureQA_val2_qa.json'} test_files = dict() test_files['FigureQA'] = {'test1': 'FigureQA_test1_qa.json', 'test2': 'FigureQA_test2_qa.json'} test_files['DVQA'] = {} transform_combo_train = dict() transform_combo_test = dict() transform_combo_train['FigureQA'] = transforms.Compose([ transforms.Resize((224, 320)), transforms.RandomCrop(size=(224, 320), padding=8), transforms.RandomRotation(2.8), transforms.ToTensor(), transforms.Normalize(mean=[0.9365, 0.9303, 0.9295], std=[1, 1, 1]) ]) transform_combo_train['DVQA'] = transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(size=(256, 256), padding=8), transforms.RandomRotation(2.8), transforms.ToTensor(), transforms.Normalize(mean=[0.8744, 0.8792, 0.8836], std=[1, 1, 1]) ])
def make_dataset(): # Small noise is added, following SN-GAN def noise(x): return x + torch.FloatTensor(x.size()).uniform_(0, 1.0 / 128) if opt.dataset == "cifar10": trans = tfs.Compose([ tfs.RandomCrop(opt.img_width, padding=4), tfs.RandomHorizontalFlip(), tfs.ToTensor(), tfs.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5]), tfs.Lambda(noise) ]) data = CIFAR10(root=opt.root, train=True, download=True, transform=trans) loader = DataLoader(data, batch_size=opt.batch_size, shuffle=True, num_workers=opt.workers) elif opt.dataset == "dog_and_cat_64": trans = tfs.Compose([ tfs.RandomResizedCrop(opt.img_width, scale=(0.8, 0.9), ratio=(1.0, 1.0)), tfs.RandomHorizontalFlip(), tfs.ToTensor(), tfs.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5]), tfs.Lambda(noise) ]) data = ImageFolder(opt.root, transform=trans) loader = DataLoader(data, batch_size=opt.batch_size, shuffle=True, num_workers=opt.workers) elif opt.dataset == "dog_and_cat_128": trans = tfs.Compose([ tfs.RandomResizedCrop(opt.img_width, scale=(0.8, 0.9), ratio=(1.0, 1.0)), tfs.RandomHorizontalFlip(), tfs.ToTensor(), tfs.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5]), tfs.Lambda(noise) ]) data = ImageFolder(opt.root, transform=trans) loader = DataLoader(data, batch_size=opt.batch_size, shuffle=True, num_workers=opt.workers) elif opt.dataset == "imagenet": trans = tfs.Compose([ tfs.RandomResizedCrop(opt.img_width, scale=(0.8, 0.9), ratio=(1.0, 1.0)), tfs.RandomHorizontalFlip(), tfs.ToTensor(), tfs.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5]), tfs.Lambda(noise) ]) data = ImageFolder(opt.root, transform=trans) loader = DataLoader(data, batch_size=opt.batch_size, shuffle=True, num_workers=opt.workers) else: raise ValueError(f"Unknown dataset: {opt.dataset}") return loader
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) if args.arch.startswith('efficientnet'): transform_train = transforms.Compose([ transforms.Resize(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.Resize(224), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) else: transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 trainset = dataloader(root=args.datapath, train=True, download=False, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root=args.datapath, train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) elif args.arch.startswith('efficientnet'): model = EfficientNet.from_pretrained(args.arch, num_classes=num_classes) else: model = models.__dict__[args.arch](num_classes=num_classes) modeltype = type(model).__name__ model, classifier = decomposeModel(model) classifier = classifier.cuda() model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) reg_net = None optimizer_reg = optim.SGD(classifier.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) if args.mtype == 'baseline': args.dcl_refsize = 0 reg_net = regressor.Net(classifier, optimizer_reg, ref_size=args.dcl_refsize, backendtype=modeltype) elif args.mtype == 'dcl': reg_net = regressor.Net(classifier, optimizer_reg, ref_size=args.dcl_refsize, backendtype=modeltype, dcl_offset=args.dcl_offset, dcl_window=args.dcl_window, QP_margin=args.dcl_QP_margin) elif args.mtype == 'gem': reg_net = gem.Net(classifier, optimizer_reg, n_memories=args.gem_memsize, backendtype=modeltype) print(args) # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Err.', 'Valid Err.', 'Cos Bef.', 'Cos Aft.', 'Mag' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate_two(optimizer, optimizer_reg, epoch) print('Epoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_top1_acc, train_top5_acc, cong_bf, cong_af, mag = train( trainloader, model, criterion, optimizer, epoch, use_cuda, reg_net=reg_net) test_loss, test_top1_acc, test_top5_acc = test(testloader, model, criterion, epoch, use_cuda, reg_net=reg_net) print( 'train_loss: {:.4f}, train_top1_err: {:.2f}, train_top5_err: {:.2f}, test_loss: {:.4f}, test_top1_err: {:.2f}, test_top5_err: {:.2f}, cong_bf: {:.4f}, cong_af: {:.4f}, mag: {:.4f}' .format(train_loss, 100 - train_top1_acc, 100 - train_top5_acc, test_loss, 100 - test_top1_acc, 100 - test_top5_acc, cong_bf, cong_af, mag)) # append logger file logger.append([ state['lr'], train_loss, test_loss, 100 - train_top1_acc, 100 - test_top1_acc, cong_bf, cong_af, mag ]) # save model is_best = test_top1_acc > best_acc best_acc = max(test_top1_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'classifier_state_dict': reg_net.state_dict(), 'acc': test_top1_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), 'cong_bf': cong_bf, 'cong_af': cong_af, 'mag': mag, }, is_best, checkpoint=args.checkpoint) logger.close() print('Best err:') print(100 - best_acc)
def main(): global args, best_prec1, min_loss args = parser.parse_args() rank, world_size = dist_init(args.port) assert (args.batch_size % world_size == 0) assert (args.workers % world_size == 0) args.batch_size = args.batch_size // world_size args.workers = args.workers // world_size # create model print("=> creating model '{}'".format(args.arch)) print("train source is:{}".format(args.train_source)) print("save_path is: {}", format(args.save_path)) if args.arch.startswith('inception_v3'): print('inception_v3 without aux_logits!') image_size = 341 input_size = 299 model = models.__dict__[args.arch](pretrained=True) else: image_size = 256 input_size = 224 model = models.__dict__[args.arch](pretrained=True) model = FineTuneModel(model, args.arch, 128) # print("model is: {}".format(model)) model.cuda() model = DistModule(model) # optionally resume from a checkpoint if args.load_path: if args.resume_opt: best_prec1, start_epoch = load_state(args.load_path, model, optimizer=optimizer) else: # print('load weights from', args.load_path) load_state(args.load_path, model) cudnn.benchmark = True # Data loading code normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = McDataset( args.train_root, args.train_source, transforms.Compose([ transforms.Resize(image_size), transforms.RandomCrop(input_size), transforms.RandomHorizontalFlip(), transforms.ToTensor(), ColorAugmentation(), normalize, ])) val_dataset = McDataset( args.val_root, args.val_source, transforms.Compose([ transforms.Resize(image_size), transforms.CenterCrop(input_size), # transforms.Resize(input_size), # transforms.CenterCrop(input_size), transforms.ToTensor(), normalize, ])) train_sampler = DistributedSampler(train_dataset) val_sampler = DistributedSampler(val_dataset) train_loader = DataLoader( train_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=False, sampler=train_sampler) val_loader = DataLoader( val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=False, sampler=val_sampler) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss() lr = 0 patience = 0 for epoch in range(args.start_epoch, args.epochs): # adjust_learning_rate(optimizer, epoch) train_sampler.set_epoch(epoch) if epoch == 3: lr = 0.00003 if patience == 2: # if epoch == 0: patience = 0 checkpoint = load_checkpoint(args.save_path+'_best.pth.tar') model.load_state_dict(checkpoint['state_dict']) print("Loading checkpoint_best.............") # lr = adjust_learning_rate(optimizer, lr) lr = lr / 10.0 if epoch < 3: lr = 0.001 for name, param in model.named_parameters(): # print("name is: {}".format(name)) if (name != 'module.classifier.0.weight' and name != 'module.classifier.0.bias'): param.requires_grad = False optimizer = torch.optim.Adam( filter(lambda p: p.requires_grad, model.parameters()), lr=lr) else: for param in model.parameters(): param.requires_grad = True optimizer = torch.optim.Adam( model.parameters(), lr=lr, weight_decay=0.0001) print("lr is: {}".format(lr)) # train for one epoch train(train_loader, model, criterion, optimizer, epoch) # evaluate on validation set val_prec1, val_losses = validate(val_loader, model, criterion) print("val_losses is: {}".format(val_losses)) # remember best prec@1 and save checkpoint if rank == 0: # remember best prec@1 and save checkpoint if val_losses < min_loss: is_best = True save_checkpoint({ 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best, args.save_path) # torch.save(model.state_dict(), 'best_val_weight.pth') print('val score improved from {:.5f} to {:.5f}. Saved!'.format(min_loss, val_losses)) min_loss = val_losses patience = 0 else: patience += 1 if rank == 1 or rank == 2 or rank == 3 or rank == 4 or rank == 5 or rank == 6 or rank == 7: if val_losses < min_loss: min_loss = val_losses patience = 0 else: patience += 1 print("patience is: {}".format(patience)) print("min_loss is: {}".format(min_loss))
help='Loss of the model') args = parser.parse_args() if torch.cuda.is_available(): device = 'cuda' else: device = 'cpu' preprocess = transforms.Compose([ transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.5), transforms.RandomHorizontalFlip(p=0.5), transforms.RandomGrayscale(p=0.1), transforms.RandomCrop(size=28, pad_if_needed=True), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) val_process = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) if args.mode == 'pretrain' or 'cont_pre': train_folder = 'train_data/medium' val_folder = 'validation_classification/medium' elif args.mode == 'finetune':
def main_worker(gpu, ngpus_per_node, args): global best_acc1 args.gpu = gpu if args.gpu is not None: print("Use GPU: {} for training".format(args.gpu)) if args.distributed: if args.dist_url == "env://" and args.rank == -1: args.rank = int(os.environ["RANK"]) if args.multiprocessing_distributed: # For multiprocessing distributed training, rank needs to be the # global rank among all the processes args.rank = args.rank * ngpus_per_node + gpu dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank) else: print("We should be distributed") return # create model model = None if args.arch == "alexnet": model = AlexNet() if args.arch == "vgg16": model = VGG16() if args.arch == "vgg19": model = VGG19() if args.arch == "vgg13": model = VGG13() if args.arch == "lenet": model = LeNet() if args.arch == "resnet50": model = resnet50() if args.arch == "goolenet": model = GoogLeNet() torch.cuda.set_device(args.gpu) model.cuda(args.gpu) # When using a single GPU per process and per # DistributedDataParallel, we need to divide the batch size # ourselves based on the total number of GPUs we have args.batch_size = int(args.batch_size / ngpus_per_node) args.workers = int(args.workers / ngpus_per_node) # model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) model = distributed_layer_sparse_whole_randperm_threshold.DistributedDataParallel( model, device_id=args.gpu, sparse_ratio=args.sparse_ratio, sparse_threshold=args.sparse_threshold) args.file_name = args.method + '_' + args.arch + "batch-size_" + str(args.batch_size) + "_sparse-ratio_" + str(args.sparse_ratio) + \ "_sparse-threshold_" + str(args.sparse_threshold) + "_memory-decay_" + str(args.memory_decay) criterion = nn.CrossEntropyLoss().cuda(args.gpu) optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_acc1 = checkpoint['best_acc1'] if args.gpu is not None: # best_acc1 may be from a checkpoint from a different GPU best_acc1 = best_acc1.to(args.gpu) model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True # Data loading code #traindir = os.path.join(args.data, 'train') #valdir = os.path.join(args.data, 'val') #normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], # std=[0.229, 0.224, 0.225]) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) train_dataset = torchvision.datasets.CIFAR10(root=args.data, train=True, download=True, transform=transform_train) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset) else: train_sampler = None train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), num_workers=args.workers, sampler=train_sampler) testset = torchvision.datasets.CIFAR10(root=args.data, train=False, download=True, transform=transform_test) val_loader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2) if args.evaluate: validate(val_loader, model, criterion, args) return epoch_start = time.time() for epoch in range(args.start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) adjust_learning_rate(optimizer, epoch, args) # train for one epoch losses = train(train_loader, model, criterion, optimizer, epoch, args) # evaluate on validation set acc1 = validate(val_loader, model, criterion, args) # remember best acc@1 and save checkpoint is_best = acc1 > best_acc1 best_acc1 = max(acc1, best_acc1) if not args.multiprocessing_distributed or ( args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_acc1': best_acc1, 'optimizer': optimizer.state_dict(), }, is_best) f = open(args.data + "/result_" + args.file_name + ".txt", "a+") f.write( str(epoch + 1) + '\t' + str(time.time() - epoch_start) + '\t' + str(losses.avg) + '\t' + str(acc1.item()) + '\n') f.close()
def load_data(args): args.batch_size = 1 train_loader = [] test_loader = [] if args.dataset_mode == "CIFAR10": transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ]) train_loader = torch.utils.data.DataLoader( datasets.CIFAR10('data', train=True, download=True, transform=transform_train), batch_size=args.batch_size, shuffle=True, num_workers=0 ) test_loader = torch.utils.data.DataLoader( datasets.CIFAR10('data', train=False, transform=transform_test), batch_size=args.batch_size, shuffle=True, num_workers=1 ) elif args.dataset_mode == "CIFAR100": transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.5071, 0.4865, 0.4409), (0.2673, 0.2564, 0.2762)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5071, 0.4865, 0.4409), (0.2673, 0.2564, 0.2762)), ]) train_loader = torch.utils.data.DataLoader( datasets.CIFAR100('data', train=True, download=True, transform=transform_train), batch_size=args.batch_size, shuffle=True, num_workers=2 ) test_loader = torch.utils.data.DataLoader( datasets.CIFAR100('data', train=False, transform=transform_test), batch_size=args.batch_size, shuffle=False, num_workers=2 ) elif args.dataset_mode == "MNIST": transform_train = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)), ]) train_loader = torch.utils.data.DataLoader( datasets.MNIST('data/newMNIST', train=True, download=True, transform=transform_train), batch_size=args.batch_size, shuffle=True, num_workers=2 ) test_loader = torch.utils.data.DataLoader( datasets.MNIST('data/newMNIST', train=False, transform=transform_test), batch_size=args.batch_size, shuffle=True, num_workers=2 ) return train_loader, test_loader
def weight_pruning_by_name(model, layer_info): parent, child = layer_info exec('model.{}.{}.gate.z = 1'.format(str(parent), str(child)) def cifar10_loader(): batch_size = 128 print("cifar10 Data Loading ...") transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean=(0.4914, 0.4824, 0.4467), std=(0.2471, 0.2436, 0.2616)) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=(0.4914, 0.4824, 0.4467), std=(0.2471, 0.2436, 0.2616)) ]) train_dataset = datasets.CIFAR10(root='../hhjung/cifar10/', train=True, transform=transform_train, download=True) test_dataset = datasets.CIFAR10(root='../hhjung/cifar10/', train=False, transform=transform_test) train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=4) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False, num_workers=4) return train_loader, test_loader def cifar100_loader(): batch_size = 128 print("cifar100 Data Loading ...") transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean=(0.5071, 0.4867, 0.4408), std=(0.2675, 0.2565, 0.2761)) ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=(0.5071, 0.4867, 0.4408), std=(0.2675, 0.2565, 0.2761)) ]) train_dataset = datasets.CIFAR100(root='../hhjung/cifar100/', train=True, transform=transform_train, download=True) test_dataset = datasets.CIFAR100(root='../hhjung/cifar100/', train=False, transform=transform_test) train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=4) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False, num_workers=4) return train_loader, test_loader
for k, ema_v in self.ema.state_dict().items(): if needs_module: k = 'module.' + k model_v = msd[k].detach() model_v = model_v.to(device) ema_v.copy_(ema_v * self.decay + (1. - self.decay) * model_v) # weight decay if 'bn' not in k: msd[k] = msd[k] * (1. - self.wd) # Data print('==> Preparing data..') if args.dataset == 'CIFAR10' or args.dataset == 'CIFAR100': transform_ori = transforms.Compose([ transforms.RandomCrop(32, padding=4, padding_mode='reflect'), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616)), ]) transform_aug = transforms.Compose([ transforms.ToTensor(), Cutout(n_holes=args.n_holes, length=args.cutout_size), transforms.ToPILImage(), transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4, padding_mode='reflect'), CIFAR10Policy(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616)),
def preprocess(self, data): ## Scales, crops, and normalizes a DICOM image for a PyTorch model, returns an Numpy array img_trsfm = transforms.Compose([ transforms.ToPILImage(), # convert to PIL image. PIL does not support multi-channel floating point data, workaround is to transform first then expand to 3 channel transforms.Resize([516, 516], interpolation=2), # resize and crop to [512x512] transforms.RandomCrop(512), transforms.ToTensor(), transforms.Lambda(lambda x: x.repeat(3, 1, 1)), # classifier is fine-tuned on pretrained imagenet using mimic-jpeg images [HxWx3] transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) for k, v in data[0].items(): logger.info('received data in post request: {0}: {1}'.format( k, v.decode())) ## download dicom from s3 bucket = data[0].get("dicombucket").decode() key = data[0].get("dicomkey").decode() if self.access_key is not None and self.secret_key is not None and self.aws_region is not None: s3_client = boto3.client('s3', region_name=self.aws_region, aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key) ddb_client = boto3.client('dynamodb', region_name=self.aws_region, aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key) else: my_session = boto3.session.Session() s3_client = my_session.client('s3') ddb_client = my_session.client('dynamodb') s3_client.download_file(bucket, key, key) dicom = pydicom.dcmread(key) os.remove(key) if dicom.get_item('ViewPosition') and len( dicom.data_element('ViewPosition').value) > 0: dicom_array = dicom.pixel_array.astype(float) ## convert dicom to png thumbnail dicom_array_scaled = (np.maximum(dicom_array, 0) / dicom_array.max()) * 255.0 dicom_array_scaled = np.uint8(dicom_array_scaled) out_png = key.replace('.dcm', '.full.png') with open(out_png, 'wb') as out_png_file: w = png.Writer(dicom_array.shape[1], dicom_array.shape[0], greyscale=True) w.write(out_png_file, dicom_array_scaled) pilimage = Image.open(out_png) os.remove(out_png) newsize = (int(pilimage.size[0] / 10), int(pilimage.size[1] / 10) ) ## thumbnail is 1/10 in size pilimage = pilimage.resize(newsize) thumbnail_png = key.replace('.dcm', '.png') pilimage.save(thumbnail_png) ## upload png thumbnail to s3 pngbucket = data[0].get("pngbucket").decode() prefix = data[0].get("prefix").decode() s3_thumbnail_png = prefix + '/' + thumbnail_png try: s3_client.head_object(Bucket=pngbucket, Key=s3_thumbnail_png) except: try: s3_client.upload_file(thumbnail_png, pngbucket, s3_thumbnail_png) os.remove(thumbnail_png) except ClientError as e: print('upload thumbnail png error: {}'.format(e)) ## dicom metadata to be saved in dynamodb metadata_dict = {"ImageId": {"S": key.replace('.dcm', '')}} metadata_dict["ViewPosition"] = { "S": dicom.data_element('ViewPosition').value } metadata_dict["Bucket"] = {"S": pngbucket} metadata_dict["Key"] = {"S": thumbnail_png} metadata_dict["ReportId"] = { "S": 's' + dicom.data_element('StudyID').value } metadata_dict["Modality"] = { "S": dicom.data_element('Modality').value } metadata_dict["BodyPartExamined"] = { "S": dicom.data_element('BodyPartExamined').value } ddb_table = data[0].get("ddb_table").decode() response = ddb_client.put_item(TableName=ddb_table, Item=metadata_dict) logger.info('Dynamodb create item status: {}'.format( response['ResponseMetadata']['HTTPStatusCode'])) ## transform dicom arrary for pytorch model featurization X = np.asarray(dicom_array, np.float32) / (2**dicom.BitsStored - 1) image = img_trsfm(X).unsqueeze(0) es_endpoint = data[0].get("es_endpoint").decode() return { 'id': key.split('.')[0], 'image': image, 'ViewPosition': dicom.data_element('ViewPosition').value, 'ES': es_endpoint } else: return None
transform_ = transforms.Compose([ #transforms.CenterCrop(size = (200, 200)), transforms.Resize(size=(192, 192), interpolation=1), ]) transform_flip = transforms.Compose([ #transforms.CenterCrop(size = (200, 200)), transforms.Resize(size=(192, 192), interpolation=1), transforms.RandomHorizontalFlip(p=1), ]) transform_crop = transforms.Compose([ #transforms.CenterCrop(size = (200, 200)), transforms.Resize(size=(192, 192), interpolation=1), transforms.RandomCrop(192, padding=7), ]) transform_flip_crop = transforms.Compose([ #transforms.CenterCrop(size = (200, 200)), transforms.Resize(size=(192, 192), interpolation=1), transforms.RandomCrop(192, padding=7), transforms.RandomHorizontalFlip(p=1), ]) def pil2np(img): arr = np.array(img) arr = arr.transpose(2, 0, 1) return arr.astype(np.float32) / 255
def make_dataloader_Pseudo(cfg): if cfg.DATASETS.HARD_AUG: train_transforms = T.Compose([ T.Resize(cfg.INPUT.SIZE_TRAIN), T.RandomHorizontalFlip(p=cfg.INPUT.PROB), T.Pad(cfg.INPUT.PADDING), T.RandomCrop(cfg.INPUT.SIZE_TRAIN), T.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.2), T.transforms.RandomAffine(0, translate=None, scale=[0.9, 1.1], shear=None, resample=False, fillcolor=128), T.ToTensor(), T.Normalize(mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD), RandomErasing(probability=cfg.INPUT.RE_PROB, mean=cfg.INPUT.PIXEL_MEAN) ]) else: train_transforms = T.Compose([ T.Resize(cfg.INPUT.SIZE_TRAIN), T.RandomHorizontalFlip(p=cfg.INPUT.PROB), T.Pad(cfg.INPUT.PADDING), T.RandomCrop(cfg.INPUT.SIZE_TRAIN), T.ToTensor(), T.Normalize(mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD), RandomErasing(probability=cfg.INPUT.RE_PROB, mean=cfg.INPUT.PIXEL_MEAN) ]) val_transforms = T.Compose([ T.Resize(cfg.INPUT.SIZE_TEST), T.ToTensor(), T.Normalize(mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD) ]) num_workers = cfg.DATALOADER.NUM_WORKERS dataset = __factory[cfg.DATASETS.NAMES](root=cfg.DATASETS.ROOT_DIR) num_classes = dataset.num_train_pids train_set = ImageDataset(dataset.train, train_transforms) if 'triplet' in cfg.DATALOADER.SAMPLER: train_loader = DataLoader( train_set, batch_size=cfg.SOLVER.IMS_PER_BATCH, sampler=RandomIdentitySampler(dataset.train, cfg.SOLVER.IMS_PER_BATCH, cfg.DATALOADER.NUM_INSTANCE), num_workers=num_workers, collate_fn=train_collate_fn, pin_memory=True ) elif cfg.DATALOADER.SAMPLER == 'softmax': print('using softmax sampler') train_loader = DataLoader( train_set, batch_size=cfg.SOLVER.IMS_PER_BATCH, shuffle=True, num_workers=num_workers, collate_fn=train_collate_fn ) else: print('unsupported sampler! expected softmax or triplet but got {}'.format(cfg.SAMPLER)) ''' val_set_green = ImageDataset(dataset.query_green + dataset.gallery_green, val_transforms) val_loader_green = DataLoader( val_set_green, batch_size=cfg.TEST.IMS_PER_BATCH, shuffle=False, num_workers=num_workers, collate_fn=val_collate_fn ) ''' #return train_loader, val_loader_green, len(dataset.query_green), num_classes, dataset, train_set, train_transforms return train_loader, num_classes, dataset, train_set, train_transforms
import numpy as np from PIL import Image import torch from torch.utils.data import Dataset import torchvision.transforms as transforms from skimage.color import rgb2lab, rgb2gray data_augmentation = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.RandomCrop(256), transforms.Resize(224) ]) class ColorDataset(Dataset): def __init__(self, phase): assert (phase in ['train', 'val', 'test']) self.phase = phase self.root_dir = '/home/wsf/Pictures/Wallpapers' self.samples = None with open('{}/labels/{}.txt'.format(self.root_dir, phase), 'r') as f: self.samples = f.readlines()[:3] print('[+] dataset `{}` loaded {} images'.format(self.phase, len(self.samples))) def __getitem__(self, idx): if self.phase == 'train' or self.phase == 'val': image_path, label = self.samples[idx].strip().split() label = np.array(int(label)) image = Image.open('{}/images/{}'.format(self.root_dir, image_path)).convert('RGB') image = data_augmentation(image)