def get_train_val_loaders(self): if self.args.dataset == 'cifar10': train_transform, valid_transform = utils._data_transforms_cifar10(self.args) train_data = dset.CIFAR10(root=self.args.data, train=True, download=True, transform=train_transform) elif self.args.dataset == 'cifar100': train_transform, valid_transform = utils._data_transforms_cifar100(self.args) train_data = dset.CIFAR100(root=self.args.data, train=True, download=True, transform=train_transform) elif self.args.dataset == 'svhn': train_transform, valid_transform = utils._data_transforms_svhn(self.args) train_data = dset.SVHN(root=self.args.data, split='train', download=True, transform=train_transform) num_train = len(train_data) indices = list(range(num_train)) split = int(np.floor(self.args.train_portion * num_train)) train_queue = torch.utils.data.DataLoader( train_data, batch_size=self.args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( train_data, batch_size=self.args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]), pin_memory=True, num_workers=2) return train_queue, valid_queue, train_transform, valid_transform
def get_train_validation_loader(args): train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) num_train = len(train_data) indices = list(range(num_train)) split = int(np.floor(args.train_portion * num_train)) # train[0:split] as training data train_sampler = torch.utils.data.sampler.SubsetRandomSampler( indices[:split]) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.train_batch_size, num_workers=args.workers, pin_memory=True, sampler=train_sampler) # train[split:] as validation data valid_sampler = torch.utils.data.sampler.SubsetRandomSampler( indices[split:]) valid_queue = torch.utils.data.DataLoader(train_data, batch_size=args.valid_batch_size, num_workers=args.workers, pin_memory=True, sampler=valid_sampler) valid_queue.name = 'valid' return train_queue, train_sampler, valid_queue
def get_test_loader(args): _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.valid_batch_size, shuffle=False, pin_memory=True, num_workers=args.workers) test_queue.name = 'test' return test_queue
def get_train_val_loaders(self): if self.args.dataset == 'cifar10': train_transform, valid_transform = utils._data_transforms_cifar10( self.args) train_data = dset.CIFAR10(root=self.args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=self.args.data, train=False, download=True, transform=valid_transform) elif self.args.dataset == 'cifar100': train_transform, valid_transform = utils._data_transforms_cifar100( self.args) train_data = dset.CIFAR100(root=self.args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=self.args.data, train=False, download=True, transform=valid_transform) elif self.args.dataset == 'svhn': train_transform, valid_transform = utils._data_transforms_svhn( self.args) train_data = dset.SVHN(root=self.args.data, split='train', download=True, transform=train_transform) valid_data = dset.SVHN(root=self.args.data, split='test', download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=self.args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=self.args.batch_size, shuffle=False, pin_memory=True, num_workers=2) return train_queue, valid_queue, train_transform, valid_transform
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model.drop_path_prob = args.drop_path_prob * 0 / args.epochs flops, params = profile(model, inputs=(torch.randn(1, 3, 32, 32), ), verbose=False) logging.info('flops = %fM', flops / 1e6) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) model = model.cuda() utils.load(model, args.model_path) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob with torch.no_grad(): test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc)
def main(): wandb.init(project="automl-gradient-based-nas", name="GDAS-" + "Opt: " + str(args.optimization) + "Search: " + str(args.arch_search_method), config=args, entity="automl") wandb.config.update(args) # adds all of the arguments as config variables global is_multi_gpu gpus = [int(i) for i in args.gpu.split(',')] logging.info('gpus = %s' % gpus) if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %s' % args.gpu) logging.info("args = %s", args) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() # default: args.init_channels = 16, CIFAR_CLASSES = 10, args.layers = 8 if args.arch_search_method == "DARTS": model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion) elif args.arch_search_method == "GDAS": model = Network_GumbelSoftmax(args.init_channels, CIFAR_CLASSES, args.layers, criterion) else: raise Exception("search space does not exist!") if len(gpus) > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs model = nn.DataParallel(model) is_multi_gpu = True model.cuda() wandb.watch(model) arch_parameters = model.module.arch_parameters( ) if is_multi_gpu else model.arch_parameters() arch_params = list(map(id, arch_parameters)) parameters = model.module.parameters( ) if is_multi_gpu else model.parameters() weight_params = filter(lambda p: id(p) not in arch_params, parameters) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) optimizer = torch.optim.SGD( weight_params, # model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) train_transform, valid_transform = utils._data_transforms_cifar10(args) # will cost time to download the data train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) num_train = len(train_data) indices = list(range(num_train)) split = int(np.floor(args.train_portion * num_train)) # split index train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( indices[split:num_train]), pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs), eta_min=args.learning_rate_min) architect = Architect(model, criterion, args) best_accuracy = 0 table = wandb.Table(columns=["Epoch", "Searched Architecture"]) for epoch in range(args.epochs): lr = scheduler.get_lr()[0] logging.info('epoch %d lr %e', epoch, lr) genotype = model.module.genotype() if is_multi_gpu else model.genotype( ) logging.info('genotype = %s', genotype) wandb.log({"genotype": str(genotype)}, step=epoch) table.add_data(str(epoch), str(genotype)) wandb.log({"Searched Architecture": table}) print( F.softmax(model.module.alphas_normal if is_multi_gpu else model.alphas_normal, dim=-1)) print( F.softmax(model.module.alphas_reduce if is_multi_gpu else model.alphas_reduce, dim=-1)) # training train_acc, train_obj = train(epoch, train_queue, valid_queue, model, architect, criterion, optimizer, lr) logging.info('train_acc %f', train_acc) wandb.log({"searching_train_acc": train_acc, "epoch": epoch}) # validation with torch.no_grad(): valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) wandb.log({"searching_valid_acc": valid_acc, "epoch": epoch}) scheduler.step() if valid_acc > best_accuracy: wandb.run.summary["best_valid_accuracy"] = valid_acc best_accuracy = valid_acc # utils.save(model, os.path.join(args.save, 'weights.pt')) utils.save(model, os.path.join(wandb.run.dir, 'weights.pt'))
def main(): if not torch.cuda.is_available(): print('No GPU device available') sys.exit(1) np.random.seed(args.seed) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) print("args = %s", args) print("unparsed args = %s", unparsed) # prepare dataset if args.cifar100: train_transform, valid_transform = utils._data_transforms_cifar100(args) else: train_transform, valid_transform = utils._data_transforms_cifar10(args) if args.cifar100: train_data = dset.CIFAR100(root=args.tmp_data_dir, train=True, download=False, transform=train_transform) valid_data = dset.CIFAR100(root=args.tmp_data_dir, train=False, download=False, transform=valid_transform) else: train_data = dset.CIFAR10(root=args.tmp_data_dir, train=True, download=False, transform=train_transform) valid_data = dset.CIFAR10(root=args.tmp_data_dir, train=False, download=False, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=args.workers) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=args.workers) # build Network criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() supernet = Network( args.init_channels, CIFAR_CLASSES, args.layers ) supernet.cuda() ckpt = torch.load(args.load_at) print(args.load_at) supernet.load_state_dict(ckpt) supernet.generate_share_alphas() alphas = supernet.cells[0].ops_alphas print(alphas) out_dir = args.save + '{}/eval_out/{}'.format(args.load_at.split('/')[2], args.seed) if not os.path.exists(out_dir): os.makedirs(out_dir) torch.save(alphas, os.path.join(out_dir, 'alphas.pt')) with open(os.path.join(out_dir, 'alphas.txt'), 'w') as f: for i in alphas.cpu().detach().numpy(): for j in i: f.write('{:d}'.format(int(j))) f.write('\n') # Getting subnet according to sample alpha subnet = supernet.get_sub_net(alphas) init_valid_acc, _ = infer(valid_queue, subnet, criterion) print('Initial Valid Acc {:.2f}'.format(init_valid_acc)) if args.fine_tune: if args.cifar100: weight_decay = 5e-4 else: weight_decay = 3e-4 # Fine tuning whole network: subnet = supernet.get_sub_net(alphas) optimizer = torch.optim.SGD( subnet.parameters(), args.finetune_lr, momentum=args.momentum, weight_decay=weight_decay, ) for epoch in range(args.epochs): # scheduler.step() print('epoch {} lr {:.4f}'.format(epoch, args.finetune_lr)) train_acc, _ = train(train_queue, subnet, criterion, optimizer) print('train_acc {:.2f}'.format(train_acc)) whole_valid_acc, _ = infer(valid_queue, subnet, criterion) print('valid_acc after whole fine-tune {:.2f}'.format(whole_valid_acc)) fly_whole_valid_acc, _ = infer(valid_queue, subnet, criterion, use_fly_bn=False) print('valid_acc after whole fine-tune {:.2f}'.format(fly_whole_valid_acc)) # Fine-tuning only classifier: subnet = supernet.get_sub_net(alphas) # Freezing other weights except classifier: for name, param in subnet.named_parameters(): if not 'classifier' in name: param.requires_grad_(requires_grad=False) optimizer = torch.optim.SGD( subnet.classifier.parameters(), args.finetune_lr, momentum=args.momentum, weight_decay=weight_decay, ) for epoch in range(args.epochs): # scheduler.step() print('epoch {} lr {:.4f}'.format(epoch, args.finetune_lr)) train_acc, _ = train(train_queue, subnet, criterion, optimizer) print('train_acc {:.2f}'.format(train_acc)) part_valid_acc, _ = infer(valid_queue, subnet, criterion) print('valid_acc after fine-tuning classifier {:.2f}'.format(part_valid_acc)) fly_part_valid_acc, _ = infer(valid_queue, subnet, criterion, use_fly_bn=False) print('valid_acc after fine-tuning classifier {:.2f}'.format(fly_part_valid_acc)) with open(os.path.join(out_dir, 'results.txt'), 'w') as f: f.write('-'.join([str(init_valid_acc), str(whole_valid_acc), str(fly_whole_valid_acc), str(part_valid_acc), str(fly_part_valid_acc)])) if not args.fine_tune: with open(os.path.join(out_dir, 'results.txt'), 'w') as f: f.write(str(init_valid_acc))
def main(): np.random.seed(args.seed) torch.manual_seed(args.seed) if args.gpu != -1: if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) torch.cuda.set_device(args.gpu) cudnn.benchmark = True cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) else: logging.info('using cpu') if args.dyno_schedule: args.threshold_divider = np.exp(-np.log(args.threshold_multiplier) * args.schedfreq) print( args.threshold_divider, -np.log(args.threshold_multiplier) / np.log(args.threshold_divider)) if args.dyno_split: args.train_portion = 1 - 1 / (1 + args.schedfreq) logging.info("args = %s", args) criterion = nn.CrossEntropyLoss() if args.gpu != -1: criterion = criterion.cuda() model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion, args.rho, args.crb, args.epochs, args.gpu, ewma=args.ewma, reg=args.reg) if args.gpu != -1: model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) datapath = os.path.join(utils.get_dir(), args.data) if args.task == "CIFAR100cf": train_transform, valid_transform = utils._data_transforms_cifar100( args) train_data = utils.CIFAR100C2F(root=datapath, train=True, download=True, transform=train_transform) num_train = len(train_data) indices = list(range(num_train)) split = int(np.floor(args.train_portion * len(indices))) orig_num_train = len(indices[:split]) orig_num_valid = len(indices[split:num_train]) train_indices = train_data.filter_by_fine(args.train_filter, indices[:split]) valid_indices = train_data.filter_by_fine(args.valid_filter, indices[split:num_train]) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=utils.FillingSubsetRandomSampler(train_indices, orig_num_train, reshuffle=True), pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=utils.FillingSubsetRandomSampler(valid_indices, orig_num_valid, reshuffle=True), pin_memory=True, num_workers=2) # TODO: extend each epoch or multiply number of epochs by 20%*args.class_filter elif args.task == "CIFAR100split": train_transform, valid_transform = utils._data_transforms_cifar100( args) train_data = utils.CIFAR100C2F(root=datapath, train=True, download=True, transform=train_transform) if not args.evensplit: train_indices, valid_indices = train_data.split(args.train_portion) else: num_train = len(train_data) indices = list(range(num_train)) split = int(np.floor(args.train_portion * num_train)) train_indices = indices[:split] valid_indices = indices[split:num_train] train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( train_indices), pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( valid_indices), pin_memory=True, num_workers=2) else: if args.task == "CIFAR100": train_transform, valid_transform = utils._data_transforms_cifar100( args) train_data = dset.CIFAR100(root=datapath, train=True, download=True, transform=train_transform) else: train_transform, valid_transform = utils._data_transforms_cifar10( args) train_data = dset.CIFAR10(root=datapath, train=True, download=True, transform=train_transform) num_train = len(train_data) indices = list(range(num_train)) split = int(np.floor(args.train_portion * num_train)) train_indices = indices[:split] valid_indices = indices[split:num_train] train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( train_indices), pin_memory=True, num_workers=4) valid_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( valid_indices), pin_memory=True, num_workers=4) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, int(args.epochs), eta_min=args.learning_rate_min) architect = Architect(model, args) loggers = { "train": { "loss": [], "acc": [], "step": [] }, "val": { "loss": [], "acc": [], "step": [] }, "infer": { "loss": [], "acc": [], "step": [] }, "ath": { "threshold": [], "step": [] }, "astep": [], "zustep": [] } alpha_threshold = args.init_alpha_threshold alpha_counter = 0 ewma = -1 for epoch in range(args.epochs): scheduler.step() lr = scheduler.get_last_lr()[0] logging.info('epoch %d lr %e', epoch, lr) genotype = model.genotype() logging.info('genotype = %s', genotype) if args.ckpt_interval > 0 and epoch > 0 and ( epoch) % args.ckpt_interval == 0: logging.info('checkpointing genotype') os.mkdir(os.path.join(args.save, 'genotypes', str(epoch))) with open( os.path.join(args.save, 'genotypes', str(epoch), 'genotype.txt'), "w") as f: f.write(str(genotype)) print(model.activate(model.alphas_normal)) print(model.activate(model.alphas_reduce)) # training train_acc, train_obj, alpha_threshold, alpha_counter, ewma = train( train_queue, valid_queue, model, architect, criterion, optimizer, loggers, alpha_threshold, alpha_counter, ewma, args) logging.info('train_acc %f', train_acc) # validation valid_acc, valid_obj = infer(valid_queue, model, criterion) utils.log_loss(loggers["infer"], valid_obj, valid_acc, model.clock) logging.info('valid_acc %f', valid_acc) utils.plot_loss_acc(loggers, args.save) utils.save_file(recoder=model.alphas_normal_history, path=os.path.join(args.save, 'Normalalpha'), steps=loggers["train"]["step"]) utils.save_file(recoder=model.alphas_reduce_history, path=os.path.join(args.save, 'Reducealpha'), steps=loggers["train"]["step"]) utils.plot_FI(loggers["train"]["step"], model.FI_history, args.save, "FI", loggers["ath"], loggers['astep']) utils.plot_FI(loggers["train"]["step"], model.FI_ewma_history, args.save, "FI_ewma", loggers["ath"], loggers['astep']) utils.save(model, os.path.join(args.save, 'weights.pt')) genotype = model.genotype() logging.info('genotype = %s', genotype) f = open(os.path.join(args.save, 'genotype.txt'), "w") f.write(str(genotype)) f.close()