def get_cifar_tuned_model(load_weights=True): network = NetworkCIFAR(40, CIFAR_CLASSES, 20, True, 0.4, CIFAR_TUNED) if load_weights: device = torch.device('cpu') state_dict = torch.load('weights/cifar_tuned.pt', map_location=device) network.load_state_dict(state_dict) return network
def main(): torch.cuda.set_device(args.gpu) cudnn.benchmark = True cudnn.enabled = True logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) model = Network(args.init_ch, 10, args.layers, True, genotype).cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) checkpoint = torch.load(args.checkpoint + '/top1.pt') model.load_state_dict(checkpoint['model_state_dict']) criterion = nn.CrossEntropyLoss().cuda() CIFAR_MEAN = [0.49139968, 0.48215827, 0.44653124] CIFAR_STD = [0.24703233, 0.24348505, 0.26158768] valid_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(CIFAR_MEAN, CIFAR_STD), ]) valid_queue = torch.utils.data.DataLoader(dset.CIFAR10( root=args.data, train=False, transform=valid_transform), batch_size=args.batch_size, shuffle=True, num_workers=2, pin_memory=True) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc: %f', valid_acc)
def main(): if not torch.cuda.is_available(): sys.exit(1) ## step 1 construct the selected network genotype = eval("genotypes.%s" % args.selected_arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) ## step 2 load pretrained model parameter if args.cifar100: model = torch.nn.DataParallel(model) model = model.cuda() model.load_state_dict(torch.load(args.model_path)['net']) else: utils.load(model, args.model_path) model = torch.nn.DataParallel(model) model = model.cuda() model.module.drop_path_prob = 0 model.drop_path_prob = 0 print("param size = %fMB" % utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() ## step 3 load test data valid_queue = load_data_cifar(args) ## step 4. inference on test data valid_acc, valid_obj = infer(valid_queue, model, criterion) print('-----------------------------------------------') print('Average Valid_acc: %f ' % valid_acc) print('-----------------------------------------------')
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() checkpoint = torch.load(args.model_path) model.load_state_dict(checkpoint['model_state_dict']) print("param size = {:.1f}MB".format( floor(utils.count_parameters_in_MB(model), 1))) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) start_epoch = 0 if os.path.isfile((os.path.join(args.save, 'checkpoint.pt'))): checkpoint = torch.load(os.path.join(args.save, 'checkpoint.pt')) start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) scheduler.load_state_dict(checkpoint['scheduler']) accuracies = checkpoint['accuracies'] print( 'Load checkpoint from {:} with start-epoch = {:}, acc: {}'.format( args.save, start_epoch, accuracies)) for epoch in range(start_epoch, args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save(epoch, args, model, optimizer, scheduler, valid_acc, os.path.join(args.save, 'checkpoint.pt'))
from darts.genotypes import DARTS from darts.operations import * import sys sys.path.append('../cnn') from model import NetworkCIFAR as Network from chainer.datasets import get_cifar10 c_model = NetworkCIFAR(DARTS) chainer.serializers.load_npz('model.npz', c_model) chainer.global_config.train = False t_model = Network(36, 10, 20, True, DARTS) t_model.drop_path_prob = 0.2 t_model.load_state_dict(torch.load('cifar10_model.pt')) t_model.eval() train, val = get_cifar10() img, _ = train[0] x = img[None] t_x = torch.autograd.Variable(torch.FloatTensor(x)) c_y = c_model(x) t_y = t_model(t_x) np.testing.assert_almost_equal(c_y[0].data, t_y[0].detach().data.numpy(), decimal=5) ##################### # c_prev_s = c_model.stem(x) # c_prev_prev_s = c_prev_s
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() if( os.path.exists(args.loadModelPath)): model.load_state_dict(torch.load(args.loadModelPath)) else: logging.info('model path not exist') return logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) train_transform, valid_transform = utils._data_transforms_cifar10(args) if args.set=='cifar100': train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform) else: train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) #train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) #valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs)) best_acc = 0.0 for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) if valid_acc > best_acc: best_acc = valid_acc logging.info('valid_acc %f, best_acc %f', valid_acc, best_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
def main(): if not torch.cuda.is_available(): logging.info('No GPU device available') sys.exit(1) np.random.seed(args.seed) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info("args = %s", args) logging.info("unparsed args = %s", unparsed) num_gpus = torch.cuda.device_count() genotype = eval("genotypes.%s" % args.arch) print('---------Genotype---------') logging.info(genotype) print('--------------------------') model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) model = torch.nn.DataParallel(model) model = model.cuda() start_epch = 0 if args.resume: MT = torch.load(os.path.join(args.save, 'weight_optimizers.pt')) model.load_state_dict(MT['net']) optimizer.load_state_dict(MT['optimizer']) start_epch = MT['epoch'] logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() if args.cifar100: train_transform, valid_transform = utils._data_transforms_cifar100( args) else: train_transform, valid_transform = utils._data_transforms_cifar10(args) if args.cifar100: train_data = dset.CIFAR100(root=args.data_dir, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data_dir, train=False, download=True, transform=valid_transform) else: train_data = dset.CIFAR10(root=args.data_dir, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data_dir, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=args.workers) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=args.workers) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) best_acc = 0.0 for epoch in range(start_epch, args.epochs): model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs model.drop_path_prob = args.drop_path_prob * epoch / args.epochs start_time = time.time() train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('Train_acc: %f', train_acc) scheduler.step() logging.info('Epoch: %d lr %e', epoch, scheduler.get_lr()[0]) valid_acc, valid_obj = infer(valid_queue, model, criterion) if valid_acc > best_acc: best_acc = valid_acc state = { 'net': model.state_dict(), 'optimizer': optimizer.state_dict(), 'epoch': epoch } torch.save(state, os.path.join(args.save, 'best_weight_optimizers.pt')) logging.info('Valid_acc: %f, best_acc: %f', valid_acc, best_acc) end_time = time.time() duration = end_time - start_time print('Epoch time: %d h.' % (duration * (args.epochs - epoch) / 3600)) if epoch % 50 == 0: state = { 'net': model.state_dict(), 'optimizer': optimizer.state_dict(), 'epoch': epoch } torch.save(state, os.path.join(args.save, 'weight_optimizers.pt'))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.to('cuda') logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.to('cuda') # not apply weight decay to BN layers if args.bn_no_decay: logging.info('BN layers are excluded from weight decay') bn_params, other_params = utils.split_bn_params(model) logging.debug('bn: %s', [p.dtype for p in bn_params]) logging.debug('other: %s', [p.dtype for p in other_params]) param_group = [{'params': bn_params, 'weight_decay': 0}, {'params': other_params}] else: param_group = model.parameters() optimizer = torch.optim.SGD( param_group, args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) logging.info('optimizer: %s', optimizer) train_transform, valid_transform = utils.data_transforms_cifar10(args) train_data = datasets.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = datasets.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=args.num_workers) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=args.num_workers) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs)) init_epoch = 0 best_acc = 0 if args.recover: states = torch.load(args.recover) model.load_state_dict(states['stete']) init_epoch = states['epoch'] + 1 best_acc = states['best_acc'] logging.info('checkpoint loaded') scheduler.step(init_epoch) logging.info('scheduler is set to epoch %d. learning rate is %s', init_epoch, scheduler.get_lr()) for epoch in range(init_epoch, args.epochs): logging.info('epoch %d lr %s', epoch, scheduler.get_lr()) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %.4f', train_acc) with torch.no_grad(): valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) logging.info('epoch %03d overall train_acc=%.4f valid_acc=%.4f', epoch, train_acc, valid_acc) scheduler.step() # gpu info utils.gpu_usage(args.debug) if valid_acc > best_acc: best_acc = valid_acc logging.info('best acc: %.4f', best_acc) utils.save_checkpoint(state={'stete': model.state_dict(), 'epoch': epoch, 'best_acc': best_acc,}, is_best=False, save=args.save)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) if args.parallel: model = nn.DataParallel(model).cuda() else: model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) start_epoch = 0 if args.resume: checkpoint = torch.load( os.path.join(args.resume_dir, 'checkpoint.pth.tar')) start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['model_state_dict']) model = model.cuda() scheduler.load_state_dict(checkpoint['scheduler_state_dict']) optimizer.load_state_dict(checkpoint['optimizer_state_dict']) print('model was loaded. Start training from epoch {}.'.format( start_epoch)) for epoch in range(start_epoch, args.epochs): logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) with open(os.path.join(args.save, 'learning_rate.txt'), mode='a') as f: f.write(str(scheduler.get_lr()[0]) + '\n') if args.parallel: model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs else: model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) with open(os.path.join(args.save, "train_acc.txt"), mode='a') as f: f.write(str(train_acc) + '\n') with open(os.path.join(args.save, "train_loss.txt"), mode='a') as f: f.write(str(train_obj) + '\n') scheduler.step() valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) with open(os.path.join(args.save, "test_acc.txt"), mode='a') as f: f.write(str(valid_acc) + '\n') with open(os.path.join(args.save, "test_loss.txt"), mode='a') as f: f.write(str(valid_obj) + '\n') utils.save(model, os.path.join(args.save, 'weights.pt')) torch.save(scheduler.state_dict(), os.path.join(args.save, 'scheduler.pth.tar'), pickle_module=dill) utils.save_checkpoint( { 'epoch': epoch + 1, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'scheduler_state_dict': scheduler.state_dict() }, False, save=args.save)
def main(): torch.cuda.set_device(args.gpu) cudnn.benchmark = True cudnn.enabled = True logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) cur_epoch = 0 net = eval(args.arch) print(net) code = gen_code_from_list(net, node_num=int((len(net) / 4))) genotype = translator([code, code], max_node=int((len(net) / 4))) print(genotype) model_ema = None if not continue_train: print('train from the scratch') model = Network(args.init_ch, 10, args.layers, args.auxiliary, genotype).cuda() print("model init params values:", flatten_params(model)) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = CutMixCrossEntropyLoss(True).cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.wd) if args.model_ema: model_ema = ModelEma( model, decay=args.model_ema_decay, device='cpu' if args.model_ema_force_cpu else '') else: print('continue train from checkpoint') model = Network(args.init_ch, 10, args.layers, args.auxiliary, genotype).cuda() criterion = CutMixCrossEntropyLoss(True).cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.wd) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) checkpoint = torch.load(args.save + '/model.pt') model.load_state_dict(checkpoint['model_state_dict']) cur_epoch = checkpoint['epoch'] optimizer.load_state_dict(checkpoint['optimizer_state_dict']) if args.model_ema: model_ema = ModelEma( model, decay=args.model_ema_decay, device='cpu' if args.model_ema_force_cpu else '', resume=args.save + '/model.pt') train_transform, valid_transform = utils._auto_data_transforms_cifar10( args) ds_train = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) args.cv = -1 if args.cv >= 0: sss = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=0) sss = sss.split(list(range(len(ds_train))), ds_train.targets) for _ in range(args.cv + 1): train_idx, valid_idx = next(sss) ds_valid = Subset(ds_train, valid_idx) ds_train = Subset(ds_train, train_idx) else: ds_valid = Subset(ds_train, []) train_queue = torch.utils.data.DataLoader(CutMix(ds_train, 10, beta=1.0, prob=0.5, num_mix=2), batch_size=args.batch_size, shuffle=True, num_workers=2, pin_memory=True) valid_queue = torch.utils.data.DataLoader(dset.CIFAR10( root=args.data, train=False, transform=valid_transform), batch_size=args.batch_size, shuffle=True, num_workers=2, pin_memory=True) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) best_acc = 0.0 if continue_train: for i in range(cur_epoch + 1): scheduler.step() for epoch in range(cur_epoch, args.epochs): print('cur_epoch is', epoch) scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs if model_ema is not None: model_ema.ema.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer, epoch, model_ema) logging.info('train_acc: %f', train_acc) if model_ema is not None and not args.model_ema_force_cpu: valid_acc_ema, valid_obj_ema = infer(valid_queue, model_ema.ema, criterion, ema=True) logging.info('valid_acc_ema %f', valid_acc_ema) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc: %f', valid_acc) if valid_acc > best_acc: best_acc = valid_acc print('this model is the best') torch.save( { 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict() }, os.path.join(args.save, 'top1.pt')) print('current best acc is', best_acc) logging.info('best_acc: %f', best_acc) if model_ema is not None: torch.save( { 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'state_dict_ema': get_state_dict(model_ema) }, os.path.join(args.save, 'model.pt')) else: torch.save( { 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict() }, os.path.join(args.save, 'model.pt')) print('saved to: trained.pt')
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) #torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype, args.residual_wei, args.shrink_channel) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) resume = os.path.join(args.save, 'checkpoint.pth.tar') if os.path.exists(resume): print("=> loading checkpoint %s" % resume) #checkpoint = torch.load(resume) checkpoint = torch.load(resume, map_location = lambda storage, loc: storage.cuda(0)) args.start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) #optimizer.load_state_dict(checkpoint['optimizer']) optimizer.state_dict()['state'] = checkpoint['optimizer']['state'] print('=> loaded checkpoint epoch %d' % args.start_epoch) if args.start_epoch >= args.epochs: print('training finished') sys.exit(0) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs)) for epoch in range(args.start_epoch): scheduler.step() for epoch in range(args.start_epoch, args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) if epoch == args.epochs - 1: valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_acc_top1': train_acc, 'optimizer' : optimizer.state_dict(), }, False, args.save)
def main(self, arch, epochs=600, gpu=0, load_weights=False, train_portion=0.7, seed=0, save='EXP'): # Set up save file and logging self.save = 'eval-{}-{}'.format(save, time.strftime("%Y%m%d-%H%M%S")) utils.create_exp_dir(self.save, scripts_to_save=glob.glob('*.py')) log_format = '%(asctime)s %(message)s' logging.basicConfig(stream=sys.stdout, level=logging.INFO, format=log_format, datefmt='%m/%d %I:%M:%S %p') fh = logging.FileHandler(os.path.join(self.save, 'log.txt')) fh.setFormatter(logging.Formatter(log_format)) logging.getLogger().addHandler(fh) self.arch = arch self.epochs = epochs self.load_weights = load_weights self.gpu = gpu self.train_portion = train_portion if self.train_portion == 1: self.validation_set = False self.seed = seed print('Train class params') print('arch: {}, epochs: {}, gpu: {}, load_weights: {}, train_portion: {}' .format(arch, epochs, gpu, load_weights, train_portion)) # cpu-gpu switch if not torch.cuda.is_available(): logging.info('no gpu device available') torch.manual_seed(self.seed) device = torch.device('cpu') else: torch.cuda.manual_seed_all(self.seed) random.seed(self.seed) torch.manual_seed(self.seed) device = torch.device(self.gpu) cudnn.benchmark = False cudnn.enabled=True cudnn.deterministic=True logging.info('gpu device = %d' % self.gpu) Genotype = namedtuple('Genotype', 'normal normal_concat reduce reduce_concat') genotype = eval(self.convert_to_genotype(arch)) model = Network(self.init_channels, self.CIFAR_CLASSES, self.layers, self.auxiliary, genotype) model = model.to(device) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) total_params = sum(x.data.nelement() for x in model.parameters()) logging.info('Model total parameters: {}'.format(total_params)) criterion = nn.CrossEntropyLoss() criterion = criterion.to(device) optimizer = torch.optim.SGD( model.parameters(), self.learning_rate, momentum=self.momentum, weight_decay=self.weight_decay ) train_transform, test_transform = utils._data_transforms_cifar10(self.cutout, self.cutout_length) train_data = dset.CIFAR10(root=self.data, train=True, download=True, transform=train_transform) test_data = dset.CIFAR10(root=self.data, train=False, download=True, transform=test_transform) num_train = len(train_data) indices = list(range(num_train)) if self.validation_set: split = int(np.floor(self.train_portion * num_train)) else: split = num_train train_queue = torch.utils.data.DataLoader( train_data, batch_size=self.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=0) if self.validation_set: valid_queue = torch.utils.data.DataLoader( train_data, batch_size=self.batch_size // 2, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]), pin_memory=True, num_workers=0) test_queue = torch.utils.data.DataLoader( test_data, batch_size=self.batch_size // 2, shuffle=False, pin_memory=True, num_workers=0) if self.load_weights: logging.info('loading saved weights') ml = 'cuda:{}'.format(self.gpu) if torch.cuda.is_available() else 'cpu' model.load_state_dict(torch.load('weights.pt', map_location = ml)) logging.info('loaded saved weights') scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(self.epochs)) valid_accs = [] test_accs = [] for epoch in range(self.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = self.drop_path_prob * epoch / self.epochs train_acc, train_obj = self.train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) if self.validation_set: valid_acc, valid_obj = self.infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) else: valid_acc, valid_obj = 0, 0 test_acc, test_obj = self.infer(test_queue, model, criterion, test_data=True) logging.info('test_acc %f', test_acc) utils.save(model, os.path.join(self.save, 'weights-new.pt')) if epoch in list(range(max(0, epochs - 5), epochs)): valid_accs.append((epoch, valid_acc)) test_accs.append((epoch, test_acc)) return valid_accs, test_accs
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) if args.is_cifar100: model = Network(args.init_channels, CIFAR100_CLASSES, args.layers, args.auxiliary, genotype) else: model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) if args.is_cifar100: train_transform, valid_transform = utils._data_transforms_cifar100( args) else: train_transform, valid_transform = utils._data_transforms_cifar10(args) if args.is_cifar100: train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform) else: train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) # train_queue = torch.utils.data.DataLoader( # train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) num_train = len(train_data) indices = list(range(num_train)) split = 45000 # int(np.floor(args.train_portion * num_train)) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=2) train_valid_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:]), pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) start_epoch = 0 if args.resume: checkpoint = torch.load(args.resume) start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) scheduler.load_state_dict(checkpoint['scheduler']) best_acc = 0 for epoch in range(start_epoch, args.epochs): logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) scheduler.step() logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(train_valid_queue, model, criterion) if valid_acc > best_acc: best_acc = valid_acc test_acc, _ = infer(valid_queue, model, criterion) logging.info('Test_acc: %f', test_acc) logging.info('Valid_acc: %f', valid_acc) logging.info('Best_acc: %f', best_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save(model, os.path.join(args.save, 'weights.pt')) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'scheduler': scheduler.state_dict(), 'optimizer': optimizer.state_dict() })
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) # torch.cuda.set_device(args.gpu) device = torch.device("cuda") cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) # read data train_transform, valid_transform = utils._data_transforms_cifar10(args) if args.dataset == 'cifar10': args.data = '/home/work/dataset/cifar' train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) classes = 10 if args.dataset == 'cifar100': args.data = '/home/work/dataset/cifar100' train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform) classes = 100 train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) # model genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, classes, args.layers, args.auxiliary, genotype) model = model.cuda() model.drop_path_prob = args.drop_path_prob flops, params = profile(model, inputs=(torch.randn(1, 3, 32, 32).cuda(),), verbose=False) logging.info('flops = %fM', flops / 1e6) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs)) best_val_acc = 0. if args.resume: # state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s3_1-20200608/weights.pt') # state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s2_factor1-20200609/weights.pt', map_location='cpu') # state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s3_factor1-20200609/weights.pt', map_location='cpu') # state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s3_0-20200608/weights.pt', map_location='cpu') # state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s2_0-20200608/weights.pt', map_location='cpu') state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s3_2-20200608/weights.pt', map_location='cpu') model.load_state_dict(state) model = model.to(device) for i in range(args.start_epoch): scheduler.step() best_val_acc = 97.19#97.34#97.32#94.92#94.6#97.2 for epoch in range(args.start_epoch, args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) with torch.no_grad(): valid_acc, valid_obj = infer(valid_queue, model, criterion) if valid_acc > best_val_acc: best_val_acc = valid_acc utils.save(model, os.path.join(args.save, 'best_weights.pt')) # logging.info('valid_acc %f\tbest_val_acc %f', valid_acc, best_val_acc) logging.info('val_acc: {:.6}, best_val_acc: \033[31m{:.6}\033[0m'.format(valid_acc, best_val_acc)) state = { 'epoch': epoch, 'model_state': model.state_dict(), 'optimizer': optimizer.state_dict(), 'best_val_acc': best_val_acc } torch.save(state, os.path.join(args.save, 'weights.pt.tar'))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) # device = torch.device("cuda") cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) # read data train_transform, valid_transform = utils._data_transforms_cifar10(args) if args.dataset == 'cifar10': args.data = '/home/work/dataset/cifar' train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) classes = 10 if args.dataset == 'cifar100': args.data = '/home/work/dataset/cifar100' train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform) classes = 100 train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) # model genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, classes, args.layers, args.auxiliary, genotype) model = model.cuda() model.drop_path_prob = args.drop_path_prob flops, params = profile(model, inputs=(torch.randn(1, 3, 32, 32).cuda(), ), verbose=False) logging.info('flops = %fM', flops / 1e6) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() state = torch.load( '/home/work/lixudong/code_work/sgas/cnn/full_train_s2_0-20200608/best_weights.pt', map_location='cpu') model.load_state_dict(state) model = model.cuda() valid_acc, valid_obj = infer(valid_queue, model, criterion) print('val acc: {}, val loss: {}'.format(valid_acc, valid_obj))
print('no gpu device available') sys.exit(1) genotype = eval('genotypes.%s' % args.arch) print('Parsing Genotypes: {}'.format(genotype)) model = Network(36, 10, 20, 0.4, genotype, args.parse_method) flops, params = profile(model, inputs=(torch.randn(1, 3, 32, 32),), verbose=False) print('flops = %fM' % (flops / 1e6)) print('param size = %fM' %( params / 1e6)) model = model.cuda() if args.model_path and os.path.isfile(args.model_path): checkpoint = torch.load(args.model_path) model.load_state_dict(checkpoint['state_dict']) else: print('The Pre-Trained Model Is InValid!') sys.exit(-1) top1 = dutils.AvgrageMeter() top5 = dutils.AvgrageMeter() model.eval() with torch.no_grad(): for step, (input, target) in enumerate(valid_queue): input = input.cuda(non_blocking=True) target = target.cuda(non_blocking=True) logits, _ = model(input) prec1, prec5 = dutils.accuracy(logits, target, topk=(1, 5)) n = input.size(0) top1.update(prec1.item(), n)