def plot(label, num_filled, freq, num_shells_range, interaction="normal"): d = utils.load_all() d = utils.filter_preferred_ml(d) d = d[(d["method"] != "imsrg[f]+eom[n]") & (d["method"] != "magnus_quads+eom") & (d["interaction"] == interaction) & (d["label"] == label) & (d["num_filled"] == num_filled) & (d["num_shells"] >= num_shells_range[0]) & (d["num_shells"] <= num_shells_range[1]) & (d["freq"] == 1.0)] num_particles = num_filled * (num_filled + 1) energy_type = {"ground": "ground state", "add": "addition", "rm": "removal"}[label] fig, ax = plt.subplots() fig.set_size_inches((4, 3)) for method, case in d.groupby("method"): case = case.sort_values("num_shells") xs = case["num_shells"].astype(int) ys = case["energy"] ax.plot(xs, ys, "-x", label=utils.METHOD_LABEL[method], color=utils.METHOD_COLOR[method]) ax.get_xaxis().set_major_locator( matplotlib.ticker.MaxNLocator(integer=True)) ax.set_xlabel("K (number of shells)") ax.set_ylabel("E (energy)") ax.legend() fig.tight_layout() utils.savefig(fig, "by-num-shells-{num_particles}-{num_filled}-" "{label}-{ml}-{interaction}" .format(**locals()))
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data loading code traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_loader = torch.utils.data.DataLoader( datasets.ImageFolder(traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])), batch_size=args.train_batch, shuffle=True, num_workers=args.workers, pin_memory=True, worker_init_fn=random.seed) val_loader = torch.utils.data.DataLoader( datasets.ImageFolder(valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=True) # create model if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) elif args.arch.startswith('resnext'): model = models.__dict__[args.arch]( baseWidth=args.base_width, cardinality=args.cardinality, ) else: print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch]() if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0)) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'ImageNet-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' # args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.TOP1', 'Valid Acc.TOP5']) lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.schedule, gamma=args.gamma, last_epoch=start_epoch - 1) if args.evaluate: # print('\nEvaluation only') test_loss, test_acc, test_acc_top5 = test(val_loader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc TOP1: %.2f, Test Acc TOP5: %.2f' % (test_loss, test_acc, test_acc_top5)) # return # Train and val for epoch in range(start_epoch, args.epochs): lr_scheduler.step() state['lr'] = optimizer.param_groups[0]['lr'] add_summary_value(tb_summary_writer, 'learning_rate', state['lr'], epoch + 1) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch+1, use_cuda) test_loss, test_acc, test_acc_top5 = test(val_loader, model, criterion, epoch+1, use_cuda) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc, test_acc_top5]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'acc_top5': test_acc_top5, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main(): global best_acc mp.set_start_method('spawn') e = mp.Event() if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) trainset_small, trainset_large = trainset, trainset # torch.utils.data.random_split(trainset, [len(trainset)//2, len(trainset) - (len(trainset)//2)]) print("len trainset_large: ", len(trainset_large)) print("len trainset_small: ", len(trainset_small)) trainloader_gpu0 = data.DataLoader(trainset_small, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) trainloader_gpu1 = data.DataLoader(trainset_large, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root='./data', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, block_name=args.block_name, ) else: model = models.__dict__[args.arch](num_classes=num_classes) print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # clone models save_model(model, 0) save_model(model, 1) model0 = load_model(0) model1 = load_model(1) queue01 = mp.Queue() queue10 = mp.Queue() # training model0.share_memory() model1.share_memory() train_gpu1 = mp.spawn(train_copy, args=(trainloader_gpu1, model1, criterion, use_cuda, gpu1, e, args, testloader, queue01, queue10), join=False) train_gpu0 = mp.spawn(train, args=(trainloader_gpu0, model0, criterion, use_cuda, gpu0, e, args, testloader, queue10, queue01), join=False) train_gpu1.join() train_gpu0.join() logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps'))
if question == '2': dataset = load_dataset('animals.pkl') X = dataset['X'].astype(float) animals = dataset['animals'] n, d = X.shape f1, f2 = np.random.choice(d, size=2, replace=False) plt.figure() plt.scatter(X[:, f1], X[:, f2]) plt.xlabel("$x_{%d}$" % f1) plt.ylabel("$x_{%d}$" % f2) for i in range(n): plt.annotate(animals[i], (X[i, f1], X[i, f2])) utils.savefig('two_random_features.png') elif question == '2.2': dataset = load_dataset('animals.pkl') X = dataset['X'].astype(float) animals = dataset['animals'] n, d = X.shape # standardize columns X = utils.standardize_cols(X) k = 20 model = PCA(k) model.fit(X) Z = model.compress(X) f1, f2 = np.random.choice(k, size=2, replace=False)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) if args.img_size == 32: if args.DA: print('use DA') transform_train = transforms.Compose([ transforms.RandomHorizontalFlip(), # with p = 0.5 transforms.ToTensor( ), # it must be this guy that makes it CHW again transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) else: print('no DA') transform_train = transforms.Compose([ transforms.ToTensor( ), # it must be this guy that makes it CHW again transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) else: if args.DA: print('use DA') transform_train = transforms.Compose([ transforms.RandomHorizontalFlip(), # with p = 0.5 transforms.Resize(args.img_size), transforms.ToTensor( ), # it must be this guy that makes it CHW again transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) else: print('no DA') transform_train = transforms.Compose([ transforms.Resize(args.img_size), transforms.ToTensor( ), # it must be this guy that makes it CHW again transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.Resize(args.img_size), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = CIFAR10 num_classes = 10 else: dataloader = CIFAR100 num_classes = 100 trainset = dataloader(root='/BS/yfan/work/data/cifar100_fp', train=True, download=False, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root='/BS/yfan/work/data/cifar100_fp', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) elif args.arch.startswith('resnet50'): model = models.__dict__[args.arch](num_classes=num_classes, include_top=True, dropout_rate=0, layer=args.layer) elif args.arch.startswith('d1_resnet50'): model = models.__dict__[args.arch]( num_classes=num_classes, include_top=True, dropout_rate=0, layer=args.layer, is_shuff=False # TODO: check ) elif args.arch.endswith('vgg16'): model = models.__dict__[args.arch](num_classes=num_classes, include_top=True, dropout_rate=0, layer=args.layer) elif args.arch.endswith('vgg16_sa'): model = models.__dict__[args.arch](num_classes=num_classes, include_top=True, dropout_rate=0, layer=args.layer) elif args.arch.endswith('vgg16_1d'): model = models.__dict__[args.arch](num_classes=num_classes, include_top=True, dropout_rate=0, layer=args.layer, is_shuff=False) elif args.arch.endswith("densenet_1d"): model = models.__dict__[args.arch](num_classes=num_classes, drop_rate=0, layer=args.layer) else: model = models.__dict__[args.arch](num_classes=num_classes) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = False # TODO: for deterministc result, this has to be false print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) # for name, param in model.named_parameters(): # print(name) # for name in model.named_modules(): # print(name) # for param in model.parameters(): # print(param) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=args.weight_decay) # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.save): mkdir_p(args.save) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 elif args.dataset == 'cifar100': dataloader = datasets.CIFAR100 else: raise ValueError( 'Expect dataset to be either CIFAR-10 or CIFAR-100 but got {}'. format(args.dataset)) trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root='./data', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) model = arch_module.__dict__[args.arch](dataset=args.dataset) cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) lr_scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.schedule, gamma=args.gamma) # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.save = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_prec1'] start_epoch = checkpoint['epoch'] model = arch_module.__dict__[args.arch](dataset=args.dataset, cfg=checkpoint['cfg']) # load the state dict of saved checkpoint # turn the flag off to train from scratch if args.load_model: print('===> Resuming the state dict of saved model') model.load_state_dict(checkpoint['state_dict']) else: print('===> Skip loading state dict of saved model') # finetune a pruned network if args.load_optimizer and ('optimizer' in checkpoint.keys()): print('===> Resuming the state dict of saved checkpoint') optimizer.load_state_dict(checkpoint['optimizer']) else: print('===> Skip loading the state dict of saved optimizer') # if the log file is already exist then append the log to it if os.path.isfile('log.txt'): logger = Logger(os.path.join(args.save, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.save, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) else: # training from scratch logger = Logger(os.path.join(args.save, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if use_cuda: model = model.cuda() # evaluate the results on test set if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) inp = torch.rand(1, 3, 32, 32) if use_cuda: inp = inp.cuda() flops, params = get_model_complexity_info(model, (3, 32, 32), as_strings=True, print_per_layer_stat=True) print('{:<30} {:<8}'.format('Computational complexity: ', flops)) print('{:<30} {:<8}'.format('Number of parameters: ', params)) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): current_lr = next(iter(optimizer.param_groups))['lr'] print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, current_lr)) train_loss, train_acc = train(trainloader, model, criterion, optimizer, lr_scheduler, epoch, use_cuda) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) # append logger file logger.append([current_lr, train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_prec1': test_acc, 'optimizer': optimizer.state_dict(), 'cfg': model.cfg }, is_best, checkpoint=args.save) logger.close() logger.plot() savefig(os.path.join(args.save, 'log.eps')) print('Best acc:') print(best_acc)
X = dataset['X'].astype(float) animals = dataset['animals'] n, d = X.shape print("n =", n) print("d =", d) f1, f2 = np.random.choice(d, size=2, replace=False) plt.figure() plt.scatter(X[:, f1], X[:, f2]) plt.xlabel("$x_{%d}$" % f1) plt.ylabel("$x_{%d}$" % f2) for i in range(n): plt.annotate(animals[i], (X[i, f1], X[i, f2])) utils.savefig('two_random_features.png') elif question == '1.3': dataset = load_dataset('animals.pkl') X = dataset['X'].astype(float) animals = dataset['animals'] n, d = X.shape model = MDS(n_components=2) Z = model.compress(X) fig, ax = plt.subplots() ax.scatter(Z[:, 0], Z[:, 1]) plt.ylabel('z2') plt.xlabel('z1')
def plot_fits(plot, data, get_fit_range, badness_threshold, x_col, x_label, y_col, y_label, absdydx_label, title_cols, get_title, get_fn, color_cols, get_color, get_color_label, get_dmc, dmc_label, dmc_yerr_col=None): # continuous x range for plotting continuous functions x_c = np.linspace(data[x_col].min() - 1, data[x_col].max() + 1, 250) [(title_key, gg)] = utils.groupby(data, title_cols) fig, ax = plt.subplots(2) fig.set_size_inches(8, 10) # otherwise the text will get obscured y_range = np.array([np.nan, np.nan]) fit_results = {} for color_key, g in utils.groupby(gg, color_cols): logging.info(f"plot_fits: method: {color_key['method']}") color = get_color(color_key["method"]) label = get_color_label(color_key["method"]) g = g.sort_values([x_col]) d = g.rename(columns={x_col: "x", y_col: "y"}) deriv_d = differentiate(d, "x", "y", "dydx") x = deriv_d["x"] dydx = deriv_d["dydx"] ax[0].plot(x, abs(dydx), "x", label=label, color=color) ax[1].plot(d["x"], d["y"], "x", label=label, color=color) utils.update_range(y_range, d["y"]) fit_range = get_fit_range(color_key["method"]) fit_range = (max(fit_range[0], d["x"].min()), min(fit_range[1], d["x"].max())) if fit_range[1] < fit_range[0]: continue fit_range = fit_range + np.array([-0.2, 0.2]) # to make it look nicer ax[0].axvspan(fit_range[0], fit_range[1], alpha=0.05, color=color) ax[1].axvspan(fit_range[0], fit_range[1], alpha=0.05, color=color) d_subset = d[d["x"].between(*fit_range)] deriv_subset = deriv_d[deriv_d["x"].between(*fit_range)] if len(deriv_subset) < 2: continue fit = do_fit(d_subset, deriv_subset, badness_threshold=badness_threshold) if fit is None: continue fit_result = { "num_points": len(d["x"]) } fit_result.update(fit) fit_results[color_key["method"]] = fit_result outliers = deriv_subset.loc[fit["logderiv"]["outliers"]] ax[0].plot(outliers["x"], abs(outliers["dydx"]), "o", markerfacecolor="none", label="", color="red") for stage, result in fit.items(): if stage == "fixedab": continue # fixedab yields the same plot here as logderiv a = result["coefficient"] b = result["exponent"] b_err = result.get("exponent_err", None) dydx_c = a * b * x_c ** (b - 1.0) ax[0].plot( x_c, abs(dydx_c), linestyle=STAGE_TO_LINESTYLE[stage], label=label + " " + fit_label(stage, b, b_err), color=color) for stage, result in fit.items(): if "constant" not in result: continue a = result["coefficient"] b = result["exponent"] c = result["constant"] b_err = result.get("exponent_err", None) y_c = a * x_c ** b + c ax[1].plot( x_c, y_c, linestyle=STAGE_TO_LINESTYLE[stage], label=label + " " + fit_label(stage, b, b_err), color=color) if b < 0: ax[1].axhline(c, linestyle=":", color=color) else: logging.warn(f"plot_fits: {stage}.b >= 0: no asymptotic result") utils.update_range(y_range, c) g = get_dmc(**title_key) if len(g): y = g[y_col].iloc[0] if dmc_yerr_col is not None: y_err = g[dmc_yerr_col].iloc[0] ax[1].axhspan(y - y_err, y + y_err, alpha=0.4, color="black", label=dmc_label) utils.update_range(y_range, [y - y_err, y + y_err]) # add an extra line to make sure it's visible ax[1].axhline(y, alpha=0.4, color="black") utils.update_range(y_range, [y]) ax[0].set_xlabel(x_label) ax[0].set_ylabel(absdydx_label) ax[0].set_xscale("log") ax[0].set_yscale("log") ax[0].set_title(get_title(**title_key)) box = ax[0].get_position() ax[0].set_position([box.x0, box.y0, box.width * 0.6, box.height]) ax[0].legend(bbox_to_anchor=(1, 1.0)) ax[1].legend() ax[1].set_xlabel(x_label) ax[1].set_ylabel(y_label) ax[1].set_ylim(*utils.expand_range(y_range, 0.05)) box = ax[1].get_position() ax[1].set_position([box.x0, box.y0, box.width * 0.6, box.height]) ax[1].legend(bbox_to_anchor=(1, 1.0)) ax[1].get_xaxis().set_major_locator( matplotlib.ticker.MaxNLocator(integer=True)) if plot: fn = get_fn(**title_key) settings_fn = os.path.join("plot_settings", fn + ".json") settings = utils.load_json(settings_fn) or {"ax1": {}, "ax2": {}} fit_results_fn = os.path.join("fit_results", fn + ".json") def save_settings(): utils.save_json(settings_fn, settings) utils.sync_axes_lims(ax[0], settings["ax1"], save_settings) utils.sync_axes_lims(ax[1], settings["ax2"], save_settings) utils.savefig(fig, fn) return fit_results
def main(): global best_loss start_epoch = 0 # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) model = models.CycleTime(class_num=params['classNum'], trans_param_num=3, pretrained=args.pretrained_imagenet, temporal_out=params['videoLen'], T=args.T, hist=args.hist) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = False print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss().cuda() if args.optim == 'adam': optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(args.momentum, 0.999), weight_decay=args.weight_decay) else: optimizer = optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.weight_decay, momentum=0.95) print('weight_decay: ' + str(args.weight_decay)) print('beta1: ' + str(args.momentum)) if len(args.pretrained) > 0: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.pretrained), 'Error: no checkpoint directory found!' checkpoint = torch.load(args.pretrained) partial_load(checkpoint['state_dict'], model) # model.load_state_dict(checkpoint['state_dict'], strict=False) del checkpoint title = 'videonet' if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) start_epoch = checkpoint['epoch'] partial_load(checkpoint['state_dict'], model) logger = Logger(os.path.join(args.checkpoint, 'log-resume.txt'), title=title) logger.set_names( ['Learning Rate', 'Train Loss', 'Theta Loss', 'Theta Skip Loss']) del checkpoint else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names( ['Learning Rate', 'Train Loss', 'Theta Loss', 'Theta Skip Loss']) train_loader = torch.utils.data.DataLoader(vlog.VlogSet( params, is_train=True, frame_gap=args.frame_gap), batch_size=params['batchSize'], shuffle=True, num_workers=args.workers, pin_memory=True) # Train and val for epoch in range(start_epoch, args.epochs): print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, theta_loss, theta_skip_loss = train( train_loader, model, criterion, optimizer, epoch, use_cuda, args) # append logger file logger.append([state['lr'], train_loss, theta_loss, theta_skip_loss]) if epoch % 1 == 0: save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), }, checkpoint=args.checkpoint) logger.close() savefig(os.path.join(args.checkpoint, 'log.eps'))
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 trainset = dataloader(root='/home/amir/cifar-10', train=True, download=True, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root='/home/amir/cifar-10', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) else: model = models.__dict__[args.arch](num_classes=num_classes) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() params_to_optimize = [p for p in model.parameters() if p.requires_grad] p1 = [] p2 = [] if type(model) is nn.DataParallel: m = model.module else: m = model for k, v in m.named_children(): if 'pooling_convolution' not in k: p1.append(v.parameters()) else: p2.append(v.parameters()) p1 = itertools.chain(*p1) p2 = itertools.chain(*p2) optimizer = optim.SGD([{ 'params': p1 }, { 'params': p2, 'lr': 1e-3 }], lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
train_loss, train_dice = run_epoch( train_loader, net, phase="train", optimizer=optimizer, criterion=criterion, epoch=epoch, args=args, ) val_loss, val_dice = run_epoch(valid_loader, net, phase="valid", criterion=criterion, epoch=epoch, args=args) # scheduler.step(val_loss) lr = optimizer.param_groups[0]["lr"] logger.append([lr, train_loss, val_loss, train_dice, val_dice]) save_ckp(net, optimizer, epoch + 1, args) if optimizer.param_groups[0]["lr"] <= (args.lr / 100): break logger.close() logger.plot(logger.names[1:3]) savefig(os.path.join(args.ckp, "0-loss-log.eps")) logger.plot(logger.names[-2:]) savefig(os.path.join(args.ckp, "0-dice-log.eps"))
def main(): ##################### # START SETUP LOGGING foldername = str(uuid.uuid4()) isplus = '+' if not args.no_augment else '' savefolder = 'results/%s/%s%s/%s' % (args.arch, args.dataset, isplus, foldername) os.system('mkdir -p %s' % savefolder) args.checkpoint = savefolder time.sleep(5) # wait for directory creation print 'folder is ', foldername # use sys.stdout to log to file orig_stdout = sys.stdout logfilename = '%s/log.txt' % (savefolder) logfile = file(logfilename, 'w') if not args.no_print: print 'Printing to file %s' % logfilename sys.stdout = logfile else: print 'Printing to stdout' backupfname = '%s/code_snapshot_%s.zip' % (savefolder, str(datetime.now())) backupfname = backupfname.replace(' ','_') backup_code(backupfname, '.', ['.py'], ['result', 'log',]) print args # END SETUP LOGGING ################### global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.no_augment: print 'NO DATA AUGMENTATION' transform_train = transform_test else: print 'USE DATA AUGMENTATION' transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) testset = dataloader(root='./data', train=False, download=False, transform=transform_test) if args.validation: # select random subset for validation N = len(trainset) train_size = int(N*.9) # use 90 % of training set for training valid_size = N-train_size print 'number of training examples is %i/%i' % (train_size,N) indices = torch.randperm(N) train_indices = indices[:train_size] valid_indices = indices[train_size:] assert(set(train_indices).isdisjoint(set(valid_indices))) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, sampler=SubsetRandomSampler(train_indices), num_workers=args.workers) testloader = data.DataLoader(trainset, batch_size=args.test_batch, sampler=SubsetRandomSampler(valid_indices), num_workers=args.workers) else: trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) else: model = models.__dict__[args.arch](num_classes=num_classes) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0)) # use OLE loss? print 'lambda_ =', args.lambda_ lambda_ = args.lambda_ global use_OLE print args if lambda_>0: use_OLE = True else: use_OLE = False if use_OLE: criterion = [nn.CrossEntropyLoss()] + [OLELoss(lambda_=args.lambda_)] else: criterion = [nn.CrossEntropyLoss()] optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.join(os.path.dirname(args.resume), 'fine_tune') if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = 0 # checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) # logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) # logger = Logger(os.path.join(args.checkpoint, 'log_finetune.txt'), title=title) # logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) else: pass # logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) # logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f, lambda_: %f' % (epoch + 1, args.epochs, state['lr'], args.lambda_)) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) # append logger file # logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) # logger.close() # logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc) ############# # END LOGGING sys.stdout = orig_stdout logfile.close() print '---' print 'saved results to ', savefolder print('Done!')
def main(): global args, best_prec1 args = parser.parse_args() conf_name = args.data conf_name += '_ortho' if args.ortho else '' conf_name += ('_pre_' + args.net_type) if args.net_type != 'default' else '' args.checkpoint = os.path.join(args.checkpoint, conf_name) if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) model = models.Net(extractor_type=args.net_type).cuda() # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() extractor_params = list(map(id, model.extractor.parameters())) classifier_params = filter(lambda p: id(p) not in extractor_params, model.parameters()) optimizer = torch.optim.SGD([{ 'params': model.extractor.parameters() }, { 'params': classifier_params, 'lr': args.lr * 10 }], lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1) # optionally resume from a checkpoint title = 'CUB' if args.resume: print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) # Data loading code normalize = transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) train_dataset = loader.ImageLoader(args.data, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]), train=True) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(loader.ImageLoader( args.data, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion) return for epoch in range(args.start_epoch, args.epochs): scheduler.step() lr = optimizer.param_groups[1]['lr'] print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, lr)) # train for one epoch train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, args) # evaluate on validation set test_loss, test_acc = validate(val_loader, model, criterion) # append logger file logger.append([lr, train_loss, test_loss, train_acc, test_acc]) # remember best prec@1 and save checkpoint is_best = test_acc > best_prec1 best_prec1 = max(test_acc, best_prec1) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_prec1)
def main(): global args, best_prec1 args = parser.parse_args() if args.seed is not None: random.seed(args.seed) torch.manual_seed(args.seed) cudnn.deterministic = True warnings.warn('You have chosen to seed training. ' 'This will turn on the CUDNN deterministic setting, ' 'which can slow down your training considerably! ' 'You may see unexpected behavior when restarting ' 'from checkpoints.') args.distributed = args.world_size > 1 if args.distributed: dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size) # create model print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch](width_mult=args.width_mult) if not args.distributed: if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model, device_ids=[0]).cuda() else: model.cuda() model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[0]) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint title = 'ImageNet-' + args.arch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) args.checkpoint = os.path.dirname(args.resume) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: print("=> no checkpoint found at '{}'".format(args.resume)) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) cudnn.benchmark = True # Data loading code if args.data_backend == 'pytorch': get_train_loader = get_pytorch_train_loader get_val_loader = get_pytorch_val_loader elif args.data_backend == 'dali-gpu': get_train_loader = get_dali_train_loader(dali_cpu=False) get_val_loader = get_dali_val_loader() elif args.data_backend == 'dali-cpu': get_train_loader = get_dali_train_loader(dali_cpu=True) get_val_loader = get_dali_val_loader() else: raise ValueError train_loader, train_loader_len = get_train_loader(args.data, args.batch_size, workers=args.workers, input_size=args.input_size) val_loader, val_loader_len = get_val_loader(args.data, args.batch_size, workers=args.workers, input_size=args.input_size) if args.evaluate: from collections import OrderedDict if os.path.isfile(args.weight): print("=> loading pretrained weight '{}'".format(args.weight)) source_state = torch.load(args.weight) target_state = OrderedDict() for k, v in source_state.items(): if k[:7] != 'module.': k = 'module.' + k target_state[k] = v model.load_state_dict(target_state) else: print("=> no weight found at '{}'".format(args.weight)) if args.prune: from xavier_lib import StatisticManager import time manager = StatisticManager() manager(model) # train_loss, train_acc = train(train_loader, train_loader_len, model, criterion, optimizer, 150) validate(val_loader, val_loader_len, model, criterion) for _ in range(1): manager.computer_score() manager.prune_local(200) manager.pruning_overview() validate(val_loader, val_loader_len, model, criterion) for i in range(30): train(train_loader, train_loader_len, model, criterion, optimizer, i*5) manager.reset() validate(val_loader, val_loader_len, model, criterion) log_time = time.strftime("%Y-%m-%d_%H-%M-%S") torch.save(model.state_dict(), './ckp_for_pruning/'+log_time+'.pth') else: validate(val_loader, val_loader_len, model, criterion) return for epoch in range(args.start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) print('\nEpoch: [%d | %d]' % (epoch + 1, args.epochs)) # train for one epoch train_loss, train_acc = train(train_loader, train_loader_len, model, criterion, optimizer, epoch) # evaluate on validation set val_loss, prec1 = validate(val_loader, val_loader_len, model, criterion) lr = optimizer.param_groups[0]['lr'] # append logger file logger.append([lr, train_loss, val_loss, train_acc, prec1]) is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) save_checkpoint({ 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best accuracy:') print(best_prec1)
def main(): args = parser.parse_args() torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) if args.remark != None: args.remark = args.remark else: args.remark = args.dataset + "-" + args.task + "-" + args.norm if args.dataset == "shapenet": args.num_class = 16 else: args.num_class = 40 def log_string(str): logger.info(str) print(str) '''HYPER PARAMETER''' os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu '''CREATE DIR''' timestr = str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')) experiment_dir = Path('/data-x/g12/zhangjie/3dIP/exp/v1') experiment_dir.mkdir(exist_ok=True) experiment_dir = experiment_dir.joinpath('pruning') experiment_dir.mkdir(exist_ok=True) experiment_dir = experiment_dir.joinpath(args.remark + "_" + timestr) experiment_dir.mkdir(exist_ok=True) checkpoints_dir = experiment_dir.joinpath('checkpoints/') checkpoints_dir.mkdir(exist_ok=True) log_dir = experiment_dir.joinpath('logs/') log_dir.mkdir(exist_ok=True) '''LOG_curve''' title = args.dataset + "-" + args.task + "-" + args.norm + "-" + "Pruning" logger_loss = Logger(os.path.join(log_dir, 'log_loss.txt'), title=title) logger_loss.set_names(['Valid Loss', ' Trigger Valid Loss']) logger_acc = Logger(os.path.join(log_dir, 'log_acc.txt'), title=title) logger_acc.set_names(['Valid Acc.', 'Trigger Valid Acc.']) '''LOG''' #创建log文件 logger = logging.getLogger("Model") #log的名字 logger.setLevel(logging.INFO) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') file_handler = logging.FileHandler('%s/%s.txt' % (log_dir, args.model)) file_handler.setLevel(logging.INFO) #log的最低等级 file_handler.setFormatter(formatter) logger.addHandler(file_handler) #log文件名 log_string('PARAMETER ...') log_string(args) '''DATA LOADING''' log_string('Load pruning test dataset ...') if args.dataset == "shapenet": testDataLoader = getData.get_dataLoader(train=False, Shapenet=True, batchsize=args.batch_size) triggerDataLoader = getData2.get_dataLoader(Shapenet=True, batchsize=args.batch_size) else: testDataLoader = getData.get_dataLoader(train=False, Shapenet=False, batchsize=args.batch_size) triggerDataLoader = getData2.get_dataLoader(Shapenet=False, batchsize=args.batch_size) log_string('Load finished ...') '''MODEL LOADING''' num_class = args.num_class MODEL = importlib.import_module(args.model) shutil.copy('./models/%s.py' % args.model, str(experiment_dir)) shutil.copy('./models/pointnet_util.py', str(experiment_dir)) shutil.copy('prun1.py', str(experiment_dir)) shutil.copytree('./models/layers', str(experiment_dir) + "/layers") shutil.copytree('./data', str(experiment_dir) + "/data") shutil.copytree('./utils', str(experiment_dir) + "/utils") classifier = MODEL.get_model(num_class, channel=3).cuda() pprint(classifier) pth_dir = '/data-x/g12/zhangjie/3dIP/exp/v1/classification/' + args.dataset + "-" \ + args.task + "-" + args.norm + "/checkpoints/best_model.pth" log_string('pre-trained model chk pth: %s' % pth_dir) checkpoint = torch.load(pth_dir) model_dict = checkpoint['model_state_dict'] print('Total : {}'.format(len(model_dict))) print("best epoch", checkpoint['epoch']) classifier.load_state_dict(model_dict) p_num = get_parameter_number(classifier) log_string('Original trainable parameter: %s' % p_num) '''TESTING ORIGINAL''' logger.info('Test original model...') with torch.no_grad(): _, instance_acc, class_acc = test(classifier, testDataLoader, num_class=args.num_class) _, instance_acc2, class_acc2 = test(classifier, triggerDataLoader, num_class=args.num_class) log_string( 'Original Test Instance Accuracy: %f, Class Accuracy: %f' % (instance_acc, class_acc)) log_string( 'Original Test Trigger Accuracy: %f, Trigger Class Accuracy: %f' % (instance_acc2, class_acc2)) '''PRUNING''' logger.info('Start testing of pruning...') for perc in [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]: time_start = datetime.datetime.now() classifier.load_state_dict(model_dict) p_num = get_parameter_number(classifier) log_string('Original trainable parameter: %s' % p_num) '''Testing pruning model''' logger.info('Testing pruning model--%d%%' % perc) pruning_net(classifier, perc) classifier.cuda() p_num = get_parameter_number(classifier) log_string('Pruning %d%% -- trainable parameter: %s' % (perc, p_num)) with torch.no_grad(): val_loss, instance_acc, class_acc = test(classifier, testDataLoader, num_class=args.num_class) val_loss2, instance_acc2, class_acc2 = test( classifier, triggerDataLoader, num_class=args.num_class) log_string( 'Pruning %d%% Test Instance Accuracy: %f, Class Accuracy: %f' % (perc, instance_acc, class_acc)) log_string( 'Pruning %d%% Test Trigger Accuracy: %f, Trigger Class Accuracy: %f' % (perc, instance_acc2, class_acc2)) logger_loss.append([val_loss, val_loss2]) logger_acc.append([instance_acc, instance_acc2]) time_end = datetime.datetime.now() time_span_str = str((time_end - time_start).seconds) log_string('Epoch time : %s S' % (time_span_str)) logger_loss.close() logger_loss.plot_prun() savefig(os.path.join(log_dir, 'log_loss.eps')) logger_acc.close() logger_acc.plot_prun() savefig(os.path.join(log_dir, 'log_acc.eps')) logger.info('End of pruning...')
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 trainset = dataloader(root=args.data_root, train=True, download=True, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root=args.data_root, train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) else: model = models.__dict__[args.arch](num_classes=num_classes) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def plot(fit_count=5, log=False, maxfev=0, plot_type="scatter", stat="err", hf=False): dorig = utils.filter_preferred_ml(utils.load_all()) d_good = utils.load_table("fits_good.txt") d_good = d_good.groupby( ["interaction", "label", "freq", "num_filled", "method"] ).first() d = gather_fit_data(fit_count=fit_count, maxfev=maxfev) d = d[d["interaction"] == "normal"] # d = d[d["label"] == "add"] # d = d[d["method"] == "imsrg"] # d = d[d["num_filled"] == 5] # d = d[d["freq"] == 0.1] d = d[d["fit_method"] != "fixedab"] doriggrps = dorig.groupby(["interaction", "label", "freq", "num_filled", "method"]) d["rel_constant_err"] = d["constant_err"] / d["constant"] d["rel_best_constant_err"] = d["best_constant_err"] / d["best_constant"] d["label_is_ground"] = d["label"] == "ground" d["good"] = d.apply(lambda r: d_good.loc[ (r["interaction"], r["label"], r["freq"], r["num_filled"], r["method"])]["good"], axis=1) d["rel_chi"] = d["chisq"]**.5 / d["constant"] d["rel_reduced_chi"] = d["rel_chi"] / (fit_count - 3) d["rel_best_chisq"] = d["best_chisq"]**.5 / d["best_constant"] d["rel_best_reduced_chisq"] = d["rel_best_chisq"] / (fit_count - 3) d = d[(d["rel_best_reduced_chisq"] < 1e-6)] d["fixedab_with_hf"] = ( ((d["fit_method"] == "fixedab") == (d["method"].isin(["hf", "hf+qdpt3"]))) | (d["fit_method"] == "full") ) color_col = "method" bin_transform = TRANSFORM_ID bin_transform = TRANSFORM_LOG_ABS if color_col in ["exponent", "rel_dist", "rel_constant_err", "rel_best_constant_err", "rel_best_chisq", "rel_chi", "rel_reduced_chi", "chi_ratio"]: num_bins = 16 d = d.replace([np.inf, -np.inf], np.nan).dropna(subset=[color_col]) binf = bin_transform[0] bininvf = bin_transform[1] color_bins = pd.cut(binf(abs(d[color_col])), num_bins) d["color_bin_start"] = color_bins.map( lambda bin: bininvf(parse_bin(bin)[0])) d["color_bin_stop"] = color_bins.map( lambda bin: bininvf(parse_bin(bin)[1])) color_bin_cols = ["color_bin_start", "color_bin_stop"] else: color_bin_cols = [color_col] max_bins = len(d[color_bin_cols[0]].unique()) fig, ax = plt.subplots() def on_pick_event(event): x = list(event.artist.get_xdata()[event.ind])[0] y = list(event.artist.get_ydata()[event.ind])[0] sel = d[(abs(d["x"] - x) <= 1e-20) & (abs(d["y"] - y) <= 1e-20)] print(sel.transpose().to_csv()) if len(sel) != 1: print('>> not found <<') return sel = sel.iloc[0] grp = doriggrps.get_group((sel["interaction"], sel["label"], sel["freq"], sel["num_filled"], sel["method"])) fig, ax = plt.subplots(2) ax[0].plot(grp["num_shells"], grp["energy"], "x") fit_start = sel["fit_stop"] + 1 - fit_count ax[0].axvspan(fit_start, sel["fit_stop"], color="#cccccc") xs = np.linspace(grp["num_shells"].min(), grp["num_shells"].max()) ax[0].plot(xs, sel["coefficient"] * xs ** sel["exponent"] + sel["constant"]) subgrp = grp[grp["num_shells"].between( fit_start-0.1, sel["fit_stop"]+0.1)] last_constant = sel["constant"] last_constant_err = sel["constant_err"] def random_weight(count): weights = np.zeros(count) for i in range(count): weights[np.random.randint(0, count)] += 1 return weights p0 = [sel["coefficient"], sel["exponent"], sel["constant"]] p = p0 x = subgrp["num_shells"] y = subgrp["energy"] constants = [] constants.append(p[2]) print(f"x = np.array({list(x)})") print(f"y = np.array({list(y)})") ax[1].plot(x, (p[0] * x ** p[1] + p[2] - y), "-x") ax[1].axhline(0.0, linestyle=":") for i in range(10): count = len(x) weights = random_weight(count) + 1e-99 if sum(weights > 0.1) <= 3: # can't fit with this few points continue try: p, cov = scipy.optimize.curve_fit( lambda x, a, b, c: a * x ** b + c, x, y, sigma=1.0 / weights ** 0.5, p0=p0, maxfev=100000) except RuntimeError as e: print(e) continue chisq = np.average((p[0] * x ** p[1] + p[2] - y) ** 2, weights=weights) * len(x) constant = p[2] constant_err = cov[2, 2] ** 0.5 constants.append(p[2]) last_constant = constant last_constant_err = constant_err print("result", np.mean(constants), np.std(constants)) print("rel", np.std(constants) / np.mean(constants)) ax[0].set_ylim([max(ax[0].get_ylim()[0], 0.0), min(ax[0].get_ylim()[1], np.max(y))]) ax[0].plot(xs, p[0] * xs ** p[1] + p[2], ":", color="lime") fig.canvas.mpl_connect("pick_event", on_pick_event) d["quality"] = np.log10(d["constant_err"]/d["chisq"]**0.5) # hf has unique behaviors (what about mp2?) if hf: d = d[d["method"] == "hf"] else: d = d[d["method"] != "hf"] if stat == "err": d = d[d["quality"] > 0] d["y"] = d["rel_discrep"] / d["rel_constant_err"] if stat == "hessian": d["x"] = d["quality"] elif stat == "err": d["x"] = np.log10(d["rel_constant_err"]) else: assert False d["y_err"] = d["rel_discrep_err"] / d["rel_discrep"] * d["y"] d = d[(d["rel_constant_err"] > 0) & (d["rel_constant_err"] < np.inf)] if plot_type == "contour": if hf: hf_suffix = "HF" else: hf_suffix = "non-HF" # ranged = mesh; lim = view if stat == "err": if hf: nx = 20 ny = 20 xrange = (-7, -1) yrange = (-5, 5) xlim = (-6, -2) ylim = (-4, 4) title = ("discrepancy vs fit uncertainty " f"({hf_suffix})") else: nx = 20 ny = 40 xrange = (-7, -1) yrange = (-50, 50) xlim = (-6, -2) ylim = (-40, 40) title = ("discrepancy vs fit uncertainty " f"(filtered: Q > 0, {hf_suffix})") xlabel = r"$\log_{10}\left(\frac{\sigma_c}{c}\right)$" ylabel = r"$\frac{\varepsilon}{\sigma_c}$" elif stat == "hessian": if hf: nx = 20 ny = 40 xrange = (-0.6, 1.5) yrange = (-10, 10) xlim = (-0.4, 1.1) ylim = (-10, 10) else: nx = 20 ny = 40 xrange = (-0.6, 2.1) yrange = (-200, 200) xlim = (-0.4, 1.7) ylim = (-150, 150) title = ("spread of “actual” discrepancy vs quality " f"({hf_suffix})") xlabel = (r"$Q = \log_{10}\left(\frac{\sigma_c}" r"{\sqrt{\mathtt{RSS}}}\right)$") ylabel = r"$\frac{\varepsilon}{\sigma_c}$" else: assert False ax.plot(d["x"], d["y"], "o", markersize=1, picker=3, color="white", markeredgewidth=0) dx = (xrange[1] - xrange[0]) / (nx - 1) h, x, y = np.histogram2d(d["x"], d["y"], bins=(nx, ny - 1), range=((xrange[0] - 0.5 * dx, xrange[1] + 0.5 * dx), yrange)) ch = np.concatenate([np.zeros((nx, 1)), np.cumsum(h, axis=1)], axis=1) z = ch / ch[..., -1, np.newaxis] x, y = np.meshgrid(0.5 * (x[1:] + x[:-1]), y, indexing="ij") levels = np.linspace(-2.0, 2.0, 5) levels = 0.5 + 0.5 * scipy.special.erf(2.0 ** -0.5 * levels) ax.axhline(-1.0, linestyle="--", color="white", linewidth=0.5) ax.axhline(1.0, linestyle="--", color="white", linewidth=0.5) ax.contour(x, y, z, levels=levels, colors="white", alpha=0.3) cs = ax.contourf(x, y, z, levels=np.linspace(0.0, 1.0, 300), # note: translucent cmaps tend to cause artifacts cmap=CMAP_FOLDED_VIRIDIS, linestyle=":") for c in cs.collections: # http://stackoverflow.com/a/32911283 c.set_edgecolor("face") fig.colorbar(cs) ax.set_title(title) ax.set_xlabel(xlabel) ax.set_ylabel(ylabel) ax.set_xlim(xlim) ax.set_ylim(ylim) elif plot_type == "scatter": cmap = utils.CMAP_RAINBOW for i, (bin, g) in enumerate(sorted(d.groupby(color_bin_cols))): color = cmap(float(i) / max_bins) ax.plot(g["x"], g["y"], ".", label=str(bin), color=color, markersize=10, markeredgewidth=0, alpha=0.5, picker=2) ax.legend() fn = f"fit-predictiveness-{plot_type}-{fit_count}-{stat}-{hf}" fig.tight_layout() utils.savefig(fig, fn)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # load data print('==> Preparing dataset %s' % args.dataset) features, labels = pickle_2_img_single(args.dataset_path) num_classes = 6 # transformers global transform_train, transform_test transform_train = transforms.Compose([ transforms.Grayscale(1), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([ 0.485, ], [ 0.229, ]) ]) transform_test = transforms.Compose([ transforms.Grayscale(1), transforms.ToTensor(), transforms.Normalize([ 0.485, ], [ 0.229, ]) ]) # transform_train = transforms.Compose([ # transforms.RandomCrop(32, padding=4), # transforms.RandomHorizontalFlip(), # transforms.ToTensor(), # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), # ]) # transform_test = transforms.Compose([ # transforms.ToTensor(), # transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), # ]) # if args.dataset == 'cifar10': # dataloader = datasets.CIFAR10 # num_classes = 10 # else: # dataloader = datasets.CIFAR100 # num_classes = 100 # trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) # trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) # testset = dataloader(root='./data', train=False, download=False, transform=transform_test) # testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) else: model = models.__dict__[args.arch](num_classes=num_classes) # model = torch.nn.DataParallel(model).cuda() model = model.cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'oululu-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) # 10-fold cross validation train_x, train_y = [], [] test_x, test_y = [], [] for id_fold in range(len(labels)): if id_fold == epoch % 10: test_x = features[id_fold] test_y = labels[id_fold] else: train_x = train_x + features[id_fold] train_y = train_y + labels[id_fold] # convert array to tensor train_x = torch.tensor(train_x, dtype=torch.float32) / 255.0 #(b_s, 128, 128) train_x = train_x.unsqueeze(1) #(b_s, 1, 128, 128) test_x = torch.tensor(test_x, dtype=torch.float32) / 255.0 test_x = test_x.unsqueeze(1) train_y, test_y = torch.tensor(train_y), torch.tensor(test_y) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(test_x, test_y, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(train_x, train_y, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(test_x, test_y, model, criterion, epoch, use_cuda) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def plot(fit_count=fits.DEFAULT_FIT_COUNT, log=False, maxfev=fits.DEFAULT_MAXFEV, plot_type="scatter", stat="err", hf=False): dorig = utils.filter_preferred_ml(utils.load_all()) d_good = utils.load_table("fits_good.txt") d_good = d_good.groupby( ["interaction", "label", "freq", "num_filled", "method"] ).first() d = fits.load_predictive_data(fit_count=fit_count, maxfev=maxfev) d = d[d["interaction"] == "normal"] # d = d[d["label"] == "add"] # d = d[d["method"] == "imsrg"] # d = d[d["num_filled"] == 5] # d = d[d["freq"] == 0.1] d = d[d["fit_method"] != "fixedab"] doriggrps = dorig.groupby(["interaction", "label", "freq", "num_filled", "method"]) d["rel_constant_err"] = d["constant_err"] / d["constant"] d["rel_best_constant_err"] = d["best_constant_err"] / d["best_constant"] d["label_is_ground"] = d["label"] == "ground" d["good"] = d.apply(functools.partial(is_good, d_good), axis=1) d["rel_chi"] = d["chisq"]**.5 / d["constant"] d["rel_reduced_chi"] = d["rel_chi"] / (fit_count - 3) d["rel_best_chisq"] = d["best_chisq"]**.5 / d["best_constant"] d["rel_best_reduced_chisq"] = d["rel_best_chisq"] / (fit_count - 3) d = d[(d["rel_best_reduced_chisq"] < 1e-6)] d["fixedab_with_hf"] = ( ((d["fit_method"] == "fixedab") == (d["method"].isin(["hf", "hf+qdpt3"]))) | (d["fit_method"] == "full") ) color_col = "method" bin_transform = TRANSFORM_ID bin_transform = TRANSFORM_LOG_ABS if color_col in ["exponent", "rel_dist", "rel_constant_err", "rel_best_constant_err", "rel_best_chisq", "rel_chi", "rel_reduced_chi", "chi_ratio"]: num_bins = 16 d = d.replace([np.inf, -np.inf], np.nan).dropna(subset=[color_col]) binf = bin_transform[0] bininvf = bin_transform[1] color_bins = pd.cut(binf(abs(d[color_col])), num_bins) d["color_bin_start"] = color_bins.map( lambda bin: bininvf(parse_bin(bin)[0])) d["color_bin_stop"] = color_bins.map( lambda bin: bininvf(parse_bin(bin)[1])) color_bin_cols = ["color_bin_start", "color_bin_stop"] else: color_bin_cols = [color_col] max_bins = len(d[color_bin_cols[0]].unique()) fig, ax = plt.subplots() def on_pick_event(event): x = list(event.artist.get_xdata()[event.ind])[0] y = list(event.artist.get_ydata()[event.ind])[0] sel = d[(abs(d["x"] - x) <= 1e-20) & (abs(d["y"] - y) <= 1e-20)] print(sel.transpose().to_csv()) if len(sel) != 1: print('>> not found <<') return sel = sel.iloc[0] grp = doriggrps.get_group((sel["interaction"], sel["label"], sel["freq"], sel["num_filled"], sel["method"])) fig, ax = plt.subplots(2) ax[0].plot(grp["num_shells"], grp["energy"], "x") fit_start = sel["fit_stop"] + 1 - fit_count ax[0].axvspan(fit_start, sel["fit_stop"], color="#cccccc") xs = np.linspace(grp["num_shells"].min(), grp["num_shells"].max()) ax[0].plot(xs, sel["coefficient"] * xs ** sel["exponent"] + sel["constant"]) subgrp = grp[grp["num_shells"].between( fit_start-0.1, sel["fit_stop"]+0.1)] last_constant = sel["constant"] last_constant_err = sel["constant_err"] def random_weight(count): weights = np.zeros(count) for i in range(count): weights[np.random.randint(0, count)] += 1 return weights p0 = [sel["coefficient"], sel["exponent"], sel["constant"]] p = p0 x = subgrp["num_shells"] y = subgrp["energy"] constants = [] constants.append(p[2]) print(f"x = np.array({list(x)})") print(f"y = np.array({list(y)})") ax[1].plot(x, (p[0] * x ** p[1] + p[2] - y), "-x") ax[1].axhline(0.0, linestyle=":") for i in range(10): count = len(x) weights = random_weight(count) + 1e-99 if sum(weights > 0.1) <= 3: # can't fit with this few points continue try: p, cov = scipy.optimize.curve_fit( lambda x, a, b, c: a * x ** b + c, x, y, sigma=1.0 / weights ** 0.5, p0=p0, maxfev=100000) except RuntimeError as e: print(e) continue chisq = np.average((p[0] * x ** p[1] + p[2] - y) ** 2, weights=weights) * len(x) constant = p[2] constant_err = cov[2, 2] ** 0.5 constants.append(p[2]) last_constant = constant last_constant_err = constant_err print("result", np.mean(constants), np.std(constants)) print("rel", np.std(constants) / np.mean(constants)) ax[0].set_ylim([max(ax[0].get_ylim()[0], 0.0), min(ax[0].get_ylim()[1], np.max(y))]) ax[0].plot(xs, p[0] * xs ** p[1] + p[2], ":", color="lime") fig.canvas.mpl_connect("pick_event", on_pick_event) d["quality"] = np.log10(d["constant_err"]/d["chisq"]**0.5) # hf has unique behaviors (what about mp2?) if hf: d = d[d["method"] == "hf"] else: d = d[d["method"] != "hf"] if stat == "err": d = d[d["quality"] > 0] d["y"] = d["rel_discrep"] / d["rel_constant_err"] if stat == "hessian": d["x"] = d["quality"] elif stat == "err": d["x"] = np.log10(d["rel_constant_err"]) else: assert False d["y_err"] = d["rel_discrep_err"] / d["rel_discrep"] * d["y"] d = d[(d["rel_constant_err"] > 0) & (d["rel_constant_err"] < np.inf)] if plot_type == "contour": if hf: hf_suffix = "HF" else: hf_suffix = "non-HF" # ranged = mesh; lim = view if stat == "err": if hf: nx = 20 ny = 20 xrange = (-7, -1) yrange = (-5, 5) xlim = (-6, -2) ylim = (-4, 4) title = ("discrepancy vs fit uncertainty " f"({hf_suffix})") else: nx = 20 ny = 40 xrange = (-7, -1) yrange = (-50, 50) xlim = (-6, -2) ylim = (-40, 40) title = ("discrepancy vs fit uncertainty " f"(filtered: Q > 0, {hf_suffix})") xlabel = r"$\log_{10}\left(\frac{\sigma_c}{c}\right)$" ylabel = r"$\frac{\varepsilon}{\sigma_c}$" elif stat == "hessian": if hf: nx = 20 ny = 40 xrange = (-0.6, 1.5) yrange = (-10, 10) xlim = (-0.4, 1.1) ylim = (-10, 10) else: nx = 20 ny = 40 xrange = (-0.6, 2.1) yrange = (-200, 200) xlim = (-0.4, 1.7) ylim = (-150, 150) title = ("spread of “actual” discrepancy vs quality " f"({hf_suffix})") xlabel = (r"$Q = \log_{10}\left(\frac{\sigma_c}" r"{\sqrt{\mathtt{RSS}}}\right)$") ylabel = r"$\frac{\varepsilon}{\sigma_c}$" else: assert False ax.plot(d["x"], d["y"], "o", markersize=1, picker=3, color="white", markeredgewidth=0) dx = (xrange[1] - xrange[0]) / (nx - 1) h, x, y = np.histogram2d(d["x"], d["y"], bins=(nx, ny - 1), range=((xrange[0] - 0.5 * dx, xrange[1] + 0.5 * dx), yrange)) ch = np.concatenate([np.zeros((nx, 1)), np.cumsum(h, axis=1)], axis=1) z = ch / ch[..., -1, np.newaxis] x, y = np.meshgrid(0.5 * (x[1:] + x[:-1]), y, indexing="ij") levels = np.linspace(-2.0, 2.0, 5) levels = 0.5 + 0.5 * scipy.special.erf(2.0 ** -0.5 * levels) ax.axhline(-1.0, linestyle="--", color="white", linewidth=0.5) ax.axhline(1.0, linestyle="--", color="white", linewidth=0.5) ax.contour(x, y, z, levels=levels, colors="white", alpha=0.3) cs = ax.contourf(x, y, z, levels=np.linspace(0.0, 1.0, 300), # note: translucent cmaps tend to cause artifacts cmap=CMAP_FOLDED_VIRIDIS, linestyle=":") for c in cs.collections: # http://stackoverflow.com/a/32911283 c.set_edgecolor("face") fig.colorbar(cs) ax.set_title(title) ax.set_xlabel(xlabel) ax.set_ylabel(ylabel) ax.set_xlim(xlim) ax.set_ylim(ylim) elif plot_type == "scatter": cmap = utils.CMAP_RAINBOW for i, (bin, g) in enumerate(sorted(d.groupby(color_bin_cols))): color = cmap(float(i) / max_bins) ax.plot(g["x"], g["y"], ".", label=str(bin), color=color, markersize=10, markeredgewidth=0, alpha=0.5, picker=2) ax.legend() fn = f"fit-predictiveness-{plot_type}-{fit_count}-{stat}-{hf}" fig.tight_layout() utils.savefig(fig, fn)
def main(): global BEST_ACC, LR_STATE start_epoch = cfg.CLS.start_epoch # start from epoch 0 or last checkpoint epoch # Create ckpt folder if not os.path.isdir(cfg.CLS.ckpt): mkdir_p(cfg.CLS.ckpt) if args.cfg_file is not None and not cfg.CLS.evaluate: shutil.copyfile(args.cfg_file, os.path.join(cfg.CLS.ckpt, args.cfg_file.split('/')[-1])) # Dataset and Loader normalize = transforms.Normalize(mean=cfg.pixel_mean, std=cfg.pixel_std) train_aug = [transforms.RandomResizedCrop(cfg.CLS.crop_size), transforms.RandomHorizontalFlip()] if len(cfg.CLS.rotation) > 0: train_aug.append(transforms.RandomRotation(cfg.CLS.rotation)) if len(cfg.CLS.pixel_jitter) > 0: train_aug.append(RandomPixelJitter(cfg.CLS.pixel_jitter)) if cfg.CLS.grayscale > 0: train_aug.append(transforms.RandomGrayscale(cfg.CLS.grayscale)) train_aug.append(transforms.ToTensor()) train_aug.append(normalize) traindir = os.path.join(cfg.CLS.data_root, cfg.CLS.train_folder) train_loader = torch.utils.data.DataLoader( datasets.ImageFolder(traindir, transforms.Compose(train_aug)), batch_size=cfg.CLS.train_batch, shuffle=True, num_workers=cfg.workers, pin_memory=True) if cfg.CLS.validate or cfg.CLS.evaluate: valdir = os.path.join(cfg.CLS.data_root, cfg.CLS.val_folder) val_loader = torch.utils.data.DataLoader( datasets.ImageFolder(valdir, transforms.Compose([ transforms.Resize(cfg.CLS.base_size), transforms.CenterCrop(cfg.CLS.crop_size), transforms.ToTensor(), normalize, ])), batch_size=cfg.CLS.test_batch, shuffle=False, num_workers=cfg.workers, pin_memory=True) # Create model model = models.__dict__[cfg.CLS.arch]() print(model) # Calculate FLOPs & Param n_flops, n_convops, n_params = measure_model(model, cfg.CLS.crop_size, cfg.CLS.crop_size) print('==> FLOPs: {:.4f}M, Conv_FLOPs: {:.4f}M, Params: {:.4f}M'. format(n_flops / 1e6, n_convops / 1e6, n_params / 1e6)) del model model = models.__dict__[cfg.CLS.arch]() # Load pre-train model if cfg.CLS.pretrained: print("==> Using pre-trained model '{}'".format(cfg.CLS.pretrained)) pretrained_dict = torch.load(cfg.CLS.pretrained) try: pretrained_dict = pretrained_dict['state_dict'] except: pretrained_dict = pretrained_dict model_dict = model.state_dict() updated_dict, match_layers, mismatch_layers = weight_filler(pretrained_dict, model_dict) model_dict.update(updated_dict) model.load_state_dict(model_dict) else: print("==> Creating model '{}'".format(cfg.CLS.arch)) # Define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() if cfg.CLS.pretrained: def param_filter(param): return param[1] new_params = map(param_filter, filter(lambda p: p[0] in mismatch_layers, model.named_parameters())) base_params = map(param_filter, filter(lambda p: p[0] in match_layers, model.named_parameters())) model_params = [{'params': base_params}, {'params': new_params, 'lr': cfg.CLS.base_lr * 10}] else: model_params = model.parameters() model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True optimizer = optim.SGD(model_params, lr=cfg.CLS.base_lr, momentum=cfg.CLS.momentum, weight_decay=cfg.CLS.weight_decay) # Evaluate model if cfg.CLS.evaluate: print('\n==> Evaluation only') test_loss, test_top1, test_top5 = test(val_loader, model, criterion, start_epoch, USE_CUDA) print('==> Test Loss: {:.8f} | Test_top1: {:.4f}% | Test_top5: {:.4f}%'.format(test_loss, test_top1, test_top5)) return # Resume training title = 'Pytorch-CLS-' + cfg.CLS.arch if cfg.CLS.resume: # Load checkpoint. print("==> Resuming from checkpoint '{}'".format(cfg.CLS.resume)) assert os.path.isfile(cfg.CLS.resume), 'Error: no checkpoint directory found!' checkpoint = torch.load(cfg.CLS.resume) BEST_ACC = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(cfg.CLS.ckpt, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(cfg.CLS.ckpt, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) # Train and val for epoch in range(start_epoch, cfg.CLS.epochs): print('\nEpoch: [{}/{}] | LR: {:.8f}'.format(epoch + 1, cfg.CLS.epochs, LR_STATE)) train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, USE_CUDA) if cfg.CLS.validate: test_loss, test_top1, test_top5 = test(val_loader, model, criterion, epoch, USE_CUDA) else: test_loss, test_top1, test_top5 = 0.0, 0.0, 0.0 # Append logger file logger.append([LR_STATE, train_loss, test_loss, train_acc, test_top1]) # Save model save_checkpoint(model, optimizer, test_top1, epoch) # Draw curve try: draw_curve(cfg.CLS.arch, cfg.CLS.ckpt) print('==> Success saving log curve...') except: print('==> Saving log curve error...') logger.close() try: savefig(os.path.join(cfg.CLS.ckpt, 'log.eps')) shutil.copyfile(os.path.join(cfg.CLS.ckpt, 'log.txt'), os.path.join(cfg.CLS.ckpt, 'log{}.txt'.format( datetime.datetime.now().strftime('%Y%m%d%H%M%S')))) except: print('Copy log error.') print('==> Training Done!') print('==> Best acc: {:.4f}%'.format(BEST_ACC))
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data loading code traindir = os.path.join(args.data, 'ILSVRC2012_img_val') valdir = os.path.join(args.data, 'ILSVRC2012_img_val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) #标准训练 train_loader = torch.utils.data.DataLoader(datasets.ImageFolder( traindir, transforms.Compose([ transforms.RandomSizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])), batch_size=args.train_batch, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(datasets.ImageFolder( valdir, transforms.Compose([ transforms.Scale(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=True) #专门用来调优AlexNetMini_LC的,使用61尺寸、RGB顺序、tensor不用归一化或者均值 #train_loader = torch.utils.data.DataLoader( # datasets.ImageFolder(traindir, transforms.Compose([ # transforms.Scale(70), # transforms.RandomSizedCrop(61), # transforms.RandomHorizontalFlip(), # transforms.ToTensor(), # ])), # batch_size=args.train_batch, shuffle=True, # num_workers=args.workers, pin_memory=True) #val_loader = torch.utils.data.DataLoader( # datasets.ImageFolder(valdir, transforms.Compose([ # transforms.Scale(70), # transforms.CenterCrop(61), # transforms.ToTensor(), # ])), # batch_size=args.test_batch, shuffle=False, # num_workers=args.workers, pin_memory=True) # create model if args.model_path: print("Model Path=", args.model_path) if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) elif args.arch.startswith('resnext'): model = models.__dict__[args.arch]( baseWidth=args.base_width, cardinality=args.cardinality, ) elif args.arch.startswith('MobileNetV2_CBAM'): model = MobileNetV2_CBAM() print("train MobileNetV2_CBAM\n") elif args.arch.startswith('MobileNetV2_LC'): model = MobileNetV2_LC() print("train MobileNetV2_LC\n") elif args.arch.startswith('MobileNetV3'): model = MobileNetV3(pretrained=True, model_path=args.model_path) print("train MobileNetV3\n") elif args.arch.startswith('AlexNetMini_LC'): model = AlexNetMini_LC() print("train AlexNetMini_LC\n") elif args.arch.startswith('shufflenet_v2_x1_0'): model = ShuffleNetV2_1(pretrained=True, model_path=args.model_path) print('train ShuffleNetV2\n') elif args.arch.startswith('EfficientNetB0'): model = EfficientNet.from_pretrained('efficientnet-b0', weights_path=args.model_path) print('train EfficientNetB0') elif args.arch.startswith('EfficientNetB1'): model = EfficientNet.from_pretrained('efficientnet-b1', weights_path=args.model_path) print('train EfficientNetB0') elif args.arch.startswith('EfficientNetB2'): model = EfficientNet.from_pretrained('efficientnet-b2', weights_path=args.model_path) print('train EfficientNetB0') elif args.arch.startswith('EfficientNet-lite0'): model = timm.create_model('efficientnet_lite0', pretrained=True) print('train efficientnetlite0') elif args.arch.startswith('GhostNet'): model = GhostNet(pretrained=True, model_path=args.model_path) print("train GhostNet\n") # elif args.arch.startswith('RegNetY_200MF'): # model = RegNetY_200MF('RegNetY_200MF',model_path=args.model_path) # print('train RegNetY_200MF') elif args.arch.startswith('RegNetY_200MF'): model = RegNetY_200MF('RegNetY_200MF', model_path=args.model_path) print('train RegNetY_200MF') elif args.arch.startswith('RegNetY_400MF'): model = RegNetY_200MF('RegNetY_400MF', model_path=args.model_path) print('train RegNetY_400MF') elif args.arch.startswith('RegNetY_600MF'): model = RegNetY_200MF('RegNetY_600MF', model_path=args.model_path) print('train RegNetY_600MF') elif args.arch.startswith('RegNetY_800MF'): model = RegNetY_200MF('RegNetY_800MF', model_path=args.model_path) print('train RegNetY_800MF') elif args.arch.startswith('RexNetV1_1'): model = RexNetV1_1(model_path=args.model_path) print('train RexNetV1_1') else: print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch]() if args.evaluate: input_2 = torch.randn(1, 3, 224, 224) flops, params = profile(model, inputs=(input_2, )) flops, params = clever_format([flops, params], "%.3f") print("*" * 10) print("floats && params:") print(flops, params) #exit() if use_cuda: if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) # define loss function (criterion) and optimizer if use_cuda: criterion = nn.CrossEntropyLoss().cuda() else: criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'ImageNet-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(val_loader, model, criterion, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) #inputs = torch.randn(1,3,224,224).cuda() #total_ops, total_params = profile(model, (inputs,), verbose=False) #print("Evaluate params and floats:") #print("%s | %.2f | %.2f" % (name, total_params / (1000 ** 2), total_ops / (1000 ** 3))) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(val_loader, model, criterion, use_cuda) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset') dataloader = load_data(args) Tensor = torch.cuda.FloatTensor print("==> creating model") title = 'Pytorch-OCGAN' enc = get_encoder().cuda() dec = get_decoder().cuda() disc_v = get_disc_visual().cuda() disc_l = get_disc_latent().cuda() cl = get_classifier().cuda() #load origianal weights disc_v.apply(weights_init) cl.apply(weights_init) enc.apply(weights_init) dec.apply(weights_init) disc_l.apply(weights_init) model = torch.nn.DataParallel(enc).cuda() cudnn.benchmark = True print(' enc Total params: %.2fM' % (sum(p.numel() for p in enc.parameters()) / 1000000.0)) print(' dec Total params: %.2fM' % (sum(p.numel() for p in dec.parameters()) / 1000000.0)) print(' disc_v Total params: %.2fM' % (sum(p.numel() for p in disc_v.parameters()) / 1000000.0)) print(' disc_l Total params: %.2fM' % (sum(p.numel() for p in disc_l.parameters()) / 1000000.0)) print(' cl Total params: %.2fM' % (sum(p.numel() for p in cl.parameters()) / 1000000.0)) #Loss Loss Loss Loss Loss Loss Loss print("==> creating optimizer") criterion_ce = torch.nn.BCELoss(size_average=True).cuda() criterion_ae = nn.MSELoss(size_average=True).cuda() l2_int = torch.empty(size=(args.train_batch, 288, 1, 1), dtype=torch.float32) optimizer_en = optim.Adam(enc.parameters(), lr=args.lr, betas=(0.9, 0.99)) optimizer_de = optim.Adam(dec.parameters(), lr=args.lr, betas=(0.9, 0.99)) optimizer_dl = optim.Adam(disc_l.parameters(), lr=args.lr, betas=(0.9, 0.99)) optimizer_dv = optim.Adam(disc_v.parameters(), lr=args.lr, betas=(0.9, 0.99)) optimizer_c = optim.Adam(cl.parameters(), lr=args.lr, betas=(0.9, 0.99)) optimizer_l2 = optim.Adam([{ 'params': l2_int }], lr=args.lr, betas=(0.9, 0.99)) if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) # optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Acc.']) # Train and val for epoch in range(start_epoch, args.epochs): # adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) # model = optimize_fore() if epoch < 20: train_loss_ae = train_ae(args, dataloader['train'], enc, dec, optimizer_en, optimizer_de, criterion_ae, epoch, use_cuda) test_acc = test(args, dataloader['test'], enc, dec, cl, disc_l, disc_v, epoch, use_cuda) else: train_loss = train(args, dataloader['train'], enc, dec, cl, disc_l, disc_v, optimizer_en, optimizer_de, optimizer_c, optimizer_dl, optimizer_dv, optimizer_l2, criterion_ae, criterion_ce, Tensor, epoch, use_cuda) test_acc = test(args, dataloader['test'], enc, dec, cl, disc_l, disc_v, epoch, use_cuda) # append logger file logger.append([state['lr'], train_loss, test_acc]) # save model is_best = train_loss < best_acc best_acc = min(train_loss, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': enc.state_dict(), 'loss': train_loss, 'best_loss': best_acc, }, is_best, checkpoint=args.checkpoint, filename='enc_model.pth.tar') save_checkpoint( { 'epoch': epoch + 1, 'state_dict': dec.state_dict(), 'loss': train_loss, 'best_loss': best_acc, }, is_best, checkpoint=args.checkpoint, filename='dec_model.pth.tar') save_checkpoint( { 'epoch': epoch + 1, 'state_dict': cl.state_dict(), 'loss': train_loss, 'best_loss': best_acc, }, is_best, checkpoint=args.checkpoint, filename='cl_model.pth.tar') save_checkpoint( { 'epoch': epoch + 1, 'state_dict': disc_l.state_dict(), 'loss': train_loss, 'best_loss': best_acc, }, is_best, checkpoint=args.checkpoint, filename='disc_l_model.pth.tar') save_checkpoint( { 'epoch': epoch + 1, 'state_dict': disc_v.state_dict(), 'loss': train_loss, 'best_loss': best_acc, }, is_best, checkpoint=args.checkpoint, filename='disc_v_model.pth.tar') logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Dataset preprocessing title = 'CIFAR-10' # Create Datasets transform_train_poisoned = transforms.Compose([ TriggerAppending(trigger=args.trigger, alpha=args.alpha), transforms.RandomHorizontalFlip(), transforms.ToTensor(), ]) transform_train_benign = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.ToTensor(), ]) transform_test_poisoned = transforms.Compose([ TriggerAppending(trigger=args.trigger, alpha=args.alpha), transforms.ToTensor(), ]) transform_test_benign = transforms.Compose([ transforms.ToTensor(), ]) print('==> Loading the dataset') dataloader = datasets.CIFAR10 poisoned_trainset = dataloader(root='./data', train=True, download=True, transform=transform_train_poisoned) benign_trainset = dataloader(root='./data', train=True, download=True, transform=transform_train_benign) poisoned_testset = dataloader(root='./data', train=False, download=True, transform=transform_test_poisoned) benign_testset = dataloader(root='./data', train=False, download=True, transform=transform_test_benign) num_training = len(poisoned_trainset) num_poisoned = int(num_training * args.poison_rate) idx = list(np.arange(num_training)) random.shuffle(idx) poisoned_idx = idx[:num_poisoned] benign_idx = idx[num_poisoned:] poisoned_img = poisoned_trainset.data[poisoned_idx, :, :, :] poisoned_target = [args.y_target] * len( poisoned_trainset.data) # Reassign their label to the target label poisoned_trainset.data, poisoned_trainset.targets = poisoned_img, poisoned_target benign_img = benign_trainset.data[benign_idx, :, :, :] benign_target = [benign_trainset.targets[i] for i in benign_idx] benign_trainset.data, benign_trainset.targets = benign_img, benign_target poisoned_target = [args.y_target] * len( poisoned_testset.data) # Reassign their label to the target label poisoned_testset.targets = poisoned_target poisoned_trainloader = torch.utils.data.DataLoader( poisoned_trainset, batch_size=int(args.train_batch * args.poison_rate), shuffle=True, num_workers=args.workers) benign_trainloader = torch.utils.data.DataLoader( benign_trainset, batch_size=int(args.train_batch * (1 - args.poison_rate) * 0.9), shuffle=True, num_workers=args.workers ) # *0.9 to prevent the iterations of benign data is less than that of poisoned data poisoned_testloader = torch.utils.data.DataLoader( poisoned_testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) benign_testloader = torch.utils.data.DataLoader(benign_testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) print( "Num of training samples %i, Num of poisoned samples %i, Num of benign samples %i" % (num_training, num_poisoned, num_training - num_poisoned)) # Model print('==> Loading the model') model = ResNet18() model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print('Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Benign Valid Loss', 'Poisoned Valid Loss', 'Train ACC.', 'Benign Valid ACC.', 'Poisoned Valid ACC.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(args, model, poisoned_trainloader, benign_trainloader, criterion, optimizer, epoch, use_cuda) test_loss_benign, test_acc_benign = test(benign_testloader, model, criterion, epoch, use_cuda) test_loss_poisoned, test_acc_poisoned = test(poisoned_testloader, model, criterion, epoch, use_cuda) # append logger file logger.append([ state['lr'], train_loss, test_loss_benign, test_loss_poisoned, train_acc, test_acc_benign, test_acc_poisoned ]) # save model is_best = test_acc_benign > best_acc best_acc = max(test_acc_benign, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc_benign, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main(): global args, best_prec1 args = parser.parse_args() if args.seed is not None: random.seed(args.seed) torch.manual_seed(args.seed) cudnn.deterministic = True warnings.warn('You have chosen to seed training. ' 'This will turn on the CUDNN deterministic setting, ' 'which can slow down your training considerably! ' 'You may see unexpected behavior when restarting ' 'from checkpoints.') args.distributed = args.world_size > 1 if args.distributed: dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size) # create model # print("=> creating model '{}'".format(args.arch)) # model = models.__dict__[args.arch](width_mult=args.width_mult) layer = int(args.arch.split("_")[-1]) model = SnetExtractor(layer) if not args.distributed: if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() else: model.cuda() model = torch.nn.parallel.DistributedDataParallel(model) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint title = 'ImageNet-' + args.arch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict'], strict=False) # optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) args.checkpoint = os.path.dirname(args.resume) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) for i in range(checkpoint['epoch']): if i in [30, 60, 90]: args.lr = args.lr * 0.1 else: print("=> no checkpoint found at '{}'".format(args.resume)) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) cudnn.benchmark = True # Data loading code if args.data_backend == 'pytorch': get_train_loader = get_pytorch_train_loader get_val_loader = get_pytorch_val_loader elif args.data_backend == 'dali-gpu': get_train_loader = get_dali_train_loader(dali_cpu=False) get_val_loader = get_dali_val_loader() elif args.data_backend == 'dali-cpu': get_train_loader = get_dali_train_loader(dali_cpu=True) get_val_loader = get_dali_val_loader() train_loader, train_loader_len = get_train_loader( args.data, args.batch_size, workers=args.workers, input_size=args.input_size) val_loader, val_loader_len = get_val_loader(args.data, args.batch_size, workers=args.workers, input_size=args.input_size) if args.evaluate: from collections import OrderedDict if os.path.isfile(args.weight): print("=> loading pretrained weight '{}'".format(args.weight)) source_state = torch.load(args.weight) target_state = OrderedDict() for k, v in source_state.items(): if k[:7] != 'module.': k = 'module.' + k target_state[k] = v model.load_state_dict(target_state) else: print("=> no weight found at '{}'".format(args.weight)) validate(val_loader, val_loader_len, model, criterion) return # visualization writer = SummaryWriter(os.path.join(args.checkpoint, 'logs')) for epoch in range(args.start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) print('\nEpoch: [%d | %d | lr: %f]' % (epoch + 1, args.epochs, args.lr)) # train for one epoch train_loss, train_acc = train(train_loader, train_loader_len, model, criterion, optimizer, epoch) # evaluate on validation set val_loss, prec1 = validate(val_loader, val_loader_len, model, criterion) lr = optimizer.param_groups[0]['lr'] # append logger file logger.append([lr, train_loss, val_loss, train_acc, prec1]) # tensorboardX writer.add_scalar('learning rate', lr, epoch + 1) writer.add_scalars('loss', { 'train loss': train_loss, 'validation loss': val_loss }, epoch + 1) writer.add_scalars('accuracy', { 'train accuracy': train_acc, 'validation accuracy': prec1 }, epoch + 1) is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) writer.close() print('Best accuracy:') print(best_prec1)
def main(): global args, best_prec1 args = parser.parse_args() args.distributed = args.world_size > 1 if args.distributed: dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size) # Validate dataset assert args.dataset == 'cifar10' or args.dataset == 'cifar100', 'Dataset can only be cifar10 or cifar100.' if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 # Use CUDA use_cuda = torch.cuda.is_available() # Random seed if args.manual_seed is None: args.manual_seed = random.randint(1, 10000) random.seed(args.manual_seed) torch.manual_seed(args.manual_seed) if use_cuda: torch.cuda.manual_seed_all(args.manual_seed) # create model print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch](num_classes=num_classes) if not args.distributed: if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() else: model.cuda() model = torch.nn.parallel.DistributedDataParallel(model) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint title = 'CIFAR-' + args.arch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) args.checkpoint = os.path.dirname(args.resume) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: print("=> no checkpoint found at '{}'".format(args.resume)) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) cudnn.benchmark = True # Data loading code normalize = transforms.Normalize(mean=[0.49139968, 0.48215827, 0.44653124], std=[0.24703233, 0.24348505, 0.26158768]) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]) transform_val = transforms.Compose([ transforms.ToTensor(), normalize, ]) train_dataset = dataloader(root='./data', train=True, download=True, transform=transform_train) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset) else: train_sampler = None train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.train_batch, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler) val_dataset = dataloader(root='./data', train=False, download=False, transform=transform_val) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion) return # visualization writer = SummaryWriter(os.path.join(args.checkpoint, 'logs')) for epoch in range(args.start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) lr = adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, lr)) if not args.mimic: # train for one epoch train_loss, train_loss_head1, train_loss_head2, train_acc, train_acc_head1, train_acc_head2 = train( train_loader, model, criterion, optimizer, epoch) # evaluate on validation set val_loss, val_loss_head1, val_loss_head2, prec1, prec1_head1, prec1_head2 = validate( val_loader, model, criterion) else: # train for one epoch train_loss, train_loss_t1_s0, train_loss_t2_s0, train_loss_head1, train_loss_t0_s1, train_loss_t2_s1, train_loss_head2, train_loss_t0_s2, train_loss_t1_s2, train_acc, train_acc_head1, train_acc_head2 = train( train_loader, model, criterion, optimizer, epoch) # evaluate on validation set val_loss, val_loss_t1_s0, val_loss_t2_s0, val_loss_head1, val_loss_t0_s1, val_loss_t2_s1, val_loss_head2, val_loss_t0_s2, val_loss_t1_s2, prec1, prec1_head1, prec1_head2 = validate( val_loader, model, criterion) # append logger file logger.append([lr, train_loss, val_loss, train_acc, prec1]) # tensorboardX writer.add_scalar('learning rate', lr, epoch + 1) writer.add_scalars('loss/head 0', { 'train loss': train_loss, 'validation loss': val_loss }, epoch + 1) writer.add_scalars('loss/head 1', { 'train loss': train_loss_head1, 'validation loss': val_loss_head1 }, epoch + 1) writer.add_scalars('loss/head 2', { 'train loss': train_loss_head2, 'validation loss': val_loss_head2 }, epoch + 1) writer.add_scalars('accuracy/head 0', { 'train accuracy': train_acc, 'validation accuracy': prec1 }, epoch + 1) writer.add_scalars('accuracy/head 1', { 'train accuracy': train_acc_head1, 'validation accuracy': prec1_head1 }, epoch + 1) writer.add_scalars('accuracy/head 2', { 'train accuracy': train_acc_head2, 'validation accuracy': prec1_head2 }, epoch + 1) #for name, param in model.named_parameters(): # writer.add_histogram(name, param.clone().cpu().data.numpy(), epoch + 1) if args.mimic: writer.add_scalars('loss/teacher1 student0', { 'train loss': train_loss_t1_s0, 'validation loss': val_loss_t1_s0 }, epoch + 1) writer.add_scalars('loss/teacher2 student0', { 'train loss': train_loss_t2_s0, 'validation loss': val_loss_t2_s0 }, epoch + 1) writer.add_scalars('loss/teacher0 student1', { 'train loss': train_loss_t0_s1, 'validation loss': val_loss_t0_s1 }, epoch + 1) writer.add_scalars('loss/teacher2 student1', { 'train loss': train_loss_t2_s1, 'validation loss': val_loss_t2_s1 }, epoch + 1) writer.add_scalars('loss/teacher0 student2', { 'train loss': train_loss_t0_s2, 'validation loss': val_loss_t0_s2 }, epoch + 1) writer.add_scalars('loss/teacher1 student2', { 'train loss': train_loss_t1_s2, 'validation loss': val_loss_t1_s2 }, epoch + 1) is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) writer.close() print('Best accuracy:') print(best_prec1)
def main(): global args, best_prec1, cp_logger, lr_schedul args = parser.parse_args() if args.seed is not None: random.seed(args.seed) #np.random.seed(args.seed) torch.manual_seed(args.seed) cudnn.deterministic = True warnings.warn('You have chosen to seed training. ' 'This will turn on the CUDNN deterministic setting, ' 'which can slow down your training considerably! ' 'You may see unexpected behavior when restarting ' 'from checkpoints.') args.distributed = args.world_size > 1 if args.distributed: dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size) print("#Weights to track '{}'".format(args.track_weights)) # create model print("=> creating model '{}'".format(args.arch)) model = None if args.arch.startswith('mobilenetv2'): model = models.__dict__[args.arch](width_mult=args.width_mult) elif args.arch.startswith('efficientnet'): print('efficientnet') model = EfficientNet.from_name(args.arch) else: model = models.__dict__[args.arch]() if not args.distributed: if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() else: model.cuda() model = torch.nn.parallel.DistributedDataParallel(model) model_weights_tot = sum([param.nelement() for param in model.parameters()]) print(model) print(model_weights_tot) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() if args.use_qe == 'True': target_quantile = 1 - (args.track_weights / model_weights_tot) print('target quantile: ', target_quantile) else: target_quantile = None print('Use exact sort without qe') print("Baseline Optimizer is used!") optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) if args.lr_decay == 'specified': lr_schedul = lr_scheduled('schedule.csv') scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.2, patience=5, verbose=False, threshold=1e-1, threshold_mode='abs', cooldown=0, min_lr=0.002, eps=1e-8) # optionally resume from a checkpoint title = 'ImageNet-' + args.arch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) args.checkpoint = os.path.dirname(args.resume) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) cp_logger = Logger(os.path.join(args.checkpoint, 'cp_log.txt'), title=title, resume=True) else: print("=> no checkpoint found at '{}'".format(args.resume)) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) cp_logger = Logger(os.path.join(args.checkpoint, 'cp_log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) cp_logger.set_names([ 'itteration in epoch', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Train Acc. top5', 'Valid Acc.', 'Valid Acc. top5', 'batch speed' ]) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) cp_logger = Logger(os.path.join(args.checkpoint, 'cp_log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) cp_logger.set_names([ 'itteration in epoch', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Train Acc. top5', 'Valid Acc.', 'Valid Acc. top5', 'batch speed' ]) cudnn.benchmark = True # Data loading code if args.data_backend == 'pytorch': get_train_loader = get_pytorch_train_loader get_val_loader = get_pytorch_val_loader elif args.data_backend == 'dali-gpu': get_train_loader = get_dali_train_loader(dali_cpu=False) get_val_loader = get_dali_val_loader() elif args.data_backend == 'dali-cpu': get_train_loader = get_dali_train_loader(dali_cpu=True) get_val_loader = get_dali_val_loader() train_loader, train_loader_len = get_train_loader( args.data, args.batch_size, workers=args.workers, input_size=args.input_size) val_loader, val_loader_len = get_val_loader(args.data, args.batch_size, workers=args.workers, input_size=args.input_size) if args.evaluate: from collections import OrderedDict if os.path.isfile(args.weight): print("=> loading pretrained weight '{}'".format(args.weight)) source_state = torch.load(args.weight) target_state = OrderedDict() for k, v in source_state.items(): if k[:7] != 'module.': k = 'module.' + k target_state[k] = v model.load_state_dict(target_state) else: print("=> no weight found at '{}'".format(args.weight)) validate(val_loader, val_loader_len, model, criterion) return # visualization writer = SummaryWriter(os.path.join(args.checkpoint, 'logs')) print("Training started!", flush=True) for epoch in range(args.start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) # the folder name to dump logs and files folder = 'baseline' best_reported_acc = best_accuracy(current_epoch=epoch, run_name='ImageNet_' + args.arch, folder=folder) print("Best reported Valid Acc. for epoch: " + str(epoch) + " is: " + str(best_reported_acc), flush=True) print('\nEpoch: [%d | %d]' % (epoch + 1, args.epochs), flush=True) # train for one epoch train_loss, train_acc = train(train_loader, train_loader_len, model, criterion, optimizer, epoch) # evaluate on validation set val_loss, prec1 = validate(val_loader, val_loader_len, model, criterion) lr = optimizer.param_groups[0]['lr'] # append logger file logger.append([lr, train_loss, val_loss, train_acc, prec1]) # tensorboardX writer.add_scalar('learning rate', lr, epoch + 1) writer.add_scalars('loss', { 'train loss': train_loss, 'validation loss': val_loss }, epoch + 1) writer.add_scalars('accuracy', { 'train accuracy': train_acc, 'validation accuracy': prec1 }, epoch + 1) is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) if (epoch > 5) & (epoch % 5 == 4): # save every 5 epoch! save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, False, checkpoint=args.checkpoint, filename=str(epoch + 1) + '_' + 'checkpoint.pth.tar') if args.lr_decay == 'plateau2': scheduler.step(val_loss) # Early terminations print("Accuracy here: " + str(prec1) + " vs. " + str(best_reported_acc) + " best accuracy reported!", flush=True) if early_terminate(epoch, prec1, best_reported_acc): print("Early Termination on epoch " + str(epoch)) break logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) cp_logger.close() #cp_logger.plot() #savefig(os.path.join(args.checkpoint, 'cp_log.eps')) writer.close() print('Best accuracy:') print(best_prec1, flush=True)
def main(): # global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) # train and val train_transformation = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) eval_transformation = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) data_dir = 'data-local/images/cifar/cifar10/by-image' num_classes = 10 if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 else: dataloader = datasets.CIFAR100 num_classes = 100 data_dir = 'data-local/images/cifar/cifar100/by-image' dataset_config = { 'train_transformation': train_transformation, 'eval_transformation': eval_transformation, 'datadir': data_dir, } trainloader, evalloader, unlabeled = create_data_loaders(**dataset_config, args=args) testset = dataloader(root='data-local/workdir/', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, block_name=args.block_name, ) else: model = models.__dict__[args.arch](num_classes=num_classes) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume if args.dataset == 'cifar10': title = 'cifar-10-' + args.arch else: title = 'cifar-100-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) # best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Train Acc.', ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val results = [] if args.rand: print('select samples randomly...') for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda) # test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) # if not args.rand and epoch >= args.epochs - args.epoch_num: # last 5, interval = 1 epoch interval = args.interval # interval !=1 epochs if not args.rand and epoch in np.arange( args.epochs)[-1:-args.epoch_num * interval - 1:-interval]: results.append(get_p(evalloader, model, epoch, use_cuda)) # append logger file # logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) logger.append([state['lr'], train_loss, train_acc]) # save model # is_best = test_acc > best_acc # best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), # 'acc': test_acc, # 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) select_data(args.labels, results, unlabeled, args.select_num)
def main(): global best_acc # Data print('==> Preparing data..') transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), ]) transform_test = transforms.Compose([ transforms.ToTensor(), ]) # data_file = './data' # data_file = 'C:/Users/dongxian/data/CIFAR' data_file = '/data/wudongxian/dataset/CIFAR/' if args.dataset == 'cifar10': data_name = 'CIFAR10' num_classes = 10 else: data_name = 'CIFAR100' num_classes = 100 trainset = getattr(torchvision.datasets, data_name)(root=data_file, train=True, download=True, transform=transform_train) trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = getattr(torchvision.datasets, data_name)(root=data_file, train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print('==> Building model..') net = models.__dict__[args.arch](num_classes=num_classes) teacher_net = models.__dict__[args.arch](num_classes=num_classes) title = 'noisy_cifar-' + args.arch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Adv Train Loss', 'Nat Train Loss', 'Nat Valid Loss', 'Adv Train Acc', 'Nat Train Loss', 'Nat Valid Acc']) if torch.cuda.is_available(): net = net.to(device) net = torch.nn.DataParallel(net) teacher_net = teacher_net.to(device) teacher_net = torch.nn.DataParallel(teacher_net) hard_update(teacher_net, net) print('Using', torch.cuda.device_count(), 'GPUs.') cudnn.benchmark = True print('Using CUDA..') criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) schedule_lr = optim.lr_scheduler.MultiStepLR(optimizer, args.schedule, gamma=args.gamma) adversary = PGDAttack(model=net, epsilon=0.031, num_steps=10, step_size=0.007) for epoch in range(args.start_epoch, args.epochs): schedule_lr.step(epoch) state['lr'] = optimizer.state_dict()['param_groups'][0]['lr'] if args.mode == 'nat': adv_train_loss, adv_train_acc = train(trainloader, net, teacher_net, criterion, optimizer, epoch) else: adv_train_loss, adv_train_acc = train_adv(trainloader, net, teacher_net, criterion, optimizer, epoch, adversary=adversary) nat_train_loss, nat_train_acc = test(trainloader, net, criterion, epoch) nat_test_loss, nat_test_acc = test(testloader, net, criterion, epoch) logger.append([state['lr'], adv_train_loss, nat_train_loss, nat_test_loss, adv_train_acc, nat_train_acc, nat_test_acc]) # what is the best? is_best = adv_train_acc > best_acc best_acc = max(adv_train_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': net.state_dict(), 'acc': adv_train_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) if (epoch + 1) % 10 == 0 and epoch > 1: save_checkpoint({ 'epoch': epoch + 1, 'state_dict': net.state_dict(), 'acc': adv_train_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint, filename='checkpoint_{:0>3d}.pth.tar'.format(epoch+1)) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': teacher_net.state_dict(), 'acc': adv_train_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint, filename='checkpoint_teacher_{:0>3d}.pth.tar'.format(epoch + 1)) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps'))
def main(): global args, best_prec1 args = parser.parse_args() args.distributed = args.world_size > 1 if args.distributed: dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size) # Use CUDA os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id use_cuda = torch.cuda.is_available() # Random seed if args.manual_seed is None: args.manual_seed = random.randint(1, 10000) random.seed(args.manual_seed) torch.manual_seed(args.manual_seed) if use_cuda: torch.cuda.manual_seed_all(args.manual_seed) # create model if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) elif args.arch.startswith('resnext'): model = models.__dict__[args.arch]( baseWidth=args.base_width, cardinality=args.cardinality, ) elif args.arch.startswith('shufflenet'): model = models.__dict__[args.arch]( groups=args.groups ) else: print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch](input_size=args.input_size, width_mult=args.width_mult) if not args.distributed: if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() else: model.cuda() model = torch.nn.parallel.DistributedDataParallel(model) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint title = 'ImageNet-' + args.arch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) args.checkpoint = os.path.dirname(args.resume) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: print("=> no checkpoint found at '{}'".format(args.resume)) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) if args.weightfile: from collections import OrderedDict if os.path.isfile(args.weightfile): print("=> loading weightfile '{}'".format(args.weightfile)) preloaded_params = torch.load(args.weightfile) new_state_dict = OrderedDict() for k, v in preloaded_params.items(): if k[:7] != 'module.': k = 'module.' + k new_state_dict[k] = v model.load_state_dict(new_state_dict) cudnn.benchmark = True # Data loading code traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = datasets.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(args.input_size), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: train_sampler = None train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.train_batch, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler) val_loader = torch.utils.data.DataLoader( datasets.ImageFolder(valdir, transforms.Compose([ transforms.Resize(int(256.0*(args.input_size/224.0))), transforms.CenterCrop(args.input_size), transforms.ToTensor(), normalize, ])), batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion) return # visualization writer = SummaryWriter(os.path.join(args.checkpoint, 'logs')) for epoch in range(args.start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) lr = adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, lr)) # train for one epoch train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch) # evaluate on validation set val_loss, prec1 = validate(val_loader, model, criterion) # append logger file logger.append([lr, train_loss, val_loss, train_acc, prec1]) # tensorboardX writer.add_scalar('learning rate', lr, epoch + 1) writer.add_scalars('loss', {'train loss': train_loss, 'validation loss': val_loss}, epoch + 1) writer.add_scalars('accuracy', {'train accuracy': train_acc, 'validation accuracy': prec1}, epoch + 1) for name, param in model.named_parameters(): writer.add_histogram(name, param.clone().cpu().data.numpy(), epoch + 1) is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) save_checkpoint({ 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best accuracy:') print(best_prec1)
# standardize columns X = utils.standardize_cols(X) model = PCA(k=2) model.fit(X) Z = model.compress(X) fig, ax = plt.subplots() ax.scatter(Z[:,0], Z[:,1]) plt.ylabel('z2') plt.xlabel('z1') plt.title('PCA') for i in range(n): ax.annotate(animals[i], (Z[i,0], Z[i,1])) utils.savefig('q3_2_PCA_animals.png') # code below isn't required. variance_explained = 1 - norm(model.expand(Z) - X, 'fro')**2 / norm(X, 'fro')**2 print('Variance explained {}'.format(variance_explained)) if question == '4.1': X = utils.load_dataset('highway')['X'].astype(float)/255 n,d = X.shape h,w = 64,64 # height and width of each image # the two variables below are parameters for the foreground/background extraction method # you should just leave these two as default. k = 5 # number of PCs threshold = 0.04 # a threshold for separating foreground from background
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data transform = get_transforms(input_size=args.image_size, test_size=args.image_size, backbone=None) print('==> Preparing dataset %s' % args.trainroot) trainset = dataset.Dataset(root=args.trainroot, transform=transform['train']) train_loader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers, pin_memory=True) valset = dataset.TestDataset(root=args.valroot, transform=transform['val']) val_loader = data.DataLoader(valset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=True) model = make_model(args) if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = get_optimizer(model, args) # 基于标准的学习率更新 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.2, patience=5, verbose=False) # Resume title = 'ImageNet-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.module.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, optimizer.param_groups[0]['lr'])) train_loss, train_acc, train_5 = train(train_loader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc, test_5 = test(val_loader, model, criterion, epoch, use_cuda) scheduler.step(test_loss) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) print( 'train_loss:%f, val_loss:%f, train_acc:%f, train_5:%f, val_acc:%f, val_5:%f' % (train_loss, test_loss, train_acc, train_5, test_acc, test_5)) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) if len(args.gpu_id) > 1: save_checkpoint( { 'fold': 0, 'epoch': epoch + 1, 'state_dict': model.module.state_dict(), 'train_acc': train_acc, 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, single=True, checkpoint=args.checkpoint) else: save_checkpoint( { 'fold': 0, 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'train_acc': train_acc, 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, single=True, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
d["system_pos"] = d.apply( lambda r: system_pos[(r["num_particles"], r["freq"])], axis=1) width = 3.2 height = 3.0 fig, ax = plt.subplots(figsize=(width, height)) for inter, g in sorted(d.groupby("interaction"), key=lambda x: interaction_pos[x[0]]): g.sort_values(["freq", "num_particles"], inplace=True, ascending=[False, True]) ax.plot(g["system_pos"], abs(g["slope"]), linestyle="", linewidth=2, marker=interaction_markers[inter], markerfacecolor=interaction_markerfacecolors[inter], color=interaction_colors[inter], label=interaction_label[inter]) ax.set_xticks(range(len(systems))) ax.set_xticklabels(systems, rotation=90) ax.set_xlabel(r"$(N, \omega)$") ax.set_ylabel(r"$|\rho_{15}|$") ax.set_yscale("log") ax.legend(loc="center left", frameon=False, title="$(\sigma_{\mathrm{A}}, \sigma_{\mathrm{B}})$", bbox_to_anchor=(1.0 - 0.3 / width, 0.5)) fig.tight_layout() fig.subplots_adjust(left=0.2, right=0.75) utils.savefig(fig, fn="../Manuscript/fig-rel-slopes2.pdf")
(d["num_shells"] == num_shells) & (d["num_filled"] == num_filled) & # filter out higher frequencies because they stretch the plot too much (d["freq"] <= 1.0) & (d["ml"] == ml)] num_particles = num_filled * (num_filled + 1) energy_type = {"ground": "ground state", "add": "addition", "rm": "removal"}[label] fig, ax = plt.subplots() fig.set_size_inches((4, 3)) for method, case in d.groupby("method"): case = case.sort_values("freq").drop_duplicates() xs = case["freq"] ys_ref = xs.map(lambda freq: d[(d["method"] == "hf") & (d["freq"] == freq)]["energy"]) ys = case["energy"] / ys_ref ax.plot(xs, ys, "-x", label=utils.METHOD_LABEL[method], color=utils.METHOD_COLOR[method]) if kwargs["title"]: ax.set_title("{energy_type} energy for {num_particles} particles " "(ML = {ml}, K = {num_shells})" .format(**locals())) ax.set_xlabel("ω") ax.set_ylabel("ε/ε[HF]") ax.legend() fig.tight_layout() utils.savefig(fig, "by-freq-{num_shells}-{num_particles}-" "{label}-{ml}-{interaction}" .format(**locals()))
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data print('==> Preparing dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 trainset = dataloader(root='./data', train=True, download=True, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testset = dataloader(root='./data', train=False, download=False, transform=transform_test) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop, ) elif args.arch.startswith('wrn'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, block_name=args.block_name, ) elif args.arch.startswith('preresnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, block_name=args.block_name, ) elif args.arch.startswith('horesnet'): model = models.__dict__[args.arch](num_classes=num_classes, depth=args.depth, eta=args.eta, block_name=args.block_name, feature_vec=args.feature_vec) elif args.arch.startswith('hopreresnet'): model = models.__dict__[args.arch](num_classes=num_classes, depth=args.depth, eta=args.eta, block_name=args.block_name, feature_vec=args.feature_vec) elif args.arch.startswith('nagpreresnet'): model = models.__dict__[args.arch](num_classes=num_classes, depth=args.depth, eta=args.eta, block_name=args.block_name, feature_vec=args.feature_vec) elif args.arch.startswith('mompreresnet'): model = models.__dict__[args.arch](num_classes=num_classes, depth=args.depth, eta=args.eta, block_name=args.block_name, feature_vec=args.feature_vec) elif args.arch.startswith('v2_preresnet'): model = models.__dict__[args.arch](block_name='basicblock', num_blocks=[2, 2, 2, 2], num_classes=num_classes) else: print('Model is specified wrongly - Use standard model') model = models.__dict__[args.arch](num_classes=num_classes) model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) criterion = nn.CrossEntropyLoss() if args.optimizer.lower() == 'sgd': optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # elif args.optimizer.lower() == 'adam': # optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay) elif args.optimizer.lower() == 'radam': optimizer = RAdam(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay) elif args.optimizer.lower() == 'adamw': optimizer = AdamW(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay, warmup=args.warmup) elif args.optimizer.lower() == 'adam': optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay) elif args.optimizer.lower() == 'srsgd': iter_count = 1 optimizer = SGD_Adaptive(model.parameters(), lr=args.lr, weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=40) elif args.optimizer.lower() == 'sradam': iter_count = 1 optimizer = SRNAdam(model.parameters(), lr=args.lr, iter_count=iter_count, restarting_iter=100) # Resume title = 'cifar-10-' + args.arch # if args.resume: # # Load checkpoint. # print('==> Resuming from checkpoint..') # assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' # args.checkpoint = os.path.dirname(args.resume) # checkpoint = torch.load(args.resume) # best_acc = checkpoint['best_acc'] # start_epoch = checkpoint['epoch'] # model.load_state_dict(checkpoint['state_dict']) # optimizer.load_state_dict(checkpoint['optimizer']) # logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) # else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.' ]) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): if args.optimizer.lower() == 'srsgd': if epoch == args.schedule[0]: optimizer = SGD_Adaptive(model.parameters(), lr=args.lr * args.gamma, weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=80) elif epoch == args.schedule[1]: optimizer = SGD_Adaptive(model.parameters(), lr=args.lr * (args.gamma**2), weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=200) elif epoch == args.schedule[2]: optimizer = SGD_Adaptive(model.parameters(), lr=args.lr * (args.gamma**3), weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=500) elif epoch == args.schedule[3]: optimizer = SGD_Adaptive(model.parameters(), lr=args.lr * (args.gamma**4), weight_decay=args.weight_decay, iter_count=iter_count, restarting_iter=1000) elif args.optimizer.lower() == 'sradam': if epoch == 80: optimizer = SRNAdam(model.parameters(), lr=args.lr * args.gamma, iter_count=iter_count, restarting_iter=500) #500 done elif epoch == 160: optimizer = SRNAdam(model.parameters(), lr=args.lr * (args.gamma**2), iter_count=iter_count, restarting_iter=1000) #1000 done elif args.arch.startswith('wrn'): if args.optimizer.lower() == 'sgd': if epoch == 60: optimizer = optim.SGD(model.parameters(), lr=0.02, momentum=args.momentum, weight_decay=args.weight_decay) elif epoch == 120: optimizer = optim.SGD(model.parameters(), lr=0.004, momentum=args.momentum, weight_decay=args.weight_decay) elif epoch == 160: optimizer = optim.SGD(model.parameters(), lr=0.0008, momentum=args.momentum, weight_decay=args.weight_decay) else: adjust_learning_rate(optimizer, epoch) logger.file.write('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) if args.optimizer.lower() == 'srsgd' or args.optimizer.lower( ) == 'sradam': train_loss, train_acc, iter_count = train(trainloader, model, criterion, optimizer, epoch, use_cuda, logger) else: train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda, logger) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda, logger) # append logger file logger.append( [state['lr'], train_loss, test_loss, train_acc, test_acc]) writer.add_scalars('train_loss', {args.model_name: train_loss}, epoch) writer.add_scalars('test_loss', {args.model_name: test_loss}, epoch) writer.add_scalars('train_acc', {args.model_name: train_acc}, epoch) writer.add_scalars('test_acc', {args.model_name: test_acc}, epoch) # writer.add_scalars('loss_tracking/train_loss', {args.model_name: train_loss}, epoch) # writer.add_scalars('loss_tracking/test_loss', {args.model_name: test_loss}, epoch) # writer.add_scalars('loss_tracking/train_acc', {args.model_name: train_acc}, epoch) # writer.add_scalars('loss_tracking/test_acc', {args.model_name: test_acc}, epoch) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.file.write('Best acc:%f' % best_acc) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data loading code traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_loader = torch.utils.data.DataLoader( datasets.ImageFolder(traindir, transforms.Compose([ transforms.RandomSizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])), batch_size=args.train_batch, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader( datasets.ImageFolder(valdir, transforms.Compose([ transforms.Scale(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.test_batch, shuffle=False, num_workers=args.workers, pin_memory=True) # create model if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) elif args.arch.startswith('resnext'): model = models.__dict__[args.arch]( baseWidth=args.base_width, cardinality=args.cardinality, ) else: print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch]() if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters())/1000000.0)) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = 'ImageNet-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(val_loader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(val_loader, model, criterion, epoch, use_cuda) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def main(): global best_top1, best_top5 args.world_size = 1 start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data loading code traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') crop_size = 224 val_size = 256 pipe = HybridTrainPipe(batch_size=args.train_batch, num_threads=args.workers, device_id=args.local_rank, data_dir=traindir, crop=crop_size, dali_cpu=args.dali_cpu) pipe.build() train_loader = DALIClassificationIterator( pipe, size=int(pipe.epoch_size("Reader") / args.world_size)) pipe = HybridValPipe(batch_size=args.test_batch, num_threads=args.workers, device_id=args.local_rank, data_dir=valdir, crop=crop_size, size=val_size) pipe.build() val_loader = DALIClassificationIterator( pipe, size=int(pipe.epoch_size("Reader") / args.world_size)) # create model if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) elif args.arch.startswith('resnext'): model = models.__dict__[args.arch]( baseWidth=args.base_width, cardinality=args.cardinality, ) else: print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch]() if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() if args.optimizer.lower() == 'sgd': optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) elif args.optimizer.lower() == 'adam': optimizer = AdamW(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay) elif args.optimizer.lower() == 'radam': optimizer = RAdam(model.parameters(), lr=args.lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay) elif args.optimizer.lower() == 'lsadam': optimizer = LSAdamW(model.parameters(), lr=args.lr * ((1. + 4. * args.sigma)**(0.25)), betas=(args.beta1, args.beta2), weight_decay=args.weight_decay, sigma=args.sigma) elif args.optimizer.lower() == 'lsradam': sigma = 0.1 optimizer = LSRAdam(model.parameters(), lr=args.lr * ((1. + 4. * args.sigma)**(0.25)), betas=(args.beta1, args.beta2), weight_decay=args.weight_decay, sigma=args.sigma) # Resume title = 'ImageNet-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' args.checkpoint = os.path.dirname(args.resume) checkpoint = torch.load(args.resume) best_top1 = checkpoint['best_top1'] best_top5 = checkpoint['best_top5'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Top1', 'Valid Top1', 'Train Top5', 'Valid Top5' ]) logger.file.write(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0)) if args.evaluate: logger.file.write('\nEvaluation only') test_loss, test_top1, test_top5 = test(val_loader, model, criterion, start_epoch, use_cuda, logger) logger.file.write( ' Test Loss: %.8f, Test Top1: %.2f, Test Top5: %.2f' % (test_loss, test_top1, test_top5)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) logger.file.write('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_top1, train_top5 = train(train_loader, model, criterion, optimizer, epoch, use_cuda, logger) test_loss, test_top1, test_top5 = test(val_loader, model, criterion, epoch, use_cuda, logger) # append logger file logger.append([ state['lr'], train_loss, test_loss, train_top1, test_top1, train_top5, test_top5 ]) writer.add_scalars('train_loss', {args.model_name: train_loss}, epoch) writer.add_scalars('test_loss', {args.model_name: test_loss}, epoch) writer.add_scalars('train_top1', {args.model_name: train_top1}, epoch) writer.add_scalars('test_top1', {args.model_name: test_top1}, epoch) writer.add_scalars('train_top5', {args.model_name: train_top5}, epoch) writer.add_scalars('test_top5', {args.model_name: test_top5}, epoch) # save model is_best = test_top1 > best_top1 best_top1 = max(test_top1, best_top1) best_top5 = max(test_top5, best_top5) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'top1': test_top1, 'top5': test_top5, 'best_top1': best_top1, 'best_top5': best_top5, 'optimizer': optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) # reset DALI iterators train_loader.reset() val_loader.reset() logger.file.write('Best top1:') logger.file.write(best_top1) logger.file.write('Best top5:') logger.file.write(best_top5) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best top1:') print(best_top1) print('Best top5:') print(best_top5)