def main(config): GPUManager.auto_chooce() engine = DefaultClassificationEngine() engine.hooks.update( dict( on_sample=on_sample_hook, on_start=on_start_hook, on_forward=on_forward_hook, ) ) net = LeNet(n_channels=config.n_channels, size=config.size) print(net) if torch.cuda.is_available(): net = net.cuda() train_loader = torch.utils.data.DataLoader( config.dataset(train=True), batch_size=config.batch_size, shuffle=True, num_workers=8 ) val_loader = torch.utils.data.DataLoader( config.dataset(train=False), batch_size=1000, shuffle=True, num_workers=8) optimizer = optim.SGD(net.parameters(), lr=config.lr, momentum=0.9) recorder = defaultdict(list) recorder.update( dict( dataset_name=config.dataset.__name__.split("_")[0], lr=config.lr, batch_size=config.batch_size ) ) pprint(recorder) for epoch in range(config.maxepoch): state=engine.train(network=net, iterator=train_loader, maxepoch=1, optimizer=optimizer) recorder["train_loss"].append(state["loss_meter"].value()) recorder["train_acc"].append(state["acc_meter"].value()) state=engine.validate(network=net, iterator=val_loader) recorder["val_loss"].append(state["loss_meter"].value()) recorder["val_acc"].append(state["acc_meter"].value()) filename = f"{recorder['dataset_name']}_" + time.strftime("%Y%m%d_%H%M%S", time.localtime()) with open(f"../result/{filename}.static", "wb") as f: pickle.dump(recorder, f)
def main(args): # Data Loader (Input Pipeline) model_str = args.dataset + '_%s_' % args.model_type + args.noise_type + '_' + str(args.noise_rate) + '_' + str(args.seed) txtfile = save_dir + "/" + model_str + ".txt" nowTime = datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S') if os.path.exists(txtfile): os.system('mv %s %s' % (txtfile, txtfile + ".bak-%s" % nowTime)) # Data Loader (Input Pipeline) print('loading dataset...') train_dataset, val_dataset, test_dataset = load_data(args) train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=args.batch_size, num_workers=args.num_workers, drop_last=False, shuffle=True) val_loader = torch.utils.data.DataLoader(dataset=val_dataset, batch_size=args.batch_size, num_workers=args.num_workers, drop_last=False, shuffle=False) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=args.batch_size, num_workers=args.num_workers, drop_last=False, shuffle=False) # Define models print('building model...') if args.dataset == 'mnist': clf1 = LeNet() optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9) scheduler1 = MultiStepLR(optimizer1, milestones=[10, 20], gamma=0.1) elif args.dataset == 'fmnist': clf1 = resnet.ResNet50(input_channel=1, num_classes=10) optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9) scheduler1 = MultiStepLR(optimizer1, milestones=[10, 20], gamma=0.1) elif args.dataset == 'cifar10': clf1 = resnet.ResNet50(input_channel=3, num_classes=10) optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9) scheduler1 = MultiStepLR(optimizer1, milestones=[40, 80], gamma=0.1) elif args.dataset == 'cifar100': clf1 = resnet.ResNet50(input_channel=3, num_classes=100) optimizer1 = torch.optim.SGD(clf1.parameters(), lr=learning_rate, weight_decay=args.weight_decay, momentum=0.9) scheduler1 = MultiStepLR(optimizer1, milestones=[40, 80], gamma=0.1) clf1.cuda() with open(txtfile, "a") as myfile: myfile.write('epoch train_acc1 val_acc1 test_acc1\n') epoch = 0 train_acc1 = 0 # evaluate models with random weights val_acc1 = evaluate(val_loader, clf1) print('Epoch [%d/%d] Val Accuracy on the %s val data: Model1 %.4f %%' % ( epoch + 1, args.n_epoch, len(val_dataset), val_acc1)) test_acc1 = evaluate(test_loader, clf1) print('Epoch [%d/%d] Test Accuracy on the %s test data: Model1 %.4f %%' % ( epoch + 1, args.n_epoch, len(test_dataset), test_acc1)) # save results with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(val_acc1) + ' ' + str(test_acc1) + "\n") val_acc_list = [] test_acc_list = [] for epoch in range(0, args.n_epoch): scheduler1.step() print(optimizer1.state_dict()['param_groups'][0]['lr']) clf1.train() train_acc1 = train(train_loader, epoch, clf1, optimizer1, args) val_acc1 = evaluate(val_loader, clf1) val_acc_list.append(val_acc1) test_acc1 = evaluate(test_loader, clf1) test_acc_list.append(test_acc1) # save results print('Epoch [%d/%d] Test Accuracy on the %s test data: Model1 %.4f %% ' % ( epoch + 1, args.n_epoch, len(test_dataset), test_acc1)) with open(txtfile, "a") as myfile: myfile.write(str(int(epoch)) + ' ' + str(train_acc1) + ' ' + str(val_acc1) + ' ' + str(test_acc1) + "\n") id = np.argmax(np.array(val_acc_list)) test_acc_max = test_acc_list[id] return test_acc_max
for p in model.parameters(): p.requires_grad_(False) #change criterion for ImageNet-1k and CIFAR100 to Top5Criterion criterion = lambda x, y: Top1Criterion(x, y, model) if args.norm == 'L2': norm = 2 elif args.norm == 'Linf': norm = np.inf elif args.norm == 'L0': norm = 0 elif args.norm == 'L1': norm = 1 if has_cuda: model = model.cuda() params = { 'bounds': (0, 1), 'dt': args.dt, 'alpha': args.alpha, 'beta': args.beta, 'gamma': args.gamma, 'max_outer': args.max_outer, 'max_inner': args.max_inner, 'T1': args.T1, 'T2': args.T2 } attack = Attack(model, **params)
pred = output.data.max(1, keepdim=True)[1] correct += pred.eq(target.data.view_as(pred)).cpu().sum() test_loss /= len(validate_loader.dataset) print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( test_loss, correct, len(validate_loader.dataset), 100. * correct / len(validate_loader.dataset))) if args.lenet: model = LeNet() else: model = Net() if args.cuda: model.cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) model.train() for epoch in range(args.epochs): for batch_idx, (data, target) in enumerate(train_loader): if args.cuda: data, target = data.cuda(), target.cuda() data, target = Variable(data), Variable(target) optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) loss.backward() optimizer.step() if batch_idx % args.log_interval == 0:
transform=transforms.ToTensor(), train=False) # record save_acc = open("record/MNIST_early_stopping_acc.txt", "w") train_loss = [] train_acc = [] test_loss = [] test_acc = [] # cuda setting train_args.cuda = not train_args.no_cuda and torch.cuda.is_available() if train_args.cuda: torch.cuda.manual_seed( train_args.seed) # set a random seed for the current GPU model.cuda() # move all model parameters to the GPU optimizer = optim.SGD(model.parameters(), lr=train_args.lr, momentum=train_args.momentum) # early stopping check_correct = 0 check_count = 0 model_save_dir = 'record/' def _iter_epoch(epoch, is_training): if is_training: model.train() images = data_train.data.tolist() labels = data_train.targets.tolist()
def train_lre(hyperparameters, data_loader, val_data, val_labels, test_loader, n_val=5): net, opt = build_model(hyperparameters) plot_step = 10 accuracy_log = dict() data_log = dict() subselect_val = n_val * val_labels.unique().size()[0] != len(val_labels) meta_net = LeNet(n_out=1) for i in range(hyperparameters['num_iterations']): t = time.time() net.train() # Line 2 get batch of data image, labels, index = next(iter(data_loader)) # since validation data is small I just fixed them instead of building an iterator # initialize a dummy network for the meta learning of the weights meta_net.load_state_dict(net.state_dict()) if torch.cuda.is_available(): meta_net.cuda() image = to_var(image, requires_grad=False) labels = to_var(labels, requires_grad=False) # Lines 4 - 5 initial forward pass to compute the initial weighted loss y_f_hat = meta_net(image) # Line 4 cost = functional.binary_cross_entropy_with_logits( y_f_hat, labels, reduce=False) # Line 5 eps = to_var(torch.zeros(cost.size())) # Line 5 l_f_meta = torch.sum(cost * eps) meta_net.zero_grad() # Line 6 perform a parameter update grads = torch.autograd.grad(l_f_meta, (meta_net.params()), create_graph=True) meta_net.update_params(hyperparameters['lr'], source_params=grads) # Line 8 - 10 2nd forward pass and getting the gradients with respect to epsilon if subselect_val: class_inds = list() for c in val_labels.unique(): matching_inds = (val_labels == c).nonzero() class_inds.append(matching_inds[np.random.permutation( len(matching_inds))[:n_val]]) class_inds = torch.cat(class_inds) val_input = val_data[class_inds].squeeze_(1) val_output = val_labels[class_inds].squeeze_(1) else: val_input = val_data val_output = val_labels y_g_hat = meta_net(val_input) l_g_meta = functional.binary_cross_entropy_with_logits( y_g_hat, val_output) grad_eps = torch.autograd.grad(l_g_meta, eps, only_inputs=True)[0] # Line 11 computing and normalizing the weights w_tilde = torch.clamp(-grad_eps, min=0) norm_c = torch.sum(w_tilde) if norm_c != 0: w = w_tilde / norm_c else: w = w_tilde # Lines 12 - 14 computing for the loss with the computed weights # and then perform a parameter update y_f_hat = net(image) cost = functional.binary_cross_entropy_with_logits(y_f_hat, labels, reduce=False) data_log[i] = pd.DataFrame( data={ 'w': w.numpy(), 'grad_eps': grad_eps.numpy(), 'pre_w_cost': cost.detach().numpy(), 'label': labels.numpy(), 'index': index.numpy(), }) l_f = torch.sum(cost * w) # print(data_log[i].head()) # print(data_log[i].tail()) opt.zero_grad() l_f.backward() opt.step() # meta_l = smoothing_alpha * meta_l + (1 - smoothing_alpha) * l_g_meta.item() # meta_losses_clean.append(meta_l / (1 - smoothing_alpha ** (i + 1))) # # net_l = smoothing_alpha * net_l + (1 - smoothing_alpha) * l_f.item() # net_losses.append(net_l / (1 - smoothing_alpha ** (i + 1))) # print(time.time() - t) if i % plot_step == 0: print(i) net.eval() acc_df = {'preds': [], 'labels': [], 'index': []} for itr, (test_img, test_label, test_idx) in enumerate(test_loader): test_img = to_var(test_img, requires_grad=False) test_label = to_var(test_label, requires_grad=False) output = net(test_img) predicted = functional.sigmoid(output) acc_df['preds'].extend(predicted.detach().numpy().tolist()) acc_df['labels'].extend(test_label.numpy().tolist()) acc_df['index'].extend(test_idx.numpy().tolist()) accuracy_log[i] = pd.DataFrame(acc_df).sort_values( ['labels', 'index']).set_index('index', drop=True) data_log = pd.concat(data_log) data_log.index.set_names('iteration', level=0, inplace=True) accuracy_log = pd.concat(accuracy_log) accuracy_log.index.set_names('iteration', level=0, inplace=True) return data_log, accuracy_log