def main(config): logger = prepare_logger(config) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # get loaders if not config.is_train_source: target_loader = get_loader(type="MNIST", train=False, batch_size=config.batch_size) source_train_loader = get_loader(type="SVHN", train=True, batch_size=config.batch_size) source_test_loader = get_loader(type="SVHN", train=False, batch_size=config.batch_size) # build source classifier model_src = LeNet(config.num_gpus).to(device) if (not config.is_train_source) or config.is_finetune: model_src.load_state_dict(torch.load(config.model_dir)) # train source classifier if config.is_train_source: logger.info("train source classifier..") train_source(model_src, source_train_loader, source_test_loader, config, logger) logger.info("evaluate source classifier..") logger.info("test accurracy in source domain: %f\n" % (evaluate(model_src, source_test_loader))) else: # initialize target classifer with source classifer model_trg = torch.load(open("./pretrained/lenet-source.pth", "rb")) # build discriminator D = Discriminator(config.num_gpus) # adaptation process logger.info("start adaptation process..") adapt_target_domain(D, model_src, model_trg, source_train_loader, target_loader, config) logger.info("evaluate target classifier..") logger.info("accurracy in target domain: %f\n" % (evaluate(model_trg, target_loader)))
]) test_dataset = DOGCAT(root='../Pytorch-Tutorial/datasets/dogcat_2', train=False, transform=transform) test_loader = Data.DataLoader(dataset=test_dataset, batch_size=100, shuffle=False, num_workers=2) net = LeNet() print(net) if os.path.isfile('saves/dogcat_lenet_params.pkl'): net.load_state_dict(torch.load('saves/dogcat_lenet_params.pkl')) else: print("dogcat_lenet_params.pkl don't exists.") exit() # Test the Model total = 0 correct = 0 for images, labels in test_loader: images = Variable(images) outputs = net(images) _, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += sum(predicted == labels) print('Test Accuracy of the model on the %d test images: %d %%' %
def main(): # Data Loader (Input Pipeline) print('loading dataset...') train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, num_workers=args.num_workers, drop_last=False, shuffle=False) val_loader = torch.utils.data.DataLoader(dataset=val_dataset, batch_size=batch_size, num_workers=args.num_workers, drop_last=False, shuffle=False) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, num_workers=args.num_workers, drop_last=False, shuffle=False) # Define models print('building model...') if args.dataset == 'mnist': clf1 = LeNet() if args.dataset == 'fashionmnist': clf1 = resnet.ResNet18_F(10) if args.dataset == 'cifar10': clf1 = resnet.ResNet34(10) if args.dataset == 'svhn': clf1 = resnet.ResNet34(10) clf1.cuda() optimizer = torch.optim.SGD(clf1.parameters(), lr=args.lr, weight_decay=args.weight_decay) with open(txtfile, "a") as myfile: myfile.write('epoch train_acc val_acc test_acc\n') epoch = 0 train_acc = 0 val_acc = 0 # evaluate models with random weights test_acc = evaluate(test_loader, clf1) print('Epoch [%d/%d] Test Accuracy on the %s test data: Model1 %.4f %%' % (epoch + 1, args.n_epoch_1, len(test_dataset), test_acc)) # save results with open(txtfile, "a") as myfile: myfile.write( str(int(epoch)) + ' ' + str(train_acc) + ' ' + str(val_acc) + ' ' + str(test_acc) + ' ' + "\n") best_acc = 0.0 # training for epoch in range(1, args.n_epoch_1): # train models clf1.train() train_acc = train(clf1, train_loader, epoch, optimizer, nn.CrossEntropyLoss()) # validation val_acc = evaluate(val_loader, clf1) # evaluate models test_acc = evaluate(test_loader, clf1) # save results print( 'Epoch [%d/%d] Train Accuracy on the %s train data: Model %.4f %%' % (epoch + 1, args.n_epoch_1, len(train_dataset), train_acc)) print('Epoch [%d/%d] Val Accuracy on the %s val data: Model %.4f %% ' % (epoch + 1, args.n_epoch_1, len(val_dataset), val_acc)) print( 'Epoch [%d/%d] Test Accuracy on the %s test data: Model %.4f %% ' % (epoch + 1, args.n_epoch_1, len(test_dataset), test_acc)) with open(txtfile, "a") as myfile: myfile.write( str(int(epoch)) + ' ' + str(train_acc) + ' ' + str(val_acc) + ' ' + str(test_acc) + ' ' + "\n") if val_acc > best_acc: best_acc = val_acc torch.save(clf1.state_dict(), model_save_dir + '/' + 'model.pth') print('Matrix Factorization is doing...') clf1.load_state_dict(torch.load(model_save_dir + '/' + 'model.pth')) A = respresentations_extract(train_loader, clf1, len(train_dataset), args.dim, batch_size) A_val = respresentations_extract(val_loader, clf1, len(val_dataset), args.dim, batch_size) A_total = np.append(A, A_val, axis=0) W_total, H_total, error = train_m(A_total, args.basis, args.iteration_nmf, 1e-5) for i in range(W_total.shape[0]): for j in range(W_total.shape[1]): if W_total[i, j] < 1e-6: W_total[i, j] = 0. W = W_total[0:len(train_dataset), :] W_val = W_total[len(train_dataset):, :] print('Transition Matrix is estimating...Wating...') logits_matrix = probability_extract(train_loader, clf1, len(train_dataset), args.num_classes, batch_size) idx_matrix_group, transition_matrix_group = estimate_matrix( logits_matrix, model_save_dir) logits_matrix_val = probability_extract(val_loader, clf1, len(val_dataset), args.num_classes, batch_size) idx_matrix_group_val, transition_matrix_group_val = estimate_matrix( logits_matrix_val, model_save_dir) func = nn.MSELoss() model = Matrix_optimize(args.basis, args.num_classes) optimizer_1 = torch.optim.Adam(model.parameters(), lr=0.001) basis_matrix_group = basis_matrix_optimize(model, optimizer_1, args.basis, args.num_classes, W, transition_matrix_group, idx_matrix_group, func, model_save_dir, args.n_epoch_4) basis_matrix_group_val = basis_matrix_optimize( model, optimizer_1, args.basis, args.num_classes, W_val, transition_matrix_group_val, idx_matrix_group_val, func, model_save_dir, args.n_epoch_4) for i in range(basis_matrix_group.shape[0]): for j in range(basis_matrix_group.shape[1]): for k in range(basis_matrix_group.shape[2]): if basis_matrix_group[i, j, k] < 1e-6: basis_matrix_group[i, j, k] = 0. optimizer_ = torch.optim.SGD(clf1.parameters(), lr=args.lr, weight_decay=args.weight_decay, momentum=args.momentum) best_acc = 0.0 for epoch in range(1, args.n_epoch_2): # train model clf1.train() train_acc = train_correction(clf1, train_loader, epoch, optimizer_, W, basis_matrix_group, batch_size, args.num_classes, args.basis) # validation val_acc = val_correction(clf1, val_loader, epoch, W_val, basis_matrix_group_val, batch_size, args.num_classes, args.basis) # evaluate models test_acc = evaluate(test_loader, clf1) if val_acc > best_acc: best_acc = val_acc torch.save(clf1.state_dict(), model_save_dir + '/' + 'model.pth') with open(txtfile, "a") as myfile: myfile.write( str(int(epoch)) + ' ' + str(train_acc) + ' ' + str(val_acc) + ' ' + str(test_acc) + ' ' + "\n") # save results print( 'Epoch [%d/%d] Train Accuracy on the %s train data: Model %.4f %%' % (epoch + 1, args.n_epoch_2, len(train_dataset), train_acc)) print('Epoch [%d/%d] Val Accuracy on the %s val data: Model %.4f %% ' % (epoch + 1, args.n_epoch_2, len(val_dataset), val_acc)) print( 'Epoch [%d/%d] Test Accuracy on the %s test data: Model %.4f %% ' % (epoch + 1, args.n_epoch_2, len(test_dataset), test_acc)) clf1.load_state_dict(torch.load(model_save_dir + '/' + 'model.pth')) optimizer_r = torch.optim.Adam(clf1.parameters(), lr=args.lr_revision, weight_decay=args.weight_decay) nn.init.constant_(clf1.T_revision.weight, 0.0) for epoch in range(1, args.n_epoch_3): # train models clf1.train() train_acc = train_revision(clf1, train_loader, epoch, optimizer_r, W, basis_matrix_group, batch_size, args.num_classes, args.basis) # validation val_acc = val_revision(clf1, val_loader, epoch, W_val, basis_matrix_group, batch_size, args.num_classes, args.basis) # evaluate models test_acc = evaluate(test_loader, clf1) with open(txtfile, "a") as myfile: myfile.write( str(int(epoch)) + ' ' + str(train_acc) + ' ' + str(val_acc) + ' ' + str(test_acc) + ' ' + "\n") # save results print( 'Epoch [%d/%d] Train Accuracy on the %s train data: Model %.4f %%' % (epoch + 1, args.n_epoch_3, len(train_dataset), train_acc)) print('Epoch [%d/%d] Val Accuracy on the %s val data: Model %.4f %% ' % (epoch + 1, args.n_epoch_3, len(val_dataset), val_acc)) print( 'Epoch [%d/%d] Test Accuracy on the %s test data: Model %.4f %% ' % (epoch + 1, args.n_epoch_3, len(test_dataset), test_acc))
def main(): if not os.path.isdir(CHECKPOINT): os.makedirs(CHECKPOINT) print('==> Preparing dataset') trainloader, testloader = load_CIFAR(batch_size=BATCH_SIZE, num_workers=NUM_WORKERS) CLASSES = [] AUROCs = [] auroc = AverageMeter() for t, cls in enumerate(ALL_CLASSES): print('\nTask: [%d | %d]\n' % (t + 1, len(ALL_CLASSES))) CLASSES = [cls] print("==> Creating model") model = LeNet(num_classes=1) if CUDA: model = model.cuda() model = nn.DataParallel(model) cudnn.benchmark = True print(' Total params: %.2fK' % (sum(p.numel() for p in model.parameters()) / 1000)) criterion = nn.BCELoss() optimizer = optim.SGD(model.parameters(), lr=LEARNING_RATE, momentum=MOMENTUM, weight_decay=WEIGHT_DECAY) print("==> Learning") best_loss = 1e10 learning_rate = LEARNING_RATE for epoch in range(EPOCHS): # decay learning rate if (epoch + 1) % EPOCHS_DROP == 0: learning_rate *= LR_DROP for param_group in optimizer.param_groups: param_group['lr'] = learning_rate print('Epoch: [%d | %d]' % (epoch + 1, EPOCHS)) train_loss = train(trainloader, model, criterion, CLASSES, CLASSES, optimizer=optimizer, use_cuda=CUDA) test_loss = train(testloader, model, criterion, CLASSES, CLASSES, test=True, use_cuda=CUDA) # save model is_best = test_loss < best_loss best_loss = min(test_loss, best_loss) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'loss': test_loss, 'optimizer': optimizer.state_dict() }, CHECKPOINT, is_best) print("==> Calculating AUROC") filepath_best = os.path.join(CHECKPOINT, "best.pt") checkpoint = torch.load(filepath_best) model.load_state_dict(checkpoint['state_dict']) new_auroc = calc_avg_AUROC(model, testloader, CLASSES, CLASSES, CUDA) auroc.update(new_auroc) print('New Task AUROC: {}'.format(new_auroc)) print('Average AUROC: {}'.format(auroc.avg)) AUROCs.append(auroc.avg) print('\nAverage Per-task Performance over number of tasks') for i, p in enumerate(AUROCs): print("%d: %f" % (i + 1, p))
def main(): use_cuda = torch.cuda.is_available() and not args.no_cuda device = torch.device('cuda' if use_cuda else 'cpu') print(device) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if use_cuda: torch.cuda.manual_seed(args.seed) torch.backends.cudnn.deterministic = True rgb = False if args.mode == 'rgb': rgb = True if args.gray_scale: rgb = False if args.tracking_data_mod is True: args.input_size = 192 # DATALOADER train_dataset = GesturesDataset(model=args.model, csv_path='csv_dataset', train=True, mode=args.mode, rgb=rgb, normalization_type=1, n_frames=args.n_frames, resize_dim=args.input_size, transform_train=args.train_transforms, tracking_data_mod=args.tracking_data_mod) train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.n_workers) validation_dataset = GesturesDataset(model=args.model, csv_path='csv_dataset', train=False, mode=args.mode, rgb=rgb, normalization_type=1, n_frames=args.n_frames, resize_dim=args.input_size, tracking_data_mod=args.tracking_data_mod) validation_loader = DataLoader(validation_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.n_workers) # paramteri per la rete in_channels = args.n_frames if not rgb else args.n_frames * 3 n_classes = args.n_classes if args.model == 'LeNet': model = LeNet(input_channels=in_channels, input_size=args.input_size, n_classes=n_classes).to(device) elif args.model == 'AlexNet': model = AlexNet(input_channels=in_channels, input_size=args.input_size, n_classes=n_classes).to(device) elif args.model == 'AlexNetBN': model = AlexNetBN(input_channels=in_channels, input_size=args.input_size, n_classes=n_classes).to(device) elif args.model == "Vgg16": model = Vgg16(input_channels=in_channels, input_size=args.input_size, n_classes=n_classes).to(device) elif args.model == "Vgg16P": model = models.vgg16(pretrained=args.pretrained) for params in model.parameters(): params.requires_grad = False model.features._modules['0'] = nn.Conv2d(in_channels=in_channels, out_channels=64, kernel_size=(3, 3), stride=1, padding=1) model.classifier._modules['6'] = nn.Linear(4096, n_classes) # model.fc = torch.nn.Linear(model.fc.in_features, n_classes) model = model.to(device) elif args.model == "ResNet18P": model = models.resnet18(pretrained=args.pretrained) for params in model.parameters(): params.requires_grad = False model._modules['conv1'] = nn.Conv2d(in_channels, 64, 7, stride=2, padding=3) model.fc = torch.nn.Linear(model.fc.in_features, n_classes) model = model.to(device) elif args.model == "ResNet34P": model = models.resnet34(pretrained=args.pretrained) for params in model.parameters(): params.requires_grad = False model._modules['conv1'] = nn.Conv2d(in_channels, 64, 7, stride=2, padding=3) model.fc = torch.nn.Linear(model.fc.in_features, n_classes) model = model.to(device) elif args.model == "DenseNet121P": model = models.densenet121(pretrained=args.pretrained) for params in model.parameters(): params.requires_grad = False model.features._modules['conv0'] = nn.Conv2d(in_channels=in_channels, out_channels=64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3)) model.classifier = nn.Linear(in_features=1024, out_features=n_classes, bias=True) model = model.to(device) elif args.model == "DenseNet161P": model = models.densenet161(pretrained=args.pretrained) # for params in model.parameters(): # params.requires_grad = False model.features._modules['conv0'] = nn.Conv2d(in_channels=in_channels, out_channels=96, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3)) model.classifier = nn.Linear(in_features=2208, out_features=n_classes, bias=True) model = model.to(device) elif args.model == "DenseNet169P": model = models.densenet169(pretrained=args.pretrained) for params in model.parameters(): params.requires_grad = False model.features._modules['conv0'] = nn.Conv2d(in_channels=in_channels, out_channels=64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3)) model.classifier = nn.Linear(in_features=1664, out_features=n_classes, bias=True) model = model.to(device) elif args.model == "DenseNet201P": model = models.densenet201(pretrained=args.pretrained) for params in model.parameters(): params.requires_grad = False model.features._modules['conv0'] = nn.Conv2d(in_channels=in_channels, out_channels=64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3)) model.classifier = nn.Linear(in_features=1920, out_features=n_classes, bias=True) model = model.to(device) # RNN elif args.model == 'LSTM' or args.model == 'GRU': model = Rnn(rnn_type=args.model, input_size=args.input_size, hidden_size=args.hidden_size, batch_size=args.batch_size, num_classes=args.n_classes, num_layers=args.n_layers, final_layer=args.final_layer).to(device) # C3D elif args.model == 'C3D': if args.pretrained: model = C3D(rgb=rgb, num_classes=args.n_classes) # modifico parametri print('ok') model.load_state_dict(torch.load('c3d_weights/c3d.pickle', map_location=device), strict=False) # # for params in model.parameters(): # # params.requires_grad = False model.conv1 = nn.Conv3d(1 if not rgb else 3, 64, kernel_size=(3, 3, 3), padding=(1, 1, 1)) # tolgo fc6 perchè 30 frames model.fc6 = nn.Linear(16384, 4096) # num classes 28672 (112*200) model.fc7 = nn.Linear(4096, 4096) # num classes model.fc8 = nn.Linear(4096, n_classes) # num classes model = model.to(device) # Conv-lstm elif args.model == 'Conv-lstm': model = ConvLSTM(input_size=(args.input_size, args.input_size), input_dim=1 if not rgb else 3, hidden_dim=[64, 64, 128], kernel_size=(3, 3), num_layers=args.n_layers, batch_first=True, ).to(device) elif args.model == 'DeepConvLstm': model = DeepConvLstm(input_channels_conv=1 if not rgb else 3, input_size_conv=args.input_size, n_classes=12, n_frames=args.n_frames, batch_size=args.batch_size).to(device) elif args.model == 'ConvGRU': model = ConvGRU(input_size=40, hidden_sizes=[64, 128], kernel_sizes=[3, 3], n_layers=2).to(device) else: raise NotImplementedError if args.opt == 'SGD': optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) # optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, momentum=args.momentum) elif args.opt == 'Adam': optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) loss_function = nn.CrossEntropyLoss().to(device) start_epoch = 0 if args.resume: checkpoint = torch.load("/projects/fabio/weights/gesture_recog_weights/checkpoint{}.pth.tar".format(args.model)) model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) start_epoch = checkpoint['epoch'] print("Resuming state:\n-epoch: {}\n{}".format(start_epoch, model)) #name experiment personal_name = "{}_{}_{}".format(args.model, args.mode, args.exp_name) info_experiment = "{}".format(personal_name) log_dir = "/projects/fabio/logs/gesture_recog_logs/exps" weight_dir = personal_name log_file = open("{}/{}.txt".format("/projects/fabio/logs/gesture_recog_logs/txt_logs", personal_name), 'w') log_file.write(personal_name + "\n\n") if personal_name: exp_name = (("exp_{}_{}".format(time.strftime("%c"), personal_name)).replace(" ", "_")).replace(":", "-") else: exp_name = (("exp_{}".format(time.strftime("%c"), personal_name)).replace(" ", "_")).replace(":", "-") writer = SummaryWriter("{}".format(os.path.join(log_dir, exp_name))) # add info experiment writer.add_text('Info experiment', "model:{}" "\n\npretrained:{}" "\n\nbatch_size:{}" "\n\nepochs:{}" "\n\noptimizer:{}" "\n\nlr:{}" "\n\ndn_lr:{}" "\n\nmomentum:{}" "\n\nweight_decay:{}" "\n\nn_frames:{}" "\n\ninput_size:{}" "\n\nhidden_size:{}" "\n\ntracking_data_mode:{}" "\n\nn_classes:{}" "\n\nmode:{}" "\n\nn_workers:{}" "\n\nseed:{}" "\n\ninfo:{}" "".format(args.model, args.pretrained, args.batch_size, args.epochs, args.opt, args.lr, args.dn_lr, args.momentum, args.weight_decay, args.n_frames, args.input_size, args.hidden_size, args.tracking_data_mod, args.n_classes, args.mode, args.n_workers, args.seed, info_experiment)) trainer = Trainer(model=model, loss_function=loss_function, optimizer=optimizer, train_loader=train_loader, validation_loader=validation_loader, batch_size=args.batch_size, initial_lr=args.lr, device=device, writer=writer, personal_name=personal_name, log_file=log_file, weight_dir=weight_dir, dynamic_lr=args.dn_lr) print("experiment: {}".format(personal_name)) start = time.time() for ep in range(start_epoch, args.epochs): trainer.train(ep) trainer.val(ep) # display classes results classes = ['g0', 'g1', 'g2', 'g3', 'g4', 'g5', 'g6', 'g7', 'g8', 'g9', 'g10', 'g11'] for i in range(args.n_classes): print('Accuracy of {} : {:.3f}%%'.format( classes[i], 100 * trainer.class_correct[i] / trainer.class_total[i])) end = time.time() h, rem = divmod(end - start, 3600) m, s, = divmod(rem, 60) print("\nelapsed time (ep.{}):{:0>2}:{:0>2}:{:05.2f}".format(args.epochs, int(h), int(m), s)) # writing accuracy on file log_file.write("\n\n") for i in range(args.n_classes): log_file.write('Accuracy of {} : {:.3f}%\n'.format( classes[i], 100 * trainer.class_correct[i] / trainer.class_total[i])) log_file.close()
w_locals = [w_glob for i in range(args.num_nodes)] for iter in range(args.epochs): loss_locals = [] for idx in range(args.num_nodes): #import pdb; pdb.set_trace() local = LocalUpdate(args=args, dataset=dataset_train, idxs=dict_users[idx]) w, loss = local.train(net=copy.deepcopy(net_glob).to(args.device), global_grad_data=w_glob_grad) w_locals[idx] = copy.deepcopy(w) loss_locals.append(copy.deepcopy(loss)) # update global weights w_glob = cal_ave_weight(w_locals) # copy weight to net_glob net_glob.load_state_dict(w_glob) # print loss loss_avg = sum(loss_locals) / len(loss_locals) writer_loss.add_scalar("train_loss", loss_avg, iter) print('Round {:3d}, Average loss {:.3f}'.format(iter, loss_avg)) #test after a round net_glob.eval() acc_test, loss_test = test_img(net_glob, dataset_test, args) writer_acc.add_scalar("test_acc", acc_test, iter) #loss_train.append(loss_avg) writer_loss.close() # testing net_glob.eval()
num_workers=2) classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') # Model print('==> Building model..') model = LeNet(3, len(classes)).to(device) if device.type == 'cuda': model = torch.nn.DataParallel(model) cudnn.benchmark = True if os.path.isfile(ckpt_file): checkpoint = torch.load(ckpt_file) model.load_state_dict(checkpoint['net']) best_acc = checkpoint['acc'] start_epoch = checkpoint['epoch'] criterion = nn.NLLLoss() optimizer = optim.SGD(model.parameters(), lr=1e-3, momentum=0.9, weight_decay=5e-4) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min') # Training def train(epoch): print('\nEpoch: %d' % epoch) model.train()
def main(): opt = Parser(train=False).get() # dataset and data loader _, val_loader, adv_val_loader, _, num_classes = \ load_dataset(opt.dataset, opt.batch_size, opt.data_root, False, 0.0, opt.num_val_samples, workers=4) # model if opt.arch == 'lenet': model = LeNet(num_classes) elif opt.arch == 'resnet': model = ResNetv2_20(num_classes) else: raise NotImplementedError # move model to device model.to(opt.device) # load trained weight try: model.load_state_dict(torch.load(opt.weight_path)) except: model_weight = convert_model_from_parallel(opt.weight_path) model.load_state_dict(model_weight) # criterion criterion = nn.CrossEntropyLoss() # advertorch attacker if opt.attack == 'pgd': attacker = PGDAttack(model, loss_fn=criterion, eps=opt.eps / 255, nb_iter=opt.num_steps, eps_iter=opt.eps_iter / 255, rand_init=True, clip_min=opt.clip_min, clip_max=opt.clip_max, ord=np.inf, targeted=False) else: raise NotImplementedError # trainer trainer = Trainer(opt, model, criterion, attacker) trainer.print_freq = -1 # validation val_losses, val_acc1s, val_acc5s = \ trainer.validate(val_loader) aval_losses, aval_acc1s, aval_acc5s = \ trainer.adv_validate(adv_val_loader) print('[model] {}'.format(opt.weight_path)) print('[standard]\n' 'loss: {:.4f} | acc1: {:.2f}% | acc5: {:.2f}%' '\n[adversarial]\n' 'loss: {:.4f} | acc1: {:.2f}% | acc5: {:.2f}%'.format( val_losses['val'].avg, val_acc1s['val'].avg, val_acc5s['val'].avg, aval_losses['aval'].avg, aval_acc1s['aval'].avg, aval_acc5s['aval'].avg))
cols=4): images = images.squeeze() # 16x28x28 plt.figure(figsize=(10, 12)) # plt.tight_layout() for i in range(batch_size): lbl = labels[i].item() true_lbl = true_labels[i].item() plt.subplot(rows, cols, i + 1) # 1 ~ 4 plt.imshow(images[i, :, :].numpy(), cmap='gray') plt.title('{} -> {}'.format(true_lbl, lbl)) plt.axis('off') plt.savefig('../plot.png') # plt.show() if __name__ == '__main__': model = LeNet() epoch = 12 ckpt = torch.load('../ckpts/LeNet_e{:02}.pt'.format(epoch)) model.load_state_dict(ckpt) _, dataset = get_dataset('mnist') imgs, true_labels = next(iter(dataset)) labels = model(imgs) labels = torch.argmax(labels, dim=1) batch_size = dataset.batch_size plot_with_label(imgs, labels, true_labels, batch_size, batch_size**0.5, batch_size**0.5)
def main(): # Training settings parser = argparse.ArgumentParser(description='Adversarial MNIST Example') parser.add_argument('--use-pretrained', action='store_true', default=False, help='uses the pretrained model') parser.add_argument('--adversarial-training', action='store_true', default=False, help='takes the adversarial training process') parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=512, metavar='N', help='input batch size for testing (default: 512)') args = parser.parse_args() # Define what device we are using use_cuda = torch.cuda.is_available() print("CUDA Available: ", use_cuda) device = torch.device("cuda" if use_cuda else "cpu") # MNIST Test dataset and dataloader declaration kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} train_loader = torch.utils.data.DataLoader(datasets.MNIST( '../data', train=True, download=True, transform=transforms.Compose([transforms.ToTensor()])), batch_size=args.batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader(datasets.MNIST( '../data', train=False, transform=transforms.Compose([transforms.ToTensor()])), batch_size=args.test_batch_size, shuffle=True, **kwargs) # Initialize the network model = LeNet().to(device) optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5) if args.use_pretrained: print('Loading the pretrained model') model.load_state_dict( torch.load('resources/lenet_mnist_model.bin', map_location='cpu')) else: print('Training on the MNIST dataset') model_train(model, train_loader, F.nll_loss, optimizer, epochs=10) print('Evaluating the neural network') # Evaluate the accuracy of the MNIST model on clean examples accuracy, _ = model_eval(model, test_loader, F.nll_loss) print('Test accuracy on clean examples: ' + str(accuracy)) # Evaluate the accuracy of the MNIST model on adversarial examples accuracy, _ = model_eval(model, test_loader, F.nll_loss, attack_method=fgsm_attack) print('Test accuracy on adversarial examples: ' + str(accuracy)) if args.adversarial_training: print("Repeating the process, with adversarial training") # Perform adversarial training model_train(model, train_loader, F.nll_loss, optimizer, epochs=10, attack_method=fgsm_attack) # Evaluate the accuracy of the adversarially trained MNIST model on # clean examples accuracy, _ = model_eval(model, test_loader, F.nll_loss) print('Test accuracy on clean examples: ' + str(accuracy)) # Evaluate the accuracy of the adversarially trained MNIST model on # adversarial examples accuracy_adv, _ = model_eval(model, test_loader, F.nll_loss, attack_method=fgsm_attack) print('Test accuracy on adversarial examples: ' + str(accuracy_adv))
def initiate_mnist(dataset, random_model=False): use_cuda = torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") kwargs = {"num_workers": 1, "pin_memory": True} if use_cuda else {} if dataset == "mnist": train_loader = torch.utils.data.DataLoader(datasets.MNIST( root='./data', train=True, download=True, transform=transforms.Compose([transforms.ToTensor()]), ), batch_size=50, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader(datasets.MNIST( './data', train=False, download=True, transform=transforms.Compose([transforms.ToTensor()]), ), batch_size=10000, shuffle=False, **kwargs) elif dataset == "fashion": train_loader = torch.utils.data.DataLoader(datasets.FashionMNIST( root='./data', train=True, download=True, transform=transforms.Compose([transforms.ToTensor()]), ), batch_size=50, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader(datasets.FashionMNIST( './data', train=False, download=True, transform=transforms.Compose([transforms.ToTensor()]), ), batch_size=10000, shuffle=False, **kwargs) classes = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0'] def show_images(images, labels): num_img = len(images) np_images = [img.numpy() for img in images] fig, axes = plt.subplots(nrows=1, ncols=num_img, figsize=(20, 45)) for i, ax in enumerate(axes.flat): ax.set_axis_off() im = ax.imshow(np_images[i], vmin=0., vmax=1.) ax.set_title(f'{labels[i]}') plt.axis("off") fig.subplots_adjust(bottom=0.1, top=0.9, left=0.1, right=0.8, wspace=0.1, hspace=0.25) plt.show() images, labels = iter(train_loader).next() num_img_to_plot = 9 images = [images[i].permute(1, 2, 0) for i in range(num_img_to_plot)] labels = [classes[i] for i in labels[:num_img_to_plot]] # show_images(images, labels) model = LeNet().to(device) model_2 = LeNet().to(device) if not random_model: if not use_cuda: model.load_state_dict( torch.load("checkpoints/" + dataset + "/CNN.pt", map_location='cpu')) model_2.load_state_dict( torch.load("checkpoints/" + dataset + "/mnist_PGD_e_20.pt", map_location='cpu')) else: model.load_state_dict( torch.load("checkpoints/" + dataset + "/CNN.pt")) model_2.load_state_dict( torch.load("checkpoints/" + dataset + "/mnist_PGD_e_20.pt")) # model.load_state_dict(torch.load("checkpoints/CNN.pt")) model.eval() model_2.eval() test_loss, test_acc = test(model, test_loader) print(f'Clean \t loss: {test_loss:.4f} \t acc: {test_acc:.4f}') return model, model_2, train_loader, test_loader
class Reptile(object): def __init__(self, args): self.args = args self._load_model() self.model.to(args.device) self.task_generator = TaskGen(args.max_num_classes) self.outer_stepsize = args.outer_stepsize self.criterion = nn.CrossEntropyLoss() # self.optimizer = optim.Adam(self.model.parameters(), lr=args.inner_stepsize) def _load_model(self): self.model = LeNet() self.current_iteration = 0 if os.path.exists(self.args.model_path): try: print("Loading model from: {}".format(self.args.model_path)) self.model.load_state_dict(torch.load(self.args.model_path)) self.current_iteration = joblib.load("{}.iter".format( self.args.model_path)) except Exception as e: print( "Exception: {}\nCould not load model from {} - starting from scratch" .format(e, self.args.model_path)) def inner_training(self, x, y, num_iterations): """ Run training on task """ x, y = shuffle_unison(x, y) self.model.train() x = torch.tensor(x, dtype=torch.float, device=self.args.device) y = torch.tensor(y, dtype=torch.float, device=self.args.device) total_loss = 0 for _ in range(num_iterations): start = np.random.randint(0, len(x) - self.args.inner_batch_size + 1) self.model.zero_grad() # self.optimizer.zero_grad() outputs = self.model(x[start:start + self.args.inner_batch_size]) # print("output: {} - y: {}".format(outputs.shape, y.shape)) loss = self.criterion( outputs, Variable(y[start:start + self.args.inner_batch_size].long())) total_loss += loss loss.backward() # self.optimizer.step() # Similar to calling optimizer.step() for param in self.model.parameters(): param.data -= self.args.inner_stepsize * param.grad.data return total_loss / self.args.inner_iterations def _meta_gradient_update(self, iteration, num_classes, weights_before): """ Interpolate between current weights and trained weights from this task I.e. (weights_before - weights_after) is the meta-gradient - iteration: current iteration - used for updating outer_stepsize - num_classes: current classifier number of classes - weights_before: state of weights before inner steps training """ weights_after = self.model.state_dict() outer_stepsize = self.outer_stepsize * ( 1 - iteration / self.args.n_iterations) # linear schedule self.model.load_state_dict({ name: weights_before[name] + (weights_after[name] - weights_before[name]) * outer_stepsize for name in weights_before }) def meta_training(self): # Reptile training loop total_loss = 0 try: while self.current_iteration < self.args.n_iterations: # Generate task data, labels, original_labels, num_classes = self.task_generator.get_train_task( args.num_classes) weights_before = deepcopy(self.model.state_dict()) loss = self.inner_training(data, labels, self.args.inner_iterations) total_loss += loss if self.current_iteration % self.args.log_every == 0: print("-----------------------------") print("iteration {}".format( self.current_iteration + 1)) print("Loss: {:.3f}".format(total_loss / (self.current_iteration + 1))) print("Current task info: ") print("\t- Number of classes: {}".format(num_classes)) print("\t- Batch size: {}".format(len(data))) print("\t- Labels: {}".format(set(original_labels))) self.test() self._meta_gradient_update(self.current_iteration, num_classes, weights_before) self.current_iteration += 1 torch.save(self.model.state_dict(), self.args.model_path) except KeyboardInterrupt: print("Manual Interrupt...") print("Saving to: {}".format(self.args.model_path)) torch.save(self.model.state_dict(), self.args.model_path) joblib.dump(self.current_iteration, "{}.iter".format(self.args.model_path), compress=1) def predict(self, x): self.model.eval() x = torch.tensor(x, dtype=torch.float, device=self.args.device) outputs = self.model(x) return outputs.cpu().data.numpy() def test(self): """ Run tests 1. Create task from test set. 2. Reload model 3. Check accuracy on test set 4. Train for one or more iterations on one task 5. Check accuracy again on test set """ test_data, test_labels, _, _ = self.task_generator.get_test_task( selected_labels=[1, 2, 3, 4, 5], num_samples=-1) # all available samples predicted_labels = np.argmax(self.predict(test_data), axis=1) accuracy = np.mean(1 * (predicted_labels == test_labels)) * 100 print( "Accuracy before few shots learning (a.k.a. zero-shot learning): {:.2f}%\n----" .format(accuracy)) weights_before = deepcopy( self.model.state_dict()) # save snapshot before evaluation for i in range(1, 5): enroll_data, enroll_labels, _, _ = self.task_generator.get_enroll_task( selected_labels=[1, 2, 3, 4, 5], num_samples=i) self.inner_training(enroll_data, enroll_labels, self.args.inner_iterations_test) predicted_labels = np.argmax(self.predict(test_data), axis=1) accuracy = np.mean(1 * (predicted_labels == test_labels)) * 100 print("Accuracy after {} shot{} learning: {:.2f}%)".format( i, "" if i == 1 else "s", accuracy)) self.model.load_state_dict(weights_before) # restore from snapshot
train=True, download=True, transform=transforms.ToTensor()) loader_train = torch.utils.data.DataLoader(train_dataset, batch_size=param['batch_size'], shuffle=True) test_dataset = datasets.MNIST(root='../data/', train=False, download=True, transform=transforms.ToTensor()) loader_test = torch.utils.data.DataLoader(test_dataset, batch_size=param['test_batch_size'], shuffle=True) model = LeNet() model.load_state_dict( torch.load('models/lenet_pretrained.pkl', map_location='cpu')) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model.to(device) print("--- Accuracy of Pretrained Model ---") test(model, loader_test) # pruning masks = lenet_prune() model.set_masks(masks) print("--- Accuracy After Pruning ---") test(model, loader_test) # Retraining criterion = nn.CrossEntropyLoss() optimizer = torch.optim.RMSprop(model.parameters(), lr=param['learning_rate'],