def main(): # check the configurations use_cuda = torch.cuda.is_available() device = torch.device('cuda' if use_cuda else 'cpu') # prepare data for training transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ]) trainset = torchvision.datasets.CIFAR10(root='../data', train=True, download=True, transform=transform) trainloader = torch.utils.data.DataLoader(trainset, shuffle=True, batch_size=128, num_workers=4, pin_memory=True) testset = torchvision.datasets.CIFAR10(root='../data', train=False, download=True, transform=transform) testloader = torch.utils.data.DataLoader(testset, shuffle=False, batch_size=128, num_workers=4, pin_memory=True) classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') # initilizae the model net = VGG().cuda() if use_cuda else VGG() criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=0.05, momentum=0.9, weight_decay=5e-4) # training or loading the neural network # model_train(net, trainloader, criterion, optimizer, epochs=5) net.load_state_dict( torch.load('resources/vgg16_cifar10.bin', map_location='cpu')) print('Neural network ready.') # evaluate the model performance accuracy, _ = model_eval(net, testloader, criterion) print('Accuracy of the network on the clean test images: %d %%' % (100 * accuracy)) accuracy, _ = model_eval(net, testloader, criterion, attack_method=illcm_attack) print('Accuracy of the network on the adversarial test images: %d %%' % (100 * accuracy))
def build_model(config): if config['model'] == 'ResNet18': model = ResNet18(color_channel=config['color_channel']) elif config['model'] == 'VGG11': model = VGG('VGG11', color_channel=config['color_channel']) elif config['model'] == 'VGG13': model = VGG('VGG13', color_channel=config['color_channel']) else: print('wrong model option') model = None loss_function = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=config['lr'], momentum=config['momentum'], weight_decay=config['weight_decay']) return model, loss_function, optimizer
def main(num_epochs=200, learning_rate=0.005, momentum=0.5, log_interval=500, *args, **kwargs): train_loader, test_loader = loaders.loader(batch_size_train=100, batch_size_test=1000) # Train the model total_step = len(train_loader) curr_lr1 = learning_rate model1 = VGG().to(device) # Loss and optimizer criterion = nn.CrossEntropyLoss() optimizer1 = torch.optim.Adam(model1.parameters(), lr=learning_rate) # Train the model total_step = len(train_loader) best_accuracy1 = 0 for epoch in range(num_epochs): for i, (images, labels) in enumerate(train_loader): images = images.to(device) labels = labels.to(device) # Forward outputs = model1(images) loss1 = criterion(outputs, labels) # Backward and optimize optimizer1.zero_grad() loss1.backward() optimizer1.step() if i == 499: print( "Ordinary Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}".format( epoch + 1, num_epochs, i + 1, total_step, loss1.item())) # Test the model model1.eval() with torch.no_grad(): correct1 = 0 total1 = 0 for images, labels in test_loader: images = images.to(device) labels = labels.to(device) outputs = model1(images) _, predicted = torch.max(outputs.data, 1) total1 += labels.size(0) correct1 += (predicted == labels).sum().item() if best_accuracy1 >= correct1 / total1: curr_lr1 = learning_rate * np.asscalar( pow(np.random.rand(1), 3)) update_lr(optimizer1, curr_lr1) print('Test Accuracy of NN: {} % Best: {} %'.format( 100 * correct1 / total1, 100 * best_accuracy1)) else: best_accuracy1 = correct1 / total1 net_opt1 = model1 print('Test Accuracy of NN: {} % (improvement)'.format( 100 * correct1 / total1)) model1.train()
def main(): parser = argparse.ArgumentParser(description='Style transfer') parser.add_argument('--image', '-i', type=str, default=None, help='image path e.g. image.jpg') parser.add_argument('--style', '-s', type=str, default=None, help='style path e.g. picasso.jpg') args = parser.parse_args() # # Initialise # # ----------------- get data ----------------------------------------------------------- prep = prep_data(512) postpa, postpb = post() # ----------------- get model ----------------------------------------------------------- vgg = VGG() vgg.load_state_dict(torch.load(model_dir + 'vgg_conv.pth')) for param in vgg.parameters(): param.requires_grad = False if torch.cuda.is_available(): vgg.cuda() # ----------------- load images ----------------------------------------------------------- img_dirs = [image_dir, image_dir] img_names = [args.style, args.image] imgs = [Image.open(img_dirs[i] + name) for i, name in enumerate(img_names)] imgs_torch = [prep(img) for img in imgs] if torch.cuda.is_available(): imgs_torch = [Variable(img.unsqueeze(0).cuda()) for img in imgs_torch] else: imgs_torch = [Variable(img.unsqueeze(0)) for img in imgs_torch] style_image, content_image = imgs_torch opt_img = Variable(content_image.data.clone(), requires_grad=True) # ----------------- define layers ----------------------------------------------------------- style_layers = ['r11', 'r21', 'r31', 'r41', 'r51'] content_layers = ['r42'] loss_layers = style_layers + content_layers loss_fns = [GramMSELoss()] * len(style_layers) + [nn.MSELoss() ] * len(content_layers) if torch.cuda.is_available(): loss_fns = [loss_fn.cuda() for loss_fn in loss_fns] #these are good weights settings: style_weights = [1e3 / n**2 for n in [64, 128, 256, 512, 512]] content_weights = [1e0] weights = style_weights + content_weights #compute optimization targets style_targets = [ GramMatrix()(A).detach() for A in vgg(style_image, style_layers) ] content_targets = [A.detach() for A in vgg(content_image, content_layers)] targets = style_targets + content_targets # # Low res # print("processing low res...") out_img = train(opt_img, vgg, weights, loss_fns, targets, optim, loss_layers, postpa, postpb, low_res=True) # # high res # print("processing high res...") #prep hr images prep_hr = prep_data(800) imgs_torch = [prep_hr(img) for img in imgs] if torch.cuda.is_available(): imgs_torch = [Variable(img.unsqueeze(0).cuda()) for img in imgs_torch] else: imgs_torch = [Variable(img.unsqueeze(0)) for img in imgs_torch] style_image, content_image = imgs_torch #now initialise with upsampled lowres result opt_img = prep_hr(out_img).unsqueeze(0) opt_img = Variable(opt_img.type_as(content_image.data), requires_grad=True) #compute hr targets style_targets = [ GramMatrix()(A).detach() for A in vgg(style_image, style_layers) ] content_targets = [A.detach() for A in vgg(content_image, content_layers)] targets = style_targets + content_targets out_img_hr = train(opt_img, vgg, weights, loss_fns, targets, optim, loss_layers, postpa, postpb, low_res=False) out_img_hr.save( f'outputs/{str(img_names[1]).split(".")[0]}_{str(img_names[0]).split(".")[0]}_out_hr.jpg' ) print( f'output saved to: outputs/{str(img_names[1]).split(".")[0]}_{str(img_names[0]).split(".")[0]}_out_hr.jpg' )
def main(): """ This code sets up the data and loads the model obtained after ADMM based training for retraining to enforce pruning constraints. The function retrains_model present in the utils file enforces the hard sparsity constraints on the weights on the model obtained after ADMM based training and then retrains the model to improve the accuracy. """ #model = LeNet5() model = VGG(n_class=10) device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') Path = 'saved_model/admm_model/cifar10_vgg_acc_0.688' # Path to the saved model after ADMM based training model.load_state_dict(torch.load(Path)) model.to(device) #data_transforms = transforms.Compose([transforms.CenterCrop(32),transforms.ToTensor(),transforms.Normalize((0.1307,), (0.3081,))]) train_data = datasets.CIFAR10('data/', train=True, download=False, transform=transforms.Compose([ transforms.Pad(4), transforms.RandomCrop(32), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ])) #train_data = datasets.MNIST(root='data/',download=False,train=True,transform=data_transforms) # Splitting the training dataset into training and validation dataset N_train = len(train_data) val_split = 0.1 N_val = int(val_split*N_train) train_data,val_data = torch.utils.data.random_split(train_data,(N_train-N_val,N_val)) ## Test data test_data = datasets.CIFAR10('data/', train=False, download=False, transform=transforms.Compose([ transforms.Pad(4), transforms.RandomCrop(32), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ])) #test_data = datasets.MNIST(root='data/',download=False,train=False,transform=data_transforms) batch_size = 128 num_epochs = 20 log_step = 100 loss_fn = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(),lr=1e-2) #optimizer = torch.optim.SGD(model.parameters(), lr =5e-4,momentum =0.9, weight_decay = 5e-4 ) scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,milestones = [10], gamma = 0.1) ####### Re-Training ############## # Parameters prune_type = 'filter' # Number of non-zero filters at each convolutional layer l = {'conv1':32,'conv2':64,'conv3':128,'conv4':128,'conv5':256,'conv6':256,'conv7':256,'conv8':256} retrain_model(model,train_data,val_data,batch_size,loss_fn,num_epochs,log_step,optimizer,scheduler,l,prune_type,device) # Check the test accuracy model.eval() test_accuracy = eval_accuracy_data(test_data,model,batch_size,device) print('Test accuracy is',test_accuracy)
transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ]) dataset = CIFAR10(root="./data/", train=True, download=True, transform=transform) train_set, val_set = torch.utils.data.random_split(dataset, [40000, 10000]) test_set = CIFAR10(root="./data/", train=False, download=True, transform=transform) #model = vgg16(pretrained = False) model = VGG('VGG16') optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9) criterion = nn.CrossEntropyLoss() trainer = trainer(train_set, val_set, test_set, model, optimizer, criterion, device='cuda') history = trainer.fit(epochs=10, liveplot=False, es_epochs=5) trainer.test() trainer.genPlots()
log = 50 train_loader = get_data_loader(train=True, batch_size=train_size, split='train', model=models[model_id]) val_loader = get_data_loader(train=False, batch_size=val_size, split='val', model=models[model_id]) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.train() model = model.to(device) optimizer = torch.optim.Adam(model.parameters(), lr) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=gamma) # weight = torch.tensor([0.2, 0.3, 0.5]).to(device) criterion = torch.nn.CrossEntropyLoss() for epoch in range(epochs): count = 0 current_loss = 0 for data, target in train_loader: data, target = data.to(device), target.to(device) optimizer.zero_grad() output = model(data)
def main(): """ This code implements the ADMM based training of a CNN. """ #model = LeNet5() model = VGG(n_class=10) device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') Path = 'saved_model/pre_train_models/cifar10_vgg_acc_0.943' # Path to the baseline model model.load_state_dict(torch.load(Path)) model.to(device) #data_transforms = transforms.Compose([transforms.CenterCrop(32),transforms.ToTensor(),transforms.Normalize((0.1307,), (0.3081,))]) train_data = datasets.CIFAR10('data/', train=True, download=False, transform=transforms.Compose([ transforms.Pad(4), transforms.RandomCrop(32), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize( (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ])) #train_data = datasets.MNIST(root='data/',download=False,train=True,transform=data_transforms) """ N_train = len(train_data) val_split = 0.1 N_val = int(val_split*N_train) train_data,val_data = torch.utils.data.random_split(train_data,(N_train-N_val,N_val)) """ ## Test data test_data = datasets.CIFAR10('data/', train=False, download=False, transform=transforms.Compose([ transforms.Pad(4), transforms.RandomCrop(32), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize( (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ])) #test_data = datasets.MNIST(root='data/',download=False,train=False,transform=data_transforms) batch_size = 128 num_epochs = 50 log_step = 100 loss_fn = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=1e-2) #optimizer = torch.optim.SGD(model.parameters(), lr =5e-4,momentum =0.9, weight_decay = 5e-4 ) scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[15, 30], gamma=0.1) ####### ADMM Training ############## ## Parameters fc_prune = False # True if the fully connected layers are also pruned prune_type = 'filter' # Type of structural pruning at the convolutional layers # Number of non zero filters at each convolutional layer l = { 'conv1': 32, 'conv2': 64, 'conv3': 128, 'conv4': 128, 'conv5': 256, 'conv6': 256, 'conv7': 256, 'conv8': 256 } # ADMM parameters rho_val = 1.5e-3 num_admm_steps = 10 Z = {} U = {} rho = {} best_accuracy = 0 all_acc = False ## Initialization of the variable Z and dual variable U for name_net in model.named_modules(): name, net = name_net if isinstance(net, nn.Conv2d): Z[name] = net.weight.clone().detach().requires_grad_(False) Z[name] = Projection_structured(Z[name], l[name], prune_type) U[name] = torch.zeros_like(net.weight, requires_grad=False) rho[name] = rho_val elif fc_prune and isinstance(net, nn.Linear): Z[name] = net.weight.clone().detach().requires_grad_(False) l_unst = int(len(net.weight.data.reshape(-1, )) * prune_ratio) Z[name], _ = Projection_unstructured(Z[name], l_unst) U[name] = torch.zeros_like(net.weight, requires_grad=False) ## ADMM loop for i in range(num_admm_steps): print('ADMM step number {}'.format(i)) # First train the VGG model train_model_admm(model, train_data, batch_size, loss_fn, optimizer, scheduler, num_epochs, log_step, Z, U, rho, fc_prune, device) # Update the variable Z for name_net in model.named_modules(): name, net = name_net if isinstance(net, nn.Conv2d): Z[name] = Projection_structured(net.weight.detach() + U[name], l[name], prune_type) elif fc_prune and isinstance(net, nn.Linear): l_unst = int(len(net.weight.data.reshape(-1, )) * prune_ratio) Z[name], _ = Projection_unstructured( net.weight.detach() + U[name], l_unst) # Updating the dual variable U for name_net in model.named_modules(): name, net = name_net if isinstance(net, nn.Conv2d): U[name] = U[name] + net.weight.detach() - Z[name] elif fc_prune and isinstance(net, nn.Linear): U[name] = U[name] + net.weight.detach() - Z[name] ## Check the test accuracy model.eval() test_accuracy = eval_accuracy_data(test_data, model, batch_size, device) print('Test accuracy is', test_accuracy) if test_accuracy > best_accuracy: print( 'Saving model with test accuracy {:.3f}'.format(test_accuracy)) torch.save( model.state_dict(), 'saved_model/admm_model/cifar10_vgg_acc_{:.3f}'.format( test_accuracy)) if all_acc: print('Removing model with test accuracy {:.3f}'.format( best_accuracy)) os.remove( 'saved_model/admm_model/cifar10_vgg_acc_{:.3f}'.format( best_accuracy)) best_accuracy = test_accuracy all_acc = True
def main(): """ This code implements the conventional training pipeline for a CNN for obtaining the baseline model for the ADMM based training """ #model = LeNet5() model = VGG(n_class=10) device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') #Path = 'saved_model/cifar10_vgg_acc_0.939' #model.load_state_dict(torch.load(Path)) model.to(device) #data_transforms = transforms.Compose([transforms.CenterCrop(32),transforms.ToTensor(),transforms.Normalize((0.1307,), (0.3081,))]) train_data = datasets.CIFAR10('data/', train=True, download=False, transform=transforms.Compose([ transforms.Pad(4), transforms.RandomCrop(32), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize( (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ])) #train_data = datasets.MNIST(root='data/',download=False,train=True,transform=data_transforms) # Splitting the training dataset into training and validation dataset N_train = len(train_data) val_split = 0.1 N_val = int(val_split * N_train) train_data, val_data = torch.utils.data.random_split( train_data, (N_train - N_val, N_val)) ## Test data test_data = datasets.CIFAR10('data/', train=False, download=False, transform=transforms.Compose([ transforms.Pad(4), transforms.RandomCrop(32), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize( (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ])) #test_data = datasets.MNIST(root='data/',download=False,train=False,transform=data_transforms) batch_size = 128 num_epochs = 50 log_step = 100 loss_fn = nn.CrossEntropyLoss() #optimizer = torch.optim.Adam(model.parameters(),lr=1e-3) optimizer = torch.optim.SGD(model.parameters(), lr=5e-4, momentum=0.9, weight_decay=5e-4) scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[15, 30], gamma=0.1) train(model, train_data, val_data, batch_size, loss_fn, optimizer, scheduler, num_epochs, log_step, device, save_model=True) #test_sample = iter(torch.utils.data.DataLoader(test_data,batch_size=len(test_data))).next() #X_test,y_test = test_sample[0].to(device),test_sample[1].to(device) print('Test accuracy is', eval_accuracy_data(test_data, model, batch_size, device)) torch.save(model)