def run(count, memory_size, device='cuda'): transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=10) testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=10) base_dir = os.path.join('cifar_' + str(memory_size), str(count)) model = nn.DataParallel(CifarClassifier(count, memory_size)) optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001, momentum=0.9, weight_decay=5e-4) trial = Trial(model, optimizer, nn.NLLLoss(), [torchbearer.metrics.CategoricalAccuracy(), 'loss'], callbacks=[ callbacks.MostRecent(os.path.join(base_dir, '{epoch:02d}.pt')), callbacks.GradientClipping(5), callbacks.MultiStepLR(milestones=[150, 250]), callbacks.TensorBoard(write_graph=False, comment=base_dir) ]).with_train_generator(trainloader).to(device) trial.run(350) trial.with_test_generator(testloader).evaluate(data_key=torchbearer.TEST_DATA)
def run(count, memory_size, file, device='cuda'): transform_train = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.ColorJitter(0.25, 0.25, 0.25, 0.25), transforms.ToTensor() ]) transform_test = transforms.Compose([transforms.ToTensor()]) trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=10) testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=10) base_dir = os.path.join('cifarss_' + str(memory_size), "16") model = CifarDraw(count, memory_size) model.load_state_dict(torch.load(file)[torchbearer.MODEL]) model = SelfTaught(count, 512, memory_size, model.memory) for param in model.memory.parameters(): param.requires_grad = False model.memory.decay.requires_grad = True model.memory.learn.requires_grad = True model.memory.learn2.requires_grad = True optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3) trial = Trial(model, optimizer, nn.NLLLoss(), ['acc', 'loss'], pass_state=True, callbacks=[ callbacks.MultiStepLR([25, 40, 45]), callbacks.MostRecent( os.path.join(base_dir, '{epoch:02d}.pt')), callbacks.GradientClipping(5) ]).with_generators( train_generator=trainloader, val_generator=testloader).for_val_steps(5).to(device) trial.run(50)
def run(count, memory_size, device='cuda'): traintransform = transforms.Compose([ transforms.RandomRotation(20), transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) trainset = torchvision.datasets.MNIST(root='./data/mnist', train=True, download=True, transform=traintransform) trainloader = torch.utils.data.DataLoader(trainset, pin_memory=True, batch_size=128, shuffle=True, num_workers=10) testtransform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) testset = torchvision.datasets.MNIST(root='./data/mnist', train=False, download=True, transform=testtransform) testloader = torch.utils.data.DataLoader(testset, pin_memory=True, batch_size=128, shuffle=False, num_workers=10) base_dir = os.path.join('mnist_' + str(memory_size), str(count)) model = MnistClassifier(count, memory_size) optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001) trial = Trial( model, optimizer, nn.NLLLoss(), ['acc', 'loss'], callbacks=[ callbacks.MostRecent(os.path.join(base_dir, '{epoch:02d}.pt')), callbacks.GradientClipping(5), callbacks.MultiStepLR(milestones=[50, 100, 150, 190, 195]), callbacks.ExponentialLR(0.99), callbacks.TensorBoard(write_graph=False, comment=base_dir) ]).with_train_generator(trainloader).to(device) trial.run(200) trial.with_test_generator(testloader).evaluate( data_key=torchbearer.TEST_DATA)
if mode == 'fmix': aug = [FMix(alpha=1, decay_power=3)] model = VAE(64, var=args.var) trial = Trial(model, optim.Adam(model.parameters(), lr=5e-2), nll, metrics=[ metrics.MeanSquaredError(pred_key=SAMPLE), metrics.mean(NLL), metrics.mean(KL), 'loss' ], callbacks=[ sample, kld(distributions.Normal(0, 1)), init.XavierNormal(targets=['Conv']), callbacks.MostRecent(args.dir + '/' + mode + '_' + str(args.i) + '.pt'), callbacks.MultiStepLR([40, 80]), callbacks.TensorBoard(write_graph=False, comment=mode + '_' + str(args.i), log_dir='vae_logs'), *aug ]) if mode in ['base', 'mix', 'cutmix']: trial = trial.load_state_dict( torch.load('vaes/' + '/' + mode + '_' + str(args.i) + '.pt')) trial.with_generators(train_loader, test_loader).to('cuda').run(args.epochs, verbose=1)
def run(iteration, device='cuda:1'): transform_train = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.ColorJitter(0.25, 0.25, 0.25, 0.25), transforms.ToTensor() ]) transform_test = transforms.Compose([transforms.ToTensor()]) trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=10) testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=10) base_dir = 'cifar_vae' model = CifarVAE() optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=5e-4) from datetime import datetime current_time = datetime.now().strftime('%b%d_%H-%M-%S') trial = Trial( model, optimizer, nn.MSELoss(reduction='sum'), ['acc', 'loss'], pass_state=True, callbacks=[ tm.kl_divergence(MU, LOGVAR, beta=2), callbacks.MultiStepLR([50, 90]), callbacks.MostRecent( os.path.join(base_dir, 'iter_' + str(iteration) + '.{epoch:02d}.pt')), callbacks.GradientClipping(5), callbacks.TensorBoardImages(comment=current_time, name='Prediction', write_each_epoch=True, key=torchbearer.Y_PRED), callbacks.TensorBoardImages(comment=current_time + '_cifar_vae', name='Target', write_each_epoch=False, key=torchbearer.Y_TRUE), ]).with_generators( train_generator=trainloader, val_generator=testloader).for_val_steps(5).to(device) trial.run(100)
optimizer = torch.optim.SGD(model.parameters(), 0.1, momentum=0.9, weight_decay=1e-4) # optimiser = optim.RMSprop(model.parameters(), alpha=0.9, lr=0.0001, weight_decay=1e-6) loss_function = nn.CrossEntropyLoss() # device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimizer, loss_function, metrics=['loss', 'acc', 'top_5_acc'], callbacks=[ callbacks.TensorBoard(write_graph=False, comment=f'resnet50_{n_bn}_{rep}'), callbacks.MultiStepLR([30, 60]), callbacks.MostRecent(dir + model_file + '_{epoch:02d}.pt') ]).to('cuda') trial.with_generators(trainloader, test_generator=testloader) pattern = re.compile(model_file + '_\d+.pt') for filepath in os.listdir(dir): if pattern.match(filepath): trial.load_state_dict(torch.load(dir + filepath)) trial.run(epochs=90) trial.evaluate(data_key=torchbearer.TEST_DATA) torch.save(model.module.state_dict(), dir + model_file + '.pt')