def draw(count, memory_size, file, device='cuda'): testtransform = transforms.Compose([transforms.ToTensor()]) testset = torchvision.datasets.MNIST(root='./data/mnist', train=False, download=True, transform=testtransform) testloader = torch.utils.data.DataLoader(testset, pin_memory=True, batch_size=128, shuffle=True, num_workers=10) base_dir = os.path.join('mnist_' + str(memory_size), "6") model = MnistDraw(count, memory_size, output_stages=True) optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=0) from datetime import datetime current_time = datetime.now().strftime('%b%d_%H-%M-%S') from visualise import StagesGrid trial = Trial(model, optimizer, nn.MSELoss(reduction='sum'), ['loss'], pass_state=True, callbacks=[ callbacks.TensorBoardImages(comment=current_time, nrow=10, num_images=20, name='Prediction', write_each_epoch=True, key=torchbearer.Y_PRED, pad_value=1), callbacks.TensorBoardImages(comment=current_time + '_mnist', nrow=10, num_images=20, name='Target', write_each_epoch=False, key=torchbearer.Y_TRUE, pad_value=1), StagesGrid('mnist_stages.png', STAGES, 20) ]).load_state_dict(torch.load(os.path.join(base_dir, file)), resume=False).with_generators(train_generator=testloader, val_generator=testloader).for_train_steps(1).for_val_steps(1).to(device) trial.run() # Evaluate doesn't work with tensorboard in torchbearer, seems to have been fixed in most recent version
def run(count, memory_size, device='cuda'): transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=10) testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=10) base_dir = os.path.join('cifar_' + str(memory_size), str(count)) model = nn.DataParallel(CifarClassifier(count, memory_size)) optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001, momentum=0.9, weight_decay=5e-4) trial = Trial(model, optimizer, nn.NLLLoss(), [torchbearer.metrics.CategoricalAccuracy(), 'loss'], callbacks=[ callbacks.MostRecent(os.path.join(base_dir, '{epoch:02d}.pt')), callbacks.GradientClipping(5), callbacks.MultiStepLR(milestones=[150, 250]), callbacks.TensorBoard(write_graph=False, comment=base_dir) ]).with_train_generator(trainloader).to(device) trial.run(350) trial.with_test_generator(testloader).evaluate(data_key=torchbearer.TEST_DATA)
def run(count, memory_size, iteration, device='cuda'): traintransform = transforms.Compose( [transforms.RandomRotation(20), transforms.ToTensor()]) trainset = torchvision.datasets.MNIST(root='./data/mnist', train=True, download=True, transform=traintransform) trainloader = torch.utils.data.DataLoader(trainset, pin_memory=True, batch_size=128, shuffle=True, num_workers=10) testtransform = transforms.Compose([transforms.ToTensor()]) testset = torchvision.datasets.MNIST(root='./data/mnist', train=False, download=True, transform=testtransform) testloader = torch.utils.data.DataLoader(testset, pin_memory=True, batch_size=128, shuffle=True, num_workers=10) base_dir = os.path.join('mnist_' + str(memory_size), "4") model = MnistDraw(count, memory_size) optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3) from datetime import datetime current_time = datetime.now().strftime('%b%d_%H-%M-%S') trial = Trial( model, optimizer, nn.MSELoss(reduction='sum'), ['loss'], pass_state=True, callbacks=[ tm.kl_divergence(MU, LOGVAR), callbacks.MostRecent( os.path.join(base_dir, 'iter_' + str(iteration) + '.{epoch:02d}.pt')), callbacks.GradientClipping(5), callbacks.ExponentialLR(0.99), callbacks.TensorBoardImages(comment=current_time, name='Prediction', write_each_epoch=True, key=torchbearer.Y_PRED), callbacks.TensorBoardImages(comment=current_time + '_mnist', name='Target', write_each_epoch=True, key=torchbearer.Y_TRUE) ]).with_generators(train_generator=trainloader, val_generator=testloader).to(device) trial.run(100)
def run(count, memory_size, file, device='cuda'): transform_train = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.ColorJitter(0.25, 0.25, 0.25, 0.25), transforms.ToTensor() ]) transform_test = transforms.Compose([transforms.ToTensor()]) trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=10) testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=10) base_dir = os.path.join('cifarss_' + str(memory_size), "16") model = CifarDraw(count, memory_size) model.load_state_dict(torch.load(file)[torchbearer.MODEL]) model = SelfTaught(count, 512, memory_size, model.memory) for param in model.memory.parameters(): param.requires_grad = False model.memory.decay.requires_grad = True model.memory.learn.requires_grad = True model.memory.learn2.requires_grad = True optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3) trial = Trial(model, optimizer, nn.NLLLoss(), ['acc', 'loss'], pass_state=True, callbacks=[ callbacks.MultiStepLR([25, 40, 45]), callbacks.MostRecent( os.path.join(base_dir, '{epoch:02d}.pt')), callbacks.GradientClipping(5) ]).with_generators( train_generator=trainloader, val_generator=testloader).for_val_steps(5).to(device) trial.run(50)
def train(self, model, **kwargs) -> (float, float): # Get transfer model and put it in training mode net = model.net net.train() # Create optimiser optimiser = optim.Adam(net.parameters(), lr=1e-4) # Check for cuda device = "cuda:0" if torch.cuda.is_available() else "cpu" print("Training using", device) # Setup loss function if self.class_weight_method != ClassWeightMethod.Unweighted: distribution = class_distribution("data/processed/train") if self.class_weight_method == ClassWeightMethod.SumBased: inv_distribution = [ 1 - x / np.sum(distribution) for x in distribution ] inv_distribution = torch.from_numpy( np.array(inv_distribution)).float() elif self.class_weight_method == ClassWeightMethod.MaxBased: inv_distribution = [ np.max(distribution) / x for x in distribution ] inv_distribution = torch.from_numpy( np.array(inv_distribution)).float() else: raise IndexError("Unknown class weight method " + str(self.class_weight_method)) loss_function = self.loss(inv_distribution.to(device)) else: loss_function = self.loss() # Setup trial trial = Trial(net, optimiser, loss_function, metrics=["loss", "accuracy"]).to(device) trial.with_generators( self.image_datasets.train_loader, test_generator=self.image_datasets.validation_loader, ) # Actually run the training trial.run(epochs=self.num_epochs) # Evaluate and show results time.sleep(0.1) # Ensure training has finished net.eval() results = trial.evaluate(data_key=torchbearer.TEST_DATA) acc = float(results["test_acc"]) loss = float(results["test_loss"]) return acc, loss
def run(count, memory_size, device='cuda'): traintransform = transforms.Compose([ transforms.RandomRotation(20), transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ]) trainset = torchvision.datasets.MNIST(root='./data/mnist', train=True, download=True, transform=traintransform) trainloader = torch.utils.data.DataLoader(trainset, pin_memory=True, batch_size=128, shuffle=True, num_workers=10) testtransform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) testset = torchvision.datasets.MNIST(root='./data/mnist', train=False, download=True, transform=testtransform) testloader = torch.utils.data.DataLoader(testset, pin_memory=True, batch_size=128, shuffle=False, num_workers=10) base_dir = os.path.join('mnist_' + str(memory_size), str(count)) model = MnistClassifier(count, memory_size) optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001) trial = Trial( model, optimizer, nn.NLLLoss(), ['acc', 'loss'], callbacks=[ callbacks.MostRecent(os.path.join(base_dir, '{epoch:02d}.pt')), callbacks.GradientClipping(5), callbacks.MultiStepLR(milestones=[50, 100, 150, 190, 195]), callbacks.ExponentialLR(0.99), callbacks.TensorBoard(write_graph=False, comment=base_dir) ]).with_train_generator(trainloader).to(device) trial.run(200) trial.with_test_generator(testloader).evaluate( data_key=torchbearer.TEST_DATA)
def draw(file, device='cuda'): transform_test = transforms.Compose([transforms.ToTensor()]) testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=10) base_dir = 'cifar_vae' model = CifarVAE() optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=0) from datetime import datetime current_time = datetime.now().strftime('%b%d_%H-%M-%S') trial = Trial( model, optimizer, nn.MSELoss(reduction='sum'), ['acc', 'loss'], pass_state=True, callbacks=[ callbacks.TensorBoardImages(comment=current_time, name='Prediction', write_each_epoch=True, key=torchbearer.Y_PRED, pad_value=1, nrow=16), callbacks.TensorBoardImages(comment=current_time + '_cifar_vae', name='Target', write_each_epoch=False, key=torchbearer.Y_TRUE, pad_value=1, nrow=16) ]).load_state_dict( torch.load(os.path.join(base_dir, file)), resume=False).with_generators( train_generator=testloader, val_generator=testloader).for_train_steps(1).to(device) trial.run( ) # Evaluate doesn't work with tensorboard in torchbearer, seems to have been fixed in most recent version
def train_model( model: nn.Module, train_loader: DataLoader, val_loader: DataLoader = None, output_path: str = None, ): # Define the loss function and the optimiser loss_function = nn.MSELoss() optimiser = optim.Adam(model.parameters()) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, metrics=["loss"]).to(device) trial.with_generators(train_loader, val_generator=val_loader) trial.run(epochs=100) if output_path: torch.save(model.state_dict(), output_path) return trial
def train_model(train, valid, test, classes, batch, num_epochs): # model = CNN(3, classes) model = models.resnet18() train_loader = DataLoader(train, batch_size=batch) val_loader = DataLoader(valid, batch_size=batch) test_loader = DataLoader(test, batch_size=batch) # define the loss function and the optimiser loss_function = nn.CrossEntropyLoss() # equation 8 in original paper optimiser = optim.Adam(model.parameters()) trial = Trial(model, optimiser, loss_function, metrics=['loss', 'accuracy'], verbose=1).to(device) trial.with_generators(train_loader, val_generator=val_loader, test_generator=test_loader) trial.run(epochs=num_epochs) results_test = trial.evaluate(data_key=torchbearer.TEST_DATA) results_val = trial.evaluate(data_key=torchbearer.VALIDATION_DATA) print(results_test) return results_test, results_val
def run(train_batch_size, val_batch_size, epochs, lr, log_interval, input_size=10, hidden_size=100, out_size=4): dataset = FuzzBuzzDataset(input_size) splitter = DatasetValidationSplitter(len(dataset), 0.1) train_set = splitter.get_train_dataset(dataset) val_set = splitter.get_val_dataset(dataset) train_loader = DataLoader(train_set, pin_memory=True, batch_size=train_batch_size, shuffle=True, num_workers=2) val_loader = DataLoader(val_set, pin_memory=True, batch_size=val_batch_size, shuffle=False, num_workers=2) model = FuzzBuzzModel(input_size, hidden_size, out_size) device = 'cpu' if torch.cuda.is_available(): device = 'cuda' optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=lr) loss = nn.CrossEntropyLoss() trial = Trial(model, optimizer, criterion=loss, metrics=['acc', 'loss']).to(device) trial = trial.with_generators(train_generator=train_loader, val_generator=val_loader) trial.run(epochs=epochs) trial.evaluate(data_key=VALIDATION_DATA)
def run(count, glimpse_size, memory_size, iteration, device='cuda'): base_dir = os.path.join('celeba_' + str(memory_size), str(glimpse_size)) if not os.path.exists(base_dir): os.makedirs(base_dir) transform_train = transforms.Compose([ transforms.ToTensor() ]) dataset = torchvision.datasets.ImageFolder(root='./cropped_celeba/', transform=transform_train) splitter = DatasetValidationSplitter(len(dataset), 0.05) trainset = splitter.get_train_dataset(dataset) # Save the ids torch.save((splitter.train_ids, splitter.valid_ids), os.path.join(base_dir, 'split.dat')) trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=10) model = CelebDraw(count, glimpse_size, memory_size) optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-4) from datetime import datetime current_time = datetime.now().strftime('%b%d_%H-%M-%S') call_a = callbacks.TensorBoardImages(comment=current_time, name='Prediction', write_each_epoch=True, key=torchbearer.Y_PRED) call_a.on_step_training = call_a.on_step_validation # Hack to make this log training samples call_b = callbacks.TensorBoardImages(comment=current_time + '_celeba', name='Target', write_each_epoch=True, key=torchbearer.Y_TRUE) call_b.on_step_training = call_b.on_step_validation # Hack to make this log training samples trial = Trial(model, optimizer, nn.MSELoss(reduction='sum'), ['acc', 'loss'], pass_state=True, callbacks=[ joint_kl_divergence(MU, LOGVAR), callbacks.MostRecent(os.path.join(base_dir, 'iter_' + str(iteration) + '.{epoch:02d}.pt')), callbacks.GradientClipping(5), call_a, call_b ]).with_generators(train_generator=trainloader).to(device) trial.run(100)
def draw(count, glimpse_size, memory_size, file, device='cuda'): base_dir = os.path.join('celeba_' + str(memory_size), str(glimpse_size)) transform = transforms.Compose([ transforms.ToTensor() ]) dataset = torchvision.datasets.ImageFolder(root='./cropped_celeba/', transform=transform) splitter = DatasetValidationSplitter(len(dataset), 0.05) # load the ids splitter.train_ids, splitter.valid_ids = torch.load(os.path.join(base_dir, 'split.dat')) testset = splitter.get_val_dataset(dataset) testloader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=True, num_workers=10) model = CelebDraw(count, glimpse_size, memory_size, output_stages=True) optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=0) from datetime import datetime current_time = datetime.now().strftime('%b%d_%H-%M-%S') from visualise import StagesGrid trial = Trial(model, optimizer, nn.MSELoss(reduction='sum'), ['loss'], pass_state=True, callbacks=[ callbacks.TensorBoardImages(comment=current_time, nrow=10, num_images=20, name='Prediction', write_each_epoch=True, key=torchbearer.Y_PRED, pad_value=1), callbacks.TensorBoardImages(comment=current_time + '_celeb', nrow=10, num_images=20, name='Target', write_each_epoch=False, key=torchbearer.Y_TRUE, pad_value=1), callbacks.TensorBoardImages(comment=current_time + '_celeb_mask', nrow=10, num_images=20, name='Masked Target', write_each_epoch=False, key=MASKED_TARGET, pad_value=1), StagesGrid('celeb_stages.png', STAGES, 20) ]).load_state_dict(torch.load(os.path.join(base_dir, file)), resume=False).with_generators(train_generator=testloader, val_generator=testloader).for_train_steps(1).for_val_steps(1).to(device) trial.run() # Evaluate doesn't work with tensorboard in torchbearer, seems to have been fixed in most recent version
def train(self): scheduler = torch_scheduler.StepLR(self.num_epochs_decay, gamma=self.decay_factor) loss_plot_plan = os.path.join( self.result_path, 'live_loss_plot%s-%d-%.4f-%d-%.4f.png' % (self.model_type, self.num_epochs, self.lr, self.num_epochs_decay, self.augmentation_prob)) callbacks = [scheduler] # imaging.FromState(torchbearer.X).on_val().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_TRUE).on_val().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_PRED).on_val().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.X).on_test().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_TRUE).on_test().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_PRED).on_test().cache(16).make_grid().to_pyplot(), # TensorBoard(write_batch_metrics=True), trial = Trial( self.unet, self.optimizer, self.criterion, metrics=['loss', 'binary_acc'], # binary_acc for debugging certain things callbacks=callbacks).to(self.device) trial.with_generators(train_generator=self.train_loader, val_generator=self.valid_loader, test_generator=self.test_loader) start = time.time() history = trial.run(epochs=self.num_epochs, verbose=2) stop = time.time() train_time = stop - start state = self.unet.state_dict() unet_path = os.path.join( self.model_path, '%s-%d-%.4f-%d-%.4f_Index_BCE_Dropout_STAREIndex.pkl' % ( self.model_type, self.num_epochs, self.lr, self.num_epochs_decay, self.augmentation_prob, )) torch.save(state, unet_path) print(history) ### Testing results = trial.evaluate(data_key=torchbearer.TEST_DATA) print("Test result:") print(results) return history, results
def run(hidden_size: int, file_prefix: str, epochs: int = 20): # Flatten 28*28 images to a 784 vector for each image transform = transforms.Compose([ transforms.ToTensor(), # convert to tensor transforms.Lambda(lambda x: x.view(-1)), # flatten into vector ]) trainset = MNIST(".", train=True, download=True, transform=transform) testset = MNIST(".", train=False, download=True, transform=transform) data_size = torch.numel(trainset[0][0]) # Create data loaders trainloader = DataLoader(trainset, batch_size=128, shuffle=True, drop_last=False) testloader = DataLoader(testset, batch_size=128, shuffle=True, drop_last=False) model = SingleHiddenLayerMLP(data_size, hidden_size, 10) loss_function = nn.CrossEntropyLoss() optimiser = optim.Adam(model.parameters()) device = "cuda" if torch.cuda.is_available() else "cpu" cm = (PyCM().on_train().with_handler( to_pandas_seaborn(normalize=True, title="Confusion Matrix: {epoch}"))) callbacks = [cm] trial = Trial( model, optimiser, loss_function, metrics=["loss", "accuracy"], callbacks=callbacks, ) trial.to(device) trial.with_generators(trainloader, val_generator=testloader, val_steps=1) history = trial.run(epochs=epochs) return history
if state[torchbearer.BATCH] % 10 == 0: w = state[torchbearer.MODEL].w[0].detach().to('cpu').numpy() b = state[torchbearer.MODEL].b[0].detach().to('cpu').numpy() z = (w.dot(xy) + b).reshape(x.shape) z[np.where(z > 1.)] = 4 z[np.where((z > 0.) & (z <= 1.))] = 3 z[np.where((z > -1.) & (z <= 0.))] = 2 z[np.where(z <= -1.)] = 1 if CONTOUR in state: for coll in state[CONTOUR].collections: coll.remove() state[CONTOUR] = plt.contourf(x, y, z, cmap=plt.cm.jet, alpha=0.5) else: state[CONTOUR] = plt.contourf(x, y, z, cmap=plt.cm.jet, alpha=0.5) plt.tight_layout() plt.show() mypause(0.001) svm = LinearSVM() model = Trial(svm, optim.SGD(svm.parameters(), 0.1), hinge_loss, ['loss'], callbacks=[scatter, draw_margin, ExponentialLR(0.999, step_on_batch=True), L2WeightDecay(0.01, params=[svm.w])]).to('cuda') model.with_train_data(X, Y, batch_size=32) model.run(epochs=50, verbose=1) plt.ioff() plt.show()
A.RGBShift(r_shift_limit=15, g_shift_limit=15, b_shift_limit=15, p=0.5), A.RandomBrightnessContrast(p=0.5), A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), ToTensorV2(), ]) # load dataset trainset = CourseworkDataset(tfms, 'train', path2phow_features) trainloader = DataLoader(trainset, batch_size=256, shuffle=True) valset = CourseworkDataset(tfms, 'test', path2phow_features) valloader = DataLoader(valset, batch_size=256, shuffle=True) testset = CourseworkDataset(tfms, 'predict', path2phow_features) testloader = DataLoader(testset, batch_size=256, shuffle=True) # define model and loss device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model = ClassifierNet().to(device) optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001) loss = nn.CrossEntropyLoss() # begin to train trial = Trial(model, optimizer, loss, metrics=['acc', 'loss']).to(device) trial.with_loader(custom_loader) trial.with_generators(train_generator=trainloader, val_generator=valloader, test_generator=testloader) history = trial.run(epochs=50, verbose=1)
def run(iteration, device='cuda:1'): transform_train = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.ColorJitter(0.25, 0.25, 0.25, 0.25), transforms.ToTensor() ]) transform_test = transforms.Compose([transforms.ToTensor()]) trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=10) testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=10) base_dir = 'cifar_vae' model = CifarVAE() optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=5e-4) from datetime import datetime current_time = datetime.now().strftime('%b%d_%H-%M-%S') trial = Trial( model, optimizer, nn.MSELoss(reduction='sum'), ['acc', 'loss'], pass_state=True, callbacks=[ tm.kl_divergence(MU, LOGVAR, beta=2), callbacks.MultiStepLR([50, 90]), callbacks.MostRecent( os.path.join(base_dir, 'iter_' + str(iteration) + '.{epoch:02d}.pt')), callbacks.GradientClipping(5), callbacks.TensorBoardImages(comment=current_time, name='Prediction', write_each_epoch=True, key=torchbearer.Y_PRED), callbacks.TensorBoardImages(comment=current_time + '_cifar_vae', name='Target', write_each_epoch=False, key=torchbearer.Y_TRUE), ]).with_generators( train_generator=trainloader, val_generator=testloader).for_val_steps(5).to(device) trial.run(100)
comparison = torch.cat([ data[:num_images], recon_batch.view(128, 1, 28, 28)[:num_images] ]) save_image(comparison.cpu(), str(folder) + 'reconstruction_' + str(state[torchbearer.EPOCH]) + '.png', nrow=num_images) return saver model = VAE() optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001) loss = binary_cross_entropy from torchbearer import Trial torchbearer_trial = Trial( model, optimizer, loss, metrics=['acc', 'loss'], callbacks=[add_kld_loss_callback, save_reconstruction_callback()], pass_state=True).to('cuda') torchbearer_trial.with_generators(train_generator=traingen, val_generator=valgen) torchbearer_trial.run(epochs=10)
# test_set = CIFAR10('./data', train=False, download=True, transform=transforms.Compose([transform_test])) test_gen = torch.utils.data.DataLoader(test_set, pin_memory=True, batch_size=128, shuffle=False, num_workers=4) optimizer = optim.SGD(net.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4) from torchbearer.callbacks import MultiStepLR, TensorBoard from torchbearer import Trial from datetime import datetime current_time = datetime.now().strftime('%b%d_%H-%M-%S') trial = Trial(net, optimizer, nn.CrossEntropyLoss(), metrics=['acc', 'loss'], callbacks=[ UnpackState(), TensorBoard(write_graph=False, comment=current_time), MultiStepLR([100, 150]) ]) trial.with_generators(train_generator=train_gen, val_generator=test_gen).to('cuda') trial.run(200, verbose=1)
def train(self): """Train encoder, generator and discriminator.""" # D_LOSS = state_key('d_loss') # # def dice_loss(prediction, target): # """Calculating the dice loss # Args: # prediction = predicted image # target = Targeted image # Output: # dice_loss""" # # smooth = 1.0 # # print(prediction.shape) # # print(target.shape) # # TODO: Used reshape() instead of view because of batch size > 1 # i_flat = prediction.reshape(-1) # t_flat = target.reshape(-1) # # intersection = (i_flat * t_flat).sum() # # return 1 - ((2. * intersection + smooth) / (i_flat.sum() + t_flat.sum() + smooth)) # # def calc_loss(prediction, target, bce_weight=0.5): # """Calculating the loss and metrics # Args: # prediction = predicted image # target = Targeted image # metrics = Metrics printed # bce_weight = 0.5 (default) # Output: # loss : dice loss of the epoch """ # bce = F.binary_cross_entropy_with_logits(prediction, target) # prediction = F.sigmoid(prediction) # dice = dice_loss(prediction, target) # # loss = bce * bce_weight + dice * (1 - bce_weight) # state[D_LOSS] = loss # # return loss # # stopping = EarlyStopping(monitor='val_acc', patience=5, mode='max') scheduler = torch_scheduler.StepLR(self.num_epochs_decay, gamma=0.1) loss_plot_plan = os.path.join(self.result_path, 'live_loss_plot%s-%d-%.4f-%d-%.4f.png' % (self.model_type, self.num_epochs, self.lr, self.num_epochs_decay, self.augmentation_prob)) callbacks = [scheduler] # imaging.FromState(torchbearer.X).on_val().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_TRUE).on_val().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_PRED).on_val().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.X).on_test().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_TRUE).on_test().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_PRED).on_test().cache(16).make_grid().to_pyplot(), # TensorBoard(write_batch_metrics=True), try: trial = Trial(self.unet, self.optimizer, self.criterion, metrics=['loss', 'binary_acc'], # binary_acc for debugging certain things callbacks=callbacks).to(self.device) trial.with_generators(train_generator=self.train_loader, val_generator=self.valid_loader, test_generator=self.test_loader) start = time.time() history = trial.run(epochs=self.num_epochs, verbose=2) stop = time.time() train_time = stop - start state = self.unet.state_dict() unet_path = os.path.join(self.model_path, '%s-%d-%.4f-%d-%.4f_Index_BCE_Dropout_STAREIndex.pkl' % (self.model_type, self.num_epochs, self.lr, self.num_epochs_decay, self.augmentation_prob,)) torch.save(state, unet_path) print(history) ### Testing results = trial.evaluate(data_key=torchbearer.TEST_DATA) print("Test result:") print(results) except (RuntimeError, OSError): state = self.unet.state_dict() unet_path = os.path.join(self.model_path, '%s-%d-%.4f-%d-%.4f_Index_BCE_Dropout_STAREIndex.pkl' % (self.model_type, self.num_epochs, self.lr, self.num_epochs_decay, self.augmentation_prob,)) torch.save(state, unet_path[:-4] + 'interrupted' + '.pkl')
optimizer = torch.optim.SGD(model.parameters(), 0.1, momentum=0.9, weight_decay=1e-4) # optimiser = optim.RMSprop(model.parameters(), alpha=0.9, lr=0.0001, weight_decay=1e-6) loss_function = nn.CrossEntropyLoss() # device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimizer, loss_function, metrics=['loss', 'acc', 'top_5_acc'], callbacks=[ callbacks.TensorBoard(write_graph=False, comment=f'resnet50_{n_bn}_{rep}'), callbacks.MultiStepLR([30, 60]), callbacks.MostRecent(dir + model_file + '_{epoch:02d}.pt') ]).to('cuda') trial.with_generators(trainloader, test_generator=testloader) pattern = re.compile(model_file + '_\d+.pt') for filepath in os.listdir(dir): if pattern.match(filepath): trial.load_state_dict(torch.load(dir + filepath)) trial.run(epochs=90) trial.evaluate(data_key=torchbearer.TEST_DATA) torch.save(model.module.state_dict(), dir + model_file + '.pt')
for key, value in regime.items(): if key <= epoch: lr = value else: break print("set lr to : ", lr) return lr scheduler = LambdaLR(lr_lambda=[lr_func]) @on_step_training def clip(serial): alexnetBin.clamp() trial = Trial(alexnetBin, optimizer, criterion=loss, metrics=['acc', 'loss'], callbacks=[clip, scheduler]).to('cuda') trial = trial.with_generators(train_generator=train_set, val_generator=valid_set, test_generator=test_set) trial.run(epochs=EPOCH_NB) trial.evaluate(data_key=TEST_DATA) alexnetBin.train()
testloader = DataLoader(testset, batch_size=128, shuffle=True) valloader = DataLoader(valset, batch_size=128, shuffle=True) # build the model model = SimpleCNN() # define the loss function and the optimiser loss_function = nn.CrossEntropyLoss() live_loss_plot = LiveLossPlot(draw_once=True) optimiser = optim.Adam(model.parameters(), lr=0.001) scheduler = StepLR(step_size=10, gamma=0.5) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, callbacks=[scheduler, live_loss_plot], metrics=['loss', 'accuracy']).to(device) trial.with_generators(trainloader, val_generator=valloader, test_generator=testloader) history = trial.run(verbose=1, epochs=30)# results = trial.evaluate(data_key=torchbearer.TEST_DATA) print(results) # build the model model = SimpleCNN() # define the loss function and the optimiser loss_function = nn.CrossEntropyLoss() live_loss_plot = LiveLossPlot(draw_once=True) optimiser = optim.Adam(model.parameters(), lr=0.001) scheduler = StepLR(step_size=10, gamma=0.5) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, callbacks=[scheduler, live_loss_plot], metrics=['loss', 'accuracy']).to(device)
super().__init__('roc_auc_score', lambda y_pred, y_true: sklearn.metrics.roc_auc_score(y_true.cpu().numpy(), y_pred.detach().sigmoid().cpu().numpy()), running=False) metrics_append = [RocAucScore()] if args.dataset == 'imagenet_a': from datasets.imagenet_a import indices_in_1k @on_forward_validation def map_preds(state): state[torchbearer.PREDICTION] = state[torchbearer.PREDICTION][:, indices_in_1k] cb.append(map_preds) print('==> Training model..') acc = 'acc' if args.dataset in ['toxic', 'toxic_bert', 'imdb', 'yelp_2']: acc = 'binary_acc' trial = Trial(net, optimizer, criterion, metrics=[acc, 'loss', 'lr'] + metrics_append, callbacks=cb) trial.with_generators(train_generator=trainloader, val_generator=valloader, train_steps=args.train_steps, test_generator=testloader).to(args.device) if args.reload: state = torch.load(args.model_file) trial.load_state_dict(state, resume=args.dataset != 'imagenet_a') trial.replay() if trainloader is not None: trial.run(args.epoch, verbose=args.verbose) trial.evaluate(data_key=torchbearer.TEST_DATA)
model = torch.load('fashionMNIST_VAE_20epochs.pytorchweights') model_parameters = filter(lambda p: p.requires_grad, model.parameters()) print(sum([np.prod(p.size()) for p in model_parameters])) optimizer = optim.Adam( filter(lambda p: p.requires_grad, model.parameters()), lr=5e-4) trial = Trial(model, optimizer, nn.MSELoss(reduction='sum'), metrics=['acc', 'loss'], callbacks=[ beta_kl(MU, LOGVAR), callbacks.ConsolePrinter(), plot_progress() ], verbose=1).with_generators(train_generator=traingen, test_generator=testgen) history = trial.run(10) trial.evaluate(verbose=0, data_key=torchbearer.TEST_DATA) torch.save(model, 'fashionMNIST_VAE_30epochs.pytorchweights') # torch.save(model, 'fashionMNIST_VAE_10epochs.pytorchweights') print(history) train_loss_list = [] val_loss_list = [] for item in history: train_loss_list.append(item['running_loss']) val_loss_list.append(item['test_loss']) fig1 = plt.figure('1') ax1 = fig1.gca() ax1.plot(train_loss_list, color='blue', alpha=0.7) ax1.plot(val_loss_list, color='red', alpha=0.7)
plt.imshow(out.detach().numpy()[0, 0, :, :]) plt.show() out = F.relu(out) out = self.conv2(out) out = F.relu(out) out = self.gmp1(out) out = out.view(out.shape[0], -1) out = self.fc1(out) out = F.relu(out) out = self.fc2(out) return out model = BetterCNN() loss_function = nn.L1Loss() optimiser = optim.Adam(model.parameters()) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, metrics=['loss', 'accuracy']).to(device) trial.with_generators(train_dl, valid_dl, test_generator=test_dl) history = trial.run(epochs=1) results = trial.evaluate(data_key=tb.TEST_DATA) print(results) loss_list = [] for item in history: loss_list.append(item['loss']) fig1 = plt.figure('1') ax1 = fig1.gca() ax1.plot(loss_list) plt.show()
if key <= epoch: lr = value else: break print("set lr to : ", lr) return lr scheduler = LambdaLR(lr_lambda=[lr_func]) @on_step_training def clip(serial): alexnetBin.clip() trial = Trial(alexnetBin, optimizer, criterion=loss, metrics=['acc', 'loss'], callbacks=[clip, scheduler]).to('cpu') trial = trial.with_generators(train_generator=train_set, val_generator=valid_set, test_generator=test_set) trial.run(epochs=150) trial.evaluate(data_key=TEST_DATA) alexnetBin.train() torch.save(alexnetBin.state_dict(), open("./models/cifar10/alexnetBin.pth", "wb"))
# state_dict.pop('fc1.weight') # state_dict.pop('fc1.bias') state_dict.pop('fc2.weight') state_dict.pop('fc2.bias') # state_dict['fc3'] = None model.load_state_dict(state_dict, strict=False) to_deactivate = ['conv1a', 'conv1b', 'conv1c', 'conv2a', 'conv2b', 'conv2c', 'conv3a', 'conv3b', 'conv3c'] for l in to_deactivate: deactivate_layer(model, l) ''' loss_function = nn.CrossEntropyLoss() optimiser = optim.Adam(model.parameters()) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, metrics=['loss', 'accuracy']).to(device) trial.with_generators(train_loader, test_generator=validation_loader) trial.run(epochs=10) results = trial.evaluate(data_key=torchbearer.TEST_DATA) print(results) model_name = 'resnet_model_3b' weights_path = './weights/{}/'.format(model_name) if not os.path.exists(weights_path): os.makedirs(weights_path) w_filepath = './weights/{}/train_{}.weights'.format(model_name, p) torch.save(model.state_dict(), w_filepath)
for d_vvs in ventral_depths: for t in range(n_trials): model_file = f'./models/{cmode}/model_{n_bn}_{d_vvs}_{t}.pt' log_file = f'./logs/{cmode}/model_{n_bn}_{d_vvs}_{t}.csv' pathlib.Path(model_file).parents[0].mkdir(parents=True, exist_ok=True) pathlib.Path(log_file).parents[0].mkdir(parents=True, exist_ok=True) model = BaselineModel(n_bn, d_vvs, nch) optimiser = optim.RMSprop(model.parameters(), alpha=0.9, lr=0.0001, weight_decay=1e-6) loss_function = nn.CrossEntropyLoss() device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial( model, optimiser, loss_function, metrics=['loss', 'accuracy'], callbacks=[ torchbearer.callbacks.csv_logger.CSVLogger(log_file) ]).to(device) trial.with_generators(trainloader, val_generator=testloader) trial.run(epochs=20) torch.save(model.state_dict(), model_file)
# ######### Data print('==> Preparing data..') classes, cifar = 10, datasets.CIFAR10 normalize = transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) transform_base = [transforms.ToTensor(), normalize] transform = [transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip()] + transform_base transform_train = transforms.Compose(transform) transform_test = transforms.Compose(transform_base) trainset = cifar(root='./data', train=True, download=True, transform=transform_train) valset = cifar(root='./data', train=False, download=True, transform=transform_test) trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=8) valloader = torch.utils.data.DataLoader(valset, batch_size=128, shuffle=True, num_workers=8) ######### Model print('==> Building model..') net = models.resnet18(False) optimizer = optim.SGD(net.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4) fmix = FMix() criterion = fmix.loss() ######### Trial print('==> Starting training..') trial = Trial(net, optimizer, criterion, metrics=['acc', 'loss'], callbacks=[fmix]) trial.with_generators(train_generator=trainloader, val_generator=valloader).to('cuda') trial.run(100, verbose=2)