def evaluate(file, device='cuda'): transform_test = transforms.Compose([transforms.ToTensor()]) testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=10) base_dir = 'cifarss_base' model = CifarVAE() model = SelfTaught(model.encoder, model.mu, 32) optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=0) trial = Trial( model, optimizer, nn.NLLLoss(), ['acc', 'loss']).load_state_dict( torch.load(os.path.join(base_dir, file)), resume=False).with_generators(val_generator=testloader).to(device) return trial.evaluate()
def test_model(model: nn.Module, test_loader: DataLoader): loss_function = nn.MSELoss() device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, None, loss_function, metrics=["loss"]).to(device) trial.with_generators(train_loader, test_generator=test_loader) results = trial.evaluate(data_key=torchbearer.TEST_DATA) return results, trial
def train_model(train, valid, test, classes, batch, num_epochs): # model = CNN(3, classes) model = models.resnet18() train_loader = DataLoader(train, batch_size=batch) val_loader = DataLoader(valid, batch_size=batch) test_loader = DataLoader(test, batch_size=batch) # define the loss function and the optimiser loss_function = nn.CrossEntropyLoss() # equation 8 in original paper optimiser = optim.Adam(model.parameters()) trial = Trial(model, optimiser, loss_function, metrics=['loss', 'accuracy'], verbose=1).to(device) trial.with_generators(train_loader, val_generator=val_loader, test_generator=test_loader) trial.run(epochs=num_epochs) results_test = trial.evaluate(data_key=torchbearer.TEST_DATA) results_val = trial.evaluate(data_key=torchbearer.VALIDATION_DATA) print(results_test) return results_test, results_val
def train(self, model, **kwargs) -> (float, float): # Get transfer model and put it in training mode net = model.net net.train() # Create optimiser optimiser = optim.Adam(net.parameters(), lr=1e-4) # Check for cuda device = "cuda:0" if torch.cuda.is_available() else "cpu" print("Training using", device) # Setup loss function if self.class_weight_method != ClassWeightMethod.Unweighted: distribution = class_distribution("data/processed/train") if self.class_weight_method == ClassWeightMethod.SumBased: inv_distribution = [ 1 - x / np.sum(distribution) for x in distribution ] inv_distribution = torch.from_numpy( np.array(inv_distribution)).float() elif self.class_weight_method == ClassWeightMethod.MaxBased: inv_distribution = [ np.max(distribution) / x for x in distribution ] inv_distribution = torch.from_numpy( np.array(inv_distribution)).float() else: raise IndexError("Unknown class weight method " + str(self.class_weight_method)) loss_function = self.loss(inv_distribution.to(device)) else: loss_function = self.loss() # Setup trial trial = Trial(net, optimiser, loss_function, metrics=["loss", "accuracy"]).to(device) trial.with_generators( self.image_datasets.train_loader, test_generator=self.image_datasets.validation_loader, ) # Actually run the training trial.run(epochs=self.num_epochs) # Evaluate and show results time.sleep(0.1) # Ensure training has finished net.eval() results = trial.evaluate(data_key=torchbearer.TEST_DATA) acc = float(results["test_acc"]) loss = float(results["test_loss"]) return acc, loss
def run(train_batch_size, val_batch_size, epochs, lr, log_interval, input_size=10, hidden_size=100, out_size=4): dataset = FuzzBuzzDataset(input_size) splitter = DatasetValidationSplitter(len(dataset), 0.1) train_set = splitter.get_train_dataset(dataset) val_set = splitter.get_val_dataset(dataset) train_loader = DataLoader(train_set, pin_memory=True, batch_size=train_batch_size, shuffle=True, num_workers=2) val_loader = DataLoader(val_set, pin_memory=True, batch_size=val_batch_size, shuffle=False, num_workers=2) model = FuzzBuzzModel(input_size, hidden_size, out_size) device = 'cpu' if torch.cuda.is_available(): device = 'cuda' optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=lr) loss = nn.CrossEntropyLoss() trial = Trial(model, optimizer, criterion=loss, metrics=['acc', 'loss']).to(device) trial = trial.with_generators(train_generator=train_loader, val_generator=val_loader) trial.run(epochs=epochs) trial.evaluate(data_key=VALIDATION_DATA)
def train(self): scheduler = torch_scheduler.StepLR(self.num_epochs_decay, gamma=self.decay_factor) loss_plot_plan = os.path.join( self.result_path, 'live_loss_plot%s-%d-%.4f-%d-%.4f.png' % (self.model_type, self.num_epochs, self.lr, self.num_epochs_decay, self.augmentation_prob)) callbacks = [scheduler] # imaging.FromState(torchbearer.X).on_val().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_TRUE).on_val().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_PRED).on_val().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.X).on_test().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_TRUE).on_test().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_PRED).on_test().cache(16).make_grid().to_pyplot(), # TensorBoard(write_batch_metrics=True), trial = Trial( self.unet, self.optimizer, self.criterion, metrics=['loss', 'binary_acc'], # binary_acc for debugging certain things callbacks=callbacks).to(self.device) trial.with_generators(train_generator=self.train_loader, val_generator=self.valid_loader, test_generator=self.test_loader) start = time.time() history = trial.run(epochs=self.num_epochs, verbose=2) stop = time.time() train_time = stop - start state = self.unet.state_dict() unet_path = os.path.join( self.model_path, '%s-%d-%.4f-%d-%.4f_Index_BCE_Dropout_STAREIndex.pkl' % ( self.model_type, self.num_epochs, self.lr, self.num_epochs_decay, self.augmentation_prob, )) torch.save(state, unet_path) print(history) ### Testing results = trial.evaluate(data_key=torchbearer.TEST_DATA) print("Test result:") print(results) return history, results
def train(self): """Train encoder, generator and discriminator.""" # D_LOSS = state_key('d_loss') # # def dice_loss(prediction, target): # """Calculating the dice loss # Args: # prediction = predicted image # target = Targeted image # Output: # dice_loss""" # # smooth = 1.0 # # print(prediction.shape) # # print(target.shape) # # TODO: Used reshape() instead of view because of batch size > 1 # i_flat = prediction.reshape(-1) # t_flat = target.reshape(-1) # # intersection = (i_flat * t_flat).sum() # # return 1 - ((2. * intersection + smooth) / (i_flat.sum() + t_flat.sum() + smooth)) # # def calc_loss(prediction, target, bce_weight=0.5): # """Calculating the loss and metrics # Args: # prediction = predicted image # target = Targeted image # metrics = Metrics printed # bce_weight = 0.5 (default) # Output: # loss : dice loss of the epoch """ # bce = F.binary_cross_entropy_with_logits(prediction, target) # prediction = F.sigmoid(prediction) # dice = dice_loss(prediction, target) # # loss = bce * bce_weight + dice * (1 - bce_weight) # state[D_LOSS] = loss # # return loss # # stopping = EarlyStopping(monitor='val_acc', patience=5, mode='max') scheduler = torch_scheduler.StepLR(self.num_epochs_decay, gamma=0.1) loss_plot_plan = os.path.join(self.result_path, 'live_loss_plot%s-%d-%.4f-%d-%.4f.png' % (self.model_type, self.num_epochs, self.lr, self.num_epochs_decay, self.augmentation_prob)) callbacks = [scheduler] # imaging.FromState(torchbearer.X).on_val().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_TRUE).on_val().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_PRED).on_val().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.X).on_test().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_TRUE).on_test().cache(16).make_grid().to_pyplot(), # imaging.FromState(torchbearer.Y_PRED).on_test().cache(16).make_grid().to_pyplot(), # TensorBoard(write_batch_metrics=True), try: trial = Trial(self.unet, self.optimizer, self.criterion, metrics=['loss', 'binary_acc'], # binary_acc for debugging certain things callbacks=callbacks).to(self.device) trial.with_generators(train_generator=self.train_loader, val_generator=self.valid_loader, test_generator=self.test_loader) start = time.time() history = trial.run(epochs=self.num_epochs, verbose=2) stop = time.time() train_time = stop - start state = self.unet.state_dict() unet_path = os.path.join(self.model_path, '%s-%d-%.4f-%d-%.4f_Index_BCE_Dropout_STAREIndex.pkl' % (self.model_type, self.num_epochs, self.lr, self.num_epochs_decay, self.augmentation_prob,)) torch.save(state, unet_path) print(history) ### Testing results = trial.evaluate(data_key=torchbearer.TEST_DATA) print("Test result:") print(results) except (RuntimeError, OSError): state = self.unet.state_dict() unet_path = os.path.join(self.model_path, '%s-%d-%.4f-%d-%.4f_Index_BCE_Dropout_STAREIndex.pkl' % (self.model_type, self.num_epochs, self.lr, self.num_epochs_decay, self.augmentation_prob,)) torch.save(state, unet_path[:-4] + 'interrupted' + '.pkl')
# build the model model = SimpleCNN() # define the loss function and the optimiser loss_function = nn.CrossEntropyLoss() live_loss_plot = LiveLossPlot(draw_once=True) optimiser = optim.Adam(model.parameters(), lr=0.001) scheduler = StepLR(step_size=10, gamma=0.5) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, callbacks=[scheduler, live_loss_plot], metrics=['loss', 'accuracy']).to(device) trial.with_generators(trainloader, val_generator=valloader, test_generator=testloader) history = trial.run(verbose=1, epochs=30)# results = trial.evaluate(data_key=torchbearer.TEST_DATA) print(results) # build the model model = SimpleCNN() # define the loss function and the optimiser loss_function = nn.CrossEntropyLoss() live_loss_plot = LiveLossPlot(draw_once=True) optimiser = optim.Adam(model.parameters(), lr=0.001) scheduler = StepLR(step_size=10, gamma=0.5) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, callbacks=[scheduler, live_loss_plot], metrics=['loss', 'accuracy']).to(device) trial.with_generators(trainloader, test_generator=testloader) history = trial.run(verbose=1, epochs=30)#
for key, value in regime.items(): if key <= epoch: lr = value else: break print("set lr to : ", lr) return lr scheduler = LambdaLR(lr_lambda=[lr_func]) @on_step_training def clip(serial): alexnetBin.clamp() trial = Trial(alexnetBin, optimizer, criterion=loss, metrics=['acc', 'loss'], callbacks=[clip, scheduler]).to('cuda') trial = trial.with_generators(train_generator=train_set, val_generator=valid_set, test_generator=test_set) trial.run(epochs=EPOCH_NB) trial.evaluate(data_key=TEST_DATA) alexnetBin.train()
nn.BatchNorm2d(16), nn.ReLU(), nn.Conv2d(16, 32, stride=2, kernel_size=3), nn.BatchNorm2d(32), nn.ReLU(), nn.Conv2d(32, 64, stride=2, kernel_size=3), nn.BatchNorm2d(64), nn.ReLU()) self.classifier = nn.Linear(576, 10) def forward(self, x): x = self.convs(x) x = x.view(-1, 576) return self.classifier(x) model = SimpleModel() optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001) loss = nn.CrossEntropyLoss() from torchbearer import Trial torchbearer_trial = Trial(model, optimizer, loss, metrics=['acc', 'loss']).to('cuda') torchbearer_trial.with_generators(train_generator=traingen, val_generator=valgen) torchbearer_trial.run(epochs=10) torchbearer_trial.with_val_generator(testgen) torchbearer_trial.evaluate()
plt.imshow(out.detach().numpy()[0, 0, :, :]) plt.show() out = F.relu(out) out = self.conv2(out) out = F.relu(out) out = self.gmp1(out) out = out.view(out.shape[0], -1) out = self.fc1(out) out = F.relu(out) out = self.fc2(out) return out model = BetterCNN() loss_function = nn.L1Loss() optimiser = optim.Adam(model.parameters()) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, metrics=['loss', 'accuracy']).to(device) trial.with_generators(train_dl, valid_dl, test_generator=test_dl) history = trial.run(epochs=1) results = trial.evaluate(data_key=tb.TEST_DATA) print(results) loss_list = [] for item in history: loss_list.append(item['loss']) fig1 = plt.figure('1') ax1 = fig1.gca() ax1.plot(loss_list) plt.show()
model = torch.load('fashionMNIST_VAE_20epochs.pytorchweights') model_parameters = filter(lambda p: p.requires_grad, model.parameters()) print(sum([np.prod(p.size()) for p in model_parameters])) optimizer = optim.Adam( filter(lambda p: p.requires_grad, model.parameters()), lr=5e-4) trial = Trial(model, optimizer, nn.MSELoss(reduction='sum'), metrics=['acc', 'loss'], callbacks=[ beta_kl(MU, LOGVAR), callbacks.ConsolePrinter(), plot_progress() ], verbose=1).with_generators(train_generator=traingen, test_generator=testgen) history = trial.run(10) trial.evaluate(verbose=0, data_key=torchbearer.TEST_DATA) torch.save(model, 'fashionMNIST_VAE_30epochs.pytorchweights') # torch.save(model, 'fashionMNIST_VAE_10epochs.pytorchweights') print(history) train_loss_list = [] val_loss_list = [] for item in history: train_loss_list.append(item['running_loss']) val_loss_list.append(item['test_loss']) fig1 = plt.figure('1') ax1 = fig1.gca() ax1.plot(train_loss_list, color='blue', alpha=0.7) ax1.plot(val_loss_list, color='red', alpha=0.7) plt.legend()
model = torch.load( 'data/boat/trainlab_6_boat_resnet50_100epochs.pytorchweights') # model = BetterCNN(3, len(train_dataset.classes)) # define the loss function and the optimiser loss_function = nn.CrossEntropyLoss() optimiser = optim.Adam(model.parameters()) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, metrics=['loss', 'accuracy']).to(device) trial.with_generators(train_loader, val_generator=val_loader, test_generator=test_loader) # history = trial.run(epochs=100) results = trial.evaluate(data_key=torchbearer.VALIDATION_DATA) print(results) # torch.save(model, path+'lab_6_boat_cnn_100epochs.pytorchweights') """ loss_list = [] for item in history: loss_list.append(item['loss']) fig1 = plt.figure('1') ax1 = fig1.gca() ax1.plot(loss_list) plt.show() """ predictions = trial.predict() predicted_classes = predictions.argmax(1).cpu() true_classes = list(x for (_, x) in test_dataset.samples)
def train(): # train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) print('start training') dataset_size = len(dataset) indices = list(range(dataset_size)) split = int(np.floor(validation_split * dataset_size)) if shuffle_dataset: np.random.seed(random_seed) np.random.shuffle(indices) train_indices, val_indices = indices[split:], indices[:split] # Creating PT data samplers and loaders: train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler) val_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler) model = resnet50(pretrained=True) model.avgpool = nn.AdaptiveAvgPool2d((1, 1)) model.fc = nn.Linear(2048, len(dataset.classes)) model.train() # Freeze layers by not tracking gradients for param in model.parameters(): param.requires_grad = False model.fc.weight.requires_grad = True #unfreeze last layer weights model.fc.bias.requires_grad = True #unfreeze last layer biases optimiser1 = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3) #only optimse non-frozen layers loss_function = nn.CrossEntropyLoss() device = "cuda:0" if torch.cuda.is_available() else "cpu" print(device) checkpointer = torchbearer.callbacks.checkpointers.Best( filepath='model.pt', monitor='loss') trial = Trial(model, optimiser1, loss_function, metrics=['loss', 'accuracy'], callbacks=[checkpointer]).to(device) trial.with_generators(train_loader, val_generator=val_loader) trial.run(epochs=10) state_dict = torch.load('model.pt') trial_reloaded = Trial(model, optimiser1, loss_function, metrics=['loss', 'accuracy'], callbacks=[checkpointer]).to(device) trial_reloaded.with_generators(train_loader, val_generator=val_loader) trial_reloaded.load_state_dict(state_dict) trial_reloaded.run(epochs=20) results = trial_reloaded.evaluate(data_key=torchbearer.VALIDATION_DATA) print() print(results) torch.save(model, 'mymodel.pth')
optimizer = torch.optim.SGD(model.parameters(), 0.1, momentum=0.9, weight_decay=1e-4) # optimiser = optim.RMSprop(model.parameters(), alpha=0.9, lr=0.0001, weight_decay=1e-6) loss_function = nn.CrossEntropyLoss() # device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimizer, loss_function, metrics=['loss', 'acc', 'top_5_acc'], callbacks=[ callbacks.TensorBoard(write_graph=False, comment=f'resnet50_{n_bn}_{rep}'), callbacks.MultiStepLR([30, 60]), callbacks.MostRecent(dir + model_file + '_{epoch:02d}.pt') ]).to('cuda') trial.with_generators(trainloader, test_generator=testloader) pattern = re.compile(model_file + '_\d+.pt') for filepath in os.listdir(dir): if pattern.match(filepath): trial.load_state_dict(torch.load(dir + filepath)) trial.run(epochs=90) trial.evaluate(data_key=torchbearer.TEST_DATA) torch.save(model.module.state_dict(), dir + model_file + '.pt')
out = self.fc1(out) out = F.relu(out) out = self.fc2(out) return out #reset the data loaders trainloader = DataLoader(train_data, batch_size=128, shuffle=True) testloader = DataLoader(test_data, batch_size=128, shuffle=True) valloader = DataLoader(val_data, batch_size=128, shuffle=True) # build the model model = SimpleCNN() # define the loss function and the optimiser loss_function = nn.MSELoss() optimiser = optim.Adam(model.parameters()) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, metrics=['loss', 'accuracy'], callbacks=[LiveLossPlot()]).to(device) trial.with_generators(trainloader, valloader, test_generator=testloader) trial.run(epochs=100) results_train = trial.evaluate(data_key=torchbearer.TRAIN_DATA) results_test = trial.evaluate(data_key=torchbearer.TEST_DATA) print(results_train) print(results_test)
class Model(nn.Module): def __init__(self): super(Model, self).__init__() self.net = torchvision.models.googlenet(True) def forward(self, input): if input is not None: return self.net(input) model = Model() from torchbearer import Trial import torchbearer.callbacks.imaging as imaging trial = Trial(model, callbacks=[ imaging.ClassAppearanceModel( 1000, (3, 224, 224), steps=10000, target=951, transform=inv_normalize).on_val().to_file('lemon.png'), imaging.ClassAppearanceModel( 1000, (3, 224, 224), steps=10000, target=968, transform=inv_normalize).on_val().to_file('cup.png') ]) trial.for_val_steps(1).to('cuda') trial.evaluate()