# state_dict.pop('fc1.weight') # state_dict.pop('fc1.bias') state_dict.pop('fc2.weight') state_dict.pop('fc2.bias') # state_dict['fc3'] = None model.load_state_dict(state_dict, strict=False) to_deactivate = ['conv1a', 'conv1b', 'conv1c', 'conv2a', 'conv2b', 'conv2c', 'conv3a', 'conv3b', 'conv3c'] for l in to_deactivate: deactivate_layer(model, l) ''' loss_function = nn.CrossEntropyLoss() optimiser = optim.Adam(model.parameters()) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, metrics=['loss', 'accuracy']).to(device) trial.with_generators(train_loader, test_generator=validation_loader) trial.run(epochs=10) results = trial.evaluate(data_key=torchbearer.TEST_DATA) print(results) model_name = 'resnet_model_3b' weights_path = './weights/{}/'.format(model_name) if not os.path.exists(weights_path): os.makedirs(weights_path) w_filepath = './weights/{}/train_{}.weights'.format(model_name, p) torch.save(model.state_dict(), w_filepath)
# for d_vvs in ventral_depths: # for t in range(n_trials): model_file = f'./models/{cmode}-mos/model_{n_bn}_{d_vvs}_{t}.pt' log_file = f'./logs/{cmode}-mos/model_{n_bn}_{d_vvs}_{t}.csv' pathlib.Path(model_file).parents[0].mkdir(parents=True, exist_ok=True) pathlib.Path(log_file).parents[0].mkdir(parents=True, exist_ok=True) model = BaselineModel(n_bn, d_vvs, nch) optimiser = optim.RMSprop(model.parameters(), alpha=0.9, lr=0.0001, weight_decay=1e-6) loss_function = nn.CrossEntropyLoss() device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial( model, optimiser, loss_function, metrics=['loss', 'accuracy'], callbacks=[ # torchbearer.callbacks.imaging.MakeGrid().on_train().to_file('sample.png'), torchbearer.callbacks.csv_logger.CSVLogger(log_file) ]).to(device) trial.with_generators(trainloader, val_generator=testloader) trial.run(epochs=20) torch.save(model.conv_dict(), model_file)
if key <= epoch: lr = value else: break print("set lr to : ", lr) return lr scheduler = LambdaLR(lr_lambda=[lr_func]) @on_step_training def clip(serial): alexnetBin.clip() trial = Trial(alexnetBin, optimizer, criterion=loss, metrics=['acc', 'loss'], callbacks=[clip, scheduler]).to('cpu') trial = trial.with_generators(train_generator=train_set, val_generator=valid_set, test_generator=test_set) trial.run(epochs=150) trial.evaluate(data_key=TEST_DATA) alexnetBin.train() torch.save(alexnetBin.state_dict(), open("./models/cifar10/alexnetBin.pth", "wb"))
def kld_Loss(mu, logvar): KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp()) return KLD # Reconstruction + KL divergence losses summed over all elements and batch def loss_function(y_pred, y_true): recon_x, mu, logvar = y_pred x = y_true BCE = bce_loss(recon_x, x) KLD = kld_Loss(mu, logvar) return BCE + KLD model = VAE() optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001) loss = loss_function from torchbearer import Trial torchbearer_trial = Trial(model, optimizer, loss, metrics=['loss']).to('cuda') torchbearer_trial.with_generators(train_generator=traingen, val_generator=valgen) torchbearer_trial.run(epochs=10)
cb.append(WarmupLR(0.1, args.lr)) if args.lr_warmup else [] cb.append(cutmix_reformat) if args.msda_mode == 'cutmix' else [] # FMix loss is equivalent to mixup loss and works for all msda in torchbearer if args.msda_mode not in [None, 'None']: bce = True if args.dataset == 'toxic' else False criterion = modes['fmix'].loss(bce) elif args.dataset == 'toxic': criterion = nn.BCEWithLogitsLoss() else: criterion = nn.CrossEntropyLoss() # from torchbearer.metrics.roc_auc_score import RocAucScore print('==> Training model..') trial = Trial(net, optimizer, criterion, metrics=['acc', 'loss', 'lr'], callbacks=cb) trial.with_generators(train_generator=trainloader, val_generator=valloader, train_steps=args.train_steps, test_generator=testloader).to(args.device) if args.reload: state = torch.load(args.model_file) trial.load_state_dict(state) trial.replay() trial.run(args.epoch, verbose=args.verbose) trial.evaluate(data_key=torchbearer.TEST_DATA)
plt.imshow(out.detach().numpy()[0, 0, :, :]) plt.show() out = F.relu(out) out = self.conv2(out) out = F.relu(out) out = self.gmp1(out) out = out.view(out.shape[0], -1) out = self.fc1(out) out = F.relu(out) out = self.fc2(out) return out model = BetterCNN() loss_function = nn.L1Loss() optimiser = optim.Adam(model.parameters()) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, metrics=['loss', 'accuracy']).to(device) trial.with_generators(train_dl, valid_dl, test_generator=test_dl) history = trial.run(epochs=1) results = trial.evaluate(data_key=tb.TEST_DATA) print(results) loss_list = [] for item in history: loss_list.append(item['loss']) fig1 = plt.figure('1') ax1 = fig1.gca() ax1.plot(loss_list) plt.show()
[transforms.ToTensor()])) test_gen = torch.utils.data.DataLoader(test_set, pin_memory=True, batch_size=128, shuffle=False, num_workers=10) comment = 'foveated5' trial = Trial( model, torch.optim.Adam(model.parameters(), lr=0.0001), nn.CrossEntropyLoss(), metrics=['acc', 'loss'], callbacks=[ callbacks.MostRecent(comment + '.{epoch:02d}.pt'), imaging.MakeGrid( key=TRANSFORMED, num_images=16, nrow=16, pad_value=1).to_tensorboard( name='transformed', comment=comment).on_val().to_file(comment + '_transformed.png'), imaging.MakeGrid( key=torchbearer.INPUT, num_images=16, nrow=16, pad_value=1).to_tensorboard( name='input', comment=comment).on_val().to_file(comment + '_inputs.png') ]) trial = trial.with_generators(train_gen, test_gen).to('cuda') trial.run(50)
print(loss_list) print(results) """ model = torch.load( 'data/boat/trainlab_6_boat_resnet50_100epochs.pytorchweights') # model = BetterCNN(3, len(train_dataset.classes)) # define the loss function and the optimiser loss_function = nn.CrossEntropyLoss() optimiser = optim.Adam(model.parameters()) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, metrics=['loss', 'accuracy']).to(device) trial.with_generators(train_loader, val_generator=val_loader, test_generator=test_loader) # history = trial.run(epochs=100) results = trial.evaluate(data_key=torchbearer.VALIDATION_DATA) print(results) # torch.save(model, path+'lab_6_boat_cnn_100epochs.pytorchweights') """ loss_list = [] for item in history: loss_list.append(item['loss']) fig1 = plt.figure('1') ax1 = fig1.gca() ax1.plot(loss_list) plt.show() """ predictions = trial.predict()
shuffle=True, num_workers=8) model = MimeVGG(vgg, {k: cfgs['A'][k]() for k in [layer]}) optimizer = optim.Adam(filter(lambda x: x.requires_grad, model.parameters()), lr=5e-4) mi_false = mi(False) @callbacks.add_to_loss def mi_no_tanh(state): state[OTHER_MI] = mi_false(state) return 0 trial = Trial(model, optimizer, mi(True), metrics=['loss', torchbearer.metrics.mean(OTHER_MI)], callbacks=[ mi_no_tanh, callbacks.TensorBoard(write_graph=False, comment='mi_' + args.model, log_dir='mi_data') ]) trial.with_generators(train_generator=trainloader, val_generator=valloader).to('cuda') trial.run(20, verbose=1)
def train(): # train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) print('start training') dataset_size = len(dataset) indices = list(range(dataset_size)) split = int(np.floor(validation_split * dataset_size)) if shuffle_dataset: np.random.seed(random_seed) np.random.shuffle(indices) train_indices, val_indices = indices[split:], indices[:split] # Creating PT data samplers and loaders: train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler) val_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler) model = resnet50(pretrained=True) model.avgpool = nn.AdaptiveAvgPool2d((1, 1)) model.fc = nn.Linear(2048, len(dataset.classes)) model.train() # Freeze layers by not tracking gradients for param in model.parameters(): param.requires_grad = False model.fc.weight.requires_grad = True #unfreeze last layer weights model.fc.bias.requires_grad = True #unfreeze last layer biases optimiser1 = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3) #only optimse non-frozen layers loss_function = nn.CrossEntropyLoss() device = "cuda:0" if torch.cuda.is_available() else "cpu" print(device) checkpointer = torchbearer.callbacks.checkpointers.Best( filepath='model.pt', monitor='loss') trial = Trial(model, optimiser1, loss_function, metrics=['loss', 'accuracy'], callbacks=[checkpointer]).to(device) trial.with_generators(train_loader, val_generator=val_loader) trial.run(epochs=10) state_dict = torch.load('model.pt') trial_reloaded = Trial(model, optimiser1, loss_function, metrics=['loss', 'accuracy'], callbacks=[checkpointer]).to(device) trial_reloaded.with_generators(train_loader, val_generator=val_loader) trial_reloaded.load_state_dict(state_dict) trial_reloaded.run(epochs=20) results = trial_reloaded.evaluate(data_key=torchbearer.VALIDATION_DATA) print() print(results) torch.save(model, 'mymodel.pth')
class eval_model(nn.Module): def __init__(self, model, loss_f, device, metrics, trial_name, opt=optim.Adam,opt_params={}, **kwargs): super().__init__() self.model = model self.loss_f = loss_f self.opt = opt(list(self.model.parameters()), **opt_params) self.metrics = metrics self.device = device self.trial_path = os.path.join('logs/', trial_name) self.call_backs = None self.best_step = 0 def init_trial(self): #initial call_back functions for the trail. if self.call_backs: self.trial = Trial(self.model,self.opt,self.loss_f,metrics=self.metrics,callbacks=self.call_backs).to(self.device) else: self.trial = Trial(self.model,self.opt,self.loss_f,metrics=self.metrics).to(self.device) def run(self, epoch, tr_loader,val_loader,t_loader=None,val_steps=None): self.trial.with_generators(tr_loader, test_generator=t_loader,val_generator=val_loader,val_steps=val_steps) return self.trial.run(epoch) def save_model(self, suffix, path=None, mkdir=True, **kwargs): if path is None: path = os.path.join(self.trial_path, "model_{}.mf".format(suffix)) if mkdir: os.makedirs(os.path.dirname(path), exist_ok=True) torch.save(OrderedDict([ ('model', self.model.state_dict(**kwargs)), ('trial', self.trial.state_dict()), ('opt', self.opt.state_dict()), ('best_step', self.best_step), ]), path) return path def load_model(self, suffix, path=None, **kwargs): if path is None: path = os.path.join(self.trial_path, "model_{}.mf".format(suffix)) checkpoint = torch.load(path) self.trial.load_state_dict(checkpoint['trial']) self.model.load_state_dict(checkpoint['model'], **kwargs) self.opt.load_state_dict(checkpoint['opt']) self.best_step = int(checkpoint['best_step']) def plot_loss(self, loss, title, xlabel, x_all=False): plt.figure() ax = plt.axes() ax.set_title(title) ax.set_ylabel("loss") ax.set_xlabel(xlabel) x = range(len(loss)) plt.plot(x,loss) if x_all: ax.xaxis.set_major_locator(MaxNLocator(integer=True)) plt.xticks(x) plt.legend() plt.show() def plot_fig(self,out,epoch,title,y_label, para=['loss','val_loss']): tem = [ [0]*epoch for i in range(len(para))] for idx, label in enumerate(para): for i in range(epoch): tem[idx][i] = out[i][label] plt.figure() ax = plt.axes() ax.set_title(title) ax.set_ylabel(y_label) ax.set_xlabel("Epoch") for i, item in enumerate(tem): plt.plot(range(epoch),item, label=para[i]) plt.legend() plt.show()