# state_dict.pop('fc1.weight')
# state_dict.pop('fc1.bias')
state_dict.pop('fc2.weight')
state_dict.pop('fc2.bias')
# state_dict['fc3'] = None
model.load_state_dict(state_dict, strict=False)
to_deactivate = ['conv1a', 'conv1b', 'conv1c', 'conv2a', 'conv2b', 'conv2c', 'conv3a', 'conv3b', 'conv3c']
for l in to_deactivate:
deactivate_layer(model, l)
'''
loss_function = nn.CrossEntropyLoss()
optimiser = optim.Adam(model.parameters())
device = "cuda:0" if torch.cuda.is_available() else "cpu"
trial = Trial(model, optimiser, loss_function, metrics=['loss',
'accuracy']).to(device)
trial.with_generators(train_loader, test_generator=validation_loader)
trial.run(epochs=10)
results = trial.evaluate(data_key=torchbearer.TEST_DATA)
print(results)
model_name = 'resnet_model_3b'
weights_path = './weights/{}/'.format(model_name)
if not os.path.exists(weights_path):
os.makedirs(weights_path)
w_filepath = './weights/{}/train_{}.weights'.format(model_name, p)
torch.save(model.state_dict(), w_filepath)
# for d_vvs in ventral_depths:
# for t in range(n_trials):
model_file = f'./models/{cmode}-mos/model_{n_bn}_{d_vvs}_{t}.pt'
log_file = f'./logs/{cmode}-mos/model_{n_bn}_{d_vvs}_{t}.csv'
pathlib.Path(model_file).parents[0].mkdir(parents=True, exist_ok=True)
pathlib.Path(log_file).parents[0].mkdir(parents=True, exist_ok=True)
model = BaselineModel(n_bn, d_vvs, nch)
optimiser = optim.RMSprop(model.parameters(),
alpha=0.9,
lr=0.0001,
weight_decay=1e-6)
loss_function = nn.CrossEntropyLoss()
device = "cuda:0" if torch.cuda.is_available() else "cpu"
trial = Trial(
model,
optimiser,
loss_function,
metrics=['loss', 'accuracy'],
callbacks=[
# torchbearer.callbacks.imaging.MakeGrid().on_train().to_file('sample.png'),
torchbearer.callbacks.csv_logger.CSVLogger(log_file)
]).to(device)
trial.with_generators(trainloader, val_generator=testloader)
trial.run(epochs=20)
torch.save(model.conv_dict(), model_file)
if key <= epoch:
lr = value
else:
break
print("set lr to : ", lr)
return lr
scheduler = LambdaLR(lr_lambda=[lr_func])
@on_step_training
def clip(serial):
alexnetBin.clip()
trial = Trial(alexnetBin,
optimizer,
criterion=loss,
metrics=['acc', 'loss'],
callbacks=[clip, scheduler]).to('cpu')
trial = trial.with_generators(train_generator=train_set,
val_generator=valid_set,
test_generator=test_set)
trial.run(epochs=150)
trial.evaluate(data_key=TEST_DATA)
alexnetBin.train()
torch.save(alexnetBin.state_dict(),
open("./models/cifar10/alexnetBin.pth", "wb"))
def kld_Loss(mu, logvar):
KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
return KLD
# Reconstruction + KL divergence losses summed over all elements and batch
def loss_function(y_pred, y_true):
recon_x, mu, logvar = y_pred
x = y_true
BCE = bce_loss(recon_x, x)
KLD = kld_Loss(mu, logvar)
return BCE + KLD
model = VAE()
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=0.001)
loss = loss_function
from torchbearer import Trial
torchbearer_trial = Trial(model, optimizer, loss, metrics=['loss']).to('cuda')
torchbearer_trial.with_generators(train_generator=traingen,
val_generator=valgen)
torchbearer_trial.run(epochs=10)
cb.append(WarmupLR(0.1, args.lr)) if args.lr_warmup else []
cb.append(cutmix_reformat) if args.msda_mode == 'cutmix' else []
# FMix loss is equivalent to mixup loss and works for all msda in torchbearer
if args.msda_mode not in [None, 'None']:
bce = True if args.dataset == 'toxic' else False
criterion = modes['fmix'].loss(bce)
elif args.dataset == 'toxic':
criterion = nn.BCEWithLogitsLoss()
else:
criterion = nn.CrossEntropyLoss()
# from torchbearer.metrics.roc_auc_score import RocAucScore
print('==> Training model..')
trial = Trial(net,
optimizer,
criterion,
metrics=['acc', 'loss', 'lr'],
callbacks=cb)
trial.with_generators(train_generator=trainloader,
val_generator=valloader,
train_steps=args.train_steps,
test_generator=testloader).to(args.device)
if args.reload:
state = torch.load(args.model_file)
trial.load_state_dict(state)
trial.replay()
trial.run(args.epoch, verbose=args.verbose)
trial.evaluate(data_key=torchbearer.TEST_DATA)
plt.imshow(out.detach().numpy()[0, 0, :, :])
plt.show()
out = F.relu(out)
out = self.conv2(out)
out = F.relu(out)
out = self.gmp1(out)
out = out.view(out.shape[0], -1)
out = self.fc1(out)
out = F.relu(out)
out = self.fc2(out)
return out
model = BetterCNN()
loss_function = nn.L1Loss()
optimiser = optim.Adam(model.parameters())
device = "cuda:0" if torch.cuda.is_available() else "cpu"
trial = Trial(model, optimiser, loss_function,
metrics=['loss', 'accuracy']).to(device)
trial.with_generators(train_dl, valid_dl, test_generator=test_dl)
history = trial.run(epochs=1)
results = trial.evaluate(data_key=tb.TEST_DATA)
print(results)
loss_list = []
for item in history:
loss_list.append(item['loss'])
fig1 = plt.figure('1')
ax1 = fig1.gca()
ax1.plot(loss_list)
plt.show()
[transforms.ToTensor()]))
test_gen = torch.utils.data.DataLoader(test_set,
pin_memory=True,
batch_size=128,
shuffle=False,
num_workers=10)
comment = 'foveated5'
trial = Trial(
model,
torch.optim.Adam(model.parameters(), lr=0.0001),
nn.CrossEntropyLoss(),
metrics=['acc', 'loss'],
callbacks=[
callbacks.MostRecent(comment + '.{epoch:02d}.pt'),
imaging.MakeGrid(
key=TRANSFORMED, num_images=16, nrow=16,
pad_value=1).to_tensorboard(
name='transformed',
comment=comment).on_val().to_file(comment +
'_transformed.png'),
imaging.MakeGrid(
key=torchbearer.INPUT, num_images=16, nrow=16,
pad_value=1).to_tensorboard(
name='input',
comment=comment).on_val().to_file(comment + '_inputs.png')
])
trial = trial.with_generators(train_gen, test_gen).to('cuda')
trial.run(50)
print(loss_list)
print(results) """
model = torch.load(
'data/boat/trainlab_6_boat_resnet50_100epochs.pytorchweights')
# model = BetterCNN(3, len(train_dataset.classes))
# define the loss function and the optimiser
loss_function = nn.CrossEntropyLoss()
optimiser = optim.Adam(model.parameters())
device = "cuda:0" if torch.cuda.is_available() else "cpu"
trial = Trial(model, optimiser, loss_function, metrics=['loss',
'accuracy']).to(device)
trial.with_generators(train_loader,
val_generator=val_loader,
test_generator=test_loader)
# history = trial.run(epochs=100)
results = trial.evaluate(data_key=torchbearer.VALIDATION_DATA)
print(results)
# torch.save(model, path+'lab_6_boat_cnn_100epochs.pytorchweights')
""" loss_list = []
for item in history:
loss_list.append(item['loss'])
fig1 = plt.figure('1')
ax1 = fig1.gca()
ax1.plot(loss_list)
plt.show()
"""
predictions = trial.predict()
shuffle=True,
num_workers=8)
model = MimeVGG(vgg, {k: cfgs['A'][k]() for k in [layer]})
optimizer = optim.Adam(filter(lambda x: x.requires_grad,
model.parameters()),
lr=5e-4)
mi_false = mi(False)
@callbacks.add_to_loss
def mi_no_tanh(state):
state[OTHER_MI] = mi_false(state)
return 0
trial = Trial(model,
optimizer,
mi(True),
metrics=['loss',
torchbearer.metrics.mean(OTHER_MI)],
callbacks=[
mi_no_tanh,
callbacks.TensorBoard(write_graph=False,
comment='mi_' + args.model,
log_dir='mi_data')
])
trial.with_generators(train_generator=trainloader,
val_generator=valloader).to('cuda')
trial.run(20, verbose=1)
def train():
# train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
print('start training')
dataset_size = len(dataset)
indices = list(range(dataset_size))
split = int(np.floor(validation_split * dataset_size))
if shuffle_dataset:
np.random.seed(random_seed)
np.random.shuffle(indices)
train_indices, val_indices = indices[split:], indices[:split]
# Creating PT data samplers and loaders:
train_sampler = SubsetRandomSampler(train_indices)
valid_sampler = SubsetRandomSampler(val_indices)
train_loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
sampler=train_sampler)
val_loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
sampler=valid_sampler)
model = resnet50(pretrained=True)
model.avgpool = nn.AdaptiveAvgPool2d((1, 1))
model.fc = nn.Linear(2048, len(dataset.classes))
model.train()
# Freeze layers by not tracking gradients
for param in model.parameters():
param.requires_grad = False
model.fc.weight.requires_grad = True #unfreeze last layer weights
model.fc.bias.requires_grad = True #unfreeze last layer biases
optimiser1 = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=1e-3) #only optimse non-frozen layers
loss_function = nn.CrossEntropyLoss()
device = "cuda:0" if torch.cuda.is_available() else "cpu"
print(device)
checkpointer = torchbearer.callbacks.checkpointers.Best(
filepath='model.pt', monitor='loss')
trial = Trial(model,
optimiser1,
loss_function,
metrics=['loss', 'accuracy'],
callbacks=[checkpointer]).to(device)
trial.with_generators(train_loader, val_generator=val_loader)
trial.run(epochs=10)
state_dict = torch.load('model.pt')
trial_reloaded = Trial(model,
optimiser1,
loss_function,
metrics=['loss', 'accuracy'],
callbacks=[checkpointer]).to(device)
trial_reloaded.with_generators(train_loader, val_generator=val_loader)
trial_reloaded.load_state_dict(state_dict)
trial_reloaded.run(epochs=20)
results = trial_reloaded.evaluate(data_key=torchbearer.VALIDATION_DATA)
print()
print(results)
torch.save(model, 'mymodel.pth')
class eval_model(nn.Module):
def __init__(self, model, loss_f, device, metrics, trial_name, opt=optim.Adam,opt_params={}, **kwargs):
super().__init__()
self.model = model
self.loss_f = loss_f
self.opt = opt(list(self.model.parameters()), **opt_params)
self.metrics = metrics
self.device = device
self.trial_path = os.path.join('logs/', trial_name)
self.call_backs = None
self.best_step = 0
def init_trial(self):
#initial call_back functions for the trail.
if self.call_backs:
self.trial = Trial(self.model,self.opt,self.loss_f,metrics=self.metrics,callbacks=self.call_backs).to(self.device)
else:
self.trial = Trial(self.model,self.opt,self.loss_f,metrics=self.metrics).to(self.device)
def run(self, epoch, tr_loader,val_loader,t_loader=None,val_steps=None):
self.trial.with_generators(tr_loader, test_generator=t_loader,val_generator=val_loader,val_steps=val_steps)
return self.trial.run(epoch)
def save_model(self, suffix, path=None, mkdir=True, **kwargs):
if path is None:
path = os.path.join(self.trial_path, "model_{}.mf".format(suffix))
if mkdir:
os.makedirs(os.path.dirname(path), exist_ok=True)
torch.save(OrderedDict([
('model', self.model.state_dict(**kwargs)),
('trial', self.trial.state_dict()),
('opt', self.opt.state_dict()),
('best_step', self.best_step),
]), path)
return path
def load_model(self, suffix, path=None, **kwargs):
if path is None:
path = os.path.join(self.trial_path, "model_{}.mf".format(suffix))
checkpoint = torch.load(path)
self.trial.load_state_dict(checkpoint['trial'])
self.model.load_state_dict(checkpoint['model'], **kwargs)
self.opt.load_state_dict(checkpoint['opt'])
self.best_step = int(checkpoint['best_step'])
def plot_loss(self, loss, title, xlabel, x_all=False):
plt.figure()
ax = plt.axes()
ax.set_title(title)
ax.set_ylabel("loss")
ax.set_xlabel(xlabel)
x = range(len(loss))
plt.plot(x,loss)
if x_all:
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
plt.xticks(x)
plt.legend()
plt.show()
def plot_fig(self,out,epoch,title,y_label, para=['loss','val_loss']):
tem = [ [0]*epoch for i in range(len(para))]
for idx, label in enumerate(para):
for i in range(epoch):
tem[idx][i] = out[i][label]
plt.figure()
ax = plt.axes()
ax.set_title(title)
ax.set_ylabel(y_label)
ax.set_xlabel("Epoch")
for i, item in enumerate(tem):
plt.plot(range(epoch),item, label=para[i])
plt.legend()
plt.show()