def evaluate(file, device='cuda'):
transform_test = transforms.Compose([transforms.ToTensor()])
testset = torchvision.datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform_test)
testloader = torch.utils.data.DataLoader(testset,
batch_size=100,
shuffle=False,
num_workers=10)
base_dir = 'cifarss_base'
model = CifarVAE()
model = SelfTaught(model.encoder, model.mu, 32)
optimizer = optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=0)
trial = Trial(
model, optimizer, nn.NLLLoss(), ['acc', 'loss']).load_state_dict(
torch.load(os.path.join(base_dir, file)),
resume=False).with_generators(val_generator=testloader).to(device)
return trial.evaluate()
def test_model(model: nn.Module, test_loader: DataLoader):
loss_function = nn.MSELoss()
device = "cuda:0" if torch.cuda.is_available() else "cpu"
trial = Trial(model, None, loss_function, metrics=["loss"]).to(device)
trial.with_generators(train_loader, test_generator=test_loader)
results = trial.evaluate(data_key=torchbearer.TEST_DATA)
return results, trial
def train_model(train, valid, test, classes, batch, num_epochs):
# model = CNN(3, classes)
model = models.resnet18()
train_loader = DataLoader(train, batch_size=batch)
val_loader = DataLoader(valid, batch_size=batch)
test_loader = DataLoader(test, batch_size=batch)
# define the loss function and the optimiser
loss_function = nn.CrossEntropyLoss() # equation 8 in original paper
optimiser = optim.Adam(model.parameters())
trial = Trial(model, optimiser, loss_function, metrics=['loss', 'accuracy'], verbose=1).to(device)
trial.with_generators(train_loader, val_generator=val_loader, test_generator=test_loader)
trial.run(epochs=num_epochs)
results_test = trial.evaluate(data_key=torchbearer.TEST_DATA)
results_val = trial.evaluate(data_key=torchbearer.VALIDATION_DATA)
print(results_test)
return results_test, results_val
def train(self, model, **kwargs) -> (float, float):
# Get transfer model and put it in training mode
net = model.net
net.train()
# Create optimiser
optimiser = optim.Adam(net.parameters(), lr=1e-4)
# Check for cuda
device = "cuda:0" if torch.cuda.is_available() else "cpu"
print("Training using", device)
# Setup loss function
if self.class_weight_method != ClassWeightMethod.Unweighted:
distribution = class_distribution("data/processed/train")
if self.class_weight_method == ClassWeightMethod.SumBased:
inv_distribution = [
1 - x / np.sum(distribution) for x in distribution
]
inv_distribution = torch.from_numpy(
np.array(inv_distribution)).float()
elif self.class_weight_method == ClassWeightMethod.MaxBased:
inv_distribution = [
np.max(distribution) / x for x in distribution
]
inv_distribution = torch.from_numpy(
np.array(inv_distribution)).float()
else:
raise IndexError("Unknown class weight method " +
str(self.class_weight_method))
loss_function = self.loss(inv_distribution.to(device))
else:
loss_function = self.loss()
# Setup trial
trial = Trial(net,
optimiser,
loss_function,
metrics=["loss", "accuracy"]).to(device)
trial.with_generators(
self.image_datasets.train_loader,
test_generator=self.image_datasets.validation_loader,
)
# Actually run the training
trial.run(epochs=self.num_epochs)
# Evaluate and show results
time.sleep(0.1) # Ensure training has finished
net.eval()
results = trial.evaluate(data_key=torchbearer.TEST_DATA)
acc = float(results["test_acc"])
loss = float(results["test_loss"])
return acc, loss
def run(train_batch_size,
val_batch_size,
epochs,
lr,
log_interval,
input_size=10,
hidden_size=100,
out_size=4):
dataset = FuzzBuzzDataset(input_size)
splitter = DatasetValidationSplitter(len(dataset), 0.1)
train_set = splitter.get_train_dataset(dataset)
val_set = splitter.get_val_dataset(dataset)
train_loader = DataLoader(train_set,
pin_memory=True,
batch_size=train_batch_size,
shuffle=True,
num_workers=2)
val_loader = DataLoader(val_set,
pin_memory=True,
batch_size=val_batch_size,
shuffle=False,
num_workers=2)
model = FuzzBuzzModel(input_size, hidden_size, out_size)
device = 'cpu'
if torch.cuda.is_available():
device = 'cuda'
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=lr)
loss = nn.CrossEntropyLoss()
trial = Trial(model, optimizer, criterion=loss,
metrics=['acc', 'loss']).to(device)
trial = trial.with_generators(train_generator=train_loader,
val_generator=val_loader)
trial.run(epochs=epochs)
trial.evaluate(data_key=VALIDATION_DATA)
def train(self):
scheduler = torch_scheduler.StepLR(self.num_epochs_decay,
gamma=self.decay_factor)
loss_plot_plan = os.path.join(
self.result_path, 'live_loss_plot%s-%d-%.4f-%d-%.4f.png' %
(self.model_type, self.num_epochs, self.lr, self.num_epochs_decay,
self.augmentation_prob))
callbacks = [scheduler]
# imaging.FromState(torchbearer.X).on_val().cache(16).make_grid().to_pyplot(),
# imaging.FromState(torchbearer.Y_TRUE).on_val().cache(16).make_grid().to_pyplot(),
# imaging.FromState(torchbearer.Y_PRED).on_val().cache(16).make_grid().to_pyplot(),
# imaging.FromState(torchbearer.X).on_test().cache(16).make_grid().to_pyplot(),
# imaging.FromState(torchbearer.Y_TRUE).on_test().cache(16).make_grid().to_pyplot(),
# imaging.FromState(torchbearer.Y_PRED).on_test().cache(16).make_grid().to_pyplot(),
# TensorBoard(write_batch_metrics=True),
trial = Trial(
self.unet,
self.optimizer,
self.criterion,
metrics=['loss', 'binary_acc'],
# binary_acc for debugging certain things
callbacks=callbacks).to(self.device)
trial.with_generators(train_generator=self.train_loader,
val_generator=self.valid_loader,
test_generator=self.test_loader)
start = time.time()
history = trial.run(epochs=self.num_epochs, verbose=2)
stop = time.time()
train_time = stop - start
state = self.unet.state_dict()
unet_path = os.path.join(
self.model_path,
'%s-%d-%.4f-%d-%.4f_Index_BCE_Dropout_STAREIndex.pkl' % (
self.model_type,
self.num_epochs,
self.lr,
self.num_epochs_decay,
self.augmentation_prob,
))
torch.save(state, unet_path)
print(history)
### Testing
results = trial.evaluate(data_key=torchbearer.TEST_DATA)
print("Test result:")
print(results)
return history, results
def train(self):
"""Train encoder, generator and discriminator."""
# D_LOSS = state_key('d_loss')
#
# def dice_loss(prediction, target):
# """Calculating the dice loss
# Args:
# prediction = predicted image
# target = Targeted image
# Output:
# dice_loss"""
#
# smooth = 1.0
# # print(prediction.shape)
# # print(target.shape)
# # TODO: Used reshape() instead of view because of batch size > 1
# i_flat = prediction.reshape(-1)
# t_flat = target.reshape(-1)
#
# intersection = (i_flat * t_flat).sum()
#
# return 1 - ((2. * intersection + smooth) / (i_flat.sum() + t_flat.sum() + smooth))
#
# def calc_loss(prediction, target, bce_weight=0.5):
# """Calculating the loss and metrics
# Args:
# prediction = predicted image
# target = Targeted image
# metrics = Metrics printed
# bce_weight = 0.5 (default)
# Output:
# loss : dice loss of the epoch """
# bce = F.binary_cross_entropy_with_logits(prediction, target)
# prediction = F.sigmoid(prediction)
# dice = dice_loss(prediction, target)
#
# loss = bce * bce_weight + dice * (1 - bce_weight)
# state[D_LOSS] = loss
#
# return loss
#
# stopping = EarlyStopping(monitor='val_acc', patience=5, mode='max')
scheduler = torch_scheduler.StepLR(self.num_epochs_decay, gamma=0.1)
loss_plot_plan = os.path.join(self.result_path,
'live_loss_plot%s-%d-%.4f-%d-%.4f.png' % (self.model_type,
self.num_epochs,
self.lr,
self.num_epochs_decay,
self.augmentation_prob))
callbacks = [scheduler]
# imaging.FromState(torchbearer.X).on_val().cache(16).make_grid().to_pyplot(),
# imaging.FromState(torchbearer.Y_TRUE).on_val().cache(16).make_grid().to_pyplot(),
# imaging.FromState(torchbearer.Y_PRED).on_val().cache(16).make_grid().to_pyplot(),
# imaging.FromState(torchbearer.X).on_test().cache(16).make_grid().to_pyplot(),
# imaging.FromState(torchbearer.Y_TRUE).on_test().cache(16).make_grid().to_pyplot(),
# imaging.FromState(torchbearer.Y_PRED).on_test().cache(16).make_grid().to_pyplot(),
# TensorBoard(write_batch_metrics=True),
try:
trial = Trial(self.unet, self.optimizer, self.criterion, metrics=['loss', 'binary_acc'],
# binary_acc for debugging certain things
callbacks=callbacks).to(self.device)
trial.with_generators(train_generator=self.train_loader,
val_generator=self.valid_loader,
test_generator=self.test_loader)
start = time.time()
history = trial.run(epochs=self.num_epochs, verbose=2)
stop = time.time()
train_time = stop - start
state = self.unet.state_dict()
unet_path = os.path.join(self.model_path,
'%s-%d-%.4f-%d-%.4f_Index_BCE_Dropout_STAREIndex.pkl' % (self.model_type,
self.num_epochs,
self.lr,
self.num_epochs_decay,
self.augmentation_prob,))
torch.save(state, unet_path)
print(history)
### Testing
results = trial.evaluate(data_key=torchbearer.TEST_DATA)
print("Test result:")
print(results)
except (RuntimeError, OSError):
state = self.unet.state_dict()
unet_path = os.path.join(self.model_path,
'%s-%d-%.4f-%d-%.4f_Index_BCE_Dropout_STAREIndex.pkl' % (self.model_type,
self.num_epochs,
self.lr,
self.num_epochs_decay,
self.augmentation_prob,))
torch.save(state, unet_path[:-4] + 'interrupted' + '.pkl')
# build the model model = SimpleCNN() # define the loss function and the optimiser loss_function = nn.CrossEntropyLoss() live_loss_plot = LiveLossPlot(draw_once=True) optimiser = optim.Adam(model.parameters(), lr=0.001) scheduler = StepLR(step_size=10, gamma=0.5) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, callbacks=[scheduler, live_loss_plot], metrics=['loss', 'accuracy']).to(device) trial.with_generators(trainloader, val_generator=valloader, test_generator=testloader) history = trial.run(verbose=1, epochs=30)# results = trial.evaluate(data_key=torchbearer.TEST_DATA) print(results) # build the model model = SimpleCNN() # define the loss function and the optimiser loss_function = nn.CrossEntropyLoss() live_loss_plot = LiveLossPlot(draw_once=True) optimiser = optim.Adam(model.parameters(), lr=0.001) scheduler = StepLR(step_size=10, gamma=0.5) device = "cuda:0" if torch.cuda.is_available() else "cpu" trial = Trial(model, optimiser, loss_function, callbacks=[scheduler, live_loss_plot], metrics=['loss', 'accuracy']).to(device) trial.with_generators(trainloader, test_generator=testloader) history = trial.run(verbose=1, epochs=30)#
for key, value in regime.items():
if key <= epoch:
lr = value
else:
break
print("set lr to : ", lr)
return lr
scheduler = LambdaLR(lr_lambda=[lr_func])
@on_step_training
def clip(serial):
alexnetBin.clamp()
trial = Trial(alexnetBin,
optimizer,
criterion=loss,
metrics=['acc', 'loss'],
callbacks=[clip, scheduler]).to('cuda')
trial = trial.with_generators(train_generator=train_set,
val_generator=valid_set,
test_generator=test_set)
trial.run(epochs=EPOCH_NB)
trial.evaluate(data_key=TEST_DATA)
alexnetBin.train()
nn.BatchNorm2d(16), nn.ReLU(),
nn.Conv2d(16, 32, stride=2, kernel_size=3),
nn.BatchNorm2d(32), nn.ReLU(),
nn.Conv2d(32, 64, stride=2, kernel_size=3),
nn.BatchNorm2d(64), nn.ReLU())
self.classifier = nn.Linear(576, 10)
def forward(self, x):
x = self.convs(x)
x = x.view(-1, 576)
return self.classifier(x)
model = SimpleModel()
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=0.001)
loss = nn.CrossEntropyLoss()
from torchbearer import Trial
torchbearer_trial = Trial(model, optimizer, loss, metrics=['acc',
'loss']).to('cuda')
torchbearer_trial.with_generators(train_generator=traingen,
val_generator=valgen)
torchbearer_trial.run(epochs=10)
torchbearer_trial.with_val_generator(testgen)
torchbearer_trial.evaluate()
plt.imshow(out.detach().numpy()[0, 0, :, :])
plt.show()
out = F.relu(out)
out = self.conv2(out)
out = F.relu(out)
out = self.gmp1(out)
out = out.view(out.shape[0], -1)
out = self.fc1(out)
out = F.relu(out)
out = self.fc2(out)
return out
model = BetterCNN()
loss_function = nn.L1Loss()
optimiser = optim.Adam(model.parameters())
device = "cuda:0" if torch.cuda.is_available() else "cpu"
trial = Trial(model, optimiser, loss_function,
metrics=['loss', 'accuracy']).to(device)
trial.with_generators(train_dl, valid_dl, test_generator=test_dl)
history = trial.run(epochs=1)
results = trial.evaluate(data_key=tb.TEST_DATA)
print(results)
loss_list = []
for item in history:
loss_list.append(item['loss'])
fig1 = plt.figure('1')
ax1 = fig1.gca()
ax1.plot(loss_list)
plt.show()
model = torch.load('fashionMNIST_VAE_20epochs.pytorchweights')
model_parameters = filter(lambda p: p.requires_grad, model.parameters())
print(sum([np.prod(p.size()) for p in model_parameters]))
optimizer = optim.Adam(
filter(lambda p: p.requires_grad, model.parameters()), lr=5e-4)
trial = Trial(model, optimizer, nn.MSELoss(reduction='sum'), metrics=['acc', 'loss'], callbacks=[
beta_kl(MU, LOGVAR),
callbacks.ConsolePrinter(),
plot_progress()
], verbose=1).with_generators(train_generator=traingen, test_generator=testgen)
history = trial.run(10)
trial.evaluate(verbose=0, data_key=torchbearer.TEST_DATA)
torch.save(model, 'fashionMNIST_VAE_30epochs.pytorchweights')
# torch.save(model, 'fashionMNIST_VAE_10epochs.pytorchweights')
print(history)
train_loss_list = []
val_loss_list = []
for item in history:
train_loss_list.append(item['running_loss'])
val_loss_list.append(item['test_loss'])
fig1 = plt.figure('1')
ax1 = fig1.gca()
ax1.plot(train_loss_list, color='blue', alpha=0.7)
ax1.plot(val_loss_list, color='red', alpha=0.7)
plt.legend()
model = torch.load(
'data/boat/trainlab_6_boat_resnet50_100epochs.pytorchweights')
# model = BetterCNN(3, len(train_dataset.classes))
# define the loss function and the optimiser
loss_function = nn.CrossEntropyLoss()
optimiser = optim.Adam(model.parameters())
device = "cuda:0" if torch.cuda.is_available() else "cpu"
trial = Trial(model, optimiser, loss_function, metrics=['loss',
'accuracy']).to(device)
trial.with_generators(train_loader,
val_generator=val_loader,
test_generator=test_loader)
# history = trial.run(epochs=100)
results = trial.evaluate(data_key=torchbearer.VALIDATION_DATA)
print(results)
# torch.save(model, path+'lab_6_boat_cnn_100epochs.pytorchweights')
""" loss_list = []
for item in history:
loss_list.append(item['loss'])
fig1 = plt.figure('1')
ax1 = fig1.gca()
ax1.plot(loss_list)
plt.show()
"""
predictions = trial.predict()
predicted_classes = predictions.argmax(1).cpu()
true_classes = list(x for (_, x) in test_dataset.samples)
def train():
# train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
print('start training')
dataset_size = len(dataset)
indices = list(range(dataset_size))
split = int(np.floor(validation_split * dataset_size))
if shuffle_dataset:
np.random.seed(random_seed)
np.random.shuffle(indices)
train_indices, val_indices = indices[split:], indices[:split]
# Creating PT data samplers and loaders:
train_sampler = SubsetRandomSampler(train_indices)
valid_sampler = SubsetRandomSampler(val_indices)
train_loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
sampler=train_sampler)
val_loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
sampler=valid_sampler)
model = resnet50(pretrained=True)
model.avgpool = nn.AdaptiveAvgPool2d((1, 1))
model.fc = nn.Linear(2048, len(dataset.classes))
model.train()
# Freeze layers by not tracking gradients
for param in model.parameters():
param.requires_grad = False
model.fc.weight.requires_grad = True #unfreeze last layer weights
model.fc.bias.requires_grad = True #unfreeze last layer biases
optimiser1 = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=1e-3) #only optimse non-frozen layers
loss_function = nn.CrossEntropyLoss()
device = "cuda:0" if torch.cuda.is_available() else "cpu"
print(device)
checkpointer = torchbearer.callbacks.checkpointers.Best(
filepath='model.pt', monitor='loss')
trial = Trial(model,
optimiser1,
loss_function,
metrics=['loss', 'accuracy'],
callbacks=[checkpointer]).to(device)
trial.with_generators(train_loader, val_generator=val_loader)
trial.run(epochs=10)
state_dict = torch.load('model.pt')
trial_reloaded = Trial(model,
optimiser1,
loss_function,
metrics=['loss', 'accuracy'],
callbacks=[checkpointer]).to(device)
trial_reloaded.with_generators(train_loader, val_generator=val_loader)
trial_reloaded.load_state_dict(state_dict)
trial_reloaded.run(epochs=20)
results = trial_reloaded.evaluate(data_key=torchbearer.VALIDATION_DATA)
print()
print(results)
torch.save(model, 'mymodel.pth')
optimizer = torch.optim.SGD(model.parameters(),
0.1,
momentum=0.9,
weight_decay=1e-4)
# optimiser = optim.RMSprop(model.parameters(), alpha=0.9, lr=0.0001, weight_decay=1e-6)
loss_function = nn.CrossEntropyLoss()
# device = "cuda:0" if torch.cuda.is_available() else "cpu"
trial = Trial(model,
optimizer,
loss_function,
metrics=['loss', 'acc', 'top_5_acc'],
callbacks=[
callbacks.TensorBoard(write_graph=False,
comment=f'resnet50_{n_bn}_{rep}'),
callbacks.MultiStepLR([30, 60]),
callbacks.MostRecent(dir + model_file + '_{epoch:02d}.pt')
]).to('cuda')
trial.with_generators(trainloader, test_generator=testloader)
pattern = re.compile(model_file + '_\d+.pt')
for filepath in os.listdir(dir):
if pattern.match(filepath):
trial.load_state_dict(torch.load(dir + filepath))
trial.run(epochs=90)
trial.evaluate(data_key=torchbearer.TEST_DATA)
torch.save(model.module.state_dict(), dir + model_file + '.pt')
out = self.fc1(out)
out = F.relu(out)
out = self.fc2(out)
return out
#reset the data loaders
trainloader = DataLoader(train_data, batch_size=128, shuffle=True)
testloader = DataLoader(test_data, batch_size=128, shuffle=True)
valloader = DataLoader(val_data, batch_size=128, shuffle=True)
# build the model
model = SimpleCNN()
# define the loss function and the optimiser
loss_function = nn.MSELoss()
optimiser = optim.Adam(model.parameters())
device = "cuda:0" if torch.cuda.is_available() else "cpu"
trial = Trial(model,
optimiser,
loss_function,
metrics=['loss', 'accuracy'],
callbacks=[LiveLossPlot()]).to(device)
trial.with_generators(trainloader, valloader, test_generator=testloader)
trial.run(epochs=100)
results_train = trial.evaluate(data_key=torchbearer.TRAIN_DATA)
results_test = trial.evaluate(data_key=torchbearer.TEST_DATA)
print(results_train)
print(results_test)
class Model(nn.Module):
def __init__(self):
super(Model, self).__init__()
self.net = torchvision.models.googlenet(True)
def forward(self, input):
if input is not None:
return self.net(input)
model = Model()
from torchbearer import Trial
import torchbearer.callbacks.imaging as imaging
trial = Trial(model,
callbacks=[
imaging.ClassAppearanceModel(
1000, (3, 224, 224),
steps=10000,
target=951,
transform=inv_normalize).on_val().to_file('lemon.png'),
imaging.ClassAppearanceModel(
1000, (3, 224, 224),
steps=10000,
target=968,
transform=inv_normalize).on_val().to_file('cup.png')
])
trial.for_val_steps(1).to('cuda')
trial.evaluate()
