f"\n\nTraining noise2noise for {n_epochs} epochs with loader {loader_name}"
)
for epoch in tqdm.tqdm(range(n_epochs), total=n_epochs):
# train
train(net,
train_loader,
optimizer,
LOSS_CRITERION,
epoch,
log_interval=25,
tb_logger=logger,
device=device)
step = epoch * len(train_loader.dataset)
# validate
validate(net,
val_loader,
LOSS_CRITERION,
EVAL_METRIC,
step=step,
tb_logger=logger,
device=device)
"""## Exercises
1. Train a separete denoising model using clean target and compare the PSNR scores with those obtained with noise2noise model. Compare results of the two models visually in tensorboard.
**Hint** the only change that needs to be done in the loader is changing the noise transformer to return a clean image during training
```
TRAIN_NOISE_TRANSFORM = lambda x: (additive_gaussian_noise_train(x), x)
```
2. Train noise2noise with different noise model, e.g. Poisson noise with varying lambda.
"""
if len(val_fold_indices) > 0:
if val_fold_index not in val_fold_indices:
val_fold_index += 1
continue
val_fold_index += 1
print("\n\n ---- Validation fold index: ", val_fold_index, "/",
n_folds)
print(len(train_id_type_list), len(val_id_type_list))
assert len(to_set(train_id_type_list)
& to_set(val_id_type_list)) == 0, "WTF"
cnn = params['network'](lr=params['lr_kwargs']['lr'],
**params,
**params['network_kwargs'])
params['save_prefix'] = params['save_prefix_template'].format(
cnn_name=cnn.name, fold_index=val_fold_index - 1)
print("\n {} - Loaded {} model ...".format(datetime.now(), cnn.name))
load_pretrained_model(cnn, **params)
params['seed'] += run_counter - 1
f2, mae = validate(cnn, val_id_type_list, verbose=0, **params)
cv_mean_scores[run_counter - 1, val_fold_index - 1] = f2
np.random.shuffle(_trainval_id_type_list)
print(cv_mean_scores)
def main():
global best_metrics
# Parse the arguments
args = parser.parse_args()
# Create the SummaryWriter for Tensorboard
args.writer = SummaryWriter('./logs/tensorboard/{}'.format(args.run_id))
# Set the RNG seegs
if args.seed is not None:
random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.deterministic = True
warnings.warn('You have chosen to seed training. \
This will turn on the CUDNN deterministic setting, \
which can slow down your training considerably! \
You may see unexpected behavior when restarting \
from checkpoints.')
# Print out the training setup
print('New training run...\n')
print(' Run ID: {}'.format(args.run_id))
print(' Architecture: {}'.format(args.arch))
print(' Batch size: {}'.format(args.batch_size))
print(' Learning rate: {}'.format(args.learning_rate))
print(' Decay rate: {}\n'.format(args.decay_rate))
# Create the model
print("=> creating model...")
device = torch.device('cuda')
model = models.__dict__[args.arch](pretrained=False,
num_classes=args.classes).to(device)
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
criterion = nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.Adam(model.parameters(), args.learning_rate)
lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, mode='min', factor=args.decay_rate, patience=10)
cudnn.benchmark = True
# Create the datasets and loaders
print('=> creating the datasets and iterators')
# Create the training dataset and loader
training_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261))
])
training_dataset = datasets.CIFAR10('./data',
train=True,
download=True,
transform=training_transform)
training_loader = torch.utils.data.DataLoader(training_dataset,
batch_size=args.batch_size,
shuffle=True)
validation_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261))
])
validation_dataset = datasets.CIFAR10('./data',
train=False,
transform=validation_transform)
validation_loader = torch.utils.data.DataLoader(validation_dataset,
batch_size=args.batch_size,
shuffle=True)
# Save the lengths of the data loaders for Tensorboard
args.train_loader_len = len(training_loader)
args.validation_loader_len = len(validation_loader)
# Train the model
print('=> starting the training\n')
for epoch in range(args.epochs):
# Set the current epoch to be used by Tensorboard
args.current_epoch = epoch
# Take a training step
train(training_loader, model, criterion, optimizer, epoch, device,
args)
# Evaluate on validation set and check if it is the current best
val_loss, metrics = validate(validation_loader, model, criterion,
device, args)
best_metrics, is_best = test_best_metrics(metrics, best_metrics)
# Take a step using the learning rate scheduler
lr_scheduler.step(val_loss)
save_checkpoint(
{
'epoch': epoch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'best_acc': best_metrics[0],
'best_pre': best_metrics[1],
'best_rec': best_metrics[2]
}, is_best, args)
# Close the Tensorboard writer
args.writer.close()
for epoch in tqdm.tqdm(range(n_epochs), total=n_epochs):
# train
train(net,
train_loader,
optimizer,
loss_function,
epoch,
tb_logger=logger,
device=device)
step = epoch * len(train_loader.dataset)
# validate
_, acc = validate(net,
val_loader,
loss_function,
metric,
step=step,
tb_logger=logger,
device=device,
optimizer=optimizer)
if acc > best_accuracy:
best_accuracy = acc
best_epoch = epoch
utils.save_checkpoint(net, optimizer, epoch, checkpoint_name)
images, _ = next(iter(train_loader))
images = images.to(device)
grid = torchvision.utils.make_grid(images)
logger.add_image('images', grid, 0)
logger.add_graph(net, images)
logger.close()
def main():
total_steps = 0
results_file = open(RESULTS_WEIGHTS_PATH.joinpath("results.txt"), "w")
age_criterion = nn.MSELoss()
sex_criterion = nn.BCELoss()
age_pred = torch.empty(0).to(DEVICE)
age_data = torch.empty(0).to(DEVICE)
sex_pred = torch.empty(0).to(DEVICE)
sex_data = torch.empty(0).to(DEVICE)
for i in range(N_FOLDS):
model = m.Dasnet().to(DEVICE)
optimizer = torch.optim.Adadelta(model.parameters(),
lr=1.0,
rho=0.95,
eps=1e-06)
if FIXED_GROUPS:
training_gen, eval_gen, test_gen = dat_ut.fixed_dataset_generator()
elif STATIC_TEST:
training_gen, eval_gen, test_gen = dat_ut.kfold_generator_simple(i)
else:
training_gen, eval_gen, test_gen = dat_ut.kfold_generator_simple(i)
best_epoch_result = [-1, -1, -1, -1]
best_epoch = 0
best_epoch_model = dict()
no_upgrade_cont = 0
for k in range(1, N_EPOCHS + 1):
train_ut.train(model, training_gen, age_criterion, sex_criterion,
optimizer)
_, _, _, _, total_loss, age_loss, sex_loss, avg_age_diff, avg_sex_diff = train_ut.validate(
model, eval_gen, age_criterion, sex_criterion)
if best_epoch_result[0] >= total_loss or best_epoch_result[0] == -1:
best_epoch_result = [
total_loss, age_loss, sex_loss, avg_age_diff, avg_sex_diff
]
best_epoch_model = model.state_dict()
best_epoch = k
no_upgrade_cont = 0
if best_epoch_result[0] < total_loss:
no_upgrade_cont += 1
if no_upgrade_cont == MAX_ITER_NO_IMPROVE:
print("UPGRADE FIN / EPOCH: {}".format(best_epoch),
file=results_file)
print("FINAL EPOCH: {}".format(k), file=results_file)
break
model.load_state_dict(best_epoch_model)
torch.save(
model.state_dict(),
RESULTS_WEIGHTS_PATH.joinpath("model_weights.pth".format(i)))
age, age_out, sex, sex_out, total_test_loss, age_test_loss, sex_test_loss, avg_age_diff, avg_sex_diff = train_ut.validate(
model, test_gen, age_criterion, sex_criterion, 'test')
print(
"TEST :: TOTAL LOSS = {} \nAGE_LOSS = {} / SEX_LOSS = {} \nAVG_AGE_DIFF = {} / AVG_SEX_DIFF = {}"
.format(total_test_loss, age_test_loss, sex_test_loss,
avg_age_diff, avg_sex_diff),
file=results_file)
age_data = torch.cat((age_data, age), 0)
age_pred = torch.cat((age_pred, age_out), 0)
sex_data = torch.cat((sex_data, sex), 0)
sex_pred = torch.cat((sex_pred, sex_out), 0)
print_metrics(age, age_out, sex, sex_out, results_file)
save_results(age_data, age_pred, sex_data, sex_pred)
results_file.close()