def validate(model,
loss,
dataloader,
epoch: int,
metrics=dict(),
summary_writer: SummaryWriter = None):
losses = AverageMeter()
valid_scores = {}
for key, _ in metrics.items():
valid_scores[key] = AverageMeter()
with torch.set_grad_enabled(False):
model.eval()
n_batches = len(dataloader)
with tqdm(total=len(dataloader)) as tq:
tq.set_description('Validation')
x = None
y = None
outputs = None
batch_loss = None
for batch_index, (x, y) in enumerate(dataloader):
x, y = x.cuda(non_blocking=True), y.cuda(non_blocking=True)
# forward + backward + optimize
outputs = model(x)
batch_loss = loss(outputs, y)
# Log train progress
batch_loss_val = batch_loss.cpu().item()
if summary_writer is not None:
summary_writer.add_scalar('val/batch/loss', batch_loss_val,
epoch * n_batches + batch_index)
losses.update(batch_loss_val)
for key, metric in metrics.items():
score = metric(outputs, y).cpu().item()
valid_scores[key].update(score)
if summary_writer is not None:
summary_writer.add_scalar(
'val/batch/' + key, score,
epoch * n_batches + batch_index)
tq.set_postfix(loss='{:.3f}'.format(losses.avg),
**valid_scores)
tq.update()
if summary_writer is not None:
summary_writer.add_image('val/image',
make_grid(x.cpu(), normalize=True),
epoch)
summary_writer.add_image('val/y_true',
make_grid(y.cpu(), normalize=True),
epoch)
summary_writer.add_image(
'val/y_pred',
make_grid(outputs.sigmoid().cpu(), normalize=True), epoch)
summary_writer.add_scalar('val/epoch/loss', losses.avg, epoch)
for key, value in valid_scores.items():
summary_writer.add_scalar('val/epoch/' + key, value.avg,
epoch)
del x, y, outputs, batch_loss
return losses, valid_scores
def train(model,
loss,
optimizer,
dataloader,
epoch: int,
metrics={},
summary_writer=None):
losses = AverageMeter()
train_scores = {}
for key, _ in metrics.items():
train_scores[key] = AverageMeter()
with torch.set_grad_enabled(True):
model.train()
n_batches = len(dataloader)
with tqdm(total=n_batches) as tq:
tq.set_description('Train')
x = None
y = None
outputs = None
batch_loss = None
for batch_index, (x, y) in enumerate(dataloader):
x, y = x.cuda(non_blocking=True), y.cuda(non_blocking=True)
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = model(x)
batch_loss = loss(outputs, y)
batch_size = x.size(0)
(batch_size * batch_loss).backward()
optimizer.step()
# Batch train end
# Log train progress
batch_loss_val = batch_loss.cpu().item()
if summary_writer is not None:
summary_writer.add_scalar('train/batch/loss',
batch_loss_val,
epoch * n_batches + batch_index)
# Plot gradient absmax to see if there are any gradient explosions
grad_max = 0
for name, param in model.named_parameters():
if param.grad is not None:
grad_max = max(grad_max,
param.grad.abs().max().cpu().item())
summary_writer.add_scalar('train/grad/global_max',
grad_max,
epoch * n_batches + batch_index)
losses.update(batch_loss_val)
for key, metric in metrics.items():
score = metric(outputs, y).cpu().item()
train_scores[key].update(score)
if summary_writer is not None:
summary_writer.add_scalar(
'train/batch/' + key, score,
epoch * n_batches + batch_index)
tq.set_postfix(loss='{:.3f}'.format(losses.avg),
**train_scores)
tq.update()
# End of train epoch
if summary_writer is not None:
summary_writer.add_image('train/image',
make_grid(x.cpu(), normalize=True),
epoch)
summary_writer.add_image('train/y_true',
make_grid(y.cpu(), normalize=True),
epoch)
summary_writer.add_image(
'train/y_pred',
make_grid(outputs.sigmoid().cpu(), normalize=True), epoch)
summary_writer.add_scalar('train/epoch/loss', losses.avg,
epoch)
for key, value in train_scores.items():
summary_writer.add_scalar('train/epoch/' + key, value.avg,
epoch)
# Plot histogram of parameters after each epoch
for name, param in model.named_parameters():
if param.grad is not None:
# Plot weighs
param_data = param.data.cpu().numpy()
summary_writer.add_histogram('model/' + name,
param_data,
epoch,
bins='doane')
# for m in model.modules():
# if isinstance(m, nn.Conv2d):
# weights = m.weights.data.numpy()
del x, y, outputs, batch_loss
return losses, train_scores
def validate(model,
loss,
dataloader,
epoch: int,
metrics=dict(),
summary_writer: SummaryWriter = None):
losses = AverageMeter()
pr_meter = PRCurveMeter()
valid_scores = {}
for key, _ in metrics.items():
valid_scores[key] = AverageMeter()
with torch.set_grad_enabled(False):
model.eval()
n_batches = len(dataloader)
with tqdm(total=len(dataloader)) as tq:
tq.set_description('Validation')
x = None
y = None
outputs = None
batch_loss = None
for batch_index, (x, y) in enumerate(dataloader):
x, y = x.cuda(non_blocking=True), y.cuda(non_blocking=True)
# forward + backward + optimize
outputs = model(x)
batch_loss = loss(outputs, y)
# Log train progress
batch_loss_val = batch_loss.cpu().item()
if summary_writer is not None:
summary_writer.add_scalar('val/batch/loss', batch_loss_val,
epoch * n_batches + batch_index)
losses.update(batch_loss_val)
for key, metric in metrics.items():
score = metric(outputs, y).cpu().item()
valid_scores[key].update(score)
if summary_writer is not None:
summary_writer.add_scalar(
'val/batch/' + key, score,
epoch * n_batches + batch_index)
tq.set_postfix(loss='{:.3f}'.format(losses.avg),
**valid_scores)
tq.update()
if summary_writer is not None:
summary_writer.add_image('val/image',
make_grid(x.cpu(), normalize=True),
epoch)
summary_writer.add_image('val/y_true',
make_grid(y.cpu(), normalize=True),
epoch)
summary_writer.add_image(
'val/y_pred',
make_grid(outputs.sigmoid().cpu(), normalize=True), epoch)
summary_writer.add_scalar('val/epoch/loss', losses.avg, epoch)
for key, value in valid_scores.items():
summary_writer.add_scalar('val/epoch/' + key, value.avg,
epoch)
# Compute PR curve only for last batch, because computing it for entire validation set is costly
pr_meter.update(outputs, y)
summary_writer.add_pr_curve_raw(
'val/pr_curve',
true_positive_counts=pr_meter.tp,
true_negative_counts=pr_meter.tn,
false_negative_counts=pr_meter.fn,
false_positive_counts=pr_meter.fp,
precision=pr_meter.precision(),
recall=pr_meter.recall(),
global_step=epoch)
del x, y, outputs, batch_loss
return losses, valid_scores