def eval_epoch(self, model, dataloader, log_prefix=""):
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
model = model.to(device)
model.eval()
metrics = defaultdict(list)
lwlrap = utils.lwlrap_accumulator()
for batch in tqdm(dataloader):
with torch.no_grad():
x = batch['logmel'].to(device)
y = batch['labels'].to(device)
out = model(x)
loss = F.binary_cross_entropy_with_logits(out, y)
probs = torch.sigmoid(out).cpu().data.numpy()
lrap = label_ranking_average_precision_score(
batch['labels'], probs)
lwlrap.accumulate_samples(batch['labels'], probs)
metrics['lrap'].append(lrap)
metrics['loss'].append(loss.item())
metrics = {key: np.mean(values) for key, values in metrics.items()}
metrics['lwlrap'] = lwlrap.overall_lwlrap()
for name, value in metrics.items():
if log_prefix != '':
name = log_prefix + '/' + name
self.eval_writer.add_scalar(name,
value,
global_step=self.global_step)
fig = plt.figure(figsize=(12, 9))
z = lwlrap.per_class_lwlrap() * lwlrap.per_class_weight()
plt.bar(np.arange(len(z)), z)
plt.hlines(np.mean(z), 0, len(z), linestyles='dashed')
plt.ylim([0, 0.013])
plt.xlim([-1, 80])
plt.grid()
self.eval_writer.add_figure('per_class_weighted_lwlrap',
fig,
global_step=self.global_step)
fig = plt.figure(figsize=(12, 9))
z = lwlrap.per_class_lwlrap()
plt.bar(np.arange(len(z)), z)
plt.hlines(np.mean(z), 0, len(z), linestyles='dashed')
plt.xlim([-1, 80])
plt.grid()
self.eval_writer.add_figure('per_class_lwlrap',
fig,
global_step=self.global_step)
return metrics
def validate(valloader, model, criterion, epoch, use_cuda, mode):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
lrap = AverageMeter()
lwlrap_acc = lwlrap_accumulator()
# switch to evaluate mode
model.eval()
end = time.time()
bar = Bar(f'{mode}', max=len(valloader))
with torch.no_grad():
for batch_idx, (inputs, targets) in enumerate(valloader):
# measure data loading time
data_time.update(time.time() - end)
if use_cuda:
inputs, targets = inputs.cuda(), targets.cuda(non_blocking=True)
# compute output
outputs = model(inputs)
loss = criterion(outputs, targets)
# measure accuracy and record loss
lwlrap_acc.accumulate_samples(targets, outputs)
losses.update(loss.item(), inputs.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.3f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f} | LRAP: {lrap: .4f}'.format(
batch=batch_idx + 1,
size=len(valloader),
data=data_time.avg,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg,
lrap=lwlrap_acc.overall_lwlrap()
)
bar.next()
bar.finish()
return (losses.avg, lwlrap_acc.overall_lwlrap())