def eval(val_loader, model):
'''
Run evaluation
'''
mses = AverageMeter()
pccs = AverageMeter()
less05s = AverageMeter()
less1s = AverageMeter()
avgs = AverageMeter()
# switch to eval mode
model.eval()
with torch.no_grad():
for i, (id, mask, target) in enumerate(val_loader):
# Forward pass
pred = model(id, mask)
# update all stats
mses.update(calculate_mse(pred, target).item(), id.size(0))
pccs.update(calculate_pcc(pred, target).item(), id.size(0))
less05s.update(calculate_less05(pred, target), id.size(0))
less1s.update(calculate_less1(pred, target), id.size(0))
avgs.update(calculate_avg(pred).item(), id.size(0))
return mses.avg, pccs.avg, less05s.avg, less1s.avg, avgs.avg
def get_single_stats(all_preds, targets):
mses = []
pccs = []
avgs = []
less05s = []
less1s = []
for preds in all_preds:
mses.append(
calculate_mse(torch.FloatTensor(preds),
torch.FloatTensor(targets)).item())
pccs.append(
calculate_pcc(torch.FloatTensor(preds),
torch.FloatTensor(targets)).item())
avgs.append(calculate_avg(torch.FloatTensor(preds)).item())
less05s.append(
calculate_less05(torch.FloatTensor(preds),
torch.FloatTensor(targets)))
less1s.append(
calculate_less1(torch.FloatTensor(preds),
torch.FloatTensor(targets)))
mse_mean = statistics.mean(mses)
mse_std = statistics.pstdev(mses)
pcc_mean = statistics.mean(pccs)
pcc_std = statistics.pstdev(pccs)
avg_mean = statistics.mean(avgs)
avg_std = statistics.pstdev(avgs)
less05_mean = statistics.mean(less05s)
less05_std = statistics.pstdev(less05s)
less1_mean = statistics.mean(less1s)
less1_std = statistics.pstdev(less1s)
return mse_mean, mse_std, pcc_mean, pcc_std, avg_mean, avg_std, less05_mean, less05_std, less1_mean, less1_std
def all_stats(self):
rmse = calculate_rmse(torch.FloatTensor(self.preds),
torch.FloatTensor(self.refs)).item()
pcc = calculate_pcc(torch.FloatTensor(self.preds),
torch.FloatTensor(self.refs)).item()
avg = calculate_avg(torch.FloatTensor(self.preds)).item()
less05 = calculate_less05(torch.FloatTensor(self.preds),
torch.FloatTensor(self.refs))
less1 = calculate_less1(torch.FloatTensor(self.preds),
torch.FloatTensor(self.refs))
return rmse, pcc, avg, less05, less1
def get_ensemble_stats(all_preds, targets):
y_sum = torch.zeros(len(all_preds[0]))
for preds in all_preds:
y_sum += torch.FloatTensor(preds)
ensemble_preds = y_sum / len(all_preds)
mse = calculate_mse(ensemble_preds, torch.FloatTensor(targets))
pcc = calculate_pcc(ensemble_preds, torch.FloatTensor(targets))
avg = calculate_avg(ensemble_preds)
less05 = calculate_less05(ensemble_preds, torch.FloatTensor(targets))
less1 = calculate_less1(ensemble_preds, torch.FloatTensor(targets))
return ensemble_preds.tolist(), mse.item(), pcc.item(), avg.item(
), less05, less1
for pred_dict in pred_dicts:
try:
pred = pred_dict[id]
pred_sum += pred
pred_counter += 1
except:
continue
pred_overall = pred_sum / pred_counter
preds.append(pred_overall)
# Get all the stats
mse = calculate_mse(torch.FloatTensor(preds),
torch.FloatTensor(refs)).item()
pcc = calculate_pcc(torch.FloatTensor(preds),
torch.FloatTensor(refs)).item()
avg = calculate_avg(torch.FloatTensor(preds)).item()
less05 = calculate_less05(torch.FloatTensor(preds),
torch.FloatTensor(refs))
less1 = calculate_less1(torch.FloatTensor(preds), torch.FloatTensor(refs))
print("ALL PARTS STATS\n")
print("MSE: ", mse)
print("PCC: ", pcc)
print("AVG: ", avg)
print("LESS05: ", less05)
print("LESS1: ", less1)
# Save the predicted scores
with open(out_file, 'w') as f:
text = 'SPEAKERID REF PRED'
f.write(text)