BertTokenizer.from_pretrained(bert_model))
evaluator = ClassificationEvaluator(dset, device)
(loss, acc, P, R,
F1), plots, (labels, logits) = evaluator.evaluate(
model,
plot_callbacks=[plot_label_distribution],
return_labels_logits=True)
accs.append(acc)
Ps.append(P)
Rs.append(R)
F1s.append(F1)
labels_all.append(labels)
logits_all.append(logits)
with open(f'{args.model_dir}/pred_lab.txt', 'a+') as f:
for p, l in zip(np.argmax(logits, axis=-1), labels):
f.write(f'{i}\t{p}\t{l}\n')
print(f"Macro avg accuracy: {sum(accs) / len(accs)}")
print(f"Macro avg P: {sum(Ps) / len(Ps)}")
print(f"Macro avg R: {sum(Rs) / len(Rs)}")
print(f"Macro avg F1: {sum(F1s) / len(F1s)}")
acc, P, R, F1 = acc_f1([l for run in logits_all for l in run],
[l for run in labels_all for l in run])
print(f"Micro avg accuracy: {acc}")
print(f"Micro avg P: {P}")
print(f"Micro avg R: {R}")
print(f"Micro avg F1: {F1}")
wandb.run.summary[f"{domain}-R"] = R
wandb.run.summary[f"{domain}-F1"] = F1
wandb.run.summary[f"{domain}-Acc"] = acc
# macro and micro F1 are only with respect to the 5 main splits
if i < len(all_dsets):
Ps.append(P)
Rs.append(R)
F1s.append(F1)
accs.append(acc)
labels_all.extend(labels)
logits_all.extend(logits)
with open(f'{args.model_dir}/{Path(wandb.run.dir).name}/pred_lab.txt',
'a+') as f:
for p, l in zip(np.argmax(logits, axis=-1), labels):
f.write(f'{domain}\t{p}\t{l}\n')
acc, P, R, F1 = acc_f1(logits_all, labels_all)
# Add to wandb
wandb.run.summary[f'test-loss'] = loss
wandb.run.summary[f'test-micro-acc'] = acc
wandb.run.summary[f'test-micro-P'] = P
wandb.run.summary[f'test-micro-R'] = R
wandb.run.summary[f'test-micro-F1'] = F1
wandb.run.summary[f'test-macro-acc'] = sum(accs) / len(accs)
wandb.run.summary[f'test-macro-P'] = sum(Ps) / len(Ps)
wandb.run.summary[f'test-macro-R'] = sum(Rs) / len(Rs)
wandb.run.summary[f'test-macro-F1'] = sum(F1s) / len(F1s)
# wandb.log({f"label-distribution-test-{i}": plots[0]})
wandb.run.summary[f"{domain}-P"] = P
wandb.run.summary[f"{domain}-R"] = R
wandb.run.summary[f"{domain}-F1"] = F1
wandb.run.summary[f"{domain}-Acc"] = acc
Ps.append(P)
Rs.append(R)
F1s.append(F1)
accs.append(acc)
labels_all.extend(labels)
logits_all.extend(logits)
with open(f'{args.model_dir}/{Path(wandb.run.dir).name}/pred_lab.txt',
'a+') as f:
for p, l in zip(np.argmax(logits, axis=-1), labels):
f.write(f'{domain}\t{p}\t{l}\n')
acc, P, R, F1 = acc_f1(logits_all, labels_all)
# Add to wandb
wandb.run.summary[f'test-loss'] = loss
wandb.run.summary[f'test-micro-acc'] = acc
wandb.run.summary[f'test-micro-P'] = P
wandb.run.summary[f'test-micro-R'] = R
wandb.run.summary[f'test-micro-F1'] = F1
wandb.run.summary[f'test-macro-acc'] = sum(accs) / len(accs)
wandb.run.summary[f'test-macro-P'] = sum(Ps) / len(Ps)
wandb.run.summary[f'test-macro-R'] = sum(Rs) / len(Rs)
wandb.run.summary[f'test-macro-F1'] = sum(F1s) / len(F1s)
acc, P, R, F1 = acc_f1(logits_all_avg, labels_all_avg)
# Add to wandb
wandb.run.summary[f'test-loss-alt'] = loss
