BertTokenizer.from_pretrained(bert_model)) evaluator = ClassificationEvaluator(dset, device) (loss, acc, P, R, F1), plots, (labels, logits) = evaluator.evaluate( model, plot_callbacks=[plot_label_distribution], return_labels_logits=True) accs.append(acc) Ps.append(P) Rs.append(R) F1s.append(F1) labels_all.append(labels) logits_all.append(logits) with open(f'{args.model_dir}/pred_lab.txt', 'a+') as f: for p, l in zip(np.argmax(logits, axis=-1), labels): f.write(f'{i}\t{p}\t{l}\n') print(f"Macro avg accuracy: {sum(accs) / len(accs)}") print(f"Macro avg P: {sum(Ps) / len(Ps)}") print(f"Macro avg R: {sum(Rs) / len(Rs)}") print(f"Macro avg F1: {sum(F1s) / len(F1s)}") acc, P, R, F1 = acc_f1([l for run in logits_all for l in run], [l for run in labels_all for l in run]) print(f"Micro avg accuracy: {acc}") print(f"Micro avg P: {P}") print(f"Micro avg R: {R}") print(f"Micro avg F1: {F1}")
wandb.run.summary[f"{domain}-R"] = R wandb.run.summary[f"{domain}-F1"] = F1 wandb.run.summary[f"{domain}-Acc"] = acc # macro and micro F1 are only with respect to the 5 main splits if i < len(all_dsets): Ps.append(P) Rs.append(R) F1s.append(F1) accs.append(acc) labels_all.extend(labels) logits_all.extend(logits) with open(f'{args.model_dir}/{Path(wandb.run.dir).name}/pred_lab.txt', 'a+') as f: for p, l in zip(np.argmax(logits, axis=-1), labels): f.write(f'{domain}\t{p}\t{l}\n') acc, P, R, F1 = acc_f1(logits_all, labels_all) # Add to wandb wandb.run.summary[f'test-loss'] = loss wandb.run.summary[f'test-micro-acc'] = acc wandb.run.summary[f'test-micro-P'] = P wandb.run.summary[f'test-micro-R'] = R wandb.run.summary[f'test-micro-F1'] = F1 wandb.run.summary[f'test-macro-acc'] = sum(accs) / len(accs) wandb.run.summary[f'test-macro-P'] = sum(Ps) / len(Ps) wandb.run.summary[f'test-macro-R'] = sum(Rs) / len(Rs) wandb.run.summary[f'test-macro-F1'] = sum(F1s) / len(F1s) # wandb.log({f"label-distribution-test-{i}": plots[0]})
wandb.run.summary[f"{domain}-P"] = P wandb.run.summary[f"{domain}-R"] = R wandb.run.summary[f"{domain}-F1"] = F1 wandb.run.summary[f"{domain}-Acc"] = acc Ps.append(P) Rs.append(R) F1s.append(F1) accs.append(acc) labels_all.extend(labels) logits_all.extend(logits) with open(f'{args.model_dir}/{Path(wandb.run.dir).name}/pred_lab.txt', 'a+') as f: for p, l in zip(np.argmax(logits, axis=-1), labels): f.write(f'{domain}\t{p}\t{l}\n') acc, P, R, F1 = acc_f1(logits_all, labels_all) # Add to wandb wandb.run.summary[f'test-loss'] = loss wandb.run.summary[f'test-micro-acc'] = acc wandb.run.summary[f'test-micro-P'] = P wandb.run.summary[f'test-micro-R'] = R wandb.run.summary[f'test-micro-F1'] = F1 wandb.run.summary[f'test-macro-acc'] = sum(accs) / len(accs) wandb.run.summary[f'test-macro-P'] = sum(Ps) / len(Ps) wandb.run.summary[f'test-macro-R'] = sum(Rs) / len(Rs) wandb.run.summary[f'test-macro-F1'] = sum(F1s) / len(F1s) acc, P, R, F1 = acc_f1(logits_all_avg, labels_all_avg) # Add to wandb wandb.run.summary[f'test-loss-alt'] = loss