def eval_epochs_done_callback(global_vars, label_ids, graph_fold=None, none_label_id=0): labels = np.asarray(global_vars['all_labels']) preds = np.asarray(global_vars['all_preds']) subtokens_mask = np.asarray(global_vars['all_subtokens_mask']) > 0.5 labels = labels[subtokens_mask] preds = preds[subtokens_mask] accuracy = sum(labels == preds) / labels.shape[0] logger.info(f'Accuracy: {accuracy}') # print predictions and labels for a small random subset of data sample_size = 20 i = 0 if preds.shape[0] > sample_size + 1: i = random.randint(0, preds.shape[0] - sample_size - 1) logger.info("Sampled preds: [%s]" % list2str(preds[i:i + sample_size])) logger.info("Sampled labels: [%s]" % list2str(labels[i:i + sample_size])) # remove labels from label_ids that don't appear in the dev set used_labels = set(labels) | set(preds) label_ids = \ {k: label_ids[k] for k, v in label_ids.items() if v in used_labels} logger.info(classification_report(labels, preds, target_names=label_ids)) # calculate and plot confusion_matrix if graph_fold: ids_to_labels = {label_ids[k]: k for k in label_ids} classes = [0] + \ [ids_to_labels[id] for id in sorted(label_ids.values())] cm = confusion_matrix(labels, preds) fig = plt.figure() ax = fig.add_subplot(111) ax.set(xticks=np.arange(cm.shape[1]), yticks=np.arange(cm.shape[0]), xticklabels=classes, yticklabels=classes, title='Confusion matrix', ylabel='True label', xlabel='Predicted label') cax = ax.matshow(cm) plt.title('Confusion matrix of the classifier') fig.colorbar(cax) plt.xlabel('Predicted') plt.ylabel('True') os.makedirs(graph_fold, exist_ok=True) plt.savefig(os.path.join(graph_fold, time.strftime('%Y%m%d-%H%M%S'))) return dict({'Accuracy': accuracy})
def _eval_epochs_done_callback(task_name, global_vars, label_ids, graph_fold=None, normalize_cm=True): labels = np.asarray(global_vars[task_name + '_all_labels']) preds = np.asarray(global_vars[task_name + '_all_preds']) subtokens_mask = np.asarray(global_vars['all_subtokens_mask']) > 0.5 labels = labels[subtokens_mask] preds = preds[subtokens_mask] accuracy = sum(labels == preds) / labels.shape[0] nemo.logging.info(f'Accuracy for task {task_name}: {accuracy}') # print predictions and labels for a small random subset of data sample_size = 20 i = 0 if preds.shape[0] > sample_size + 1: i = random.randint(0, preds.shape[0] - sample_size - 1) nemo.logging.info("Sampled preds: [%s]" % list2str(preds[i:i + sample_size])) nemo.logging.info("Sampled labels: [%s]" % list2str(labels[i:i + sample_size])) # remove labels from label_ids that don't appear in the dev set used_labels = set(labels) | set(preds) label_ids = \ {k: label_ids[k] for k, v in label_ids.items() if v in used_labels} nemo.logging.info( classification_report(labels, preds, target_names=label_ids)) # calculate and plot confusion_matrix if graph_fold: plot_confusion_matrix(label_ids, labels, preds, graph_fold, normalize=normalize_cm, prefix=task_name) return accuracy