def eval_epochs_done_callback(global_vars,
label_ids,
graph_fold=None,
none_label_id=0):
labels = np.asarray(global_vars['all_labels'])
preds = np.asarray(global_vars['all_preds'])
subtokens_mask = np.asarray(global_vars['all_subtokens_mask']) > 0.5
labels = labels[subtokens_mask]
preds = preds[subtokens_mask]
accuracy = sum(labels == preds) / labels.shape[0]
logger.info(f'Accuracy: {accuracy}')
# print predictions and labels for a small random subset of data
sample_size = 20
i = 0
if preds.shape[0] > sample_size + 1:
i = random.randint(0, preds.shape[0] - sample_size - 1)
logger.info("Sampled preds: [%s]" % list2str(preds[i:i + sample_size]))
logger.info("Sampled labels: [%s]" % list2str(labels[i:i + sample_size]))
# remove labels from label_ids that don't appear in the dev set
used_labels = set(labels) | set(preds)
label_ids = \
{k: label_ids[k] for k, v in label_ids.items() if v in used_labels}
logger.info(classification_report(labels, preds, target_names=label_ids))
# calculate and plot confusion_matrix
if graph_fold:
ids_to_labels = {label_ids[k]: k for k in label_ids}
classes = [0] + \
[ids_to_labels[id] for id in sorted(label_ids.values())]
cm = confusion_matrix(labels, preds)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set(xticks=np.arange(cm.shape[1]),
yticks=np.arange(cm.shape[0]),
xticklabels=classes,
yticklabels=classes,
title='Confusion matrix',
ylabel='True label',
xlabel='Predicted label')
cax = ax.matshow(cm)
plt.title('Confusion matrix of the classifier')
fig.colorbar(cax)
plt.xlabel('Predicted')
plt.ylabel('True')
os.makedirs(graph_fold, exist_ok=True)
plt.savefig(os.path.join(graph_fold, time.strftime('%Y%m%d-%H%M%S')))
return dict({'Accuracy': accuracy})
def _eval_epochs_done_callback(task_name,
global_vars,
label_ids,
graph_fold=None,
normalize_cm=True):
labels = np.asarray(global_vars[task_name + '_all_labels'])
preds = np.asarray(global_vars[task_name + '_all_preds'])
subtokens_mask = np.asarray(global_vars['all_subtokens_mask']) > 0.5
labels = labels[subtokens_mask]
preds = preds[subtokens_mask]
accuracy = sum(labels == preds) / labels.shape[0]
nemo.logging.info(f'Accuracy for task {task_name}: {accuracy}')
# print predictions and labels for a small random subset of data
sample_size = 20
i = 0
if preds.shape[0] > sample_size + 1:
i = random.randint(0, preds.shape[0] - sample_size - 1)
nemo.logging.info("Sampled preds: [%s]" %
list2str(preds[i:i + sample_size]))
nemo.logging.info("Sampled labels: [%s]" %
list2str(labels[i:i + sample_size]))
# remove labels from label_ids that don't appear in the dev set
used_labels = set(labels) | set(preds)
label_ids = \
{k: label_ids[k] for k, v in label_ids.items() if v in used_labels}
nemo.logging.info(
classification_report(labels, preds, target_names=label_ids))
# calculate and plot confusion_matrix
if graph_fold:
plot_confusion_matrix(label_ids,
labels,
preds,
graph_fold,
normalize=normalize_cm,
prefix=task_name)
return accuracy