def cb_multi_fi(cls, items):
preds, golds = zip(*items)
preds = np.array(preds)
golds = np.array(golds)
f11 = sklearn.metrics.f1_score(y_true=golds == 0, y_pred=preds == 0)
f12 = sklearn.metrics.f1_score(y_true=golds == 1, y_pred=preds == 1)
f13 = sklearn.metrics.f1_score(y_true=golds == 2, y_pred=preds == 2)
avg_f1 = mean([f11, f12, f13])
return avg_f1
def calc_em(self, items):
# Calculate exact matches - i.e. all in a pair of 5 are correct
preds_sort = sorted(items, key=lambda x: x[0])
em_sums = [
int(preds_sort[5 * i][1]) + int(preds_sort[5 * i + 1][1]) +
int(preds_sort[5 * i + 2][1]) + int(preds_sort[5 * i + 3][1]) +
int(preds_sort[5 * i + 4][1]) for i in range(len(preds_sort) // 5)
]
em_cors = [em_sums[i] == 5 for i in range(len(em_sums))]
return mean(em_cors)
def calc_em(self, items):
# Calculate exact matches - i.e. all in a pair of 4 are correct
# NOTE: `items` is a tuple of (doc["group_id"], is_correct)
preds_sort = sorted(items, key=lambda x: x[0])
em_sums = [
int(preds_sort[4 * i][1])
+ int(preds_sort[4 * i + 1][1])
+ int(preds_sort[4 * i + 2][1])
+ int(preds_sort[4 * i + 3][1])
for i in range(len(preds_sort) // 4)
]
em_cors = [em_sums[i] == 4 for i in range(len(em_sums))]
return mean(em_cors)
