def cb_multi_fi(cls, items): preds, golds = zip(*items) preds = np.array(preds) golds = np.array(golds) f11 = sklearn.metrics.f1_score(y_true=golds == 0, y_pred=preds == 0) f12 = sklearn.metrics.f1_score(y_true=golds == 1, y_pred=preds == 1) f13 = sklearn.metrics.f1_score(y_true=golds == 2, y_pred=preds == 2) avg_f1 = mean([f11, f12, f13]) return avg_f1
def calc_em(self, items): # Calculate exact matches - i.e. all in a pair of 5 are correct preds_sort = sorted(items, key=lambda x: x[0]) em_sums = [ int(preds_sort[5 * i][1]) + int(preds_sort[5 * i + 1][1]) + int(preds_sort[5 * i + 2][1]) + int(preds_sort[5 * i + 3][1]) + int(preds_sort[5 * i + 4][1]) for i in range(len(preds_sort) // 5) ] em_cors = [em_sums[i] == 5 for i in range(len(em_sums))] return mean(em_cors)
def calc_em(self, items): # Calculate exact matches - i.e. all in a pair of 4 are correct # NOTE: `items` is a tuple of (doc["group_id"], is_correct) preds_sort = sorted(items, key=lambda x: x[0]) em_sums = [ int(preds_sort[4 * i][1]) + int(preds_sort[4 * i + 1][1]) + int(preds_sort[4 * i + 2][1]) + int(preds_sort[4 * i + 3][1]) for i in range(len(preds_sort) // 4) ] em_cors = [em_sums[i] == 4 for i in range(len(em_sums))] return mean(em_cors)