# Main running script import glob import os import params import Run import sys from score import KappaScore, MeanKappaScore RUN_VAL = True RUN_KAGGLE = False if RUN_VAL: train_mean_kappa = MeanKappaScore() val_mean_kappa = MeanKappaScore() for essay_set in params.ESSAY_SETS: total_domains = 1 if essay_set == 2: total_domains = 2 for domain in range(1, total_domains+1): run = Run.Run() run.setup('data/c_train.utf8ignore.tsv', 'data/c_val.utf8ignore.tsv', essay_set, domain) run.extract() run.learn() run.predict() train_score, test_score = run.eval() train_mean_kappa.add(train_score, weight=1.0/total_domains) val_mean_kappa.add(test_score, weight=1.0/total_domains)
def test_mean_quadratic_weighted_kappa(self): class MockKappa(KappaScore): def __init__(self, val): self.val = val def quadratic_weighted_kappa(self): return self.val mks = MeanKappaScore() mks.add(MockKappa(1)) mks.add(MockKappa(1)) kappa = mks.mean_quadratic_weighted_kappa() self.assertAlmostEqual(kappa, 0.999) mks = MeanKappaScore() mks.add(MockKappa(0.5)) mks.add(MockKappa(0.8)) kappa = mks.mean_quadratic_weighted_kappa([1,.5]) self.assertAlmostEqual(kappa, 0.624536446425734) mks = MeanKappaScore() mks.add(MockKappa(-1)) mks.add(MockKappa(1)) kappa = mks.mean_quadratic_weighted_kappa() self.assertAlmostEqual(kappa, 0.0)