# Main running script
import glob
import os
import params
import Run
import sys
from score import KappaScore, MeanKappaScore
RUN_VAL = True
RUN_KAGGLE = False
if RUN_VAL:
train_mean_kappa = MeanKappaScore()
val_mean_kappa = MeanKappaScore()
for essay_set in params.ESSAY_SETS:
total_domains = 1
if essay_set == 2:
total_domains = 2
for domain in range(1, total_domains+1):
run = Run.Run()
run.setup('data/c_train.utf8ignore.tsv', 'data/c_val.utf8ignore.tsv', essay_set, domain)
run.extract()
run.learn()
run.predict()
train_score, test_score = run.eval()
train_mean_kappa.add(train_score, weight=1.0/total_domains)
val_mean_kappa.add(test_score, weight=1.0/total_domains)
def test_mean_quadratic_weighted_kappa(self):
class MockKappa(KappaScore):
def __init__(self, val):
self.val = val
def quadratic_weighted_kappa(self):
return self.val
mks = MeanKappaScore()
mks.add(MockKappa(1))
mks.add(MockKappa(1))
kappa = mks.mean_quadratic_weighted_kappa()
self.assertAlmostEqual(kappa, 0.999)
mks = MeanKappaScore()
mks.add(MockKappa(0.5))
mks.add(MockKappa(0.8))
kappa = mks.mean_quadratic_weighted_kappa([1,.5])
self.assertAlmostEqual(kappa, 0.624536446425734)
mks = MeanKappaScore()
mks.add(MockKappa(-1))
mks.add(MockKappa(1))
kappa = mks.mean_quadratic_weighted_kappa()
self.assertAlmostEqual(kappa, 0.0)
