def test_mean_quadratic_weighted_kappa(self):
class MockKappa(KappaScore):
def __init__(self, val):
self.val = val
def quadratic_weighted_kappa(self):
return self.val
mks = MeanKappaScore()
mks.add(MockKappa(1))
mks.add(MockKappa(1))
kappa = mks.mean_quadratic_weighted_kappa()
self.assertAlmostEqual(kappa, 0.999)
mks = MeanKappaScore()
mks.add(MockKappa(0.5))
mks.add(MockKappa(0.8))
kappa = mks.mean_quadratic_weighted_kappa([1,.5])
self.assertAlmostEqual(kappa, 0.624536446425734)
mks = MeanKappaScore()
mks.add(MockKappa(-1))
mks.add(MockKappa(1))
kappa = mks.mean_quadratic_weighted_kappa()
self.assertAlmostEqual(kappa, 0.0)
val_mean_kappa = MeanKappaScore()
for essay_set in params.ESSAY_SETS:
total_domains = 1
if essay_set == 2:
total_domains = 2
for domain in range(1, total_domains+1):
run = Run.Run()
run.setup('data/c_train.utf8ignore.tsv', 'data/c_val.utf8ignore.tsv', essay_set, domain)
run.extract()
run.learn()
run.predict()
train_score, test_score = run.eval()
train_mean_kappa.add(train_score, weight=1.0/total_domains)
val_mean_kappa.add(test_score, weight=1.0/total_domains)
print "--\n"
print "Overall Train / Test"
print "Kappa Score %f" %train_mean_kappa.mean_quadratic_weighted_kappa()
print "Kappa Score %f" %val_mean_kappa.mean_quadratic_weighted_kappa()
if RUN_KAGGLE:
fd = open('data/kaggle_out.tsv', 'w')
fd.write('prediction_id\tessay_id\tessay_set\tessay_weight\tpredicted_score\n')
for essay_set in params.ESSAY_SETS:
total_domains = 1
if essay_set == 2:
total_domains = 2
