def get_QWkappa(input):
head,body = fio.ReadMatrix(input, True)
metric = Metric()
data = {}
for i,row in enumerate(body):
for coder, label in enumerate(row):
if label == 'a': label = '0'
label = int(label)
if head[coder] not in data:
data[ head[coder] ] = []
data[ head[coder] ].append(label)
print 'annototor 1', '\t','annototor 2', '\t', 'accuracy', '\t', 'kappa', '\t', 'QWkappa'
print head[0], '\t', head[1], '\t', metric.accuracy(data[head[0]], data[head[1]]), '\t', metric.kappa(data[head[0]], data[head[1]]), '\t', metric.QWkappa(data[head[0]], data[head[1]])
print head[0], '\t', head[2], '\t', metric.accuracy(data[head[0]], data[head[2]]), '\t', metric.kappa(data[head[0]], data[head[2]]), '\t', metric.QWkappa(data[head[0]], data[head[2]])
print head[1], '\t', head[2], '\t', metric.accuracy(data[head[1]], data[head[2]]), '\t', metric.kappa(data[head[1]], data[head[2]]), '\t', metric.QWkappa(data[head[1]], data[head[2]])
print '', '\t', 'Average', '\t', np.mean([metric.accuracy(data[head[0]], data[head[1]]), metric.accuracy(data[head[0]], data[head[2]]), metric.accuracy(data[head[1]], data[head[2]])]), '\t',\
np.mean([metric.kappa(data[head[0]], data[head[1]]), metric.kappa(data[head[0]], data[head[2]]), metric.kappa(data[head[1]], data[head[2]])]), '\t',\
np.mean([metric.QWkappa(data[head[0]], data[head[1]]), metric.QWkappa(data[head[0]], data[head[2]]), metric.QWkappa(data[head[1]], data[head[2]])])
print metric.confusion_matrix(data[head[0]], data[head[1]])
return 0
def get_metrics_H0(datadir):
metric = Metric()
body = []
for feature in ['quality_rubric', 'quality_binary_model', 'quality_New', 'quality_firstnode']:
file = feature+'.label'
print file
labels, predicts, _ = load_label(datadir + file)
fio.WriteMatrix(datadir + file + '.cm', metric.confusion_matrix(labels, predicts), None)
row = [file]
row.append(metric.accuracy(labels, predicts))
row.append(metric.kappa(labels, predicts))
row.append(metric.QWkappa(labels, predicts))
# row += metric.cv_accuracy(labels, predicts)
# row += metric.cv_kappa(labels, predicts)
# row += metric.cv_QWkappa(labels, predicts)
body.append(row)
output = datadir+'H0.txt'
print output
fio.WriteMatrix(output, body, header=None)
