def get_QWkappa(input): head,body = fio.ReadMatrix(input, True) metric = Metric() data = {} for i,row in enumerate(body): for coder, label in enumerate(row): if label == 'a': label = '0' label = int(label) if head[coder] not in data: data[ head[coder] ] = [] data[ head[coder] ].append(label) print 'annototor 1', '\t','annototor 2', '\t', 'accuracy', '\t', 'kappa', '\t', 'QWkappa' print head[0], '\t', head[1], '\t', metric.accuracy(data[head[0]], data[head[1]]), '\t', metric.kappa(data[head[0]], data[head[1]]), '\t', metric.QWkappa(data[head[0]], data[head[1]]) print head[0], '\t', head[2], '\t', metric.accuracy(data[head[0]], data[head[2]]), '\t', metric.kappa(data[head[0]], data[head[2]]), '\t', metric.QWkappa(data[head[0]], data[head[2]]) print head[1], '\t', head[2], '\t', metric.accuracy(data[head[1]], data[head[2]]), '\t', metric.kappa(data[head[1]], data[head[2]]), '\t', metric.QWkappa(data[head[1]], data[head[2]]) print '', '\t', 'Average', '\t', np.mean([metric.accuracy(data[head[0]], data[head[1]]), metric.accuracy(data[head[0]], data[head[2]]), metric.accuracy(data[head[1]], data[head[2]])]), '\t',\ np.mean([metric.kappa(data[head[0]], data[head[1]]), metric.kappa(data[head[0]], data[head[2]]), metric.kappa(data[head[1]], data[head[2]])]), '\t',\ np.mean([metric.QWkappa(data[head[0]], data[head[1]]), metric.QWkappa(data[head[0]], data[head[2]]), metric.QWkappa(data[head[1]], data[head[2]])]) print metric.confusion_matrix(data[head[0]], data[head[1]]) return 0
def get_metrics_H0(datadir): metric = Metric() body = [] for feature in ['quality_rubric', 'quality_binary_model', 'quality_New', 'quality_firstnode']: file = feature+'.label' print file labels, predicts, _ = load_label(datadir + file) fio.WriteMatrix(datadir + file + '.cm', metric.confusion_matrix(labels, predicts), None) row = [file] row.append(metric.accuracy(labels, predicts)) row.append(metric.kappa(labels, predicts)) row.append(metric.QWkappa(labels, predicts)) # row += metric.cv_accuracy(labels, predicts) # row += metric.cv_kappa(labels, predicts) # row += metric.cv_QWkappa(labels, predicts) body.append(row) output = datadir+'H0.txt' print output fio.WriteMatrix(output, body, header=None)