def csv_agreement(file1, file2):
f1 = pandas.read_csv(file1)
d1 = dict(zip(f1.image, f1.level))
f2 = pandas.read_csv(file2)
d2 = dict(zip(f2.image, f2.level))
overlap_images = set(f1.image).intersection(set(f2.image))
overlaps = {}
y1 = []
y2 = []
for image in overlap_images:
y1.append(d1[image])
y2.append(d2[image])
overlaps[image] = [d1[image], d2[image]]
kappa, M = QWK(numpy.array(y1), numpy.array(y2))
print "Kappa = %.5f" % kappa
print_confusion_matrix(M)
return overlaps
def csv_agreement(file1, file2):
f1 = pandas.read_csv(file1)
d1 = dict(zip(f1.image, f1.level))
f2 = pandas.read_csv(file2)
d2 = dict(zip(f2.image, f2.level))
overlap_images = set(f1.image).intersection(set(f2.image))
overlaps = {}
y1 = []
y2 = []
for image in overlap_images:
y1.append(d1[image])
y2.append(d2[image])
overlaps[image] = [d1[image], d2[image]]
K = max(numpy.array(y1).max(), numpy.array(y2).max()) + 1
kappa, M = QWK(numpy.array(y1), numpy.array(y2), K)
print "Kappa = %.5f" % kappa
print "Accuracy = %.5f" % (numpy.diag(M).sum() / float(M.sum()))
print_confusion_matrix(M)
save_confusion_matrix_plot(M)
return overlaps
def validate(self, print_confusion_mat):
"""
Iterates through validation minibatches
"""
batch_valid_losses = []
valid_predictions = []
for j in range(self.n_valid_batches):
batch_valid_loss, prediction = self.validate_batch(j)
batch_valid_losses.append(batch_valid_loss)
valid_predictions.extend(prediction)
[kappa, M] = QWK(self.valid_y.get_value(borrow=True), numpy.array(valid_predictions), K=self.ds.K)
val_loss = numpy.mean(batch_valid_losses)
# housekeeping
self.historical_val_losses.append([self.iter, val_loss])
self.historical_val_kappas.append([self.iter, kappa])
print(' epoch %i, minibatch %i/%i, validation error %f %%' %
(self.epoch, (self.iter + 1) % self.n_train_batches, self.n_train_batches,
val_loss * 100.))
print(' kappa on validation set is: %f' % kappa)
if print_confusion_mat:
print_confusion_matrix(M)
return [val_loss, kappa]
def testPerfect(self):
y = numpy.array([0, 1, 2, 3, 4])
k, M = QWK(y, y)
# print_confusion_matrix(M)
self.failUnless(k == 1)
def testAwful(self):
y = numpy.array([0, 1, 2, 3, 4])
yrev = numpy.array(list(reversed(y)))
k, M = QWK(y, yrev)
self.failUnless(round(k, 4) == -1)
def testAlmostPerfect(self):
y_true = numpy.array([4, 0, 2, 3, 1, 2, 2, 3])
y_pred = numpy.array([4, 0, 2, 3, 1, 2, 3, 2])
k, M = QWK(y_true, y_pred)
self.failUnless(round(k, 4) == 0.9080)