def testLogisticError():
k = 5
data = Data(k, 0, 0)
data.importDataFromMat()
data.normalize()
lg = LogisticLinearClassifier(0.03, 0.03, 576, k, data)
err_train, miss_train, err_val, miss_val = lg.train(30)
mis_fig = plt.figure()
ax2 = mis_fig.add_subplot(111)
ax2.plot(err_val, label="error (validation)")
ax2.plot(err_train, label="error (training)")
title = "std(val)=%f std(err)=%f" % (sp.std(err_val), sp.std(err_train))
mis_fig.suptitle(title)
ax2.set_ylabel("error")
ax2.set_xlabel("epoch")
plt.legend()
mis_fig = plt.figure()
ax2 = mis_fig.add_subplot(111)
ax2.plot(miss_val, label="misclassification ratio (validation)")
ax2.plot(miss_train, label="misclassification ratio (training)")
mis_fig.suptitle(title)
ax2.set_ylabel("misclassification ratio")
ax2.set_xlabel("epoch")
plt.legend()
results, cat = lg.classify(data.test_left, data.test_right)
lg.confusion_matrix(cat, data.test_cat.argmax(axis=0))
err = Error()
err, misclass = err.norm_total_error(results.T, data.test_cat, k)
print "Error on the test set " + str(err)
print "Misclassification ratio on the test set " + str(misclass)
def testSquaredError():
k = 5
data = Data(k, 0, 0)
data.importDataFromMat()
data.normalize()
sq = SquaredErrorLinearClassifier(2 ** 10, k)
sq.train(data.train_left, data.train_right, data.train_cat)
results, cat = sq.classify(data.test_left, data.test_right)
sq.confusion_matrix(cat, data.test_cat.argmax(axis=0))
err = Error()
err, misclass = err.norm_total_error(results.T, data.test_cat, k)
print "Error on the test set " + str(err)
print "Misclassification ratio on the test set " + str(misclass)
class Test:
def __init__(self, mlp, data, k):
self.mlp = mlp
self.data = data
self.k = k
self.error = Error()
def classify(self) :
self.test_val, self.test_classif = self.mlp.classify(self.data.test_left, self.data.test_right)
test_error, misclassified_test = self.error.norm_total_error(self.test_val, self.data.test_cat, self.k)
print "TEST ERROR :"+str(test_error)
print "TEST MISCLASSIFICATION RATIO :"+str(misclassified_test)
#Find a good example (large positive t*a), a bad one (large negative t*a) and a slightly bad one (negative t*a close to 0)
def examples(self) :
if self.k == 2 :
misclass = self.test_val * self.data.test_cat
large_pos = sp.argmax(misclass)
print "Largest positive t*a : "+str(max(misclass[misclass > 0]))
print "Category : "+str(self.data.test_cat[:,large_pos])
print "Found : "+str(self.test_classif[:,large_pos])
large_neg = sp.argmin(misclass)
print "Largest negative t*a : "+str(min(misclass[misclass < 0]))
print "Category : "+str(self.data.test_cat[:,large_neg])
print "Found : "+str(self.test_classif[:,large_neg])
misclass_neg = misclass
misclass_neg[misclass_neg>0] = -float("inf")
close_0 = sp.argmax(misclass_neg)
print "Closest to zero negative t*a : "+str(max(misclass_neg[misclass_neg < 0]))
print "Category : "+str(self.data.test_cat[:,close_0])
print "Found : "+str(self.test_classif[:,close_0])
#Show image
best_fig = plt.figure()
ax5 = best_fig.add_subplot(111)
ax5.imshow(sp.reshape(self.data.test_left[:,large_pos],(24,24)), cmap=plt.gray())
worst_fig = plt.figure()
ax3 = worst_fig.add_subplot(111)
ax3.imshow(sp.reshape(self.data.test_left[:,large_neg],(24,24)), cmap=plt.gray())
almost_fig = plt.figure()
ax4 = almost_fig.add_subplot(111)
ax4.imshow(sp.reshape(self.data.test_left[:,close_0],(24,24)), cmap=plt.gray())
else :
err = Error()
maxerr = -float("inf")
minerr = float("inf")
besterr = float("inf")
for i in range(self.test_val.shape[1]) :
error = 0.5*sp.sum(LA.norm(self.test_val[:,i]-self.data.test_cat[:,i])**2)
if self.test_classif[i] != self.data.test_cat[:,i].argmax() :
if error > maxerr :
maxerr = error
maxindex = i
if error < minerr :
minerr = error
minindex = i
else :
if error < besterr :
besterr = error
bestindex = i
print "Best : "+str(besterr)
print "Category : "+str(self.data.test_cat[:,bestindex].argmax())
print "Found : "+str(self.test_classif[bestindex])
print "Largest negative error : "+str(maxerr)
print "Category : "+str(self.data.test_cat[:,maxindex].argmax())
print "Found : "+str(self.test_classif[maxindex])
print "Smallest negative error : "+str(minerr)
print "Category : "+str(self.data.test_cat[:,minindex].argmax())
print "Found : "+str(self.test_classif[minindex])
#Show image
best_fig = plt.figure()
ax5 = best_fig.add_subplot(111)
ax5.imshow(sp.reshape(self.data.test_left[:,bestindex],(24,24)), cmap=plt.gray())
worst_fig = plt.figure()
ax3 = worst_fig.add_subplot(111)
ax3.imshow(sp.reshape(self.data.test_left[:,maxindex],(24,24)), cmap=plt.gray())
almost_fig = plt.figure()
ax4 = almost_fig.add_subplot(111)
ax4.imshow(sp.reshape(self.data.test_left[:,minindex],(24,24)), cmap=plt.gray())
def plot_confusion_matrix(self) :
if self.k == 2:
classif = self.test_classif
cat = self.data.test_cat
classif[self.test_classif == -1] = 0
cat[self.data.test_cat == -1] = 0
self.confusion_matrix(classif[0,:], cat[0,:])
else :
self.confusion_matrix(self.test_classif, self.data.test_cat.argmax(axis=0))
def confusion_matrix(self, result, expected) :
confusion_matrix = sp.zeros((self.k, self.k))
for i in range(result.shape[0]) :
confusion_matrix[result[i]][expected[i]] += 1
print confusion_matrix
conf = plt.figure()
ax = conf.add_subplot(111)
ax.imshow(confusion_matrix, cmap=cm.get_cmap(name='gray_r'), interpolation='nearest')
class TrainerValidator:
def __init__(self, k, nb_epochs, H1, H2, nu, mu, batchsize, data):
self.k = k
self.data = data
self.H1 = H1
self.H2 = H2
self.mu = mu
self.nu = nu
self.batchsize = batchsize
self.mlp = MLP(H1,H2,576, nu, mu, batchsize, self.k)
self.error = Error()
self.NUM_EPOCH = nb_epochs
self.validation_error = sp.zeros(self.NUM_EPOCH+1)
self.misclassified_val = sp.zeros(self.NUM_EPOCH+1)
self.training_error = sp.zeros(self.NUM_EPOCH+1)
self.misclassified_train = sp.zeros(self.NUM_EPOCH+1)
def trainAndClassify(self):
converge = 0
a = 4
var_thresh = 0.005
early_stopping = 0
for i in range(self.NUM_EPOCH+1):
self.data.shuffleData()
self.mlp.train(self.data.train_left, self.data.train_right, self.data.train_cat)
_, _, _, _, _, results_train, _, _, _, _, _, _ = self.mlp.forward_pass(self.data.train_left, self.data.train_right)
results_val, results_classif = self.mlp.classify(self.data.val_left, self.data.val_right)
self.training_error[i], self.misclassified_train[i] = self.error.norm_total_error(results_train, self.data.train_cat, self.k)
self.validation_error[i], self.misclassified_val[i] = self.error.norm_total_error(results_val, self.data.val_cat, self.k)
print "Epoch #"+str(i)+" Ratio of misclassified: "+str(self.misclassified_val[i])+" - Error: "+str(self.validation_error[i])
# Early stopping
if early_stopping :
if i > 0 :
if (self.validation_error[i]>(self.validation_error[i-1]*(1-var_thresh))) :
converge += 1
else :
if converge > 0 :
converge -= 1/2
if converge>=a :
print "Triggering early stopping - Cause : increasing(overfitting) or convergence of the error has been detected"
break
#self.mlp.test_gradient(self.data.val_left, self.data.val_right, self.data.val_cat)
def plotResults(self):
error_fig = plt.figure()
ax1 = error_fig.add_subplot(111)
ax1.plot(self.validation_error, label='validation error')
ax1.plot(self.training_error, label='training error')
ax1.set_ylabel('error')
ax1.set_xlabel('epoch')
title = "k=%d H1=%d H2=%d mu=%f nu=%f batchsize=%d std(val)=%f std(err)=%f" % (self.k, self.H1, self.H2, self.mu, self.nu, self.batchsize, sp.std(self.validation_error), sp.std(self.training_error) )
error_fig.suptitle(title)
plt.legend()
filename = "k=%d-H1=%d-H2=%d-mu=%f-nu=%f-batchsize=%d-nb_epoch=%d" % (self.k,self.H1, self.H2, self.mu, self.nu, self.batchsize, self.NUM_EPOCH)
plt.savefig('results/'+filename+"-error.png")
mis_fig = plt.figure()
ax2 = mis_fig.add_subplot(111)
ax2.plot(self.misclassified_val, label='misclassified ratio (validation)')
ax2.plot(self.misclassified_train, label='misclassified ratio (training)')
title = "k=%d H1=%d H2=%d mu=%f nu=%f batchsize=%d std(val)=%f std(err)=%f" % (self.k, self.H1, self.H2, self.mu, self.nu, self.batchsize, sp.std(self.misclassified_val), sp.std(self.misclassified_train) )
mis_fig.suptitle(title)
#ax2.set_xlim([1,self.NUM_EPOCH])
ax2.set_ylabel('misclassified')
ax2.set_xlabel('epoch')
plt.legend()
plt.savefig('results/'+filename+"-misclassified.png")
#plt.show()
def getMLP(self) :
return self.mlp
