def testRootMeanSqError(self):
y = numpy.array([1,2,3])
predY = numpy.array([1,2,3])
self.assertEquals(Evaluator.rootMeanSqError(y, predY), 0.0)
y = numpy.array([1,2,3])
predY = numpy.array([1,2,2])
self.assertEquals(Evaluator.rootMeanSqError(y, predY), float(1)/numpy.sqrt(3))
predY = numpy.array([1,2])
self.assertRaises(ValueError, Evaluator.rootMeanSqError, y, predY)
def testRootMeanSqError(self):
y = numpy.array([1, 2, 3])
predY = numpy.array([1, 2, 3])
self.assertEquals(Evaluator.rootMeanSqError(y, predY), 0.0)
y = numpy.array([1, 2, 3])
predY = numpy.array([1, 2, 2])
self.assertEquals(Evaluator.rootMeanSqError(y, predY),
float(1) / numpy.sqrt(3))
predY = numpy.array([1, 2])
self.assertRaises(ValueError, Evaluator.rootMeanSqError, y, predY)
def testCvPrune(self):
numExamples = 500
X, y = data.make_regression(numExamples)
y = Standardiser().standardiseArray(y)
numTrain = numpy.round(numExamples * 0.33)
numValid = numpy.round(numExamples * 0.33)
trainX = X[0:numTrain, :]
trainY = y[0:numTrain]
validX = X[numTrain:numTrain+numValid, :]
validY = y[numTrain:numTrain+numValid]
testX = X[numTrain+numValid:, :]
testY = y[numTrain+numValid:]
learner = DecisionTreeLearner()
learner.learnModel(trainX, trainY)
error1 = Evaluator.rootMeanSqError(learner.predict(testX), testY)
#print(learner.getTree())
unprunedTree = learner.tree.copy()
learner.setGamma(1000)
learner.cvPrune(trainX, trainY)
self.assertEquals(unprunedTree.getNumVertices(), learner.tree.getNumVertices())
learner.setGamma(100)
learner.cvPrune(trainX, trainY)
#Test if pruned tree is subtree of current:
for vertexId in learner.tree.getAllVertexIds():
self.assertTrue(vertexId in unprunedTree.getAllVertexIds())
#The error should be better after pruning
learner.learnModel(trainX, trainY)
#learner.cvPrune(validX, validY, 0.0, 5)
learner.repPrune(validX, validY)
error2 = Evaluator.rootMeanSqError(learner.predict(testX), testY)
self.assertTrue(error1 >= error2)
def testModelSelect(self):
"""
We test the results on some data and compare to SVR.
"""
numExamples = 200
X, y = data.make_regression(numExamples, noise=0.5)
X = Standardiser().standardiseArray(X)
y = Standardiser().standardiseArray(y)
trainX = X[0:100, :]
trainY = y[0:100]
testX = X[100:, :]
testY = y[100:]
learner = DecisionTreeLearner(maxDepth=20, minSplit=10, pruneType="REP-CV")
learner.setPruneCV(8)
paramDict = {}
paramDict["setGamma"] = numpy.linspace(0.0, 1.0, 10)
paramDict["setPruneCV"] = numpy.arange(6, 11, 2, numpy.int)
folds = 5
idx = Sampling.crossValidation(folds, trainX.shape[0])
bestTree, cvGrid = learner.parallelModelSelect(trainX, trainY, idx, paramDict)
predY = bestTree.predict(testX)
error = Evaluator.rootMeanSqError(testY, predY)
print(error)
learner = DecisionTreeLearner(maxDepth=20, minSplit=5, pruneType="CART")
paramDict = {}
paramDict["setGamma"] = numpy.linspace(0.0, 1.0, 50)
folds = 5
idx = Sampling.crossValidation(folds, trainX.shape[0])
bestTree, cvGrid = learner.parallelModelSelect(trainX, trainY, idx, paramDict)
predY = bestTree.predict(testX)
error = Evaluator.rootMeanSqError(testY, predY)
print(error)
return
#Let's compare to the SVM
learner2 = LibSVM(kernel='gaussian', type="Epsilon_SVR")
paramDict = {}
paramDict["setC"] = 2.0**numpy.arange(-10, 14, 2, dtype=numpy.float)
paramDict["setGamma"] = 2.0**numpy.arange(-10, 4, 2, dtype=numpy.float)
paramDict["setEpsilon"] = learner2.getEpsilons()
idx = Sampling.crossValidation(folds, trainX.shape[0])
bestSVM, cvGrid = learner2.parallelModelSelect(trainX, trainY, idx, paramDict)
predY = bestSVM.predict(testX)
error = Evaluator.rootMeanSqError(testY, predY)
print(error)
#Figure out why the penalty is increasing
X = trainX
y = trainY
for i in range(foldsSet.shape[0]):
folds = foldsSet[i]
idx = Sampling.crossValidation(folds, validX.shape[0])
penalty = 0
fullError = 0
trainError = 0
learner.learnModel(validX, validY)
predY = learner.predict(X)
predValidY = learner.predict(validX)
idealPenalty = Evaluator.rootMeanSqError(predY, y) - Evaluator.rootMeanSqError(predValidY, validY)
for trainInds, testInds in idx:
trainX = validX[trainInds, :]
trainY = validY[trainInds]
#learner.setGamma(gamma)
#learner.setC(C)
learner.learnModel(trainX, trainY)
predY = learner.predict(validX)
predTrainY = learner.predict(trainX)
fullError += Evaluator.rootMeanSqError(predY, validY)
trainError += Evaluator.rootMeanSqError(predTrainY, trainY)
penalty += Evaluator.rootMeanSqError(predY, validY) - Evaluator.rootMeanSqError(predTrainY, trainY)
print((folds-1)*fullError/folds, (folds-1)*trainError/folds, (folds-1)*penalty/folds)
minAlpha = alpha
if alpha > maxAlpha:
maxAlpha = alpha
numAlphas = 100
alphas = numpy.linspace(maxAlpha+0.1, minAlpha, numAlphas)
errors = numpy.zeros(numAlphas)
for i in range(alphas.shape[0]):
#learner.learnModel(trainX, trainY)
learner.setAlphaThreshold(alphas[i])
learner.cvPrune(trainX, trainY)
#learner.cvPrune(validX, validY, alphas[numpy.argmin(errors)])
#learner.prune(validX, validY, alphas[i])
predY = learner.predict(testX)
errors[i] = Evaluator.rootMeanSqError(predY, testY)
plt.figure(3)
plt.scatter(alphas, errors)
#Now plot best tree
plt.figure(4)
learner.learnModel(trainX, trainY)
#learner.cvPrune(validX, validY, alphas[numpy.argmin(errors)])
learner.setAlphaThreshold(alphas[numpy.argmin(errors)])
learner.cvPrune(trainX, trainY)
rootId = learner.tree.getRootId()
displayTree(learner, rootId, 0, 1, 0, 1, colormap)
plt.show()
