def testCrossValidation(self):
numExamples = 10
folds = 2
indices = Sampling.crossValidation(folds, numExamples)
self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([5, 6, 7, 8, 9], [0, 1, 2, 3, 4]))
self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 1, 2, 3, 4], [5, 6, 7, 8, 9]))
indices = Sampling.crossValidation(3, numExamples)
self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([3, 4, 5, 6, 7, 8, 9], [0, 1, 2]))
self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 1, 2, 6, 7, 8, 9], [3, 4, 5]))
self.assertEquals((list(indices[2][0]), list(indices[2][1])), ([0, 1, 2, 3, 4, 5], [6, 7, 8, 9]))
indices = Sampling.crossValidation(4, numExamples)
self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([2, 3, 4, 5, 6, 7, 8, 9], [0, 1]))
self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 1, 5, 6, 7, 8, 9], [2, 3, 4]))
self.assertEquals((list(indices[2][0]), list(indices[2][1])), ([0, 1, 2, 3, 4, 7, 8, 9], [5, 6]))
self.assertEquals((list(indices[3][0]), list(indices[3][1])), ([0, 1, 2, 3, 4, 5, 6], [7, 8, 9]))
indices = Sampling.crossValidation(numExamples, numExamples)
self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([1, 2, 3, 4, 5, 6, 7, 8, 9], [0]))
self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 2, 3, 4, 5, 6, 7, 8, 9], [1]))
self.assertEquals((list(indices[2][0]), list(indices[2][1])), ([0, 1, 3, 4, 5, 6, 7, 8, 9], [2]))
self.assertEquals((list(indices[3][0]), list(indices[3][1])), ([0, 1, 2, 4, 5, 6, 7, 8, 9], [3]))
self.assertEquals((list(indices[4][0]), list(indices[4][1])), ([0, 1, 2, 3, 5, 6, 7, 8, 9], [4]))
self.assertRaises(ValueError, Sampling.crossValidation, numExamples+1, numExamples)
self.assertRaises(ValueError, Sampling.crossValidation, 0, numExamples)
self.assertRaises(ValueError, Sampling.crossValidation, -1, numExamples)
self.assertRaises(ValueError, Sampling.crossValidation, folds, 1)
def testParallelPen(self):
#Check if penalisation == inf when treeSize < gamma
numExamples = 100
X, y = data.make_regression(numExamples)
learner = DecisionTreeLearner(pruneType="CART", maxDepth=10, minSplit=2)
paramDict = {}
paramDict["setGamma"] = numpy.array(numpy.round(2**numpy.arange(1, 10, 0.5)-1), dtype=numpy.int)
folds = 3
alpha = 1.0
Cvs = numpy.array([(folds-1)*alpha])
idx = Sampling.crossValidation(folds, X.shape[0])
resultsList = learner.parallelPen(X, y, idx, paramDict, Cvs)
learner, trainErrors, currentPenalties = resultsList[0]
learner.setGamma(2**10)
treeSize = 0
#Let's work out the size of the unpruned tree
for trainInds, testInds in idx:
trainX = X[trainInds, :]
trainY = y[trainInds]
learner.learnModel(trainX, trainY)
treeSize += learner.tree.size
treeSize /= float(folds)
self.assertTrue(numpy.isinf(currentPenalties[paramDict["setGamma"]>treeSize]).all())
self.assertTrue(not numpy.isinf(currentPenalties[paramDict["setGamma"]<treeSize]).all())
def testParallelVfPenRbf2(self):
#Test support vector regression
folds = 3
Cv = numpy.array([4.0])
idx = Sampling.crossValidation(folds, self.X.shape[0])
svm = self.svm
svm.setKernel("gaussian")
svm.setSvmType("Epsilon_SVR")
resultsList = svm.parallelVfPenRbf(self.X,
self.y,
idx,
Cv,
type="Epsilon_SVR")
tol = 10**-6
bestError = 100
meanErrors2 = numpy.zeros(
(svm.gammas.shape[0], svm.epsilons.shape[0], svm.Cs.shape[0]))
for i in range(svm.Cs.shape[0]):
C = svm.Cs[i]
for j in range(svm.gammas.shape[0]):
gamma = svm.gammas[j]
for k in range(svm.epsilons.shape[0]):
epsilon = svm.epsilons[k]
penalty = 0
for trainInds, testInds in idx:
trainX = self.X[trainInds, :]
trainY = self.y[trainInds]
svm.setGamma(gamma)
svm.setC(C)
svm.setEpsilon(epsilon)
svm.learnModel(trainX, trainY)
predY = svm.predict(self.X)
predTrainY = svm.predict(trainX)
penalty += svm.getMetricMethod()(
predY, self.y) - svm.getMetricMethod()(predTrainY,
trainY)
penalty = penalty * Cv[0] / len(idx)
svm.learnModel(self.X, self.y)
predY = svm.predict(self.X)
meanErrors2[j, k, i] = svm.getMetricMethod()(
predY, self.y) + penalty
if meanErrors2[j, k, i] < bestError:
bestC = C
bestGamma = gamma
bestEpsilon = epsilon
bestError = meanErrors2[j, k, i]
bestSVM, trainErrors, currentPenalties = resultsList[0]
meanErrors = trainErrors + currentPenalties
self.assertEquals(bestC, bestSVM.getC())
self.assertEquals(bestGamma, bestSVM.getGamma())
self.assertEquals(bestEpsilon, bestSVM.getEpsilon())
self.assertTrue(numpy.linalg.norm(meanErrors2 - meanErrors) < tol)
def cvModelSelection(self, graph, paramList, paramFunc, folds, errorFunc):
"""
ParamList is a list of lists of parameters and paramFunc
is a list of the corresponding functions to call with the parameters
as arguments. Note that a parameter can also be a tuple which is expanded
out before the function is called.
e.g.
paramList = [[1, 2], [2, 1], [12, 1]]
paramFunc = [predictor.setC, predictor.setD]
"""
inds = Sampling.crossValidation(folds, graph.getNumEdges())
errors = numpy.zeros((len(paramList), folds))
allEdges = graph.getAllEdges()
for i in range(len(paramList)):
paramSet = paramList[i]
logging.debug("Using paramSet=" + str(paramSet))
for j in range(len(paramSet)):
if type(paramSet[j]) == tuple:
paramFunc[j](*paramSet[j])
else:
paramFunc[j](paramSet[j])
predY = numpy.zeros(0)
y = numpy.zeros(0)
j = 0
for (trainInds, testInds) in inds:
trainEdges = allEdges[trainInds, :]
testEdges = allEdges[testInds, :]
trainGraph = SparseGraph(graph.getVertexList(),
graph.isUndirected())
trainGraph.addEdges(trainEdges,
graph.getEdgeValues(trainEdges))
testGraph = SparseGraph(graph.getVertexList(),
graph.isUndirected())
testGraph.addEdges(testEdges, graph.getEdgeValues(testEdges))
self.learnModel(trainGraph)
predY = self.predictEdges(testGraph, testGraph.getAllEdges())
y = testGraph.getEdgeValues(testGraph.getAllEdges())
#Note that the order the edges is different in testGraphs as
#opposed to graph when calling getAllEdges()
errors[i, j] = errorFunc(y, predY)
j = j + 1
logging.info("Error of current fold: " +
str(numpy.mean(errors[i, :])))
meanErrors = numpy.mean(errors, 1)
strErrors = numpy.std(errors, 1)
return meanErrors, strErrors
def testParallelPen(self):
folds = 3
Cv = numpy.array([4.0])
idx = Sampling.crossValidation(folds, self.X.shape[0])
svm = self.svm
svm.setKernel("gaussian")
paramDict = {}
paramDict["setC"] = svm.getCs()
paramDict["setGamma"] = svm.getGammas()
resultsList = svm.parallelPen(self.X, self.y, idx, paramDict, Cv)
tol = 10**-6
bestError = 1
trainErrors2 = numpy.zeros((svm.Cs.shape[0], svm.gammas.shape[0]))
penalties2 = numpy.zeros((svm.Cs.shape[0], svm.gammas.shape[0]))
meanErrors2 = numpy.zeros((svm.Cs.shape[0], svm.gammas.shape[0]))
for i in range(svm.Cs.shape[0]):
C = svm.Cs[i]
for j in range(svm.gammas.shape[0]):
gamma = svm.gammas[j]
penalty = 0
for trainInds, testInds in idx:
trainX = self.X[trainInds, :]
trainY = self.y[trainInds]
svm.setGamma(gamma)
svm.setC(C)
svm.learnModel(trainX, trainY)
predY = svm.predict(self.X)
predTrainY = svm.predict(trainX)
penalty += Evaluator.binaryError(
predY, self.y) - Evaluator.binaryError(
predTrainY, trainY)
penalty = penalty * Cv[0] / len(idx)
svm.learnModel(self.X, self.y)
predY = svm.predict(self.X)
trainErrors2[i, j] = Evaluator.binaryError(predY, self.y)
penalties2[i, j] = penalty
meanErrors2[i,
j] = Evaluator.binaryError(predY, self.y) + penalty
if meanErrors2[i, j] < bestError:
bestC = C
bestGamma = gamma
bestError = meanErrors2[i, j]
bestSVM, trainErrors, currentPenalties = resultsList[0]
meanErrors = trainErrors + currentPenalties
self.assertEquals(bestC, bestSVM.getC())
self.assertEquals(bestGamma, bestSVM.getGamma())
self.assertTrue(numpy.linalg.norm(meanErrors2.T - meanErrors) < tol)
self.assertTrue(numpy.linalg.norm(trainErrors2.T - trainErrors) < tol)
self.assertTrue(
numpy.linalg.norm(penalties2.T - currentPenalties) < tol)
def cvPrune(self, validX, validY):
"""
We do something like reduced error pruning but we use cross validation
to decide which nodes to prune.
"""
#First set the value of the vertices using the training set.
#Reset all alphas to zero
inds = Sampling.crossValidation(self.folds, validX.shape[0])
for i in self.tree.getAllVertexIds():
self.tree.getVertex(i).setAlpha(0.0)
self.tree.getVertex(i).setTestError(0.0)
for trainInds, testInds in inds:
rootId = (0,)
root = self.tree.getVertex(rootId)
root.setTrainInds(trainInds)
root.setTestInds(testInds)
root.tempValue = numpy.mean(validY[trainInds])
nodeStack = [(rootId, root.tempValue)]
while len(nodeStack) != 0:
(nodeId, value) = nodeStack.pop()
node = self.tree.getVertex(nodeId)
tempTrainInds = node.getTrainInds()
tempTestInds = node.getTestInds()
node.setTestError(numpy.sum((validY[tempTestInds] - node.tempValue)**2) + node.getTestError())
childIds = [self.getLeftChildId(nodeId), self.getRightChildId(nodeId)]
for childId in childIds:
if self.tree.vertexExists(childId):
child = self.tree.getVertex(childId)
if childId[-1] == 0:
childInds = validX[tempTrainInds, node.getFeatureInd()] < node.getThreshold()
else:
childInds = validX[tempTrainInds, node.getFeatureInd()] >= node.getThreshold()
if childInds.sum() !=0:
value = numpy.mean(validY[tempTrainInds[childInds]])
child.tempValue = value
child.setTrainInds(tempTrainInds[childInds])
nodeStack.append((childId, value))
if childId[-1] == 0:
childInds = validX[tempTestInds, node.getFeatureInd()] < node.getThreshold()
else:
childInds = validX[tempTestInds, node.getFeatureInd()] >= node.getThreshold()
child.setTestInds(tempTestInds[childInds])
self.computeAlphas()
self.prune()
def cvModelSelection(self, graph, paramList, paramFunc, folds, errorFunc):
"""
ParamList is a list of lists of parameters and paramFunc
is a list of the corresponding functions to call with the parameters
as arguments. Note that a parameter can also be a tuple which is expanded
out before the function is called.
e.g.
paramList = [[1, 2], [2, 1], [12, 1]]
paramFunc = [predictor.setC, predictor.setD]
"""
inds = Sampling.crossValidation(folds, graph.getNumEdges())
errors = numpy.zeros((len(paramList), folds))
allEdges = graph.getAllEdges()
for i in range(len(paramList)):
paramSet = paramList[i]
logging.debug("Using paramSet=" + str(paramSet))
for j in range(len(paramSet)):
if type(paramSet[j]) == tuple:
paramFunc[j](*paramSet[j])
else:
paramFunc[j](paramSet[j])
predY = numpy.zeros(0)
y = numpy.zeros(0)
j = 0
for (trainInds, testInds) in inds:
trainEdges = allEdges[trainInds, :]
testEdges = allEdges[testInds, :]
trainGraph = SparseGraph(graph.getVertexList(), graph.isUndirected())
trainGraph.addEdges(trainEdges, graph.getEdgeValues(trainEdges))
testGraph = SparseGraph(graph.getVertexList(), graph.isUndirected())
testGraph.addEdges(testEdges, graph.getEdgeValues(testEdges))
self.learnModel(trainGraph)
predY = self.predictEdges(testGraph, testGraph.getAllEdges())
y = testGraph.getEdgeValues(testGraph.getAllEdges())
#Note that the order the edges is different in testGraphs as
#opposed to graph when calling getAllEdges()
errors[i, j] = errorFunc(y, predY)
j = j+1
logging.info("Error of current fold: " + str(numpy.mean(errors[i, :])))
meanErrors = numpy.mean(errors, 1)
strErrors = numpy.std(errors, 1)
return meanErrors, strErrors
def testParallelPen(self):
folds = 3
Cv = numpy.array([4.0])
idx = Sampling.crossValidation(folds, self.X.shape[0])
svm = self.svm
svm.setKernel("gaussian")
paramDict = {}
paramDict["setC"] = svm.getCs()
paramDict["setGamma"] = svm.getGammas()
resultsList = svm.parallelPen(self.X, self.y, idx, paramDict, Cv)
tol = 10**-6
bestError = 1
trainErrors2 = numpy.zeros((svm.Cs.shape[0], svm.gammas.shape[0]))
penalties2 = numpy.zeros((svm.Cs.shape[0], svm.gammas.shape[0]))
meanErrors2 = numpy.zeros((svm.Cs.shape[0], svm.gammas.shape[0]))
for i in range(svm.Cs.shape[0]):
C = svm.Cs[i]
for j in range(svm.gammas.shape[0]):
gamma = svm.gammas[j]
penalty = 0
for trainInds, testInds in idx:
trainX = self.X[trainInds, :]
trainY = self.y[trainInds]
svm.setGamma(gamma)
svm.setC(C)
svm.learnModel(trainX, trainY)
predY = svm.predict(self.X)
predTrainY = svm.predict(trainX)
penalty += Evaluator.binaryError(predY, self.y) - Evaluator.binaryError(predTrainY, trainY)
penalty = penalty*Cv[0]/len(idx)
svm.learnModel(self.X, self.y)
predY = svm.predict(self.X)
trainErrors2[i, j] = Evaluator.binaryError(predY, self.y)
penalties2[i, j] = penalty
meanErrors2[i, j] = Evaluator.binaryError(predY, self.y) + penalty
if meanErrors2[i, j] < bestError:
bestC = C
bestGamma = gamma
bestError = meanErrors2[i, j]
bestSVM, trainErrors, currentPenalties = resultsList[0]
meanErrors = trainErrors + currentPenalties
self.assertEquals(bestC, bestSVM.getC())
self.assertEquals(bestGamma, bestSVM.getGamma())
self.assertTrue(numpy.linalg.norm(meanErrors2.T - meanErrors) < tol)
self.assertTrue(numpy.linalg.norm(trainErrors2.T - trainErrors) < tol)
self.assertTrue(numpy.linalg.norm(penalties2.T - currentPenalties) < tol)
def generateLearner(self, X, y):
"""
Train using the given examples and labels, and use model selection to
find the best parameters.
"""
if numpy.unique(y).shape[0] != 2:
print(y)
raise ValueError("Can only operate on binary data")
#Do model selection first
if self.sampleSize == None:
idx = Sampling.crossValidation(self.folds, X.shape[0])
learner, meanErrors = self.parallelModelSelect(X, y, idx, self.paramDict)
else:
idx = Sampling.crossValidation(self.folds, self.sampleSize)
inds = numpy.random.permutation(X.shape[0])[0:self.sampleSize]
learner, meanErrors = self.parallelModelSelect(X[inds, :], y[inds], idx, self.paramDict)
learner = self.getBestLearner(meanErrors, self.paramDict, X, y)
return learner
def evaluateCv(self, X, y, folds, metricMethod=Evaluator.binaryError):
"""
Compute the cross validation according to a given metric.
"""
Parameter.checkInt(folds, 2, float('inf'))
idx = Sampling.crossValidation(folds, y.shape[0])
metrics = AbstractPredictor.evaluateLearn(X, y, idx, self.learnModel, self.predict, metricMethod)
mean = numpy.mean(metrics, 0)
var = numpy.var(metrics, 0)
return (mean, var)
def testParallelVfPenRbf2(self):
#Test support vector regression
folds = 3
Cv = numpy.array([4.0])
idx = Sampling.crossValidation(folds, self.X.shape[0])
svm = self.svm
svm.setKernel("gaussian")
svm.setSvmType("Epsilon_SVR")
resultsList = svm.parallelVfPenRbf(self.X, self.y, idx, Cv, type="Epsilon_SVR")
tol = 10**-6
bestError = 100
meanErrors2 = numpy.zeros((svm.gammas.shape[0], svm.epsilons.shape[0], svm.Cs.shape[0]))
for i in range(svm.Cs.shape[0]):
C = svm.Cs[i]
for j in range(svm.gammas.shape[0]):
gamma = svm.gammas[j]
for k in range(svm.epsilons.shape[0]):
epsilon = svm.epsilons[k]
penalty = 0
for trainInds, testInds in idx:
trainX = self.X[trainInds, :]
trainY = self.y[trainInds]
svm.setGamma(gamma)
svm.setC(C)
svm.setEpsilon(epsilon)
svm.learnModel(trainX, trainY)
predY = svm.predict(self.X)
predTrainY = svm.predict(trainX)
penalty += svm.getMetricMethod()(predY, self.y) - svm.getMetricMethod()(predTrainY, trainY)
penalty = penalty*Cv[0]/len(idx)
svm.learnModel(self.X, self.y)
predY = svm.predict(self.X)
meanErrors2[j, k, i] = svm.getMetricMethod()(predY, self.y) + penalty
if meanErrors2[j, k, i] < bestError:
bestC = C
bestGamma = gamma
bestEpsilon = epsilon
bestError = meanErrors2[j, k, i]
bestSVM, trainErrors, currentPenalties = resultsList[0]
meanErrors = trainErrors + currentPenalties
self.assertEquals(bestC, bestSVM.getC())
self.assertEquals(bestGamma, bestSVM.getGamma())
self.assertEquals(bestEpsilon, bestSVM.getEpsilon())
self.assertTrue(numpy.linalg.norm(meanErrors2 - meanErrors) < tol)
def evaluateCv(self, X, y, folds, metricMethod=Evaluator.binaryError):
"""
Compute the cross validation according to a given metric.
"""
Parameter.checkInt(folds, 2, float('inf'))
idx = Sampling.crossValidation(folds, y.shape[0])
metrics = AbstractPredictor.evaluateLearn(X, y, idx, self.learnModel,
self.predict, metricMethod)
mean = numpy.mean(metrics, 0)
var = numpy.var(metrics, 0)
return (mean, var)
def testParallelPenaltyGrid(self):
folds = 3
idx = Sampling.crossValidation(folds, self.X.shape[0])
randomForest = RandomForest()
trainX = self.X[0:40, :]
trainY = self.y[0:40]
paramDict = {}
paramDict["setMinSplit"] = randomForest.getMinSplits()
paramDict["setMaxDepth"] = randomForest.getMaxDepths()
idealPenalties = randomForest.parallelPenaltyGrid(trainX, trainY, self.X, self.y, paramDict)
def generateLearner(self, X, y):
"""
Train using the given examples and labels, and use model selection to
find the best parameters.
"""
if numpy.unique(y).shape[0] != 2:
print(y)
raise ValueError("Can only operate on binary data")
#Do model selection first
if self.sampleSize == None:
idx = Sampling.crossValidation(self.folds, X.shape[0])
learner, meanErrors = self.parallelModelSelect(
X, y, idx, self.paramDict)
else:
idx = Sampling.crossValidation(self.folds, self.sampleSize)
inds = numpy.random.permutation(X.shape[0])[0:self.sampleSize]
learner, meanErrors = self.parallelModelSelect(
X[inds, :], y[inds], idx, self.paramDict)
learner = self.getBestLearner(meanErrors, self.paramDict, X, y)
return learner
def testParallelVfcvRbf2(self):
#In this test we try SVM regression
folds = 3
idx = Sampling.crossValidation(folds, self.X.shape[0])
svm = self.svm
svm.setKernel("gaussian")
svm.setSvmType("Epsilon_SVR")
bestSVM, meanErrors = svm.parallelVfcvRbf(self.X,
self.y,
idx,
type="Epsilon_SVR")
tol = 10**-6
bestError = 100
meanErrors2 = numpy.zeros(
(svm.gammas.shape[0], svm.epsilons.shape[0], svm.Cs.shape[0]))
for i in range(svm.Cs.shape[0]):
C = svm.Cs[i]
for j in range(svm.gammas.shape[0]):
gamma = svm.gammas[j]
for k in range(svm.epsilons.shape[0]):
epsilon = svm.epsilons[k]
error = 0
for trainInds, testInds in idx:
trainX = self.X[trainInds, :]
trainY = self.y[trainInds]
testX = self.X[testInds, :]
testY = self.y[testInds]
svm.setGamma(gamma)
svm.setC(C)
svm.setEpsilon(epsilon)
svm.learnModel(trainX, trainY)
predY = svm.predict(testX)
error += svm.getMetricMethod()(predY, testY)
meanErrors2[j, k, i] = error / len(idx)
if error < bestError:
bestC = C
bestGamma = gamma
bestError = error
bestEpsilon = epsilon
self.assertEquals(bestC, bestSVM.getC())
self.assertEquals(bestGamma, bestSVM.getGamma())
self.assertEquals(bestEpsilon, bestSVM.getEpsilon())
self.assertTrue(numpy.linalg.norm(meanErrors2 - meanErrors) < tol)
def testParallelPenaltyGrid(self):
folds = 3
idx = Sampling.crossValidation(folds, self.X.shape[0])
decisionTree = DecisionTree()
bestLearner, meanErrors = decisionTree.parallelVfcv(self.X, self.y, idx)
trainX = self.X[0:40, :]
trainY = self.y[0:40]
paramDict = {}
paramDict["setMinSplit"] = decisionTree.getMinSplits()
paramDict["setMaxDepth"] = decisionTree.getMaxDepths()
idealPenalties = decisionTree.parallelPenaltyGrid(trainX, trainY, self.X, self.y, paramDict)
def testParallelPenaltyGrid(self):
folds = 3
idx = Sampling.crossValidation(folds, self.X.shape[0])
randomForest = RandomForest()
trainX = self.X[0:40, :]
trainY = self.y[0:40]
paramDict = {}
paramDict["setMinSplit"] = randomForest.getMinSplits()
paramDict["setMaxDepth"] = randomForest.getMaxDepths()
idealPenalties = randomForest.parallelPenaltyGrid(
trainX, trainY, self.X, self.y, paramDict)
def testParallelVfcvRbf2(self):
#In this test we try SVM regression
folds = 3
idx = Sampling.crossValidation(folds, self.X.shape[0])
svm = self.svm
svm.setKernel("gaussian")
svm.setSvmType("Epsilon_SVR")
bestSVM, meanErrors = svm.parallelVfcvRbf(self.X, self.y, idx, type="Epsilon_SVR")
tol = 10**-6
bestError = 100
meanErrors2 = numpy.zeros((svm.gammas.shape[0], svm.epsilons.shape[0], svm.Cs.shape[0]))
for i in range(svm.Cs.shape[0]):
C = svm.Cs[i]
for j in range(svm.gammas.shape[0]):
gamma = svm.gammas[j]
for k in range(svm.epsilons.shape[0]):
epsilon = svm.epsilons[k]
error = 0
for trainInds, testInds in idx:
trainX = self.X[trainInds, :]
trainY = self.y[trainInds]
testX = self.X[testInds, :]
testY = self.y[testInds]
svm.setGamma(gamma)
svm.setC(C)
svm.setEpsilon(epsilon)
svm.learnModel(trainX, trainY)
predY = svm.predict(testX)
error += svm.getMetricMethod()(predY, testY)
meanErrors2[j, k, i] = error/len(idx)
if error < bestError:
bestC = C
bestGamma = gamma
bestError = error
bestEpsilon = epsilon
self.assertEquals(bestC, bestSVM.getC())
self.assertEquals(bestGamma, bestSVM.getGamma())
self.assertEquals(bestEpsilon, bestSVM.getEpsilon())
self.assertTrue(numpy.linalg.norm(meanErrors2 - meanErrors) < tol)
def testParallelModelSelect(self):
folds = 3
idx = Sampling.crossValidation(folds, self.X.shape[0])
svm = self.svm
svm.setKernel("gaussian")
paramDict = {}
paramDict["setC"] = svm.getCs()
paramDict["setGamma"] = svm.getGammas()
bestSVM, meanErrors = svm.parallelModelSelect(self.X, self.y, idx,
paramDict)
tol = 10**-6
bestError = 1
meanErrors2 = numpy.zeros((svm.Cs.shape[0], svm.gammas.shape[0]))
print("Computing real grid")
for i in range(svm.Cs.shape[0]):
C = svm.Cs[i]
for j in range(svm.gammas.shape[0]):
gamma = svm.gammas[j]
error = 0
for trainInds, testInds in idx:
trainX = self.X[trainInds, :]
trainY = self.y[trainInds]
testX = self.X[testInds, :]
testY = self.y[testInds]
svm.setGamma(gamma)
svm.setC(C)
svm.learnModel(trainX, trainY)
predY = svm.predict(testX)
error += Evaluator.binaryError(predY, testY)
meanErrors2[i, j] = error / len(idx)
if error < bestError:
bestC = C
bestGamma = gamma
bestError = error
self.assertEquals(bestC, bestSVM.getC())
self.assertEquals(bestGamma, bestSVM.getGamma())
self.assertTrue(numpy.linalg.norm(meanErrors2.T - meanErrors) < tol)
def testParallelModelSelect(self):
folds = 3
idx = Sampling.crossValidation(folds, self.X.shape[0])
svm = self.svm
svm.setKernel("gaussian")
paramDict = {}
paramDict["setC"] = svm.getCs()
paramDict["setGamma"] = svm.getGammas()
bestSVM, meanErrors = svm.parallelModelSelect(self.X, self.y, idx, paramDict)
tol = 10**-6
bestError = 1
meanErrors2 = numpy.zeros((svm.Cs.shape[0], svm.gammas.shape[0]))
print("Computing real grid")
for i in range(svm.Cs.shape[0]):
C = svm.Cs[i]
for j in range(svm.gammas.shape[0]):
gamma = svm.gammas[j]
error = 0
for trainInds, testInds in idx:
trainX = self.X[trainInds, :]
trainY = self.y[trainInds]
testX = self.X[testInds, :]
testY = self.y[testInds]
svm.setGamma(gamma)
svm.setC(C)
svm.learnModel(trainX, trainY)
predY = svm.predict(testX)
error += Evaluator.binaryError(predY, testY)
meanErrors2[i, j] = error/len(idx)
if error < bestError:
bestC = C
bestGamma = gamma
bestError = error
self.assertEquals(bestC, bestSVM.getC())
self.assertEquals(bestGamma, bestSVM.getGamma())
self.assertTrue(numpy.linalg.norm(meanErrors2.T - meanErrors) < tol)
def testGetBestLearner(self):
svm = self.svm
paramDict = {}
paramDict["setC"] = svm.getCs()
paramDict["setGamma"] = svm.getGammas()
errors = numpy.random.rand(svm.getCs().shape[0],
svm.getGammas().shape[0])
folds = 5
idx = Sampling.crossValidation(folds, self.X.shape[0])
svm.normModelSelect = True
svm.setKernel("gaussian")
learner = svm.getBestLearner(errors, paramDict, self.X, self.y, idx)
bestC = learner.getC()
#Find the best norm
bestInds = numpy.unravel_index(numpy.argmin(errors), errors.shape)
learner.setC(svm.getCs()[bestInds[0]])
learner.setGamma(svm.getGammas()[bestInds[1]])
norms = []
for trainInds, testInds in idx:
validX = self.X[trainInds, :]
validY = self.y[trainInds]
learner.learnModel(validX, validY)
norms.append(learner.weightNorm())
bestNorm = numpy.array(norms).mean()
norms = numpy.zeros(paramDict["setC"].shape[0])
for i, C in enumerate(paramDict["setC"]):
learner.setC(C)
learner.learnModel(self.X, self.y)
norms[i] = learner.weightNorm()
bestC2 = paramDict["setC"][numpy.abs(norms - bestNorm).argmin()]
self.assertEquals(bestC, bestC2)
def testGetBestLearner(self):
svm = self.svm
paramDict = {}
paramDict["setC"] = svm.getCs()
paramDict["setGamma"] = svm.getGammas()
errors = numpy.random.rand(svm.getCs().shape[0], svm.getGammas().shape[0])
folds = 5
idx = Sampling.crossValidation(folds, self.X.shape[0])
svm.normModelSelect = True
svm.setKernel("gaussian")
learner = svm.getBestLearner(errors, paramDict, self.X, self.y, idx)
bestC = learner.getC()
#Find the best norm
bestInds = numpy.unravel_index(numpy.argmin(errors), errors.shape)
learner.setC(svm.getCs()[bestInds[0]])
learner.setGamma(svm.getGammas()[bestInds[1]])
norms = []
for trainInds, testInds in idx:
validX = self.X[trainInds, :]
validY = self.y[trainInds]
learner.learnModel(validX, validY)
norms.append(learner.weightNorm())
bestNorm = numpy.array(norms).mean()
norms = numpy.zeros(paramDict["setC"].shape[0])
for i, C in enumerate(paramDict["setC"]):
learner.setC(C)
learner.learnModel(self.X, self.y)
norms[i] = learner.weightNorm()
bestC2 = paramDict["setC"][numpy.abs(norms-bestNorm).argmin()]
self.assertEquals(bestC, bestC2)
def learningRate(self, X, y, foldsSet, paramDict):
"""
Find a matrix beta which has the same dimensions as the parameter grid.
Each value in the grid represents the learning rate with respect to
those particular parameters.
:param X: The examples as rows
:type X: :class:`numpy.ndarray`
:param y: The binary -1/+1 labels
:type y: :class:`numpy.ndarray`
:param foldsSet: A list of folds to try.
:param paramDict: A dictionary index by the method name and with value as an array of values
:type X: :class:`dict`
"""
try:
from sklearn import linear_model
except ImportError:
raise
gridSize = []
gridInds = []
for key in paramDict.keys():
gridSize.append(paramDict[key].shape[0])
gridInds.append(numpy.arange(paramDict[key].shape[0]))
betaGrid = numpy.ones(tuple(gridSize))
gridSize.insert(0, foldsSet.shape[0])
penalties = numpy.zeros(tuple(gridSize))
Cvs = numpy.array([1])
for i in range(foldsSet.shape[0]):
folds = foldsSet[i]
logging.debug("Folds " + str(folds))
idx = Sampling.crossValidation(folds, X.shape[0])
resultsList = self.parallelPen(X, y, idx, paramDict, Cvs)
bestLearner, trainErrors, currentPenalties = resultsList[0]
penalties[i, :] = currentPenalties
indexIter = itertools.product(*gridInds)
for inds in indexIter:
inds2 = [slice(0, penalties.shape[0])]
inds2.extend(inds)
inds2 = tuple(inds2)
tempPenalties = penalties[inds2]
penInds = numpy.logical_and(numpy.isfinite(tempPenalties),
tempPenalties > 0)
penInds = numpy.squeeze(penInds)
tempPenalties = tempPenalties[penInds].flatten()
tempfoldsSet = numpy.array(foldsSet, numpy.float)[penInds]
if tempPenalties.shape[0] > 1:
xp = numpy.log((tempfoldsSet - 1) / tempfoldsSet * X.shape[0])
yp = numpy.log(tempPenalties) + numpy.log(tempfoldsSet)
clf = linear_model.LinearRegression()
clf.fit(numpy.array([xp]).T, yp)
betaGrid[inds] = clf.coef_[0]
return -betaGrid
def cvPrune(self, validX, validY):
"""
We do something like reduced error pruning but we use cross validation
to decide which nodes to prune.
"""
#First set the value of the vertices using the training set.
#Reset all alphas to zero
inds = Sampling.crossValidation(self.folds, validX.shape[0])
for i in self.tree.getAllVertexIds():
self.tree.getVertex(i).setAlpha(0.0)
self.tree.getVertex(i).setTestError(0.0)
for trainInds, testInds in inds:
rootId = (0, )
root = self.tree.getVertex(rootId)
root.setTrainInds(trainInds)
root.setTestInds(testInds)
root.tempValue = numpy.mean(validY[trainInds])
nodeStack = [(rootId, root.tempValue)]
while len(nodeStack) != 0:
(nodeId, value) = nodeStack.pop()
node = self.tree.getVertex(nodeId)
tempTrainInds = node.getTrainInds()
tempTestInds = node.getTestInds()
node.setTestError(
numpy.sum((validY[tempTestInds] - node.tempValue)**2) +
node.getTestError())
childIds = [
self.getLeftChildId(nodeId),
self.getRightChildId(nodeId)
]
for childId in childIds:
if self.tree.vertexExists(childId):
child = self.tree.getVertex(childId)
if childId[-1] == 0:
childInds = validX[
tempTrainInds,
node.getFeatureInd()] < node.getThreshold()
else:
childInds = validX[
tempTrainInds,
node.getFeatureInd()] >= node.getThreshold()
if childInds.sum() != 0:
value = numpy.mean(
validY[tempTrainInds[childInds]])
child.tempValue = value
child.setTrainInds(tempTrainInds[childInds])
nodeStack.append((childId, value))
if childId[-1] == 0:
childInds = validX[
tempTestInds,
node.getFeatureInd()] < node.getThreshold()
else:
childInds = validX[
tempTestInds,
node.getFeatureInd()] >= node.getThreshold()
child.setTestInds(tempTestInds[childInds])
self.computeAlphas()
self.prune()
def run():
for i in range(2):
print("Iteration " + str(i))
idx = Sampling.crossValidation(self.folds, numExamples)
learner.parallelPen(X, Y, idx, self.paramDict, Cvs)
idx = sampleMethod(folds, validY.shape[0])
svmGridResults = learner.parallelPen(validX, validY, idx, paramDict, Cvs)
for result in svmGridResults:
learner, trainErrors, currentPenalties = result
print(numpy.mean(trainErrors), numpy.mean(currentPenalties))
"""
#Figure out why the penalty is increasing
X = trainX
y = trainY
for i in range(foldsSet.shape[0]):
folds = foldsSet[i]
idx = Sampling.crossValidation(folds, validX.shape[0])
penalty = 0
fullError = 0
trainError = 0
learner.learnModel(validX, validY)
predY = learner.predict(X)
predValidY = learner.predict(validX)
idealPenalty = Evaluator.rootMeanSqError(predY, y) - Evaluator.rootMeanSqError(predValidY, validY)
for trainInds, testInds in idx:
trainX = validX[trainInds, :]
trainY = validY[trainInds]
#learner.setGamma(gamma)
def testParallelVfcv(self):
folds = 3
idx = Sampling.crossValidation(folds, self.X.shape[0])
decisionTree = DecisionTree()
bestLearner, meanErrors = decisionTree.parallelVfcv(self.X, self.y, idx)
def learningRate(self, X, y, foldsSet, paramDict):
"""
Find a matrix beta which has the same dimensions as the parameter grid.
Each value in the grid represents the learning rate with respect to
those particular parameters.
:param X: The examples as rows
:type X: :class:`numpy.ndarray`
:param y: The binary -1/+1 labels
:type y: :class:`numpy.ndarray`
:param foldsSet: A list of folds to try.
:param paramDict: A dictionary index by the method name and with value as an array of values
:type X: :class:`dict`
"""
try:
from sklearn import linear_model
except ImportError:
raise
gridSize = []
gridInds = []
for key in paramDict.keys():
gridSize.append(paramDict[key].shape[0])
gridInds.append(numpy.arange(paramDict[key].shape[0]))
betaGrid = numpy.ones(tuple(gridSize))
gridSize.insert(0, foldsSet.shape[0])
penalties = numpy.zeros(tuple(gridSize))
Cvs = numpy.array([1])
for i in range(foldsSet.shape[0]):
folds = foldsSet[i]
logging.debug("Folds " + str(folds))
idx = Sampling.crossValidation(folds, X.shape[0])
resultsList = self.parallelPen(X, y, idx, paramDict, Cvs)
bestLearner, trainErrors, currentPenalties = resultsList[0]
penalties[i, :] = currentPenalties
indexIter = itertools.product(*gridInds)
for inds in indexIter:
inds2 = [slice(0, penalties.shape[0])]
inds2.extend(inds)
inds2 = tuple(inds2)
tempPenalties = penalties[inds2]
penInds = numpy.logical_and(numpy.isfinite(tempPenalties), tempPenalties>0)
penInds = numpy.squeeze(penInds)
tempPenalties = tempPenalties[penInds].flatten()
tempfoldsSet = numpy.array(foldsSet, numpy.float)[penInds]
if tempPenalties.shape[0] > 1:
xp = numpy.log((tempfoldsSet-1)/tempfoldsSet*X.shape[0])
yp = numpy.log(tempPenalties)+numpy.log(tempfoldsSet)
clf = linear_model.LinearRegression()
clf.fit(numpy.array([xp]).T, yp)
betaGrid[inds] = clf.coef_[0]
return -betaGrid
def testModelSelect(self):
"""
We test the results on some data and compare to SVR.
"""
numExamples = 200
X, y = data.make_regression(numExamples, noise=0.5)
X = Standardiser().standardiseArray(X)
y = Standardiser().standardiseArray(y)
trainX = X[0:100, :]
trainY = y[0:100]
testX = X[100:, :]
testY = y[100:]
learner = DecisionTreeLearner(maxDepth=20, minSplit=10, pruneType="REP-CV")
learner.setPruneCV(8)
paramDict = {}
paramDict["setGamma"] = numpy.linspace(0.0, 1.0, 10)
paramDict["setPruneCV"] = numpy.arange(6, 11, 2, numpy.int)
folds = 5
idx = Sampling.crossValidation(folds, trainX.shape[0])
bestTree, cvGrid = learner.parallelModelSelect(trainX, trainY, idx, paramDict)
predY = bestTree.predict(testX)
error = Evaluator.rootMeanSqError(testY, predY)
print(error)
learner = DecisionTreeLearner(maxDepth=20, minSplit=5, pruneType="CART")
paramDict = {}
paramDict["setGamma"] = numpy.linspace(0.0, 1.0, 50)
folds = 5
idx = Sampling.crossValidation(folds, trainX.shape[0])
bestTree, cvGrid = learner.parallelModelSelect(trainX, trainY, idx, paramDict)
predY = bestTree.predict(testX)
error = Evaluator.rootMeanSqError(testY, predY)
print(error)
return
#Let's compare to the SVM
learner2 = LibSVM(kernel='gaussian', type="Epsilon_SVR")
paramDict = {}
paramDict["setC"] = 2.0**numpy.arange(-10, 14, 2, dtype=numpy.float)
paramDict["setGamma"] = 2.0**numpy.arange(-10, 4, 2, dtype=numpy.float)
paramDict["setEpsilon"] = learner2.getEpsilons()
idx = Sampling.crossValidation(folds, trainX.shape[0])
bestSVM, cvGrid = learner2.parallelModelSelect(trainX, trainY, idx, paramDict)
predY = bestSVM.predict(testX)
error = Evaluator.rootMeanSqError(testY, predY)
print(error)
def run():
for i in range(5):
print("Iteration " + str(i))
idx = Sampling.crossValidation(folds, numExamples)
learner.parallelPen(X, Y, idx, paramDict, Cvs)
def run():
for i in range(5):
print("Iteration " + str(i))
idx = Sampling.crossValidation(folds, numExamples)
learner.parallelModelSelect(X, Y, idx, paramDict)
def run():
for i in range(5):
print("Iteration " + str(i))
idx = Sampling.crossValidation(self.folds, numExamples)
learner.parallelModelSelect(X, Y, idx, self.paramDict)
meanPenalties = numpy.zeros((numGammas, numEpsilons, numCs))
meanBetaPenalties = numpy.zeros((numGammas, numEpsilons, numCs))
meanIdealPenalities = numpy.zeros((numGammas, numEpsilons, numCs))
for j in range(numRealisations):
print("")
logging.debug("j=" + str(j))
trainX, trainY, testX, testY = loadMethod(dataDir, datasetName, j)
logging.debug("Loaded dataset with " + str(trainX.shape) + " train and " + str(testX.shape) + " test examples")
trainInds = numpy.random.permutation(trainX.shape[0])[0:sampleSize]
trainX = trainX[trainInds,:]
trainY = trainY[trainInds]
idx = Sampling.crossValidation(folds, trainX.shape[0])
Cvs = [(folds-1)*alpha, beta[j, sampleSizeInd, :]]
#Now try penalisation
methodInd = 0
resultsList = learner.parallelPen(trainX, trainY, idx, paramDict, Cvs)
bestLearner, trainErrors, currentPenalties = resultsList[0]
meanPenalties += currentPenalties
predY = bestLearner.predict(testX)
#Learning rate penalisation
methodInd = 1
bestLearner, trainErrors, currentPenalties = resultsList[1]
meanBetaPenalties += currentPenalties
predY = bestLearner.predict(testX)
