def testCrossValidation(self): numExamples = 10 folds = 2 indices = Sampling.crossValidation(folds, numExamples) self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([5, 6, 7, 8, 9], [0, 1, 2, 3, 4])) self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 1, 2, 3, 4], [5, 6, 7, 8, 9])) indices = Sampling.crossValidation(3, numExamples) self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([3, 4, 5, 6, 7, 8, 9], [0, 1, 2])) self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 1, 2, 6, 7, 8, 9], [3, 4, 5])) self.assertEquals((list(indices[2][0]), list(indices[2][1])), ([0, 1, 2, 3, 4, 5], [6, 7, 8, 9])) indices = Sampling.crossValidation(4, numExamples) self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([2, 3, 4, 5, 6, 7, 8, 9], [0, 1])) self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 1, 5, 6, 7, 8, 9], [2, 3, 4])) self.assertEquals((list(indices[2][0]), list(indices[2][1])), ([0, 1, 2, 3, 4, 7, 8, 9], [5, 6])) self.assertEquals((list(indices[3][0]), list(indices[3][1])), ([0, 1, 2, 3, 4, 5, 6], [7, 8, 9])) indices = Sampling.crossValidation(numExamples, numExamples) self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([1, 2, 3, 4, 5, 6, 7, 8, 9], [0])) self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 2, 3, 4, 5, 6, 7, 8, 9], [1])) self.assertEquals((list(indices[2][0]), list(indices[2][1])), ([0, 1, 3, 4, 5, 6, 7, 8, 9], [2])) self.assertEquals((list(indices[3][0]), list(indices[3][1])), ([0, 1, 2, 4, 5, 6, 7, 8, 9], [3])) self.assertEquals((list(indices[4][0]), list(indices[4][1])), ([0, 1, 2, 3, 5, 6, 7, 8, 9], [4])) self.assertRaises(ValueError, Sampling.crossValidation, numExamples+1, numExamples) self.assertRaises(ValueError, Sampling.crossValidation, 0, numExamples) self.assertRaises(ValueError, Sampling.crossValidation, -1, numExamples) self.assertRaises(ValueError, Sampling.crossValidation, folds, 1)
def testParallelPen(self): #Check if penalisation == inf when treeSize < gamma numExamples = 100 X, y = data.make_regression(numExamples) learner = DecisionTreeLearner(pruneType="CART", maxDepth=10, minSplit=2) paramDict = {} paramDict["setGamma"] = numpy.array(numpy.round(2**numpy.arange(1, 10, 0.5)-1), dtype=numpy.int) folds = 3 alpha = 1.0 Cvs = numpy.array([(folds-1)*alpha]) idx = Sampling.crossValidation(folds, X.shape[0]) resultsList = learner.parallelPen(X, y, idx, paramDict, Cvs) learner, trainErrors, currentPenalties = resultsList[0] learner.setGamma(2**10) treeSize = 0 #Let's work out the size of the unpruned tree for trainInds, testInds in idx: trainX = X[trainInds, :] trainY = y[trainInds] learner.learnModel(trainX, trainY) treeSize += learner.tree.size treeSize /= float(folds) self.assertTrue(numpy.isinf(currentPenalties[paramDict["setGamma"]>treeSize]).all()) self.assertTrue(not numpy.isinf(currentPenalties[paramDict["setGamma"]<treeSize]).all())
def cvPrune(self, validX, validY): """ We do something like reduced error pruning but we use cross validation to decide which nodes to prune. """ #First set the value of the vertices using the training set. #Reset all alphas to zero inds = Sampling.crossValidation(self.folds, validX.shape[0]) for i in self.tree.getAllVertexIds(): self.tree.getVertex(i).setAlpha(0.0) self.tree.getVertex(i).setTestError(0.0) for trainInds, testInds in inds: rootId = (0,) root = self.tree.getVertex(rootId) root.setTrainInds(trainInds) root.setTestInds(testInds) root.tempValue = numpy.mean(validY[trainInds]) nodeStack = [(rootId, root.tempValue)] while len(nodeStack) != 0: (nodeId, value) = nodeStack.pop() node = self.tree.getVertex(nodeId) tempTrainInds = node.getTrainInds() tempTestInds = node.getTestInds() node.setTestError(numpy.sum((validY[tempTestInds] - node.tempValue)**2) + node.getTestError()) childIds = [self.getLeftChildId(nodeId), self.getRightChildId(nodeId)] for childId in childIds: if self.tree.vertexExists(childId): child = self.tree.getVertex(childId) if childId[-1] == 0: childInds = validX[tempTrainInds, node.getFeatureInd()] < node.getThreshold() else: childInds = validX[tempTrainInds, node.getFeatureInd()] >= node.getThreshold() if childInds.sum() !=0: value = numpy.mean(validY[tempTrainInds[childInds]]) child.tempValue = value child.setTrainInds(tempTrainInds[childInds]) nodeStack.append((childId, value)) if childId[-1] == 0: childInds = validX[tempTestInds, node.getFeatureInd()] < node.getThreshold() else: childInds = validX[tempTestInds, node.getFeatureInd()] >= node.getThreshold() child.setTestInds(tempTestInds[childInds]) self.computeAlphas() self.prune()
def cvModelSelection(self, graph, paramList, paramFunc, folds, errorFunc): """ ParamList is a list of lists of parameters and paramFunc is a list of the corresponding functions to call with the parameters as arguments. Note that a parameter can also be a tuple which is expanded out before the function is called. e.g. paramList = [[1, 2], [2, 1], [12, 1]] paramFunc = [predictor.setC, predictor.setD] """ inds = Sampling.crossValidation(folds, graph.getNumEdges()) errors = numpy.zeros((len(paramList), folds)) allEdges = graph.getAllEdges() for i in range(len(paramList)): paramSet = paramList[i] logging.debug("Using paramSet=" + str(paramSet)) for j in range(len(paramSet)): if type(paramSet[j]) == tuple: paramFunc[j](*paramSet[j]) else: paramFunc[j](paramSet[j]) predY = numpy.zeros(0) y = numpy.zeros(0) j = 0 for (trainInds, testInds) in inds: trainEdges = allEdges[trainInds, :] testEdges = allEdges[testInds, :] trainGraph = SparseGraph(graph.getVertexList(), graph.isUndirected()) trainGraph.addEdges(trainEdges, graph.getEdgeValues(trainEdges)) testGraph = SparseGraph(graph.getVertexList(), graph.isUndirected()) testGraph.addEdges(testEdges, graph.getEdgeValues(testEdges)) self.learnModel(trainGraph) predY = self.predictEdges(testGraph, testGraph.getAllEdges()) y = testGraph.getEdgeValues(testGraph.getAllEdges()) #Note that the order the edges is different in testGraphs as #opposed to graph when calling getAllEdges() errors[i, j] = errorFunc(y, predY) j = j+1 logging.info("Error of current fold: " + str(numpy.mean(errors[i, :]))) meanErrors = numpy.mean(errors, 1) strErrors = numpy.std(errors, 1) return meanErrors, strErrors
def generateLearner(self, X, y): """ Train using the given examples and labels, and use model selection to find the best parameters. """ if numpy.unique(y).shape[0] != 2: print(y) raise ValueError("Can only operate on binary data") #Do model selection first if self.sampleSize == None: idx = Sampling.crossValidation(self.folds, X.shape[0]) learner, meanErrors = self.parallelModelSelect(X, y, idx, self.paramDict) else: idx = Sampling.crossValidation(self.folds, self.sampleSize) inds = numpy.random.permutation(X.shape[0])[0:self.sampleSize] learner, meanErrors = self.parallelModelSelect(X[inds, :], y[inds], idx, self.paramDict) learner = self.getBestLearner(meanErrors, self.paramDict, X, y) return learner
def evaluateCv(self, X, y, folds, metricMethod=Evaluator.binaryError): """ Compute the cross validation according to a given metric. """ Parameter.checkInt(folds, 2, float('inf')) idx = Sampling.crossValidation(folds, y.shape[0]) metrics = AbstractPredictor.evaluateLearn(X, y, idx, self.learnModel, self.predict, metricMethod) mean = numpy.mean(metrics, 0) var = numpy.var(metrics, 0) return (mean, var)
def testParallelPenaltyGrid(self): folds = 3 idx = Sampling.crossValidation(folds, self.X.shape[0]) randomForest = RandomForest() trainX = self.X[0:40, :] trainY = self.y[0:40] paramDict = {} paramDict["setMinSplit"] = randomForest.getMinSplits() paramDict["setMaxDepth"] = randomForest.getMaxDepths() idealPenalties = randomForest.parallelPenaltyGrid(trainX, trainY, self.X, self.y, paramDict)
def testParallelPenaltyGrid(self): folds = 3 idx = Sampling.crossValidation(folds, self.X.shape[0]) decisionTree = DecisionTree() bestLearner, meanErrors = decisionTree.parallelVfcv(self.X, self.y, idx) trainX = self.X[0:40, :] trainY = self.y[0:40] paramDict = {} paramDict["setMinSplit"] = decisionTree.getMinSplits() paramDict["setMaxDepth"] = decisionTree.getMaxDepths() idealPenalties = decisionTree.parallelPenaltyGrid(trainX, trainY, self.X, self.y, paramDict)
def recommend(learner): """ Take a list of coauthors and read in the complete graph into a sparse matrix X such that X_ij = k means author i has worked with j, k times. Then do matrix factorisation on the resulting methods. """ outputDir = PathDefaults.getOutputDir() + "erasm/" matrixFileName = outputDir + "Toy" numExamples = 50 numFolds = 5 X = scipy.io.mmread(matrixFileName) X = scipy.sparse.csr_matrix(X) logging.debug("Loaded matrix " + str(X.shape) + " with " + str(X.getnnz()) + " non zeros") X = X.tocsr() X = X[0:numExamples ,:] X, maxS = preprocess(X) #Take out some ratings to form a training set rowInds, colInds = X.nonzero() randInds = numpy.random.permutation(rowInds.shape[0]) indexList = Sampling.crossValidation(numFolds, rowInds.shape[0]) paramList = [] for j, (trnIdx, tstIdx) in enumerate(indexList): trainInds = randInds[trnIdx] testInds = randInds[tstIdx] trainX = SparseUtils.selectMatrix(X, rowInds[trainInds], colInds[trainInds]).tocsr() testX = SparseUtils.selectMatrix(X, rowInds[testInds], colInds[testInds]).tocsr() paramList.append((trainX, testX, learner)) pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()) results = pool.map(computeTestError, paramList) #results = map(computeTestError, paramList) testErrors = numpy.array(results) meanTestErrors = testErrors.mean() logging.debug("Test errors = " + str(meanTestErrors)) errorFileName = outputDir + "results_" + learner.name() numpy.savez(errorFileName, meanTestErrors) logging.debug("Saved results as " + errorFileName)
def run(): for i in range(5): print("Iteration " + str(i)) idx = Sampling.crossValidation(folds, numExamples) learner.parallelModelSelect(X, Y, idx, paramDict)
def learningRate(self, X, y, foldsSet, paramDict): """ Find a matrix beta which has the same dimensions as the parameter grid. Each value in the grid represents the learning rate with respect to those particular parameters. :param X: The examples as rows :type X: :class:`numpy.ndarray` :param y: The binary -1/+1 labels :type y: :class:`numpy.ndarray` :param foldsSet: A list of folds to try. :param paramDict: A dictionary index by the method name and with value as an array of values :type X: :class:`dict` """ try: from sklearn import linear_model except ImportError: raise gridSize = [] gridInds = [] for key in paramDict.keys(): gridSize.append(paramDict[key].shape[0]) gridInds.append(numpy.arange(paramDict[key].shape[0])) betaGrid = numpy.ones(tuple(gridSize)) gridSize.insert(0, foldsSet.shape[0]) penalties = numpy.zeros(tuple(gridSize)) Cvs = numpy.array([1]) for i in range(foldsSet.shape[0]): folds = foldsSet[i] logging.debug("Folds " + str(folds)) idx = Sampling.crossValidation(folds, X.shape[0]) resultsList = self.parallelPen(X, y, idx, paramDict, Cvs) bestLearner, trainErrors, currentPenalties = resultsList[0] penalties[i, :] = currentPenalties indexIter = itertools.product(*gridInds) for inds in indexIter: inds2 = [slice(0, penalties.shape[0])] inds2.extend(inds) inds2 = tuple(inds2) tempPenalties = penalties[inds2] penInds = numpy.logical_and(numpy.isfinite(tempPenalties), tempPenalties>0) penInds = numpy.squeeze(penInds) tempPenalties = tempPenalties[penInds].flatten() tempfoldsSet = numpy.array(foldsSet, numpy.float)[penInds] if tempPenalties.shape[0] > 1: xp = numpy.log((tempfoldsSet-1)/tempfoldsSet*X.shape[0]) yp = numpy.log(tempPenalties)+numpy.log(tempfoldsSet) clf = linear_model.LinearRegression() clf.fit(numpy.array([xp]).T, yp) betaGrid[inds] = clf.coef_[0] return -betaGrid
meanPenalties = numpy.zeros((numGammas, numEpsilons, numCs)) meanBetaPenalties = numpy.zeros((numGammas, numEpsilons, numCs)) meanIdealPenalities = numpy.zeros((numGammas, numEpsilons, numCs)) for j in range(numRealisations): print("") logging.debug("j=" + str(j)) trainX, trainY, testX, testY = loadMethod(dataDir, datasetName, j) logging.debug("Loaded dataset with " + str(trainX.shape) + " train and " + str(testX.shape) + " test examples") trainInds = numpy.random.permutation(trainX.shape[0])[0:sampleSize] trainX = trainX[trainInds,:] trainY = trainY[trainInds] idx = Sampling.crossValidation(folds, trainX.shape[0]) Cvs = [(folds-1)*alpha, beta[j, sampleSizeInd, :]] #Now try penalisation methodInd = 0 resultsList = learner.parallelPen(trainX, trainY, idx, paramDict, Cvs) bestLearner, trainErrors, currentPenalties = resultsList[0] meanPenalties += currentPenalties predY = bestLearner.predict(testX) #Learning rate penalisation methodInd = 1 bestLearner, trainErrors, currentPenalties = resultsList[1] meanBetaPenalties += currentPenalties predY = bestLearner.predict(testX)
def testModelSelect(self): """ We test the results on some data and compare to SVR. """ numExamples = 200 X, y = data.make_regression(numExamples, noise=0.5) X = Standardiser().standardiseArray(X) y = Standardiser().standardiseArray(y) trainX = X[0:100, :] trainY = y[0:100] testX = X[100:, :] testY = y[100:] learner = DecisionTreeLearner(maxDepth=20, minSplit=10, pruneType="REP-CV") learner.setPruneCV(8) paramDict = {} paramDict["setGamma"] = numpy.linspace(0.0, 1.0, 10) paramDict["setPruneCV"] = numpy.arange(6, 11, 2, numpy.int) folds = 5 idx = Sampling.crossValidation(folds, trainX.shape[0]) bestTree, cvGrid = learner.parallelModelSelect(trainX, trainY, idx, paramDict) predY = bestTree.predict(testX) error = Evaluator.rootMeanSqError(testY, predY) print(error) learner = DecisionTreeLearner(maxDepth=20, minSplit=5, pruneType="CART") paramDict = {} paramDict["setGamma"] = numpy.linspace(0.0, 1.0, 50) folds = 5 idx = Sampling.crossValidation(folds, trainX.shape[0]) bestTree, cvGrid = learner.parallelModelSelect(trainX, trainY, idx, paramDict) predY = bestTree.predict(testX) error = Evaluator.rootMeanSqError(testY, predY) print(error) return #Let's compare to the SVM learner2 = LibSVM(kernel='gaussian', type="Epsilon_SVR") paramDict = {} paramDict["setC"] = 2.0**numpy.arange(-10, 14, 2, dtype=numpy.float) paramDict["setGamma"] = 2.0**numpy.arange(-10, 4, 2, dtype=numpy.float) paramDict["setEpsilon"] = learner2.getEpsilons() idx = Sampling.crossValidation(folds, trainX.shape[0]) bestSVM, cvGrid = learner2.parallelModelSelect(trainX, trainY, idx, paramDict) predY = bestSVM.predict(testX) error = Evaluator.rootMeanSqError(testY, predY) print(error)
def saveResults(self, labelIndex): """ Compute the results and save them for a particular hormone. Does so for all leafranks """ folds = 5 if type(self.X) == numpy.ndarray: X = self.X[self.YList[labelIndex][1], :] else: X = self.X[labelIndex][self.YList[labelIndex][1], :] X = numpy.c_[X, self.ages[self.YList[labelIndex][1]]] Y = self.YList[labelIndex][0] numExamples = X.shape[0] logging.debug("Shape of examples: " + str(X.shape)) standardiserX = Standardiser() X = standardiserX.standardiseArray(X) standardiserY = Standardiser() Y = standardiserY.standardiseArray(Y) #We need to include the ROC curves indexList = Sampling.crossValidation(folds, numExamples) splitFunction = lambda trainX, trainY: Sampling.crossValidation(folds, trainX.shape[0]) #We need a metric to minimise def invMeanAUC(predY, testY): return 1 - self.meanAUC(predY, testY, labelIndex, standardiserY) metricMethods = [invMeanAUC] #Now create a learnerIterator based on the SVM Cs = 2**numpy.arange(-8, 2, dtype=numpy.float) gammas = 2**numpy.arange(-10, 0, dtype=numpy.float) epsilons = 2**numpy.arange(-5, 0, dtype=numpy.float) fileName = self.resultsDir + self.labelNames[labelIndex] + "-svr_rbf-" + self.featuresName + ".dat" learnerIterator = [] for C in Cs: for gamma in gammas: for epsilon in epsilons: learner = svm.SVR(C=C, gamma=gamma, epsilon=epsilon) learner.learnModel = learner.fit learnerIterator.append(learner) self.saveResult(X, Y, indexList, splitFunction, learnerIterator, metricMethods, fileName, labelIndex, standardiserY) #Try the polynomial SVM fileName = self.resultsDir + self.labelNames[labelIndex] + "-svr_poly-" + self.featuresName + ".dat" degrees = numpy.array([2, 3]) for C in Cs: for degree in degrees: for epsilon in epsilons: learner = svm.SVR(kernel='poly', C=C, degree=degree, epsilon=epsilon) learner.learnModel = learner.fit learnerIterator.append(learner) self.saveResult(X, Y, indexList, splitFunction, learnerIterator, metricMethods, fileName, labelIndex, standardiserY) #Now try Lasso and ElasticNet fileName = self.resultsDir + self.labelNames[labelIndex] + "-lasso-" + self.featuresName + ".dat" alphas = 2**numpy.arange(-9, 0, dtype=numpy.float) learnerIterator = [] for alpha in alphas: learner = linear_model.Lasso(alpha = alpha) learner.learnModel = learner.fit learnerIterator.append(learner) self.saveResult(X, Y, indexList, splitFunction, learnerIterator, metricMethods, fileName, labelIndex, standardiserY)
def testParallelVfcv(self): folds = 3 idx = Sampling.crossValidation(folds, self.X.shape[0]) decisionTree = DecisionTree() bestLearner, meanErrors = decisionTree.parallelVfcv(self.X, self.y, idx)
def run(): for i in range(2): print("Iteration " + str(i)) idx = Sampling.crossValidation(self.folds, numExamples) learner.parallelPen(X, Y, idx, self.paramDict, Cvs)
idx = sampleMethod(folds, validY.shape[0]) svmGridResults = learner.parallelPen(validX, validY, idx, paramDict, Cvs) for result in svmGridResults: learner, trainErrors, currentPenalties = result print(numpy.mean(trainErrors), numpy.mean(currentPenalties)) """ #Figure out why the penalty is increasing X = trainX y = trainY for i in range(foldsSet.shape[0]): folds = foldsSet[i] idx = Sampling.crossValidation(folds, validX.shape[0]) penalty = 0 fullError = 0 trainError = 0 learner.learnModel(validX, validY) predY = learner.predict(X) predValidY = learner.predict(validX) idealPenalty = Evaluator.rootMeanSqError(predY, y) - Evaluator.rootMeanSqError(predValidY, validY) for trainInds, testInds in idx: trainX = validX[trainInds, :] trainY = validY[trainInds] #learner.setGamma(gamma)