def crossValidate(exampleBuilder, corpusElements, examples, options, timer): parameterOptimizationSet = None constantParameterOptimizationSet = None if options.paramOptData != None: print >> sys.stderr, "Separating parameter optimization set" parameterOptimizationDivision = Example.makeCorpusDivision(corpusElements, float(options.paramOptData)) exampleSets = Example.divideExamples(examples, parameterOptimizationDivision) constantParameterOptimizationSet = exampleSets[0] parameterOptimizationSet = constantParameterOptimizationSet optDocs = 0 for k,v in parameterOptimizationDivision.iteritems(): if v == 0: del corpusElements.documentsById[k] optDocs += 1 print >> sys.stderr, " Documents for parameter optimization:", optDocs discardedParameterCombinations = [] print >> sys.stderr, "Dividing data into folds" corpusFolds = Example.makeCorpusFolds(corpusElements, options.folds[0]) exampleSets = Example.divideExamples(examples, corpusFolds) keys = exampleSets.keys() keys.sort() evaluations = [] for key in keys: testSet = exampleSets[key] for example in testSet: example[3]["visualizationSet"] = key + 1 trainSet = [] for key2 in keys: if key != key2: trainSet.extend(exampleSets[key2]) print >> sys.stderr, "Fold", str(key + 1) # Create classifier object if options.output != None: if not os.path.exists(options.output+"/fold"+str(key+1)): os.mkdir(options.output+"/fold"+str(key+1)) # if not os.path.exists(options.output+"/fold"+str(key+1)+"/classifier"): # os.mkdir(options.output+"/fold"+str(key+1)+"/classifier") classifier = Classifier(workDir = options.output + "/fold"+str(key + 1)) else: classifier = Classifier() classifier.featureSet = exampleBuilder.featureSet # Optimize #################### # Check whether there is need for included param opt set if parameterOptimizationSet == None and options.folds[1] == 0: # 8-1-1 folds assert(len(keys) > 1) if keys.index(key) == 0: parameterOptimizationSetKey = keys[-1] else: parameterOptimizationSetKey = keys[keys.index(key)-1] parameterOptimizationSet = exampleSets[parameterOptimizationSetKey] trainSet = [] for key2 in keys: if key2 != key and key2 != parameterOptimizationSetKey: trainSet.extend(exampleSets[key2]) if parameterOptimizationSet != None: # constant external parameter optimization set evaluationArgs = {"classSet":exampleBuilder.classSet} if options.parameters != None: paramDict = splitParameters(options.parameters) bestResults = classifier.optimize([trainSet], [parameterOptimizationSet], paramDict, Evaluation, evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) else: bestResults = classifier.optimize([trainSet], [parameterOptimizationSet], evaluationClass=Evaluation, evaluationArgs=evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) else: # nested x-fold parameter optimization assert (options.folds[1] >= 2) optimizationFolds = Example.makeExampleFolds(trainSet, options.folds[1]) optimizationSets = Example.divideExamples(trainSet, optimizationFolds) optimizationSetList = [] optSetKeys = optimizationSets.keys() optSetKeys.sort() for optSetKey in optSetKeys: optimizationSetList.append(optimizationSets[optSetKey]) evaluationArgs = {"classSet":exampleBuilder.classSet} if options.parameters != None: paramDict = splitParameters(options.parameters) bestResults = classifier.optimize(optimizationSetList, optimizationSetList, paramDict, Evaluation, evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) else: bestResults = classifier.optimize(optimizationSetList, optimizationSetList, evaluationClass=Evaluation, evaluationArgs=evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) # Classify print >> sys.stderr, "Classifying test data" bestParams = bestResults[2] if bestParams.has_key("timeout"): del bestParams["timeout"] print >> sys.stderr, "Parameters:", bestParams print >> sys.stderr, "Training", startTime = time.time() classifier.train(trainSet, bestParams) print >> sys.stderr, "(Time spent:", time.time() - startTime, "s)" print >> sys.stderr, "Testing", startTime = time.time() predictions = classifier.classify(testSet) if options.output != None: pdict = [] fieldnames = ["class","prediction","id","fold"] for p in predictions: if "typed" in exampleBuilder.styles: pdict.append( {"class":exampleBuilder.classSet.getName(p[0][1]), "prediction":exampleBuilder.classSet.getName(p[1]), "id":p[0][0], "fold":key} ) else: pdict.append( {"class":p[0][1], "prediction":p[1], "id":p[0][0], "fold":key} ) TableUtils.addToCSV(pdict, options.output +"/predictions.csv", fieldnames) print >> sys.stderr, "(Time spent:", time.time() - startTime, "s)" # Calculate statistics evaluation = Evaluation(predictions, classSet=exampleBuilder.classSet) print >> sys.stderr, evaluation.toStringConcise() print >> sys.stderr, timer.toString() evaluations.append(evaluation) # Save example sets if options.output != None: print >> sys.stderr, "Saving example sets to", options.output Example.writeExamples(exampleSets[0], options.output +"/fold"+str(key+1) + "/examplesTest.txt") Example.writeExamples(exampleSets[1], options.output +"/fold"+str(key+1) + "/examplesTrain.txt") if parameterOptimizationSet == None: for k,v in optimizationSets.iteritems(): Example.writeExamples(v, options.output +"/fold"+str(key+1) + "/examplesOptimizationSet" + str(k) + ".txt") else: Example.writeExamples(parameterOptimizationSet, options.output +"/fold"+str(key+1) + "/examplesOptimizationSetPredefined.txt") TableUtils.writeCSV(bestResults[2], options.output +"/fold"+str(key+1) + "/parameters.csv") evaluation.saveCSV(options.output +"/fold"+str(key+1) + "/results.csv") print >> sys.stderr, "Compressing folder" zipTree(options.output, "fold"+str(key+1)) parameterOptimizationSet = constantParameterOptimizationSet print >> sys.stderr, "Cross-validation Results" for i in range(len(evaluations)): print >> sys.stderr, evaluations[i].toStringConcise(" Fold "+str(i)+": ") averageResult = Evaluation.average(evaluations) print >> sys.stderr, averageResult.toStringConcise(" Avg: ") pooledResult = Evaluation.pool(evaluations) print >> sys.stderr, pooledResult.toStringConcise(" Pool: ") if options.output != None: for i in range(len(evaluations)): evaluations[i].saveCSV(options.output+"/results.csv", i) averageResult.saveCSV(options.output+"/results.csv", "Avg") pooledResult.saveCSV(options.output+"/results.csv", "Pool") averageResult.saveCSV(options.output+"/resultsAverage.csv") pooledResult.saveCSV(options.output+"/resultsPooled.csv") # Visualize if options.visualization != None: visualize(sentences, pooledResult.classifications, options, exampleBuilder) # Save interactionXML if options.resultsToXML != None: classSet = None if "typed" in exampleBuilder.styles: classSet = exampleBuilder.classSet Example.writeToInteractionXML(pooledResult.classifications, corpusElements, options.resultsToXML, classSet)
def crossValidate(exampleBuilder, corpusElements, examples, options, timer): parameterOptimizationSet = None constantParameterOptimizationSet = None if options.paramOptData != None: print >> sys.stderr, "Separating parameter optimization set" parameterOptimizationDivision = Example.makeCorpusDivision( corpusElements, float(options.paramOptData)) exampleSets = Example.divideExamples(examples, parameterOptimizationDivision) constantParameterOptimizationSet = exampleSets[0] parameterOptimizationSet = constantParameterOptimizationSet optDocs = 0 for k, v in parameterOptimizationDivision.iteritems(): if v == 0: del corpusElements.documentsById[k] optDocs += 1 print >> sys.stderr, " Documents for parameter optimization:", optDocs discardedParameterCombinations = [] print >> sys.stderr, "Dividing data into folds" corpusFolds = Example.makeCorpusFolds(corpusElements, options.folds[0]) exampleSets = Example.divideExamples(examples, corpusFolds) keys = exampleSets.keys() keys.sort() evaluations = [] for key in keys: testSet = exampleSets[key] for example in testSet: example[3]["visualizationSet"] = key + 1 trainSet = [] for key2 in keys: if key != key2: trainSet.extend(exampleSets[key2]) print >> sys.stderr, "Fold", str(key + 1) # Create classifier object if options.output != None: if not os.path.exists(options.output + "/fold" + str(key + 1)): os.mkdir(options.output + "/fold" + str(key + 1)) # if not os.path.exists(options.output+"/fold"+str(key+1)+"/classifier"): # os.mkdir(options.output+"/fold"+str(key+1)+"/classifier") classifier = Classifier(workDir=options.output + "/fold" + str(key + 1)) else: classifier = Classifier() classifier.featureSet = exampleBuilder.featureSet # Optimize #################### # Check whether there is need for included param opt set if parameterOptimizationSet == None and options.folds[ 1] == 0: # 8-1-1 folds assert (len(keys) > 1) if keys.index(key) == 0: parameterOptimizationSetKey = keys[-1] else: parameterOptimizationSetKey = keys[keys.index(key) - 1] parameterOptimizationSet = exampleSets[parameterOptimizationSetKey] trainSet = [] for key2 in keys: if key2 != key and key2 != parameterOptimizationSetKey: trainSet.extend(exampleSets[key2]) if parameterOptimizationSet != None: # constant external parameter optimization set evaluationArgs = {"classSet": exampleBuilder.classSet} if options.parameters != None: paramDict = splitParameters(options.parameters) bestResults = classifier.optimize( [trainSet], [parameterOptimizationSet], paramDict, Evaluation, evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) else: bestResults = classifier.optimize( [trainSet], [parameterOptimizationSet], evaluationClass=Evaluation, evaluationArgs=evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) else: # nested x-fold parameter optimization assert (options.folds[1] >= 2) optimizationFolds = Example.makeExampleFolds( trainSet, options.folds[1]) optimizationSets = Example.divideExamples(trainSet, optimizationFolds) optimizationSetList = [] optSetKeys = optimizationSets.keys() optSetKeys.sort() for optSetKey in optSetKeys: optimizationSetList.append(optimizationSets[optSetKey]) evaluationArgs = {"classSet": exampleBuilder.classSet} if options.parameters != None: paramDict = splitParameters(options.parameters) bestResults = classifier.optimize( optimizationSetList, optimizationSetList, paramDict, Evaluation, evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) else: bestResults = classifier.optimize( optimizationSetList, optimizationSetList, evaluationClass=Evaluation, evaluationArgs=evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) # Classify print >> sys.stderr, "Classifying test data" bestParams = bestResults[2] if bestParams.has_key("timeout"): del bestParams["timeout"] print >> sys.stderr, "Parameters:", bestParams print >> sys.stderr, "Training", startTime = time.time() classifier.train(trainSet, bestParams) print >> sys.stderr, "(Time spent:", time.time() - startTime, "s)" print >> sys.stderr, "Testing", startTime = time.time() predictions = classifier.classify(testSet) if options.output != None: pdict = [] fieldnames = ["class", "prediction", "id", "fold"] for p in predictions: if "typed" in exampleBuilder.styles: pdict.append({ "class": exampleBuilder.classSet.getName(p[0][1]), "prediction": exampleBuilder.classSet.getName(p[1]), "id": p[0][0], "fold": key }) else: pdict.append({ "class": p[0][1], "prediction": p[1], "id": p[0][0], "fold": key }) TableUtils.addToCSV(pdict, options.output + "/predictions.csv", fieldnames) print >> sys.stderr, "(Time spent:", time.time() - startTime, "s)" # Calculate statistics evaluation = Evaluation(predictions, classSet=exampleBuilder.classSet) print >> sys.stderr, evaluation.toStringConcise() print >> sys.stderr, timer.toString() evaluations.append(evaluation) # Save example sets if options.output != None: print >> sys.stderr, "Saving example sets to", options.output Example.writeExamples( exampleSets[0], options.output + "/fold" + str(key + 1) + "/examplesTest.txt") Example.writeExamples( exampleSets[1], options.output + "/fold" + str(key + 1) + "/examplesTrain.txt") if parameterOptimizationSet == None: for k, v in optimizationSets.iteritems(): Example.writeExamples( v, options.output + "/fold" + str(key + 1) + "/examplesOptimizationSet" + str(k) + ".txt") else: Example.writeExamples( parameterOptimizationSet, options.output + "/fold" + str(key + 1) + "/examplesOptimizationSetPredefined.txt") TableUtils.writeCSV( bestResults[2], options.output + "/fold" + str(key + 1) + "/parameters.csv") evaluation.saveCSV(options.output + "/fold" + str(key + 1) + "/results.csv") print >> sys.stderr, "Compressing folder" zipTree(options.output, "fold" + str(key + 1)) parameterOptimizationSet = constantParameterOptimizationSet print >> sys.stderr, "Cross-validation Results" for i in range(len(evaluations)): print >> sys.stderr, evaluations[i].toStringConcise(" Fold " + str(i) + ": ") averageResult = Evaluation.average(evaluations) print >> sys.stderr, averageResult.toStringConcise(" Avg: ") pooledResult = Evaluation.pool(evaluations) print >> sys.stderr, pooledResult.toStringConcise(" Pool: ") if options.output != None: for i in range(len(evaluations)): evaluations[i].saveCSV(options.output + "/results.csv", i) averageResult.saveCSV(options.output + "/results.csv", "Avg") pooledResult.saveCSV(options.output + "/results.csv", "Pool") averageResult.saveCSV(options.output + "/resultsAverage.csv") pooledResult.saveCSV(options.output + "/resultsPooled.csv") # Visualize if options.visualization != None: visualize(sentences, pooledResult.classifications, options, exampleBuilder) # Save interactionXML if options.resultsToXML != None: classSet = None if "typed" in exampleBuilder.styles: classSet = exampleBuilder.classSet Example.writeToInteractionXML(pooledResult.classifications, corpusElements, options.resultsToXML, classSet)
# Build examples trainExamples = buildExamples(exampleBuilder, sentences, options) exampleSets[0] = trainExamples # Create classifier object classifier = Classifier() #if options.output != None: # classifier = Classifier(workDir = options.output + "/classifier") #else: # classifier = Classifier() classifier.featureSet = exampleBuilder.featureSet if hasattr(exampleBuilder,"classSet"): classifier.classSet = None # Optimize optimizationSets = Example.divideExamples(exampleSets[0]) evaluationArgs = {"classSet":exampleBuilder.classSet} if options.parameters != None: paramDict = splitParameters(options.parameters) bestResults = classifier.optimize([optimizationSets[0]], [optimizationSets[1]], paramDict, Evaluation, evaluationArgs) else: bestResults = classifier.optimize([optimizationSets[0]], [optimizationSets[1]], evaluationClass=Evaluation, evaluationArgs=evaluationArgs) else: print >> sys.stderr, "Using predefined model" bestResults = [None,None,{}] for k,v in classifierParamDict.iteritems(): bestResults[2][k] = v featureSet = IdSet() featureSet.load(os.path.join(classifierParamDict["predefined"][0], "feature_names.txt")) classSet = None if os.path.exists(os.path.join(classifierParamDict["predefined"][0], "class_names.txt")):
# Build examples trainExamples = buildExamples(exampleBuilder, sentences, options) exampleSets[0] = trainExamples # Create classifier object classifier = Classifier() #if options.output != None: # classifier = Classifier(workDir = options.output + "/classifier") #else: # classifier = Classifier() classifier.featureSet = exampleBuilder.featureSet if hasattr(exampleBuilder, "classSet"): classifier.classSet = None # Optimize optimizationSets = Example.divideExamples(exampleSets[0]) evaluationArgs = {"classSet": exampleBuilder.classSet} if options.parameters != None: paramDict = splitParameters(options.parameters) bestResults = classifier.optimize([optimizationSets[0]], [optimizationSets[1]], paramDict, Evaluation, evaluationArgs) else: bestResults = classifier.optimize([optimizationSets[0]], [optimizationSets[1]], evaluationClass=Evaluation, evaluationArgs=evaluationArgs) else: print >> sys.stderr, "Using predefined model" bestResults = [None, None, {}] for k, v in classifierParamDict.iteritems():