def saveCSV(self, filename, fold=None): global g_evaluatorFieldnames import sys sys.path.append("..") import Utils.TableUtils as TableUtils dicts = self.toDict() if fold != None: for d in dicts: d["fold"] = fold #TableUtils.addToCSV(dicts, filename, g_evaluatorFieldnames) TableUtils.writeCSV(dicts, filename, g_evaluatorFieldnames, writeTitles=True)
def resultsToCSV(results, filename=None): rows = [] for k1 in sorted(results.keys()): for k2 in sorted(results[k1].keys()): rows.append({}) rows[-1]["eval"] = k1 rows[-1]["event_class"] = k2 for k3 in sorted(results[k1][k2].keys()): rows[-1][k3] = results[k1][k2][k3] if filename != None: fieldnames = ["eval", "event_class", "gold", "gold_match", "answer", "answer_match", "recall", "precision", "fscore"] TableUtils.writeCSV(rows, filename, fieldnames) return rows
def crossValidate(exampleBuilder, corpusElements, examples, options, timer): parameterOptimizationSet = None constantParameterOptimizationSet = None if options.paramOptData != None: print >> sys.stderr, "Separating parameter optimization set" parameterOptimizationDivision = Example.makeCorpusDivision(corpusElements, float(options.paramOptData)) exampleSets = Example.divideExamples(examples, parameterOptimizationDivision) constantParameterOptimizationSet = exampleSets[0] parameterOptimizationSet = constantParameterOptimizationSet optDocs = 0 for k,v in parameterOptimizationDivision.iteritems(): if v == 0: del corpusElements.documentsById[k] optDocs += 1 print >> sys.stderr, " Documents for parameter optimization:", optDocs discardedParameterCombinations = [] print >> sys.stderr, "Dividing data into folds" corpusFolds = Example.makeCorpusFolds(corpusElements, options.folds[0]) exampleSets = Example.divideExamples(examples, corpusFolds) keys = exampleSets.keys() keys.sort() evaluations = [] for key in keys: testSet = exampleSets[key] for example in testSet: example[3]["visualizationSet"] = key + 1 trainSet = [] for key2 in keys: if key != key2: trainSet.extend(exampleSets[key2]) print >> sys.stderr, "Fold", str(key + 1) # Create classifier object if options.output != None: if not os.path.exists(options.output+"/fold"+str(key+1)): os.mkdir(options.output+"/fold"+str(key+1)) # if not os.path.exists(options.output+"/fold"+str(key+1)+"/classifier"): # os.mkdir(options.output+"/fold"+str(key+1)+"/classifier") classifier = Classifier(workDir = options.output + "/fold"+str(key + 1)) else: classifier = Classifier() classifier.featureSet = exampleBuilder.featureSet # Optimize #################### # Check whether there is need for included param opt set if parameterOptimizationSet == None and options.folds[1] == 0: # 8-1-1 folds assert(len(keys) > 1) if keys.index(key) == 0: parameterOptimizationSetKey = keys[-1] else: parameterOptimizationSetKey = keys[keys.index(key)-1] parameterOptimizationSet = exampleSets[parameterOptimizationSetKey] trainSet = [] for key2 in keys: if key2 != key and key2 != parameterOptimizationSetKey: trainSet.extend(exampleSets[key2]) if parameterOptimizationSet != None: # constant external parameter optimization set evaluationArgs = {"classSet":exampleBuilder.classSet} if options.parameters != None: paramDict = splitParameters(options.parameters) bestResults = classifier.optimize([trainSet], [parameterOptimizationSet], paramDict, Evaluation, evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) else: bestResults = classifier.optimize([trainSet], [parameterOptimizationSet], evaluationClass=Evaluation, evaluationArgs=evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) else: # nested x-fold parameter optimization assert (options.folds[1] >= 2) optimizationFolds = Example.makeExampleFolds(trainSet, options.folds[1]) optimizationSets = Example.divideExamples(trainSet, optimizationFolds) optimizationSetList = [] optSetKeys = optimizationSets.keys() optSetKeys.sort() for optSetKey in optSetKeys: optimizationSetList.append(optimizationSets[optSetKey]) evaluationArgs = {"classSet":exampleBuilder.classSet} if options.parameters != None: paramDict = splitParameters(options.parameters) bestResults = classifier.optimize(optimizationSetList, optimizationSetList, paramDict, Evaluation, evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) else: bestResults = classifier.optimize(optimizationSetList, optimizationSetList, evaluationClass=Evaluation, evaluationArgs=evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) # Classify print >> sys.stderr, "Classifying test data" bestParams = bestResults[2] if bestParams.has_key("timeout"): del bestParams["timeout"] print >> sys.stderr, "Parameters:", bestParams print >> sys.stderr, "Training", startTime = time.time() classifier.train(trainSet, bestParams) print >> sys.stderr, "(Time spent:", time.time() - startTime, "s)" print >> sys.stderr, "Testing", startTime = time.time() predictions = classifier.classify(testSet) if options.output != None: pdict = [] fieldnames = ["class","prediction","id","fold"] for p in predictions: if "typed" in exampleBuilder.styles: pdict.append( {"class":exampleBuilder.classSet.getName(p[0][1]), "prediction":exampleBuilder.classSet.getName(p[1]), "id":p[0][0], "fold":key} ) else: pdict.append( {"class":p[0][1], "prediction":p[1], "id":p[0][0], "fold":key} ) TableUtils.addToCSV(pdict, options.output +"/predictions.csv", fieldnames) print >> sys.stderr, "(Time spent:", time.time() - startTime, "s)" # Calculate statistics evaluation = Evaluation(predictions, classSet=exampleBuilder.classSet) print >> sys.stderr, evaluation.toStringConcise() print >> sys.stderr, timer.toString() evaluations.append(evaluation) # Save example sets if options.output != None: print >> sys.stderr, "Saving example sets to", options.output Example.writeExamples(exampleSets[0], options.output +"/fold"+str(key+1) + "/examplesTest.txt") Example.writeExamples(exampleSets[1], options.output +"/fold"+str(key+1) + "/examplesTrain.txt") if parameterOptimizationSet == None: for k,v in optimizationSets.iteritems(): Example.writeExamples(v, options.output +"/fold"+str(key+1) + "/examplesOptimizationSet" + str(k) + ".txt") else: Example.writeExamples(parameterOptimizationSet, options.output +"/fold"+str(key+1) + "/examplesOptimizationSetPredefined.txt") TableUtils.writeCSV(bestResults[2], options.output +"/fold"+str(key+1) + "/parameters.csv") evaluation.saveCSV(options.output +"/fold"+str(key+1) + "/results.csv") print >> sys.stderr, "Compressing folder" zipTree(options.output, "fold"+str(key+1)) parameterOptimizationSet = constantParameterOptimizationSet print >> sys.stderr, "Cross-validation Results" for i in range(len(evaluations)): print >> sys.stderr, evaluations[i].toStringConcise(" Fold "+str(i)+": ") averageResult = Evaluation.average(evaluations) print >> sys.stderr, averageResult.toStringConcise(" Avg: ") pooledResult = Evaluation.pool(evaluations) print >> sys.stderr, pooledResult.toStringConcise(" Pool: ") if options.output != None: for i in range(len(evaluations)): evaluations[i].saveCSV(options.output+"/results.csv", i) averageResult.saveCSV(options.output+"/results.csv", "Avg") pooledResult.saveCSV(options.output+"/results.csv", "Pool") averageResult.saveCSV(options.output+"/resultsAverage.csv") pooledResult.saveCSV(options.output+"/resultsPooled.csv") # Visualize if options.visualization != None: visualize(sentences, pooledResult.classifications, options, exampleBuilder) # Save interactionXML if options.resultsToXML != None: classSet = None if "typed" in exampleBuilder.styles: classSet = exampleBuilder.classSet Example.writeToInteractionXML(pooledResult.classifications, corpusElements, options.resultsToXML, classSet)
def optimize(self, trainSets, classifySets, parameters=defaultOptimizationParameters, evaluationClass=None, evaluationArgs={}, combinationsThatTimedOut=None): if parameters.has_key("predefined"): print >> sys.stderr, "Predefined model, skipping parameter estimation" return {"predefined": parameters["predefined"]} print >> sys.stderr, "Optimizing parameters" parameterNames = parameters.keys() parameterNames.sort() # for p in self.notOptimizedParameters: # if p in parameterNames: # parameterNames.remove(p) parameterValues = [] for parameterName in parameterNames: parameterValues.append([]) for value in parameters[parameterName]: parameterValues[-1].append((parameterName, value)) combinationLists = combine.combine(*parameterValues) combinations = [] for combinationList in combinationLists: combinations.append({}) for value in combinationList: combinations[-1][value[0]] = value[1] if combinationsThatTimedOut == None: combinationsThatTimedOut = [] # # re-add non-optimized parameters to combinations # for p in self.notOptimizedParameters: # if parameters.has_key(p): # for combination in combinations: # combination[p] = parameters[p] bestResult = None combinationCount = 1 if hasattr(self, "tempDir"): mainTempDir = self.tempDir mainDebugFile = self.debugFile for combination in combinations: print >> sys.stderr, " Parameters " + str( combinationCount) + "/" + str( len(combinations)) + ":", str(combination), skip = False #print combinationsThatTimedOut for discarded in combinationsThatTimedOut: if self._dictIsIdentical(combination, discarded): print >> sys.stderr print >> sys.stderr, " Discarded before, skipping" skip = True break if skip: continue # Make copies of examples in case they are modified fold = 1 foldResults = [] for classifyExamples in classifySets: if type(trainSets[0]) == types.StringType: trainExamples = trainSets[0] else: trainExamples = [] for trainSet in trainSets: if trainSet != classifyExamples: trainExamples.extend(trainSet) trainExamplesCopy = trainExamples if type(trainExamples) == types.ListType: trainExamplesCopy = trainExamples #ExampleUtils.copyExamples(trainExamples) classifyExamplesCopy = classifyExamples if type(classifyExamples) == types.ListType: classifyExamplesCopy = classifyExamples #ExampleUtils.copyExamples(classifyExamples) if hasattr(self, "tempDir"): self.tempDir = mainTempDir + "/parameters" + str( combinationCount) + "/optimization" + str(fold) if not os.path.exists(self.tempDir): os.makedirs(self.tempDir) self.debugFile = open(self.tempDir + "/debug.txt", "wt") timer = Timer() #trainStartTime = time.time() trainRV = self.train(trainExamplesCopy, combination) #trainTime = time.time() - trainStartTime #print >> sys.stderr, " Time spent:", trainTime, "s" print >> sys.stderr, " Time spent:", timer.elapsedTimeToString( ) if trainRV == 0: predictions = self.classify(classifyExamplesCopy) evaluation = evaluationClass(predictions, **evaluationArgs) if len(classifySets) == 1: print >> sys.stderr, evaluation.toStringConcise(" ") else: print >> sys.stderr, evaluation.toStringConcise( indent=" ", title="Fold " + str(fold)) foldResults.append(evaluation) if hasattr(self, "tempDir"): evaluation.saveCSV(self.tempDir + "/results.csv") else: combinationsThatTimedOut.append(combination) print >> sys.stderr, " Timed out" fold += 1 if len(foldResults) > 0: averageResult = evaluationClass.average(foldResults) poolResult = evaluationClass.pool(foldResults) if hasattr(self, "tempDir"): TableUtils.writeCSV( combination, mainTempDir + "/parameters" + str(combinationCount) + ".csv") averageResult.saveCSV(mainTempDir + "/parameters" + str(combinationCount) + "/resultsAverage.csv") poolResult.saveCSV(mainTempDir + "/parameters" + str(combinationCount) + "/resultsPooled.csv") if len(classifySets) > 1: print >> sys.stderr, averageResult.toStringConcise( " Avg: ") print >> sys.stderr, poolResult.toStringConcise(" Pool: ") if bestResult == None or poolResult.compare( bestResult[1] ) > 0: #: averageResult.fScore > bestResult[1].fScore: #bestResult = (predictions, averageResult, combination) bestResult = (None, poolResult, combination) # Make sure memory is released, especially important since some of the previous steps # copy examples bestResult[1].classifications = None bestResult[1].predictions = None combinationCount += 1 if hasattr(self, "tempDir"): self.debugFile.close() if hasattr(self, "tempDir"): self.tempDir = mainTempDir self.debugFile = mainDebugFile return bestResult
bestResults[2][k] = v featureSet = IdSet() featureSet.load(os.path.join(classifierParamDict["predefined"][0], "feature_names.txt")) classSet = None if os.path.exists(os.path.join(classifierParamDict["predefined"][0], "class_names.txt")): classSet = IdSet() classSet.load(os.path.join(classifierParamDict["predefined"][0], "class_names.txt")) exampleBuilder = ExampleBuilder(featureSet=featureSet, classSet=classSet, **splitParameters(options.exampleBuilderParameters)) # Save training sets if options.output != None: print >> sys.stderr, "Saving example sets to", options.output Example.writeExamples(exampleSets[0], options.output + "/examplesTrain.txt") if not classifierParamDict.has_key("predefined"): Example.writeExamples(optimizationSets[0], options.output + "/examplesOptimizationTest.txt") Example.writeExamples(optimizationSets[1], options.output + "/examplesOptimizationTrain.txt") TableUtils.writeCSV(bestResults[2], options.output +"/best_parameters.csv") # Optimize and train if options.output != None: classifier = Classifier(workDir = options.output + "/classifier") else: classifier = Classifier() classifier.featureSet = exampleBuilder.featureSet if hasattr(exampleBuilder,"classSet"): classifier.classSet = exampleBuilder.classSet print >> sys.stderr, "Classifying test data" if bestResults[2].has_key("timeout"): del bestResults[2]["timeout"] print >> sys.stderr, "Parameters:", bestResults[2] print >> sys.stderr, "Training", startTime = time.time()
def crossValidate(exampleBuilder, corpusElements, examples, options, timer): parameterOptimizationSet = None constantParameterOptimizationSet = None if options.paramOptData != None: print >> sys.stderr, "Separating parameter optimization set" parameterOptimizationDivision = Example.makeCorpusDivision( corpusElements, float(options.paramOptData)) exampleSets = Example.divideExamples(examples, parameterOptimizationDivision) constantParameterOptimizationSet = exampleSets[0] parameterOptimizationSet = constantParameterOptimizationSet optDocs = 0 for k, v in parameterOptimizationDivision.iteritems(): if v == 0: del corpusElements.documentsById[k] optDocs += 1 print >> sys.stderr, " Documents for parameter optimization:", optDocs discardedParameterCombinations = [] print >> sys.stderr, "Dividing data into folds" corpusFolds = Example.makeCorpusFolds(corpusElements, options.folds[0]) exampleSets = Example.divideExamples(examples, corpusFolds) keys = exampleSets.keys() keys.sort() evaluations = [] for key in keys: testSet = exampleSets[key] for example in testSet: example[3]["visualizationSet"] = key + 1 trainSet = [] for key2 in keys: if key != key2: trainSet.extend(exampleSets[key2]) print >> sys.stderr, "Fold", str(key + 1) # Create classifier object if options.output != None: if not os.path.exists(options.output + "/fold" + str(key + 1)): os.mkdir(options.output + "/fold" + str(key + 1)) # if not os.path.exists(options.output+"/fold"+str(key+1)+"/classifier"): # os.mkdir(options.output+"/fold"+str(key+1)+"/classifier") classifier = Classifier(workDir=options.output + "/fold" + str(key + 1)) else: classifier = Classifier() classifier.featureSet = exampleBuilder.featureSet # Optimize #################### # Check whether there is need for included param opt set if parameterOptimizationSet == None and options.folds[ 1] == 0: # 8-1-1 folds assert (len(keys) > 1) if keys.index(key) == 0: parameterOptimizationSetKey = keys[-1] else: parameterOptimizationSetKey = keys[keys.index(key) - 1] parameterOptimizationSet = exampleSets[parameterOptimizationSetKey] trainSet = [] for key2 in keys: if key2 != key and key2 != parameterOptimizationSetKey: trainSet.extend(exampleSets[key2]) if parameterOptimizationSet != None: # constant external parameter optimization set evaluationArgs = {"classSet": exampleBuilder.classSet} if options.parameters != None: paramDict = splitParameters(options.parameters) bestResults = classifier.optimize( [trainSet], [parameterOptimizationSet], paramDict, Evaluation, evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) else: bestResults = classifier.optimize( [trainSet], [parameterOptimizationSet], evaluationClass=Evaluation, evaluationArgs=evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) else: # nested x-fold parameter optimization assert (options.folds[1] >= 2) optimizationFolds = Example.makeExampleFolds( trainSet, options.folds[1]) optimizationSets = Example.divideExamples(trainSet, optimizationFolds) optimizationSetList = [] optSetKeys = optimizationSets.keys() optSetKeys.sort() for optSetKey in optSetKeys: optimizationSetList.append(optimizationSets[optSetKey]) evaluationArgs = {"classSet": exampleBuilder.classSet} if options.parameters != None: paramDict = splitParameters(options.parameters) bestResults = classifier.optimize( optimizationSetList, optimizationSetList, paramDict, Evaluation, evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) else: bestResults = classifier.optimize( optimizationSetList, optimizationSetList, evaluationClass=Evaluation, evaluationArgs=evaluationArgs, combinationsThatTimedOut=discardedParameterCombinations) # Classify print >> sys.stderr, "Classifying test data" bestParams = bestResults[2] if bestParams.has_key("timeout"): del bestParams["timeout"] print >> sys.stderr, "Parameters:", bestParams print >> sys.stderr, "Training", startTime = time.time() classifier.train(trainSet, bestParams) print >> sys.stderr, "(Time spent:", time.time() - startTime, "s)" print >> sys.stderr, "Testing", startTime = time.time() predictions = classifier.classify(testSet) if options.output != None: pdict = [] fieldnames = ["class", "prediction", "id", "fold"] for p in predictions: if "typed" in exampleBuilder.styles: pdict.append({ "class": exampleBuilder.classSet.getName(p[0][1]), "prediction": exampleBuilder.classSet.getName(p[1]), "id": p[0][0], "fold": key }) else: pdict.append({ "class": p[0][1], "prediction": p[1], "id": p[0][0], "fold": key }) TableUtils.addToCSV(pdict, options.output + "/predictions.csv", fieldnames) print >> sys.stderr, "(Time spent:", time.time() - startTime, "s)" # Calculate statistics evaluation = Evaluation(predictions, classSet=exampleBuilder.classSet) print >> sys.stderr, evaluation.toStringConcise() print >> sys.stderr, timer.toString() evaluations.append(evaluation) # Save example sets if options.output != None: print >> sys.stderr, "Saving example sets to", options.output Example.writeExamples( exampleSets[0], options.output + "/fold" + str(key + 1) + "/examplesTest.txt") Example.writeExamples( exampleSets[1], options.output + "/fold" + str(key + 1) + "/examplesTrain.txt") if parameterOptimizationSet == None: for k, v in optimizationSets.iteritems(): Example.writeExamples( v, options.output + "/fold" + str(key + 1) + "/examplesOptimizationSet" + str(k) + ".txt") else: Example.writeExamples( parameterOptimizationSet, options.output + "/fold" + str(key + 1) + "/examplesOptimizationSetPredefined.txt") TableUtils.writeCSV( bestResults[2], options.output + "/fold" + str(key + 1) + "/parameters.csv") evaluation.saveCSV(options.output + "/fold" + str(key + 1) + "/results.csv") print >> sys.stderr, "Compressing folder" zipTree(options.output, "fold" + str(key + 1)) parameterOptimizationSet = constantParameterOptimizationSet print >> sys.stderr, "Cross-validation Results" for i in range(len(evaluations)): print >> sys.stderr, evaluations[i].toStringConcise(" Fold " + str(i) + ": ") averageResult = Evaluation.average(evaluations) print >> sys.stderr, averageResult.toStringConcise(" Avg: ") pooledResult = Evaluation.pool(evaluations) print >> sys.stderr, pooledResult.toStringConcise(" Pool: ") if options.output != None: for i in range(len(evaluations)): evaluations[i].saveCSV(options.output + "/results.csv", i) averageResult.saveCSV(options.output + "/results.csv", "Avg") pooledResult.saveCSV(options.output + "/results.csv", "Pool") averageResult.saveCSV(options.output + "/resultsAverage.csv") pooledResult.saveCSV(options.output + "/resultsPooled.csv") # Visualize if options.visualization != None: visualize(sentences, pooledResult.classifications, options, exampleBuilder) # Save interactionXML if options.resultsToXML != None: classSet = None if "typed" in exampleBuilder.styles: classSet = exampleBuilder.classSet Example.writeToInteractionXML(pooledResult.classifications, corpusElements, options.resultsToXML, classSet)
"feature_names.txt")) classSet = None if os.path.exists( os.path.join(classifierParamDict["predefined"][0], "class_names.txt")): classSet = IdSet() classSet.load( os.path.join(classifierParamDict["predefined"][0], "class_names.txt")) exampleBuilder = ExampleBuilder(featureSet=featureSet, classSet=classSet, **splitParameters( options.exampleBuilderParameters)) # Save training sets if options.output != None: TableUtils.writeCSV(bestResults[2], options.output + "/best_parameters.csv") # Optimize and train if options.output != None: classifier = Classifier(workDir=options.output + "/classifier") else: classifier = Classifier() classifier.featureSet = exampleBuilder.featureSet if hasattr(exampleBuilder, "classSet"): classifier.classSet = exampleBuilder.classSet print >> sys.stderr, "Classifying test data" if bestResults[2].has_key("timeout"): del bestResults[2]["timeout"] print >> sys.stderr, "Parameters:", bestResults[2] print >> sys.stderr, "Training", startTime = time.time()