def __init__(self, examples=None, predictions=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
self.keep = set(["CPR:3", "CPR:4", "CPR:5", "CPR:6", "CPR:9"])
self.classSet = classSet
self.results = None
self.internal = None
if predictions != None:
for example in examples:
if example[3] != None:
print >> sys.stderr, "ChemProt Evaluator:"
self._calculateExamples(examples, predictions)
else:
print >> sys.stderr, "No example extra info, skipping ChemProt evaluation"
break
self.internal = AveragingMultiClassEvaluator(
examples, predictions, classSet)
print >> sys.stderr, "AveragingMultiClassEvaluator:"
print >> sys.stderr, self.internal.toStringConcise()
def classifyToXML(self, data, model, exampleFileName=None, tag="", classifierModel=None, goldData=None, parse=None, recallAdjust=None, compressExamples=True, exampleStyle=None):
model = self.openModel(model, "r")
if parse == None:
parse = self.getStr(self.tag+"parse", model)
if exampleFileName == None:
exampleFileName = tag+self.tag+"examples"
if compressExamples:
exampleFileName += ".gz"
self.buildExamples(model, [data], [exampleFileName], [goldData], parse=parse, exampleStyle=exampleStyle)
if classifierModel == None:
classifierModel = model.get(self.tag+"classifier-model", defaultIfNotExist=None)
#else:
# assert os.path.exists(classifierModel), classifierModel
classifier = self.getClassifier(model.getStr(self.tag+"classifier-parameter", defaultIfNotExist=None))()
classifier.classify(exampleFileName, tag+self.tag+"classifications", classifierModel, finishBeforeReturn=True)
threshold = model.getStr(self.tag+"threshold", defaultIfNotExist=None, asType=float)
predictions = ExampleUtils.loadPredictions(tag+self.tag+"classifications", recallAdjust, threshold=threshold)
evaluator = self.evaluator.evaluate(exampleFileName, predictions, model.get(self.tag+"ids.classes"))
#outputFileName = tag+"-"+self.tag+"pred.xml.gz"
#exampleStyle = self.exampleBuilder.getParameters(model.getStr(self.tag+"example-style"))
if exampleStyle == None:
exampleStyle = Parameters.get(model.getStr(self.tag+"example-style")) # no checking, but these should already have passed the ExampleBuilder
self.structureAnalyzer.load(model)
return self.exampleWriter.write(exampleFileName, predictions, data, tag+self.tag+"pred.xml.gz", model.get(self.tag+"ids.classes"), parse, exampleStyle=exampleStyle, structureAnalyzer=self.structureAnalyzer)
# if evaluator.getData().getTP() + evaluator.getData().getFP() > 0:
# return self.exampleWriter.write(exampleFileName, predictions, data, outputFileName, model.get(self.tag+"ids.classes"), parse)
# else:
# # TODO: e.g. interactions must be removed if task does unmerging
# print >> sys.stderr, "No positive", self.tag + "predictions, XML file", outputFileName, "unchanged from input"
# if type(data) in types.StringTypes: # assume its a file
# shutil.copy(data, outputFileName)
# else: # assume its an elementtree
# ETUtils.write(data, outputFileName)
# #print >> sys.stderr, "No positive predictions, XML file", tag+self.tag+"pred.xml", "not written"
# return data #None
def __init__(self, examples, predictions=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
SharedTaskEvaluator.corpusElements = Core.SentenceGraph.loadCorpus(
SharedTaskEvaluator.corpusFilename, SharedTaskEvaluator.parse,
SharedTaskEvaluator.tokenization)
# Build interaction xml
xml = BioTextExampleWriter.write(
examples, predictions, SharedTaskEvaluator.corpusElements, None,
SharedTaskEvaluator.ids + ".class_names",
SharedTaskEvaluator.parse, SharedTaskEvaluator.tokenization)
#xml = ExampleUtils.writeToInteractionXML(examples, predictions, SharedTaskEvaluator.corpusElements, None, "genia-direct-event-ids.class_names", SharedTaskEvaluator.parse, SharedTaskEvaluator.tokenization)
# Convert to GENIA format
gifxmlToGenia(xml,
SharedTaskEvaluator.geniaDir,
task=SharedTaskEvaluator.task,
verbose=False)
# Use GENIA evaluation tool
self.results = evaluateSharedTask(SharedTaskEvaluator.geniaDir,
task=SharedTaskEvaluator.task,
evaluations=["approximate"],
verbose=False)
def __init__(self, examples, predictions=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
self.classSet = classSet
# define class ids in alphabetical order
self.classSet = classSet
if classSet != None:
classNames = sorted(classSet.Ids.keys())
else:
classNames = []
# make an ordered list of class ids
self.classes = []
for className in classNames:
self.classes.append(classSet.getId(className))
# create data structures for per-class evaluation
self.dataByClass = {}
for cls in self.classes:
self.dataByClass[cls] = EvaluationData()
# hack for unnamed classes
if len(self.dataByClass) == 0:
self.dataByClass[1] = EvaluationData()
self.dataByClass[2] = EvaluationData()
#self.untypedUndirected = None
self.untypedCurrentMajorId = None
self.untypedPredictionQueue = []
self.untypedUndirected = EvaluationData()
#self.AUC = None
if predictions != None:
self._calculate(examples, predictions)
def __init__(self, examples, predictions=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
self.classSet = classSet
# define class ids in alphabetical order
self.classSet = classSet
if classSet != None:
classNames = sorted(classSet.Ids.keys())
else:
classNames = []
# make an ordered list of class ids
self.classes = []
for className in classNames:
self.classes.append(classSet.getId(className))
# create data structures for per-class evaluation
self.dataByClass = {}
for cls in self.classes:
self.dataByClass[cls] = EvaluationData()
# hack for unnamed classes
if len(self.dataByClass) == 0:
self.dataByClass[1] = EvaluationData()
self.dataByClass[2] = EvaluationData()
#self.untypedUndirected = None
self.untypedCurrentMajorId = None
self.untypedPredictionQueue = []
self.untypedUndirected = EvaluationData()
#self.AUC = None
if predictions != None:
self._calculate(examples, predictions)
def classifyToXML(self, data, model, exampleFileName=None, tag="", classifierModel=None, goldData=None, parse=None, recallAdjust=None, compressExamples=True):
model = self.openModel(model, "r")
if parse == None:
parse = self.getStr(self.tag+"parse", model)
if exampleFileName == None:
exampleFileName = tag+self.tag+"examples"
if compressExamples:
exampleFileName += ".gz"
self.buildExamples(model, [data], [exampleFileName], [goldData], parse=parse)
if classifierModel == None:
classifierModel = model.get(self.tag+"classifier-model")
else:
assert os.path.exists(classifierModel), classifierModel
classifier = self.Classifier()
classifier.classify(exampleFileName, tag+self.tag+"classifications", classifierModel, finishBeforeReturn=True)
predictions = ExampleUtils.loadPredictions(tag+self.tag+"classifications", recallAdjust)
evaluator = self.evaluator.evaluate(exampleFileName, predictions, model.get(self.tag+"ids.classes"))
#outputFileName = tag+"-"+self.tag+"pred.xml.gz"
return self.exampleWriter.write(exampleFileName, predictions, data, tag+self.tag+"pred.xml.gz", model.get(self.tag+"ids.classes"), parse)
# if evaluator.getData().getTP() + evaluator.getData().getFP() > 0:
# return self.exampleWriter.write(exampleFileName, predictions, data, outputFileName, model.get(self.tag+"ids.classes"), parse)
# else:
# # TODO: e.g. interactions must be removed if task does unmerging
# print >> sys.stderr, "No positive", self.tag + "predictions, XML file", outputFileName, "unchanged from input"
# if type(data) in types.StringTypes: # assume its a file
# shutil.copy(data, outputFileName)
# else: # assume its an elementtree
# ETUtils.write(data, outputFileName)
# #print >> sys.stderr, "No positive predictions, XML file", tag+self.tag+"pred.xml", "not written"
# return data #None
def loadExamples(self, examples, predictions):
if type(predictions) == types.StringType:
print >> sys.stderr, "Reading predictions from", predictions
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType:
print >> sys.stderr, "Reading examples from", examples
examples = ExampleUtils.readExamples(examples, False)
return examples, predictions
def loadExamples(self, examples, predictions):
if type(predictions) == types.StringType:
print >> sys.stderr, "Reading predictions from", predictions
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType:
print >> sys.stderr, "Reading examples from", examples
examples = ExampleUtils.readExamples(examples, False)
return examples, predictions
def optimize(self, examples, outDir, parameters, classifyExamples, classIds, step="BOTH", evaluator=None, determineThreshold=False, timeout=None, downloadAllModels=False):
assert step in ["BOTH", "SUBMIT", "RESULTS"], step
outDir = os.path.abspath(outDir)
# Initialize training (or reconnect to existing jobs)
combinations = Parameters.getCombinations(Parameters.get(parameters, valueListKey="c")) #Core.OptimizeParameters.getParameterCombinations(parameters)
trained = []
for combination in combinations:
trained.append( self.train(examples, outDir, combination, classifyExamples, replaceRemoteExamples=(len(trained) == 0), dummy=(step == "RESULTS")) )
if step == "SUBMIT": # Return already
classifier = copy.copy(self)
classifier.setState("OPTIMIZE")
return classifier
# Wait for the training to finish
finalJobStatus = self.connection.waitForJobs([x.getJob() for x in trained])
# Evaluate the results
print >> sys.stderr, "Evaluating results"
#Stream.setIndent(" ")
bestResult = None
if evaluator == None:
evaluator = self.defaultEvaluator
for i in range(len(combinations)):
id = trained[i].parameterIdStr
#Stream.setIndent(" ")
# Get predictions
predictions = None
if trained[i].getStatus() == "FINISHED":
predictions = trained[i].downloadPredictions()
else:
print >> sys.stderr, "No results for combination" + id
continue
if downloadAllModels:
trained[i].downloadModel()
# Compare to other results
print >> sys.stderr, "*** Evaluating results for combination" + id + " ***"
threshold = None
if determineThreshold:
print >> sys.stderr, "Thresholding, original micro =",
evaluation = evaluator.evaluate(classifyExamples, predictions, classIds, os.path.join(outDir, "evaluation-before-threshold" + id + ".csv"), verbose=False)
print >> sys.stderr, evaluation.microF.toStringConcise()
threshold, bestF = evaluator.threshold(classifyExamples, predictions)
print >> sys.stderr, "threshold =", threshold, "at binary fscore", str(bestF)[0:6]
evaluation = evaluator.evaluate(classifyExamples, ExampleUtils.loadPredictions(predictions, threshold=threshold), classIds, os.path.join(outDir, "evaluation" + id + ".csv"))
if bestResult == None or evaluation.compare(bestResult[0]) > 0: #: averageResult.fScore > bestResult[1].fScore:
bestResult = [evaluation, trained[i], combinations[i], threshold]
if not self.connection.isLocal():
os.remove(predictions) # remove predictions to save space
#Stream.setIndent()
if bestResult == None:
raise Exception("No results for any parameter combination")
print >> sys.stderr, "*** Evaluation complete", finalJobStatus, "***"
print >> sys.stderr, "Selected parameters", bestResult[2]
classifier = copy.copy(bestResult[1])
classifier.threshold = bestResult[3]
classifier.downloadModel()
return classifier
def classifyToXML(self,
data,
model,
exampleFileName=None,
tag="",
classifierModel=None,
goldData=None,
parse=None,
recallAdjust=None,
compressExamples=True):
model = self.openModel(model, "r")
if parse == None:
parse = self.getStr(self.tag + "parse", model)
if exampleFileName == None:
exampleFileName = tag + self.tag + "examples"
if compressExamples:
exampleFileName += ".gz"
self.buildExamples(model, [data], [exampleFileName], [goldData],
parse=parse)
if classifierModel == None:
classifierModel = model.get(self.tag + "classifier-model")
else:
assert os.path.exists(classifierModel), classifierModel
classifier = self.Classifier()
classifier.classify(exampleFileName,
tag + self.tag + "classifications",
classifierModel,
finishBeforeReturn=True)
predictions = ExampleUtils.loadPredictions(
tag + self.tag + "classifications", recallAdjust)
evaluator = self.evaluator.evaluate(
exampleFileName, predictions, model.get(self.tag + "ids.classes"))
#outputFileName = tag+"-"+self.tag+"pred.xml.gz"
return self.exampleWriter.write(exampleFileName, predictions, data,
tag + self.tag + "pred.xml.gz",
model.get(self.tag + "ids.classes"),
parse)
# if evaluator.getData().getTP() + evaluator.getData().getFP() > 0:
# return self.exampleWriter.write(exampleFileName, predictions, data, outputFileName, model.get(self.tag+"ids.classes"), parse)
# else:
# # TODO: e.g. interactions must be removed if task does unmerging
# print >> sys.stderr, "No positive", self.tag + "predictions, XML file", outputFileName, "unchanged from input"
# if type(data) in types.StringTypes: # assume its a file
# shutil.copy(data, outputFileName)
# else: # assume its an elementtree
# ETUtils.write(data, outputFileName)
# #print >> sys.stderr, "No positive predictions, XML file", tag+self.tag+"pred.xml", "not written"
# return data #None
def __init__(self, examples, predictions=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
SharedTaskEvaluator.corpusElements = Core.SentenceGraph.loadCorpus(SharedTaskEvaluator.corpusFilename, SharedTaskEvaluator.parse, SharedTaskEvaluator.tokenization)
# Build interaction xml
xml = BioTextExampleWriter.write(examples, predictions, SharedTaskEvaluator.corpusElements, None, SharedTaskEvaluator.ids+".class_names", SharedTaskEvaluator.parse, SharedTaskEvaluator.tokenization)
#xml = ExampleUtils.writeToInteractionXML(examples, predictions, SharedTaskEvaluator.corpusElements, None, "genia-direct-event-ids.class_names", SharedTaskEvaluator.parse, SharedTaskEvaluator.tokenization)
# Convert to GENIA format
gifxmlToGenia(xml, SharedTaskEvaluator.geniaDir, task=SharedTaskEvaluator.task, verbose=False)
# Use GENIA evaluation tool
self.results = evaluateSharedTask(SharedTaskEvaluator.geniaDir, task=SharedTaskEvaluator.task, evaluations=["approximate"], verbose=False)
def __init__(self, examples, predictions=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
self.classSet = classSet
self.dataByClass = defaultdict(EvaluationData)
#self.untypedUndirected = None
self.untypedCurrentMajorId = None
self.untypedPredictionQueue = []
self.untypedUndirected = EvaluationData()
#self.AUC = None
if predictions != None:
self._calculate(examples, predictions)
def __init__(self, examples, predictions=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
self.classSet = classSet
self.dataByClass = defaultdict(EvaluationData)
#self.untypedUndirected = None
self.untypedCurrentMajorId = None
self.untypedPredictionQueue = []
self.untypedUndirected = EvaluationData()
#self.AUC = None
if predictions != None:
self._calculate(examples, predictions)
def __init__(self, examples=None, predictions=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
#self.examples = examples
#self.predictions = predictions
self.truePositives = 0
self.falsePositives = 0
self.trueNegatives = 0
self.falseNegatives = 0
self.precision = None
self.recall = None
self.fScore = None
self.AUC = None
self.type = "binary"
if predictions != None:
self._calculate(examples, predictions)
def __init__(self, examples, predictions=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
corpusElements = Core.SentenceGraph.loadCorpus(BXEvaluator.corpusFilename, BXEvaluator.parse, BXEvaluator.tokenization)
# Build interaction xml
xml = BioTextExampleWriter.write(examples, predictions, corpusElements, None, BXEvaluator.ids+".class_names", BXEvaluator.parse, BXEvaluator.tokenization)
xml = ix.splitMergedElements(xml, None)
xml = ix.recalculateIds(xml, None, True)
#xml = ExampleUtils.writeToInteractionXML(examples, predictions, SharedTaskEvaluator.corpusElements, None, "genia-direct-event-ids.class_names", SharedTaskEvaluator.parse, SharedTaskEvaluator.tokenization)
# Convert to GENIA format
STFormat.ConvertXML.toSTFormat(xml, BXEvaluator.geniaDir, outputTag="a2")
#gifxmlToGenia(xml, BXEvaluator.geniaDir, task=SharedTaskEvaluator.task, verbose=False)
# Use GENIA evaluation tool
self.results = BioNLP11GeniaTools.evaluateBX(BXEvaluator.geniaDir, corpusName=BXEvaluator.corpusTag)
corpusElements = None
def __init__(self, examples=None, predictions=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
# self.examples = examples
# self.predictions = predictions
self.truePositives = 0
self.falsePositives = 0
self.trueNegatives = 0
self.falseNegatives = 0
self.precision = None
self.recall = None
self.fScore = None
self.AUC = None
self.type = "binary"
if predictions != None:
self._calculate(examples, predictions)
def threshold(cls, examples, predictions):
# Make negative confidence score / true class pairs
if type(examples) in types.StringTypes:
examples = ExampleUtils.readExamples(examples, False)
if type(predictions) in types.StringTypes:
predictions = ExampleUtils.loadPredictions(predictions)
pairs = []
realPositives = 0
for example, prediction in itertools.izip(examples, predictions):
trueClass = example[1]
assert (trueClass > 0
) # multiclass classification uses non-negative integers
if trueClass > 1:
realPositives += 1
negClassValue = prediction[1]
pairs.append((negClassValue, trueClass))
pairs.sort(reverse=True)
realNegatives = len(pairs) - realPositives
# When starting thresholding, all examples are considered positive
binaryF = EvaluationData()
binaryF._tp = realPositives
binaryF._fp = realNegatives
binaryF._fn = 0
binaryF.calculateFScore()
fscore = binaryF.fscore
threshold = pairs[0][0] - 1.
# Turn one example negative at a time
for pair in pairs:
if pair[1] == 1: # the real class is negative
binaryF._fp -= 1 # false positive -> true negative
else: # the real class is a positive class
binaryF._tp -= 1 # true positive -> ...
binaryF._fn += 1 # ... false negative
binaryF.calculateFScore()
if binaryF.fscore > fscore:
fscore = binaryF.fscore
threshold = pair[0] + 0.00000001
return threshold, fscore
def threshold(cls, examples, predictions):
# Make negative confidence score / true class pairs
if type(examples) in types.StringTypes:
examples = ExampleUtils.readExamples(examples, False)
if type(predictions) in types.StringTypes:
predictions = ExampleUtils.loadPredictions(predictions)
pairs = []
realPositives = 0
for example, prediction in itertools.izip(examples, predictions):
trueClass = example[1]
assert(trueClass > 0) # multiclass classification uses non-negative integers
if trueClass > 1:
realPositives += 1
negClassValue = prediction[1]
pairs.append( (negClassValue, trueClass) )
pairs.sort(reverse=True)
realNegatives = len(pairs) - realPositives
# When starting thresholding, all examples are considered positive
binaryF = EvaluationData()
binaryF._tp = realPositives
binaryF._fp = realNegatives
binaryF._fn = 0
binaryF.calculateFScore()
fscore = binaryF.fscore
threshold = pairs[0][0]-1.
# Turn one example negative at a time
for pair in pairs:
if pair[1] == 1: # the real class is negative
binaryF._fp -= 1 # false positive -> true negative
else: # the real class is a positive class
binaryF._tp -= 1 # true positive -> ...
binaryF._fn += 1 # ... false negative
binaryF.calculateFScore()
if binaryF.fscore > fscore:
fscore = binaryF.fscore
threshold = pair[0]+0.00000001
return threshold, fscore
def __init__(self, examples=None, predictions=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
self.keep = set(["CPR:3", "CPR:4", "CPR:5", "CPR:6", "CPR:9"])
self.classSet = classSet
self.results = None
self.internal = None
if predictions != None:
for example in examples:
if example[3] != None:
print >> sys.stderr, "ChemProt Evaluator:"
self._calculateExamples(examples, predictions)
else:
print >> sys.stderr, "No example extra info, skipping ChemProt evaluation"
break
self.internal = AveragingMultiClassEvaluator(examples, predictions, classSet)
print >> sys.stderr, "AveragingMultiClassEvaluator:"
print >> sys.stderr, self.internal.toStringConcise()
def threshold(examples, predictionsDir=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
classIds = set()
if type(examples) == types.StringType: # examples are in file
examplesTemp = ExampleUtils.readExamples(examples, False)
examples = []
for example in examplesTemp:
examples.append(example)
classIds.add(example[1])
classIds = list(classIds)
classIds.sort()
#multilabel = MultiLabelMultiClassEvaluator(None, None, classSet)
#multilabel._calculate(examples, predictions)
#print multilabel.toStringConcise(title="multilabel")
bestThrF = [0]
bestBaseF = [0]
predFileNames = []
for filename in os.listdir(predictionsDir):
if "predictions" in filename:
predFileNames.append( (int(filename.rsplit("_")[-1]), filename) )
predFileNames.sort()
for predFileName in predFileNames:
predictionsTemp = ExampleUtils.loadPredictions(os.path.join(predictionsDir, predFileName[1]))
predictions = []
for prediction in predictionsTemp:
predictions.append(prediction)
baseEv = AveragingMultiClassEvaluator(None, None, classSet)
baseEv._calculate(examples, predictions)
print "============================"
print predFileName[1]
print "============================"
#print baseEv.toStringConcise(title="baseline")
baseLineF = baseEv.microF.fscore
for step in [0]:
for classId in [1]: #classIds:
cls = None
if classSet != None:
cls = classSet.getName(classId)
else:
cls = str(classId)
bestF = thresholdClass(examples, predictions, classId, baseLineF)
for prediction in predictions:
prediction[classId] -= bestF[2][0] + 0.00000001
changed = 0
for prediction in predictions:
maxVal = -999999
maxClass = None
for i in range(1, len(prediction)):
if prediction[i] > maxVal:
maxVal = prediction[i]
maxClass = i
if maxClass != prediction[0]:
prediction[0] = maxClass
changed += 1
print step, cls, "changed", changed, bestF[0]
baseLineF = bestF[0]
if bestF[0] > bestThrF[0]:
bestThrF = (bestF[0], predFileName[1], bestF[1], bestF[2], bestF[3])
if baseEv.microF.fscore > bestBaseF[0]:
bestBaseF = (baseEv.microF.fscore, predFileName[1], baseEv.microF.toStringConcise())
print "-------- Baseline ------------"
print baseEv.toStringConcise()
print "-------- Best ------------"
print bestF[0], bestF[1], bestF[2]
print bestF[3]
thEv = AveragingMultiClassEvaluator(None, None, classSet)
thEv._calculate(examples, predictions)
print thEv.toStringConcise()
print "=============== All Best ==============="
print "Threshold", bestThrF
print "Base", bestBaseF
sys.exit()
memPredictions = []
bestEv = baseEv
bestPair = [None, None, None]
for p in predictions:
memPredictions.append(p)
for pair in pairs:
modifier = pair[0] + 0.00000001
changedClass = 0
for pred in memPredictions:
negPred = pred[1] - modifier
maxVal = negPred
maxClass = 1
for i in range(2, len(pred)):
if pred[i] > maxVal:
maxVal = pred[i]
maxClass = i
if pred[0] != maxClass:
changedClass += 1
pred[0] = maxClass
ev = AveragingMultiClassEvaluator(None)
ev._calculate(examples, memPredictions)
print pair[0], pair[2], changedClass
print ev.toStringConcise()
if ev.compare(bestEv) == 1:
print "Improved"
bestPair = pair
bestEv = ev
print "---------------------------------------------"
print baseEv.toStringConcise(title="baseline")
print bestPair[0], bestPair[2]
print bestEv.toStringConcise(title="best")
def classifyToXML(self,
data,
model,
exampleFileName=None,
tag="",
classifierModel=None,
goldData=None,
parse=None,
recallAdjust=None,
compressExamples=True,
exampleStyle=None,
useExistingExamples=False):
model = self.openModel(model, "r")
if parse == None:
parse = self.getStr(self.tag + "parse", model)
if useExistingExamples:
assert exampleFileName != None
assert os.path.exists(exampleFileName)
if exampleFileName == None:
exampleFileName = tag + self.tag + "examples"
if compressExamples:
exampleFileName += ".gz"
if not useExistingExamples:
self.buildExamples(model, [data], [exampleFileName], [goldData],
parse=parse,
exampleStyle=exampleStyle)
if classifierModel == None:
classifierModel = model.get(self.tag + "classifier-model",
defaultIfNotExist=None)
#else:
# assert os.path.exists(classifierModel), classifierModel
classifier = self.getClassifier(
model.getStr(self.tag + "classifier-parameter",
defaultIfNotExist=None))()
classifier.classify(exampleFileName,
tag + self.tag + "classifications",
classifierModel,
finishBeforeReturn=True)
threshold = model.getStr(self.tag + "threshold",
defaultIfNotExist=None,
asType=float)
predictions = ExampleUtils.loadPredictions(tag + self.tag +
"classifications",
recallAdjust,
threshold=threshold)
evaluator = self.evaluator.evaluate(
exampleFileName, predictions, model.get(self.tag + "ids.classes"))
#outputFileName = tag+"-"+self.tag+"pred.xml.gz"
#exampleStyle = self.exampleBuilder.getParameters(model.getStr(self.tag+"example-style"))
if exampleStyle == None:
exampleStyle = Parameters.get(
model.getStr(self.tag + "example-style")
) # no checking, but these should already have passed the ExampleBuilder
self.structureAnalyzer.load(model)
return self.exampleWriter.write(
exampleFileName,
predictions,
data,
tag + self.tag + "pred.xml.gz",
model.get(self.tag + "ids.classes"),
parse,
exampleStyle=exampleStyle,
structureAnalyzer=self.structureAnalyzer)
# if evaluator.getData().getTP() + evaluator.getData().getFP() > 0:
# return self.exampleWriter.write(exampleFileName, predictions, data, outputFileName, model.get(self.tag+"ids.classes"), parse)
# else:
# # TODO: e.g. interactions must be removed if task does unmerging
# print >> sys.stderr, "No positive", self.tag + "predictions, XML file", outputFileName, "unchanged from input"
# if type(data) in types.StringTypes: # assume its a file
# shutil.copy(data, outputFileName)
# else: # assume its an elementtree
# ETUtils.write(data, outputFileName)
# #print >> sys.stderr, "No positive predictions, XML file", tag+self.tag+"pred.xml", "not written"
# return data #None
def determineThreshold(self, examples, predictions):
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
examplesByClass = {}
for cls in self.classes:
examplesByClass[cls] = []
# prepare examples
for example, prediction in itertools.izip(examples, predictions):
# Check true class for multilabel
trueClass = example[1]
trueClassName = self.classSet.getName(trueClass)
assert (trueClass > 0
) # multiclass classification uses non-negative integers
if "---" in trueClassName:
trueClass = set()
for name in trueClassName.split("---"):
trueClass.add(self.classSet.getId(name))
else:
trueClass = [trueClass]
# Check prediction for multilabel
predictedClasses = prediction[0]
if type(predictedClasses) == types.IntType:
predictedClasses = [predictedClasses]
for predType in predictedClasses:
if predType != 1:
exTrueClass = 1
if predType in trueClass:
exTrueClass = 2
examplesByClass[predType].append(
(prediction[predType], exTrueClass, 2))
# positives are negatives for other classes
for cls in self.classes:
if cls not in predictedClasses:
exTrueClass = 1
if cls in trueClass:
exTrueClass = 2
examplesByClass[cls].append(
(prediction[cls], exTrueClass, 1))
# do the thresholding
thresholdByClass = {}
for cls in self.classes:
if cls == 1:
continue
thresholdByClass[cls] = 0.0
examplesByClass[cls].sort()
# Start with all below zero being negative, and all above it being what is predicted
ev = EvaluationData()
for example in examplesByClass[cls]:
#print example
if example[0] < 0.0:
updateF(ev, example[1], 2, 1) # always negative
else:
updateF(ev, example[1], example[2], 1) # what is predicted
count = 0
bestF = [self.dataByClass[cls].fscore, None, (0.0, None), None]
for example in examplesByClass[cls]:
if example[0] < 0.0:
# Remove original example
updateF(ev, example[1], 2, -1)
# Add new example
updateF(ev, example[1], example[2], 1)
# Calculate F for this point
else:
# Remove original example
updateF(ev, example[1], example[2], -1)
# Add new example
updateF(ev, example[1], 1, 1)
# Calculate F for this point
ev.calculateFScore()
#print example, ev.toStringConcise()
count += 1
#if self.classSet.getName(cls) == "Binding":
# print count, example, ev.toStringConcise()
if ev.fscore > bestF[0]:
bestF = (ev.fscore, count, example, ev.toStringConcise())
self.dataByClass[cls] = copy.copy(ev)
print >> sys.stderr, "Threshold", self.classSet.getName(cls), bestF
if bestF[2][0] != 0.0:
thresholdByClass[cls] = bestF[2][0] + 0.00000001
else:
thresholdByClass[cls] = 0.0
#print thresholdByClass
self.thresholds = thresholdByClass
#self._calculate(examples, predictions, thresholdByClass)
#print >> sys.stderr, "Optimal", self.toStringConcise()
return thresholdByClass
def optimize(self,
examples,
outDir,
parameters,
classifyExamples,
classIds,
step="BOTH",
evaluator=None,
determineThreshold=False,
timeout=None,
downloadAllModels=False):
assert step in ["BOTH", "SUBMIT", "RESULTS"], step
outDir = os.path.abspath(outDir)
# Initialize training (or reconnect to existing jobs)
combinations = Parameters.getCombinations(
Parameters.get(parameters, valueListKey="c")
) #Core.OptimizeParameters.getParameterCombinations(parameters)
trained = []
for combination in combinations:
trained.append(
self.train(examples,
outDir,
combination,
classifyExamples,
replaceRemoteExamples=(len(trained) == 0),
dummy=(step == "RESULTS")))
if step == "SUBMIT": # Return already
classifier = copy.copy(self)
classifier.setState("OPTIMIZE")
return classifier
# Wait for the training to finish
finalJobStatus = self.connection.waitForJobs(
[x.getJob() for x in trained])
# Evaluate the results
print >> sys.stderr, "Evaluating results"
#Stream.setIndent(" ")
bestResult = None
if evaluator == None:
evaluator = self.defaultEvaluator
for i in range(len(combinations)):
id = trained[i].parameterIdStr
#Stream.setIndent(" ")
# Get predictions
predictions = None
if trained[i].getStatus() == "FINISHED":
predictions = trained[i].downloadPredictions()
else:
print >> sys.stderr, "No results for combination" + id
continue
if downloadAllModels:
trained[i].downloadModel()
# Compare to other results
print >> sys.stderr, "*** Evaluating results for combination" + id + " ***"
threshold = None
if determineThreshold:
print >> sys.stderr, "Thresholding, original micro =",
evaluation = evaluator.evaluate(
classifyExamples,
predictions,
classIds,
os.path.join(outDir,
"evaluation-before-threshold" + id + ".csv"),
verbose=False)
print >> sys.stderr, evaluation.microF.toStringConcise()
threshold, bestF = evaluator.threshold(classifyExamples,
predictions)
print >> sys.stderr, "threshold =", threshold, "at binary fscore", str(
bestF)[0:6]
evaluation = evaluator.evaluate(
classifyExamples,
ExampleUtils.loadPredictions(predictions, threshold=threshold),
classIds, os.path.join(outDir, "evaluation" + id + ".csv"))
if bestResult == None or evaluation.compare(
bestResult[0]
) > 0: #: averageResult.fScore > bestResult[1].fScore:
bestResult = [
evaluation, trained[i], combinations[i], threshold
]
if not self.connection.isLocal():
os.remove(predictions) # remove predictions to save space
#Stream.setIndent()
if bestResult == None:
raise Exception("No results for any parameter combination")
print >> sys.stderr, "*** Evaluation complete", finalJobStatus, "***"
print >> sys.stderr, "Selected parameters", bestResult[2]
classifier = copy.copy(bestResult[1])
classifier.threshold = bestResult[3]
classifier.downloadModel()
return classifier
def threshold(examples, predictionsDir=None, classSet=None):
if type(classSet) == types.StringType: # class names are in file
classSet = IdSet(filename=classSet)
classIds = set()
if type(examples) == types.StringType: # examples are in file
examplesTemp = ExampleUtils.readExamples(examples, False)
examples = []
for example in examplesTemp:
examples.append(example)
classIds.add(example[1])
classIds = list(classIds)
classIds.sort()
#multilabel = MultiLabelMultiClassEvaluator(None, None, classSet)
#multilabel._calculate(examples, predictions)
#print multilabel.toStringConcise(title="multilabel")
bestThrF = [0]
bestBaseF = [0]
predFileNames = []
for filename in os.listdir(predictionsDir):
if "predictions" in filename:
predFileNames.append((int(filename.rsplit("_")[-1]), filename))
predFileNames.sort()
for predFileName in predFileNames:
predictionsTemp = ExampleUtils.loadPredictions(
os.path.join(predictionsDir, predFileName[1]))
predictions = []
for prediction in predictionsTemp:
predictions.append(prediction)
baseEv = AveragingMultiClassEvaluator(None, None, classSet)
baseEv._calculate(examples, predictions)
print "============================"
print predFileName[1]
print "============================"
#print baseEv.toStringConcise(title="baseline")
baseLineF = baseEv.microF.fscore
for step in [0]:
for classId in [1]: #classIds:
cls = None
if classSet != None:
cls = classSet.getName(classId)
else:
cls = str(classId)
bestF = thresholdClass(examples, predictions, classId,
baseLineF)
for prediction in predictions:
prediction[classId] -= bestF[2][0] + 0.00000001
changed = 0
for prediction in predictions:
maxVal = -999999
maxClass = None
for i in range(1, len(prediction)):
if prediction[i] > maxVal:
maxVal = prediction[i]
maxClass = i
if maxClass != prediction[0]:
prediction[0] = maxClass
changed += 1
print step, cls, "changed", changed, bestF[0]
baseLineF = bestF[0]
if bestF[0] > bestThrF[0]:
bestThrF = (bestF[0], predFileName[1], bestF[1], bestF[2],
bestF[3])
if baseEv.microF.fscore > bestBaseF[0]:
bestBaseF = (baseEv.microF.fscore, predFileName[1],
baseEv.microF.toStringConcise())
print "-------- Baseline ------------"
print baseEv.toStringConcise()
print "-------- Best ------------"
print bestF[0], bestF[1], bestF[2]
print bestF[3]
thEv = AveragingMultiClassEvaluator(None, None, classSet)
thEv._calculate(examples, predictions)
print thEv.toStringConcise()
print "=============== All Best ==============="
print "Threshold", bestThrF
print "Base", bestBaseF
sys.exit()
memPredictions = []
bestEv = baseEv
bestPair = [None, None, None]
for p in predictions:
memPredictions.append(p)
for pair in pairs:
modifier = pair[0] + 0.00000001
changedClass = 0
for pred in memPredictions:
negPred = pred[1] - modifier
maxVal = negPred
maxClass = 1
for i in range(2, len(pred)):
if pred[i] > maxVal:
maxVal = pred[i]
maxClass = i
if pred[0] != maxClass:
changedClass += 1
pred[0] = maxClass
ev = AveragingMultiClassEvaluator(None)
ev._calculate(examples, memPredictions)
print pair[0], pair[2], changedClass
print ev.toStringConcise()
if ev.compare(bestEv) == 1:
print "Improved"
bestPair = pair
bestEv = ev
print "---------------------------------------------"
print baseEv.toStringConcise(title="baseline")
print bestPair[0], bestPair[2]
print bestEv.toStringConcise(title="best")
def determineThreshold(self, examples, predictions):
if type(predictions) == types.StringType: # predictions are in file
predictions = ExampleUtils.loadPredictions(predictions)
if type(examples) == types.StringType: # examples are in file
examples = ExampleUtils.readExamples(examples, False)
examplesByClass = {}
for cls in self.classes:
examplesByClass[cls] = []
# prepare examples
for example, prediction in itertools.izip(examples, predictions):
# Check true class for multilabel
trueClass = example[1]
trueClassName = self.classSet.getName(trueClass)
assert(trueClass > 0) # multiclass classification uses non-negative integers
if "---" in trueClassName:
trueClass = set()
for name in trueClassName.split("---"):
trueClass.add(self.classSet.getId(name))
else:
trueClass = [trueClass]
# Check prediction for multilabel
predictedClasses = prediction[0]
if type(predictedClasses) == types.IntType:
predictedClasses = [predictedClasses]
for predType in predictedClasses:
if predType != 1:
exTrueClass = 1
if predType in trueClass:
exTrueClass = 2
examplesByClass[predType].append( (prediction[predType], exTrueClass, 2) )
# positives are negatives for other classes
for cls in self.classes:
if cls not in predictedClasses:
exTrueClass = 1
if cls in trueClass:
exTrueClass = 2
examplesByClass[cls].append( (prediction[cls], exTrueClass, 1) )
# do the thresholding
thresholdByClass = {}
for cls in self.classes:
if cls == 1:
continue
thresholdByClass[cls] = 0.0
examplesByClass[cls].sort()
# Start with all below zero being negative, and all above it being what is predicted
ev = EvaluationData()
for example in examplesByClass[cls]:
#print example
if example[0] < 0.0:
updateF(ev, example[1], 2, 1) # always negative
else:
updateF(ev, example[1], example[2], 1) # what is predicted
count = 0
bestF = [self.dataByClass[cls].fscore, None, (0.0, None), None]
for example in examplesByClass[cls]:
if example[0] < 0.0:
# Remove original example
updateF(ev, example[1], 2, -1)
# Add new example
updateF(ev, example[1], example[2], 1)
# Calculate F for this point
else:
# Remove original example
updateF(ev, example[1], example[2], -1)
# Add new example
updateF(ev, example[1], 1, 1)
# Calculate F for this point
ev.calculateFScore()
#print example, ev.toStringConcise()
count += 1
#if self.classSet.getName(cls) == "Binding":
# print count, example, ev.toStringConcise()
if ev.fscore > bestF[0]:
bestF = (ev.fscore, count, example, ev.toStringConcise())
self.dataByClass[cls] = copy.copy(ev)
print >> sys.stderr, "Threshold", self.classSet.getName(cls), bestF
if bestF[2][0] != 0.0:
thresholdByClass[cls] = bestF[2][0] + 0.00000001
else:
thresholdByClass[cls] = 0.0
#print thresholdByClass
self.thresholds = thresholdByClass
#self._calculate(examples, predictions, thresholdByClass)
#print >> sys.stderr, "Optimal", self.toStringConcise()
return thresholdByClass
