def evaluate(documents, sortMethod, verbose, cutoffs=[], task="GE.2"):
workdir = tempfile.gettempdir()
outdir = os.path.join(workdir, "events")
cutoffs.sort()
eventList = sortByScore(documents, sortMethod)
results = {}
startTime = time.time()
for cutoff in cutoffs:
print "Cutoff", cutoff, str(
datetime.timedelta(seconds=time.time() - startTime))
markForRemoval(eventList, cutoff)
STTools.writeSet(
documents, outdir,
validate=True) # validation will remove events with 0 arguments
#results[cutoff] = getResults(BioNLP11GeniaTools.evaluateGE(outdir, task=2, evaluations=["approximate"], verbose=False, silent=not verbose))
if "REL" not in task:
results[cutoff] = getResults(
BioNLP11GeniaTools.evaluate(outdir, task=task)[1])
else:
results[cutoff] = {}
print results
#print results[cutoff]["approximate"]["ALL-TOTAL"]
#shutil.rmtree(workdir)
#maxEvents = results[0.0]["approximate"]["ALL-TOTAL"]["answer"]
maxEvents = results[0.0]["answer"]
print "Max events", maxEvents
return results, maxEvents
def evaluate(documents, sortMethod, verbose, cutoffs=[], task="GE.2"):
workdir = tempfile.gettempdir()
outdir = os.path.join(workdir, "events")
cutoffs.sort()
eventList = sortByScore(documents, sortMethod)
results = {}
startTime = time.time()
for cutoff in cutoffs:
print "Cutoff", cutoff, str(datetime.timedelta(seconds=time.time()-startTime))
markForRemoval(eventList, cutoff)
STTools.writeSet(documents, outdir, validate=True) # validation will remove events with 0 arguments
#results[cutoff] = getResults(BioNLP11GeniaTools.evaluateGE(outdir, task=2, evaluations=["approximate"], verbose=False, silent=not verbose))
if "REL" not in task:
results[cutoff] = getResults(BioNLP11GeniaTools.evaluate(outdir, task=task)[1])
else:
results[cutoff] = {}
print results
#print results[cutoff]["approximate"]["ALL-TOTAL"]
#shutil.rmtree(workdir)
#maxEvents = results[0.0]["approximate"]["ALL-TOTAL"]["answer"]
maxEvents = results[0.0]["answer"]
print "Max events", maxEvents
return results, maxEvents
counts["t2-entities-removed"] += 1
else:
triggersToKeep.append(trigger)
document.triggers = triggersToKeep
if __name__=="__main__":
import sys
import STTools
from optparse import OptionParser
# Import Psyco if available
try:
import psyco
psyco.full()
print >> sys.stderr, "Found Psyco, using"
except ImportError:
print >> sys.stderr, "Psyco not installed"
optparser = OptionParser(usage="%prog [options]\n")
optparser.add_option("-i", "--input", default=None, dest="input", help="", metavar="FILE")
optparser.add_option("-o", "--output", default=None, dest="output", help="")
optparser.add_option("--debug", default=False, action="store_true", dest="debug", help="")
optparser.add_option("--noScores", default=False, action="store_true", dest="noScores", help="")
(options, args) = optparser.parse_args()
if options.output == None:
options.output = options.input + "-validated.tar.gz"
print >> sys.stderr, "Reading documents"
documents = STTools.loadSet(options.input, readScores=(not options.noScores))
print >> sys.stderr, "Writing documents"
STTools.writeSet(documents, options.output, validate=True, writeScores=(not options.noScores), task=2, debug=options.debug)
help="",
metavar="FILE")
optparser.add_option("-o",
"--output",
default=None,
dest="output",
help="")
optparser.add_option("--debug",
default=False,
action="store_true",
dest="debug",
help="")
optparser.add_option("--noScores",
default=False,
action="store_true",
dest="noScores",
help="")
(options, args) = optparser.parse_args()
if options.output == None:
options.output = options.input + "-validated.tar.gz"
print >> sys.stderr, "Reading documents"
documents = STTools.loadSet(options.input,
readScores=(not options.noScores))
print >> sys.stderr, "Writing documents"
STTools.writeSet(documents,
options.output,
validate=True,
writeScores=(not options.noScores),
task=2,
debug=options.debug)