def evaluate(documents, sortMethod, verbose, cutoffs=[], task="GE.2"): workdir = tempfile.gettempdir() outdir = os.path.join(workdir, "events") cutoffs.sort() eventList = sortByScore(documents, sortMethod) results = {} startTime = time.time() for cutoff in cutoffs: print "Cutoff", cutoff, str( datetime.timedelta(seconds=time.time() - startTime)) markForRemoval(eventList, cutoff) STTools.writeSet( documents, outdir, validate=True) # validation will remove events with 0 arguments #results[cutoff] = getResults(BioNLP11GeniaTools.evaluateGE(outdir, task=2, evaluations=["approximate"], verbose=False, silent=not verbose)) if "REL" not in task: results[cutoff] = getResults( BioNLP11GeniaTools.evaluate(outdir, task=task)[1]) else: results[cutoff] = {} print results #print results[cutoff]["approximate"]["ALL-TOTAL"] #shutil.rmtree(workdir) #maxEvents = results[0.0]["approximate"]["ALL-TOTAL"]["answer"] maxEvents = results[0.0]["answer"] print "Max events", maxEvents return results, maxEvents
def evaluate(documents, sortMethod, verbose, cutoffs=[], task="GE.2"): workdir = tempfile.gettempdir() outdir = os.path.join(workdir, "events") cutoffs.sort() eventList = sortByScore(documents, sortMethod) results = {} startTime = time.time() for cutoff in cutoffs: print "Cutoff", cutoff, str(datetime.timedelta(seconds=time.time()-startTime)) markForRemoval(eventList, cutoff) STTools.writeSet(documents, outdir, validate=True) # validation will remove events with 0 arguments #results[cutoff] = getResults(BioNLP11GeniaTools.evaluateGE(outdir, task=2, evaluations=["approximate"], verbose=False, silent=not verbose)) if "REL" not in task: results[cutoff] = getResults(BioNLP11GeniaTools.evaluate(outdir, task=task)[1]) else: results[cutoff] = {} print results #print results[cutoff]["approximate"]["ALL-TOTAL"] #shutil.rmtree(workdir) #maxEvents = results[0.0]["approximate"]["ALL-TOTAL"]["answer"] maxEvents = results[0.0]["answer"] print "Max events", maxEvents return results, maxEvents
counts["t2-entities-removed"] += 1 else: triggersToKeep.append(trigger) document.triggers = triggersToKeep if __name__=="__main__": import sys import STTools from optparse import OptionParser # Import Psyco if available try: import psyco psyco.full() print >> sys.stderr, "Found Psyco, using" except ImportError: print >> sys.stderr, "Psyco not installed" optparser = OptionParser(usage="%prog [options]\n") optparser.add_option("-i", "--input", default=None, dest="input", help="", metavar="FILE") optparser.add_option("-o", "--output", default=None, dest="output", help="") optparser.add_option("--debug", default=False, action="store_true", dest="debug", help="") optparser.add_option("--noScores", default=False, action="store_true", dest="noScores", help="") (options, args) = optparser.parse_args() if options.output == None: options.output = options.input + "-validated.tar.gz" print >> sys.stderr, "Reading documents" documents = STTools.loadSet(options.input, readScores=(not options.noScores)) print >> sys.stderr, "Writing documents" STTools.writeSet(documents, options.output, validate=True, writeScores=(not options.noScores), task=2, debug=options.debug)
help="", metavar="FILE") optparser.add_option("-o", "--output", default=None, dest="output", help="") optparser.add_option("--debug", default=False, action="store_true", dest="debug", help="") optparser.add_option("--noScores", default=False, action="store_true", dest="noScores", help="") (options, args) = optparser.parse_args() if options.output == None: options.output = options.input + "-validated.tar.gz" print >> sys.stderr, "Reading documents" documents = STTools.loadSet(options.input, readScores=(not options.noScores)) print >> sys.stderr, "Writing documents" STTools.writeSet(documents, options.output, validate=True, writeScores=(not options.noScores), task=2, debug=options.debug)