def autotuneCutoffBinarySearch(tx, tunable, n, min=0, max=-1): progress.push() progress.status("* optimize " + nameof(tx) + " tunable " + nameof(tunable)) val, impact = optimizeParam(tx, n, nameof(tunable), min, max, best=(-1, goottimelimit())) print "* optimize " + nameof(tx) + " tunable " + nameof(tunable) + " = %d "%val + "(impact=%.2f)"%impact setConfigVal(tunable, val) progress.pop()
def autotuneAlgchoice(tx, site, ctx, n, cutoffs): progress.push() cmd = mkcmd([ "--transform=" + nameof(ctx), "--autotune", "--autotune-transform=" + nameof(tx), "--autotune-site=%d" % site, "--max=%d" % n, "--max-sec=%d" % goodtimelimit() ]) for x in cutoffs: cmd.append("--autotune-tunable=" + nameof(x)) if options.debug or options.justprint: print ' '.join(cmd) if options.justprint: progress.pop() return True runsCur = 1 runsLast = 1 inputCur = 1 #progress formula assumes linear runtime of program calcprog = lambda: 1.0 - (math.log(inputCur, 2) + min( 1.0, runsCur / float(runsLast))) / (math.log(n, 2) + 1) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=substderr) pfx = "tuning " + nameof(tx) + ":%d - " % site if site == -1: pfx = "tuning %d cutoffs in %s - " % (len(cutoffs), nameof(tx)) str = "" progress.status(pfx) while True: line = p.stdout.readline() if line == "": break #BEGIN ITERATION .... / 4096 m = iterRe.match(line) if m: inputCur = int(m.group(1)) runsLast = runsCur runsCur = 0 progress.remaining(calcprog()) #SLOT[0] KEEP .... = 1.25 m = slotRe.match(line) if m: if m.group(1) == "0": str = m.group(3) progress.status(pfx + str) # * TRY ... m = runRe.match(line) if m: runsCur += 1 progress.remaining(calcprog()) progress.pop() if p.wait() == 0: print "* " + pfx + str return True else: print 'FAILURE OF TUNING STEP:', ' '.join(cmd) return False
def runTimingTest(tx): progress.push() progress.remaining(1) progress.status("running timing test") t=timingRun(tx, options.n) progress.remaining(0) if len(results)>0: speedup=results[-1]/t print "* timing test... %.4f (%.2fx speedup)"%(t, speedup) else: print "* initial timing test... %.4lf s"%t results.append(t) progress.pop()
def runTimingTest(tx): progress.push() progress.remaining(1) progress.status("running timing test") t = timingRun(tx, options.n) progress.remaining(0) if len(results) > 0: speedup = results[-1] / t print "* timing test... %.4f (%.2fx speedup)" % (t, speedup) else: print "* initial timing test... %.4lf s" % t results.append(t) progress.pop()
def autotuneCutoffBinarySearch(tx, tunable, n, min=0, max=-1): progress.push() progress.status("* optimize " + nameof(tx) + " tunable " + nameof(tunable)) val, impact = optimizeParam(tx, n, nameof(tunable), min, max, best=(-1, goottimelimit())) print "* optimize " + nameof(tx) + " tunable " + nameof( tunable) + " = %d " % val + "(impact=%.2f)" % impact setConfigVal(tunable, val) progress.pop()
def autotuneAlgchoice(tx, site, ctx, n, cutoffs): progress.push() cmd=mkcmd(["--transform="+nameof(ctx), "--autotune", "--autotune-transform="+nameof(tx), "--autotune-site=%d"%site, "--max=%d"%n, "--max-sec=%d"%goodtimelimit()]) for x in cutoffs: cmd.append("--autotune-tunable="+nameof(x)) if options.debug or options.justprint: print ' '.join(cmd) if options.justprint: progress.pop() return True runsCur=1 runsLast=1 inputCur=1 #progress formula assumes linear runtime of program calcprog=lambda: 1.0 - (math.log(inputCur,2) + min(1.0,runsCur/float(runsLast)))/(math.log(n,2)+1) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=substderr) pfx="tuning "+nameof(tx)+":%d - "%site if site == -1: pfx="tuning %d cutoffs in %s - " % (len(cutoffs), nameof(tx)) str="" progress.status(pfx) while True: line=p.stdout.readline() if line == "": break #BEGIN ITERATION .... / 4096 m=iterRe.match(line) if m: inputCur=int(m.group(1)) runsLast=runsCur runsCur=0 progress.remaining(calcprog()) #SLOT[0] KEEP .... = 1.25 m=slotRe.match(line) if m: if m.group(1)=="0": str=m.group(3) progress.status(pfx+str) # * TRY ... m=runRe.match(line) if m: runsCur+=1 progress.remaining(calcprog()) progress.pop() if p.wait()==0: print "* "+pfx+str return True else: print 'FAILURE OF TUNING STEP:', ' '.join(cmd) return False
def generation(self): progress.push() try: self.roundNumber += 1 self.triedConfigs = set(map(lambda x: x.config, self.members)) self.removed=[] self.notadded=[] self.test(config.max_trials) if len(self.members): for z in xrange(config.rounds_per_input_size): progress.subtask(config.rounds_per_input_size-z, lambda: self.randomMutation(config.population_high_size)) if not self.accuracyTargetsMet(): self.guidedMutation() self.prune(config.population_low_size, False) self.prune(config.population_low_size, True) self.firstRound=False elif self.firstRound and len(self.failed) and config.min_input_size_nocrash>=self.inputSize(): self.members = list(self.failed) if config.print_log: print "skip generation n = ",self.inputSize(),"(program run failed)" else: warnings.warn(tunerwarnings.AlwaysCrashes()) if config.print_log: self.printPopulation() if not config.delete_output_dir: for m in self.members+self.removed: m.writestats(self.inputSize()) if config.mutatorlog: mutators = set() for c in self.members: mutators |= set(c.mutators) for m in mutators: m.writelog(self.roundNumber, self.inputSize()) except candidatetester.InputGenerationException, e: if e.testNumber==0 and self.inputSize()<=config.min_input_size_nocrash: if config.print_log: print "skip generation n = ",self.inputSize(),"(input generation failure)" else: warnings.warn(tunerwarnings.AlwaysCrashes())
def main(): warnings.simplefilter('ignore', tunerwarnings.NewProgramCrash) warnings.simplefilter('ignore', tunerwarnings.TargetNotMet) warnings.simplefilter('ignore', tunerwarnings.NanAccuracy) progress.push() progress.status("compiling benchmarks") pbutil.chdirToPetabricksRoot() pbutil.compilePetabricks(); r, lines = pbutil.loadAndCompileBenchmarks("./scripts/pbbenchmark.tests") if filter(lambda x: x.rv!=0, r): print "compile failed" sys.exit(1) print print "All scores are relative performance to a baseline system." print "Higher is better." print baselines = dict() for line in csv.reader(open("./testdata/configs/baselines.csv")): if len(line)>=3: baselines[line[0]] = line[1:] benchmarks=[] for benchmark, cfg, n, accTarg in lines: try: baseline = baselines[benchmark] except KeyError: baseline = (1.0, 1.0) benchmarks.append(Benchmark(benchmark, cfg, n, accTarg, baseline[0], baseline[1])) print LONGBAR print "Fixed (no autotuning) scores:" print SHORTBAR progress.remainingTicks(len(benchmarks)+3) progress.tick() for b in benchmarks: progress.status("running fixed "+fmtCfg(b.cfg)) b.runFixed() b.printFixed() progress.tick() score_fixed = geomean(map(Benchmark.scoreFixed, benchmarks)) print SHORTBAR print "Fixed Score (pbbenchmark v%s): %.2f" % (VERSION, geomean(map(Benchmark.scoreFixed, benchmarks))) print LONGBAR print print LONGBAR print "Tuned scores:" print SHORTBAR for b in benchmarks: progress.status("running tuned "+fmtCfg(b.cfg)) progress.status("autotuning") b.autotune() b.runTuned() b.printTuned() progress.tick() score_tuned = geomean(map(Benchmark.scoreTuned, benchmarks)) score_training_time = geomean(map(Benchmark.scoreTrainingTime, benchmarks)) print SHORTBAR print "Tuned Score (pbbenchmark v%s): %.2f" % (VERSION, score_tuned) print "Training Time Score (pbbenchmark v%s): %.2f" % (VERSION, score_training_time) print LONGBAR print if DEBUG: print LONGBAR print "Debug:" print SHORTBAR for b in benchmarks: b.printDebug() print LONGBAR print fd = open("./testdata/configs/baselines.csv.latest", "w") for b in benchmarks: print >>fd, "%s, %f, %f" % (b.benchmark, b.tuned_perf['average'], b.tuning_time) fd.close() if not os.path.isdir(LOGDIR): os.mkdir(LOGDIR) for b in benchmarks: writelog(expandLog(b.cfg), b.logEntry()) writelog(expandLog('scores.log'), { 'version' : VERSION, 'score_fixed' : score_fixed, 'score_tuned' : score_tuned, 'score_training_time' : score_training_time, 'hostname' : socket.gethostname(), 'timestamp' : TIMESTAMP, }) progress.tick() progress.status("done") progress.pop()
if failed>0: s += "%d failed, "%failed s += "%d running, "%len(jobs_running) s += "%d pending"%len(jobs_pending) return s def updatestatus(fast=False): progress.remaining(2*len(jobs_pending)+len(jobs_running)) if not fast: for j in jobs_done[maxprinted[0]:]: if j.id==maxprinted[0]: print j.getmsg() maxprinted[0]+=1 else: break progress.push() progress.status(mkstatus) updatestatus() try: while len(jobs_pending)>0 or len(jobs_running)>0: #spawn new jobs while len(jobs_pending)>0 and len(jobs_running)<NCPU: jobs_running.append(jobs_pending.pop(0).forkrun()) updatestatus() #wait for an event rj, wj, xj = select.select(jobs_running, [], jobs_running) #handle pending data for j in rj:
def main(): warnings.simplefilter('ignore', tunerwarnings.NewProgramCrash) warnings.simplefilter('ignore', tunerwarnings.SmallInputProgramCrash) warnings.simplefilter('ignore', tunerwarnings.TargetNotMet) warnings.simplefilter('ignore', tunerwarnings.NanAccuracy) #Parse input options from optparse import OptionParser parser = OptionParser(usage="usage: pbbenchmark [options]") parser.add_option("--learning", action="store_true", dest="learning", default=False, help="enable heuristics learning") parser.add_option( "--heuristics", type="string", help= "name of the file containing the set of heuristics to use. Automatically enables --learning", default=None) (options, args) = parser.parse_args() if options.heuristics: options.learning = True if options.learning: print "Learning of heuristics is ACTIVE" if options.heuristics: print "Using heuristics file: " + str(options.heuristics) else: print "Using only heuristics in the database" progress.push() progress.status("compiling benchmarks") pbutil.chdirToPetabricksRoot() pbutil.compilePetabricks() global REV try: REV = pbutil.getRevision() except: pass r, lines = pbutil.loadAndCompileBenchmarks( "./scripts/pbbenchmark.tests", searchterms=sys.argv[1:], learning=options.learning, heuristicSetFileName=options.heuristics) if filter(lambda x: x.rv != 0, r): print "compile failed" sys.exit(1) print print "All scores are relative performance to a baseline system." print "Higher is better." print baselines = dict() for line in csv.reader(open("./testdata/configs/baselines.csv")): if len(line) >= 3: baselines[line[0]] = line[1:] benchmarks = [] for benchmark, cfg, n, accTarg in lines: try: baseline = baselines[benchmark] except KeyError: baseline = (1.0, 1.0) benchmarks.append( Benchmark(benchmark, cfg, n, accTarg, baseline[0], baseline[1])) progress.remainingTicks(len(benchmarks)) #print LONGBAR #print "Fixed (no autotuning) scores:" #print SHORTBAR #for b in benchmarks: # progress.status("running fixed "+fmtCfg(b.cfg)) # b.runFixed() # b.printFixed() #score_fixed = geomean(map(Benchmark.scoreFixed, benchmarks)) #print SHORTBAR #print "Fixed Score (pbbenchmark v%s): %.2f" % (VERSION, geomean(map(Benchmark.scoreFixed, benchmarks))) #print LONGBAR #print print LONGBAR print "Tuned scores:" print SHORTBAR for b in benchmarks: progress.status("running tuned " + fmtCfg(b.cfg)) progress.status("autotuning") b.autotune() b.runTuned() b.printTuned() progress.tick() score_tuned = geomean(map(Benchmark.scoreTuned, benchmarks)) score_training_time = geomean(map(Benchmark.scoreTrainingTime, benchmarks)) print SHORTBAR print "Tuned Score (pbbenchmark v%s): %.2f" % (VERSION, score_tuned) print "Training Time Score (pbbenchmark v%s): %.2f" % (VERSION, score_training_time) print LONGBAR print if DEBUG: print LONGBAR print "Debug:" print SHORTBAR for b in benchmarks: b.printDebug() print LONGBAR print fd = open("./testdata/configs/baselines.csv.latest", "w") for b in benchmarks: print >> fd, "%s, %f, %f" % (b.benchmark, b.tuned_perf['average'], b.tuning_time) fd.close() if not os.path.isdir(LOGDIR): os.mkdir(LOGDIR) for b in benchmarks: writelog(expandLog(b.cfg), b.logEntry()) writelog( expandLog('scores.log'), { 'version': VERSION, 'score_fixed': -1, #score_fixed, 'score_tuned': score_tuned, 'score_training_time': score_training_time, 'hostname': socket.gethostname(), 'timestamp': TIMESTAMP, 'revision': REV, }) progress.status("done") progress.pop()
def main(): warnings.simplefilter('ignore', tunerwarnings.NewProgramCrash) warnings.simplefilter('ignore', tunerwarnings.SmallInputProgramCrash) warnings.simplefilter('ignore', tunerwarnings.TargetNotMet) warnings.simplefilter('ignore', tunerwarnings.NanAccuracy) #Parse input options from optparse import OptionParser parser = OptionParser(usage="usage: pbbenchmark [options]") parser.add_option("--learning", action="store_true", dest="learning", default=False, help="enable heuristics learning") parser.add_option("--heuristics", type="string", help="name of the file containing the set of heuristics to use. Automatically enables --learning", default=None) (options, args) = parser.parse_args() if options.heuristics: options.learning = True if options.learning: print "Learning of heuristics is ACTIVE" if options.heuristics: print "Using heuristics file: "+ str(options.heuristics) else: print "Using only heuristics in the database" progress.push() progress.status("compiling benchmarks") pbutil.chdirToPetabricksRoot() pbutil.compilePetabricks() global REV try: REV=pbutil.getRevision() except: pass r, lines = pbutil.loadAndCompileBenchmarks("./scripts/pbbenchmark.tests", searchterms=sys.argv[1:], learning=options.learning, heuristicSetFileName=options.heuristics) if filter(lambda x: x.rv!=0, r): print "compile failed" sys.exit(1) print print "All scores are relative performance to a baseline system." print "Higher is better." print baselines = dict() for line in csv.reader(open("./testdata/configs/baselines.csv")): if len(line)>=3: baselines[line[0]] = line[1:] benchmarks=[] for benchmark, cfg, n, accTarg in lines: try: baseline = baselines[benchmark] except KeyError: baseline = (1.0, 1.0) benchmarks.append(Benchmark(benchmark, cfg, n, accTarg, baseline[0], baseline[1])) progress.remainingTicks(len(benchmarks)) #print LONGBAR #print "Fixed (no autotuning) scores:" #print SHORTBAR #for b in benchmarks: # progress.status("running fixed "+fmtCfg(b.cfg)) # b.runFixed() # b.printFixed() #score_fixed = geomean(map(Benchmark.scoreFixed, benchmarks)) #print SHORTBAR #print "Fixed Score (pbbenchmark v%s): %.2f" % (VERSION, geomean(map(Benchmark.scoreFixed, benchmarks))) #print LONGBAR #print print LONGBAR print "Tuned scores:" print SHORTBAR for b in benchmarks: progress.status("running tuned "+fmtCfg(b.cfg)) progress.status("autotuning") b.autotune() b.runTuned() b.printTuned() progress.tick() score_tuned = geomean(map(Benchmark.scoreTuned, benchmarks)) score_training_time = geomean(map(Benchmark.scoreTrainingTime, benchmarks)) print SHORTBAR print "Tuned Score (pbbenchmark v%s): %.2f" % (VERSION, score_tuned) print "Training Time Score (pbbenchmark v%s): %.2f" % (VERSION, score_training_time) print LONGBAR print if DEBUG: print LONGBAR print "Debug:" print SHORTBAR for b in benchmarks: b.printDebug() print LONGBAR print fd = open("./testdata/configs/baselines.csv.latest", "w") for b in benchmarks: print >>fd, "%s, %f, %f" % (b.benchmark, b.tuned_perf['average'], b.tuning_time) fd.close() if not os.path.isdir(LOGDIR): os.mkdir(LOGDIR) for b in benchmarks: writelog(expandLog(b.cfg), b.logEntry()) writelog(expandLog('scores.log'), { 'version' : VERSION, 'score_fixed' : -1,#score_fixed, 'score_tuned' : score_tuned, 'score_training_time' : score_training_time, 'hostname' : socket.gethostname(), 'timestamp' : TIMESTAMP, 'revision' : REV, }) progress.status("done") progress.pop()
s += "%d failed, " % failed s += "%d running, " % len(jobs_running) s += "%d pending" % len(jobs_pending) return s def updatestatus(fast=False): progress.remaining(2 * len(jobs_pending) + len(jobs_running)) if not fast: for j in jobs_done[maxprinted[0]:]: if j.id == maxprinted[0]: print j.getmsg() maxprinted[0] += 1 else: break progress.push() progress.status(mkstatus) updatestatus() try: while len(jobs_pending) > 0 or len(jobs_running) > 0: #spawn new jobs while len(jobs_pending) > 0 and len(jobs_running) < nParallelJobs: jobs_running.append(jobs_pending.pop(0).forkrun()) updatestatus() #wait for an event rj, wj, xj = select.select(jobs_running, [], jobs_running) #handle pending data for j in rj:
def autotuneInner(benchmark): progress.push() config.benchmark = benchmark candidate, tester = init(benchmark) try: pop = Population(candidate, tester, None) if not pop.isVariableAccuracy() and config.accuracy_target: logging.info("clearing accuracy_target") config.accuracy_target = None stats = storagedirs.openCsvStats("roundstats", ("round", "input_size", "cumulative_sec", "incremental_sec", "testing_sec", "inputgen_sec")+pop.statsHeader()) timers.total.start() config.end_time = time.time() + config.max_time try: progress.remaining(config.max_input_size*(1+config.final_rounds)) while pop.inputSize() < config.max_input_size: progress.status("autotuning %s: input %d of %d" % (config.benchmark, pop.inputSize(), config.max_input_size)) pop.generation() stats.writerow((pop.roundNumber, pop.inputSize(), timers.total.total(), timers.total.lap(), timers.testing.lap(), timers.inputgen.lap())+pop.stats()) pop.nextInputSize() progress.remaining(config.max_input_size - pop.inputSize() + config.max_input_size*config.final_rounds) for z in xrange(config.final_rounds): pop.generation() stats.writerow((pop.roundNumber, pop.inputSize(), timers.total.total(), timers.total.lap(), timers.testing.lap(), timers.inputgen.lap())+pop.stats()) progress.remaining((config.final_rounds - z)*config.max_input_size) except TrainingTimeout: pass timers.total.stop() #check to make sure we did something: if pop.firstRound: warnings.warn(tunerwarnings.AlwaysCrashes()) logging.info("TODO: using acc target: "+str(config.accuracy_target)) return pop.best finally: if pop.best and config.output_cfg: print pop.best.cfgfile(),"=>" , config.output_cfg shutil.copyfile(pop.best.cfgfile(), config.output_cfg) at = storagedirs.getactivetimers() if len(at): storagedirs.openCsvStats("timers", at.keys()).writerow(at.values()) if tester: tester.cleanup() progress.pop()