def autotuneCutoffBinarySearch(tx, tunable, n, min=0, max=-1):
progress.push()
progress.status("* optimize " + nameof(tx) + " tunable " + nameof(tunable))
val, impact = optimizeParam(tx, n, nameof(tunable), min, max, best=(-1, goottimelimit()))
print "* optimize " + nameof(tx) + " tunable " + nameof(tunable) + " = %d "%val + "(impact=%.2f)"%impact
setConfigVal(tunable, val)
progress.pop()
def autotuneAlgchoice(tx, site, ctx, n, cutoffs):
progress.push()
cmd = mkcmd([
"--transform=" + nameof(ctx), "--autotune",
"--autotune-transform=" + nameof(tx),
"--autotune-site=%d" % site,
"--max=%d" % n,
"--max-sec=%d" % goodtimelimit()
])
for x in cutoffs:
cmd.append("--autotune-tunable=" + nameof(x))
if options.debug or options.justprint:
print ' '.join(cmd)
if options.justprint:
progress.pop()
return True
runsCur = 1
runsLast = 1
inputCur = 1
#progress formula assumes linear runtime of program
calcprog = lambda: 1.0 - (math.log(inputCur, 2) + min(
1.0, runsCur / float(runsLast))) / (math.log(n, 2) + 1)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=substderr)
pfx = "tuning " + nameof(tx) + ":%d - " % site
if site == -1:
pfx = "tuning %d cutoffs in %s - " % (len(cutoffs), nameof(tx))
str = ""
progress.status(pfx)
while True:
line = p.stdout.readline()
if line == "":
break
#BEGIN ITERATION .... / 4096
m = iterRe.match(line)
if m:
inputCur = int(m.group(1))
runsLast = runsCur
runsCur = 0
progress.remaining(calcprog())
#SLOT[0] KEEP .... = 1.25
m = slotRe.match(line)
if m:
if m.group(1) == "0":
str = m.group(3)
progress.status(pfx + str)
# * TRY ...
m = runRe.match(line)
if m:
runsCur += 1
progress.remaining(calcprog())
progress.pop()
if p.wait() == 0:
print "* " + pfx + str
return True
else:
print 'FAILURE OF TUNING STEP:', ' '.join(cmd)
return False
def runTimingTest(tx):
progress.push()
progress.remaining(1)
progress.status("running timing test")
t=timingRun(tx, options.n)
progress.remaining(0)
if len(results)>0:
speedup=results[-1]/t
print "* timing test... %.4f (%.2fx speedup)"%(t, speedup)
else:
print "* initial timing test... %.4lf s"%t
results.append(t)
progress.pop()
def runTimingTest(tx):
progress.push()
progress.remaining(1)
progress.status("running timing test")
t = timingRun(tx, options.n)
progress.remaining(0)
if len(results) > 0:
speedup = results[-1] / t
print "* timing test... %.4f (%.2fx speedup)" % (t, speedup)
else:
print "* initial timing test... %.4lf s" % t
results.append(t)
progress.pop()
def autotuneCutoffBinarySearch(tx, tunable, n, min=0, max=-1):
progress.push()
progress.status("* optimize " + nameof(tx) + " tunable " + nameof(tunable))
val, impact = optimizeParam(tx,
n,
nameof(tunable),
min,
max,
best=(-1, goottimelimit()))
print "* optimize " + nameof(tx) + " tunable " + nameof(
tunable) + " = %d " % val + "(impact=%.2f)" % impact
setConfigVal(tunable, val)
progress.pop()
def autotuneAlgchoice(tx, site, ctx, n, cutoffs):
progress.push()
cmd=mkcmd(["--transform="+nameof(ctx), "--autotune", "--autotune-transform="+nameof(tx), "--autotune-site=%d"%site,
"--max=%d"%n, "--max-sec=%d"%goodtimelimit()])
for x in cutoffs:
cmd.append("--autotune-tunable="+nameof(x))
if options.debug or options.justprint:
print ' '.join(cmd)
if options.justprint:
progress.pop()
return True
runsCur=1
runsLast=1
inputCur=1
#progress formula assumes linear runtime of program
calcprog=lambda: 1.0 - (math.log(inputCur,2) + min(1.0,runsCur/float(runsLast)))/(math.log(n,2)+1)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=substderr)
pfx="tuning "+nameof(tx)+":%d - "%site
if site == -1:
pfx="tuning %d cutoffs in %s - " % (len(cutoffs), nameof(tx))
str=""
progress.status(pfx)
while True:
line=p.stdout.readline()
if line == "":
break
#BEGIN ITERATION .... / 4096
m=iterRe.match(line)
if m:
inputCur=int(m.group(1))
runsLast=runsCur
runsCur=0
progress.remaining(calcprog())
#SLOT[0] KEEP .... = 1.25
m=slotRe.match(line)
if m:
if m.group(1)=="0":
str=m.group(3)
progress.status(pfx+str)
# * TRY ...
m=runRe.match(line)
if m:
runsCur+=1
progress.remaining(calcprog())
progress.pop()
if p.wait()==0:
print "* "+pfx+str
return True
else:
print 'FAILURE OF TUNING STEP:', ' '.join(cmd)
return False
def generation(self):
progress.push()
try:
self.roundNumber += 1
self.triedConfigs = set(map(lambda x: x.config, self.members))
self.removed=[]
self.notadded=[]
self.test(config.max_trials)
if len(self.members):
for z in xrange(config.rounds_per_input_size):
progress.subtask(config.rounds_per_input_size-z,
lambda: self.randomMutation(config.population_high_size))
if not self.accuracyTargetsMet():
self.guidedMutation()
self.prune(config.population_low_size, False)
self.prune(config.population_low_size, True)
self.firstRound=False
elif self.firstRound and len(self.failed) and config.min_input_size_nocrash>=self.inputSize():
self.members = list(self.failed)
if config.print_log:
print "skip generation n = ",self.inputSize(),"(program run failed)"
else:
warnings.warn(tunerwarnings.AlwaysCrashes())
if config.print_log:
self.printPopulation()
if not config.delete_output_dir:
for m in self.members+self.removed:
m.writestats(self.inputSize())
if config.mutatorlog:
mutators = set()
for c in self.members:
mutators |= set(c.mutators)
for m in mutators:
m.writelog(self.roundNumber, self.inputSize())
except candidatetester.InputGenerationException, e:
if e.testNumber==0 and self.inputSize()<=config.min_input_size_nocrash:
if config.print_log:
print "skip generation n = ",self.inputSize(),"(input generation failure)"
else:
warnings.warn(tunerwarnings.AlwaysCrashes())
def main():
warnings.simplefilter('ignore', tunerwarnings.NewProgramCrash)
warnings.simplefilter('ignore', tunerwarnings.TargetNotMet)
warnings.simplefilter('ignore', tunerwarnings.NanAccuracy)
progress.push()
progress.status("compiling benchmarks")
pbutil.chdirToPetabricksRoot()
pbutil.compilePetabricks();
r, lines = pbutil.loadAndCompileBenchmarks("./scripts/pbbenchmark.tests")
if filter(lambda x: x.rv!=0, r):
print "compile failed"
sys.exit(1)
print
print "All scores are relative performance to a baseline system."
print "Higher is better."
print
baselines = dict()
for line in csv.reader(open("./testdata/configs/baselines.csv")):
if len(line)>=3:
baselines[line[0]] = line[1:]
benchmarks=[]
for benchmark, cfg, n, accTarg in lines:
try:
baseline = baselines[benchmark]
except KeyError:
baseline = (1.0, 1.0)
benchmarks.append(Benchmark(benchmark, cfg, n, accTarg, baseline[0], baseline[1]))
print LONGBAR
print "Fixed (no autotuning) scores:"
print SHORTBAR
progress.remainingTicks(len(benchmarks)+3)
progress.tick()
for b in benchmarks:
progress.status("running fixed "+fmtCfg(b.cfg))
b.runFixed()
b.printFixed()
progress.tick()
score_fixed = geomean(map(Benchmark.scoreFixed, benchmarks))
print SHORTBAR
print "Fixed Score (pbbenchmark v%s): %.2f" % (VERSION, geomean(map(Benchmark.scoreFixed, benchmarks)))
print LONGBAR
print
print LONGBAR
print "Tuned scores:"
print SHORTBAR
for b in benchmarks:
progress.status("running tuned "+fmtCfg(b.cfg))
progress.status("autotuning")
b.autotune()
b.runTuned()
b.printTuned()
progress.tick()
score_tuned = geomean(map(Benchmark.scoreTuned, benchmarks))
score_training_time = geomean(map(Benchmark.scoreTrainingTime, benchmarks))
print SHORTBAR
print "Tuned Score (pbbenchmark v%s): %.2f" % (VERSION, score_tuned)
print "Training Time Score (pbbenchmark v%s): %.2f" % (VERSION, score_training_time)
print LONGBAR
print
if DEBUG:
print LONGBAR
print "Debug:"
print SHORTBAR
for b in benchmarks:
b.printDebug()
print LONGBAR
print
fd = open("./testdata/configs/baselines.csv.latest", "w")
for b in benchmarks:
print >>fd, "%s, %f, %f" % (b.benchmark, b.tuned_perf['average'], b.tuning_time)
fd.close()
if not os.path.isdir(LOGDIR):
os.mkdir(LOGDIR)
for b in benchmarks:
writelog(expandLog(b.cfg), b.logEntry())
writelog(expandLog('scores.log'), {
'version' : VERSION,
'score_fixed' : score_fixed,
'score_tuned' : score_tuned,
'score_training_time' : score_training_time,
'hostname' : socket.gethostname(),
'timestamp' : TIMESTAMP,
})
progress.tick()
progress.status("done")
progress.pop()
if failed>0:
s += "%d failed, "%failed
s += "%d running, "%len(jobs_running)
s += "%d pending"%len(jobs_pending)
return s
def updatestatus(fast=False):
progress.remaining(2*len(jobs_pending)+len(jobs_running))
if not fast:
for j in jobs_done[maxprinted[0]:]:
if j.id==maxprinted[0]:
print j.getmsg()
maxprinted[0]+=1
else:
break
progress.push()
progress.status(mkstatus)
updatestatus()
try:
while len(jobs_pending)>0 or len(jobs_running)>0:
#spawn new jobs
while len(jobs_pending)>0 and len(jobs_running)<NCPU:
jobs_running.append(jobs_pending.pop(0).forkrun())
updatestatus()
#wait for an event
rj, wj, xj = select.select(jobs_running, [], jobs_running)
#handle pending data
for j in rj:
def main():
warnings.simplefilter('ignore', tunerwarnings.NewProgramCrash)
warnings.simplefilter('ignore', tunerwarnings.SmallInputProgramCrash)
warnings.simplefilter('ignore', tunerwarnings.TargetNotMet)
warnings.simplefilter('ignore', tunerwarnings.NanAccuracy)
#Parse input options
from optparse import OptionParser
parser = OptionParser(usage="usage: pbbenchmark [options]")
parser.add_option("--learning",
action="store_true",
dest="learning",
default=False,
help="enable heuristics learning")
parser.add_option(
"--heuristics",
type="string",
help=
"name of the file containing the set of heuristics to use. Automatically enables --learning",
default=None)
(options, args) = parser.parse_args()
if options.heuristics:
options.learning = True
if options.learning:
print "Learning of heuristics is ACTIVE"
if options.heuristics:
print "Using heuristics file: " + str(options.heuristics)
else:
print "Using only heuristics in the database"
progress.push()
progress.status("compiling benchmarks")
pbutil.chdirToPetabricksRoot()
pbutil.compilePetabricks()
global REV
try:
REV = pbutil.getRevision()
except:
pass
r, lines = pbutil.loadAndCompileBenchmarks(
"./scripts/pbbenchmark.tests",
searchterms=sys.argv[1:],
learning=options.learning,
heuristicSetFileName=options.heuristics)
if filter(lambda x: x.rv != 0, r):
print "compile failed"
sys.exit(1)
print
print "All scores are relative performance to a baseline system."
print "Higher is better."
print
baselines = dict()
for line in csv.reader(open("./testdata/configs/baselines.csv")):
if len(line) >= 3:
baselines[line[0]] = line[1:]
benchmarks = []
for benchmark, cfg, n, accTarg in lines:
try:
baseline = baselines[benchmark]
except KeyError:
baseline = (1.0, 1.0)
benchmarks.append(
Benchmark(benchmark, cfg, n, accTarg, baseline[0], baseline[1]))
progress.remainingTicks(len(benchmarks))
#print LONGBAR
#print "Fixed (no autotuning) scores:"
#print SHORTBAR
#for b in benchmarks:
# progress.status("running fixed "+fmtCfg(b.cfg))
# b.runFixed()
# b.printFixed()
#score_fixed = geomean(map(Benchmark.scoreFixed, benchmarks))
#print SHORTBAR
#print "Fixed Score (pbbenchmark v%s): %.2f" % (VERSION, geomean(map(Benchmark.scoreFixed, benchmarks)))
#print LONGBAR
#print
print LONGBAR
print "Tuned scores:"
print SHORTBAR
for b in benchmarks:
progress.status("running tuned " + fmtCfg(b.cfg))
progress.status("autotuning")
b.autotune()
b.runTuned()
b.printTuned()
progress.tick()
score_tuned = geomean(map(Benchmark.scoreTuned, benchmarks))
score_training_time = geomean(map(Benchmark.scoreTrainingTime, benchmarks))
print SHORTBAR
print "Tuned Score (pbbenchmark v%s): %.2f" % (VERSION, score_tuned)
print "Training Time Score (pbbenchmark v%s): %.2f" % (VERSION,
score_training_time)
print LONGBAR
print
if DEBUG:
print LONGBAR
print "Debug:"
print SHORTBAR
for b in benchmarks:
b.printDebug()
print LONGBAR
print
fd = open("./testdata/configs/baselines.csv.latest", "w")
for b in benchmarks:
print >> fd, "%s, %f, %f" % (b.benchmark, b.tuned_perf['average'],
b.tuning_time)
fd.close()
if not os.path.isdir(LOGDIR):
os.mkdir(LOGDIR)
for b in benchmarks:
writelog(expandLog(b.cfg), b.logEntry())
writelog(
expandLog('scores.log'),
{
'version': VERSION,
'score_fixed': -1, #score_fixed,
'score_tuned': score_tuned,
'score_training_time': score_training_time,
'hostname': socket.gethostname(),
'timestamp': TIMESTAMP,
'revision': REV,
})
progress.status("done")
progress.pop()
def main():
warnings.simplefilter('ignore', tunerwarnings.NewProgramCrash)
warnings.simplefilter('ignore', tunerwarnings.SmallInputProgramCrash)
warnings.simplefilter('ignore', tunerwarnings.TargetNotMet)
warnings.simplefilter('ignore', tunerwarnings.NanAccuracy)
#Parse input options
from optparse import OptionParser
parser = OptionParser(usage="usage: pbbenchmark [options]")
parser.add_option("--learning", action="store_true", dest="learning", default=False, help="enable heuristics learning")
parser.add_option("--heuristics", type="string", help="name of the file containing the set of heuristics to use. Automatically enables --learning", default=None)
(options, args) = parser.parse_args()
if options.heuristics:
options.learning = True
if options.learning:
print "Learning of heuristics is ACTIVE"
if options.heuristics:
print "Using heuristics file: "+ str(options.heuristics)
else:
print "Using only heuristics in the database"
progress.push()
progress.status("compiling benchmarks")
pbutil.chdirToPetabricksRoot()
pbutil.compilePetabricks()
global REV
try:
REV=pbutil.getRevision()
except:
pass
r, lines = pbutil.loadAndCompileBenchmarks("./scripts/pbbenchmark.tests", searchterms=sys.argv[1:], learning=options.learning, heuristicSetFileName=options.heuristics)
if filter(lambda x: x.rv!=0, r):
print "compile failed"
sys.exit(1)
print
print "All scores are relative performance to a baseline system."
print "Higher is better."
print
baselines = dict()
for line in csv.reader(open("./testdata/configs/baselines.csv")):
if len(line)>=3:
baselines[line[0]] = line[1:]
benchmarks=[]
for benchmark, cfg, n, accTarg in lines:
try:
baseline = baselines[benchmark]
except KeyError:
baseline = (1.0, 1.0)
benchmarks.append(Benchmark(benchmark, cfg, n, accTarg, baseline[0], baseline[1]))
progress.remainingTicks(len(benchmarks))
#print LONGBAR
#print "Fixed (no autotuning) scores:"
#print SHORTBAR
#for b in benchmarks:
# progress.status("running fixed "+fmtCfg(b.cfg))
# b.runFixed()
# b.printFixed()
#score_fixed = geomean(map(Benchmark.scoreFixed, benchmarks))
#print SHORTBAR
#print "Fixed Score (pbbenchmark v%s): %.2f" % (VERSION, geomean(map(Benchmark.scoreFixed, benchmarks)))
#print LONGBAR
#print
print LONGBAR
print "Tuned scores:"
print SHORTBAR
for b in benchmarks:
progress.status("running tuned "+fmtCfg(b.cfg))
progress.status("autotuning")
b.autotune()
b.runTuned()
b.printTuned()
progress.tick()
score_tuned = geomean(map(Benchmark.scoreTuned, benchmarks))
score_training_time = geomean(map(Benchmark.scoreTrainingTime, benchmarks))
print SHORTBAR
print "Tuned Score (pbbenchmark v%s): %.2f" % (VERSION, score_tuned)
print "Training Time Score (pbbenchmark v%s): %.2f" % (VERSION, score_training_time)
print LONGBAR
print
if DEBUG:
print LONGBAR
print "Debug:"
print SHORTBAR
for b in benchmarks:
b.printDebug()
print LONGBAR
print
fd = open("./testdata/configs/baselines.csv.latest", "w")
for b in benchmarks:
print >>fd, "%s, %f, %f" % (b.benchmark, b.tuned_perf['average'], b.tuning_time)
fd.close()
if not os.path.isdir(LOGDIR):
os.mkdir(LOGDIR)
for b in benchmarks:
writelog(expandLog(b.cfg), b.logEntry())
writelog(expandLog('scores.log'), {
'version' : VERSION,
'score_fixed' : -1,#score_fixed,
'score_tuned' : score_tuned,
'score_training_time' : score_training_time,
'hostname' : socket.gethostname(),
'timestamp' : TIMESTAMP,
'revision' : REV,
})
progress.status("done")
progress.pop()
s += "%d failed, " % failed
s += "%d running, " % len(jobs_running)
s += "%d pending" % len(jobs_pending)
return s
def updatestatus(fast=False):
progress.remaining(2 * len(jobs_pending) + len(jobs_running))
if not fast:
for j in jobs_done[maxprinted[0]:]:
if j.id == maxprinted[0]:
print j.getmsg()
maxprinted[0] += 1
else:
break
progress.push()
progress.status(mkstatus)
updatestatus()
try:
while len(jobs_pending) > 0 or len(jobs_running) > 0:
#spawn new jobs
while len(jobs_pending) > 0 and len(jobs_running) < nParallelJobs:
jobs_running.append(jobs_pending.pop(0).forkrun())
updatestatus()
#wait for an event
rj, wj, xj = select.select(jobs_running, [], jobs_running)
#handle pending data
for j in rj:
def autotuneInner(benchmark):
progress.push()
config.benchmark = benchmark
candidate, tester = init(benchmark)
try:
pop = Population(candidate, tester, None)
if not pop.isVariableAccuracy() and config.accuracy_target:
logging.info("clearing accuracy_target")
config.accuracy_target = None
stats = storagedirs.openCsvStats("roundstats",
("round",
"input_size",
"cumulative_sec",
"incremental_sec",
"testing_sec",
"inputgen_sec")+pop.statsHeader())
timers.total.start()
config.end_time = time.time() + config.max_time
try:
progress.remaining(config.max_input_size*(1+config.final_rounds))
while pop.inputSize() < config.max_input_size:
progress.status("autotuning %s: input %d of %d" % (config.benchmark, pop.inputSize(), config.max_input_size))
pop.generation()
stats.writerow((pop.roundNumber,
pop.inputSize(),
timers.total.total(),
timers.total.lap(),
timers.testing.lap(),
timers.inputgen.lap())+pop.stats())
pop.nextInputSize()
progress.remaining(config.max_input_size - pop.inputSize() + config.max_input_size*config.final_rounds)
for z in xrange(config.final_rounds):
pop.generation()
stats.writerow((pop.roundNumber,
pop.inputSize(),
timers.total.total(),
timers.total.lap(),
timers.testing.lap(),
timers.inputgen.lap())+pop.stats())
progress.remaining((config.final_rounds - z)*config.max_input_size)
except TrainingTimeout:
pass
timers.total.stop()
#check to make sure we did something:
if pop.firstRound:
warnings.warn(tunerwarnings.AlwaysCrashes())
logging.info("TODO: using acc target: "+str(config.accuracy_target))
return pop.best
finally:
if pop.best and config.output_cfg:
print pop.best.cfgfile(),"=>" , config.output_cfg
shutil.copyfile(pop.best.cfgfile(), config.output_cfg)
at = storagedirs.getactivetimers()
if len(at):
storagedirs.openCsvStats("timers", at.keys()).writerow(at.values())
if tester:
tester.cleanup()
progress.pop()