def generate_plots(self):
if subprocess.call(['which', 'gnuplot'], stdout=open('/dev/null', 'w')):
say("Gnuplot not found. Skipping plot generation...")
return
with say("Generating plots for %s..." % self.name):
self.generate_times()
self.generate_speedup()
def generate(name, instances):
if not any(instances):
instances = [1]
bmark = get_benchmark(name)()
for instance in instances:
with say("Generating input for instance %d..." % instance):
input = bmark.generate_input(instance)
say("Generated: %s" % input)
def check_differences(self):
if self.diff_script is None:
for candidate in self._candidates:
with say("Checking differences between %s and %s..." % (self._original, candidate)):
for input in self.inputs():
self.diff(input, candidate)
else:
with say("Executing differences script: %s" % self.diff_script):
if subprocess.call(["./%s" % self.diff_script]):
raise RuntimeError("Differences detected!")
def generate_times(self):
say('Generating plot: %s vs Time (s)' % self.xlabel)
plots = ["'%s' title '%s' pt %d" % (f, f.executable, pt) for f, pt in zip(self.files, cycle(self.POINTS))]
gnuplot([
'set ylabel "Time (s)"',
'set xlabel "%s"' % self.xlabel,
'set key below',
"set term pdf color",
'set output "%s_time.pdf"' % self.name,
"plot %s" % ', '.join(plots)
])
def _generate_stats(self, executable):
stats_file = Output(executable=executable, benchmark=self.name, label=self.instances, extension='.stats')
with say("Generating stats for %s..." % executable), stats_file.open('w') as f:
say("%s%s" % (self.xlabel.ljust(20), "Time (s)"))
for input in self.inputs():
time = executable.average(input, self.executions)
f.write("%d %.4f\n" % (input.label, time))
say("%s%.4f" % (str(input.label).ljust(20), time))
return stats_file
def clean(stats):
"""
Cleans fast-generated files of the current directory
"""
ends = ['.in', '.out']
if stats:
ends.append('.stats')
remove_files = (f for f in os.listdir('.') if os.path.splitext(f)[-1] in ends)
for f in remove_files:
os.remove(f)
say('Removed: %s' % f)
def checkpoint(names):
"""
Creates a checkpoint for the current targets
"""
benchmarks = load_benchmarks()
if not any(names):
names = 'all'
for benchmark_class in benchmarks:
if names == 'all' or benchmark_class.name in names:
bmark = benchmark_class()
bmark.checkpoint()
say('Done.')
def benchmark(names, check_diffs):
"""
Performs benchmarks to the specified program
"""
benchmarks = load_benchmarks()
if not any(names):
names = 'all'
for benchmark_class in benchmarks:
if names == 'all' or benchmark_class.name in names:
bmark = benchmark_class()
bmark.full(check_diffs=check_diffs)
say('Done.')
def generate_speedup(self):
if len(self.files) < 2:
return say("Too few stats. Skipping speedup plot...")
original = self.files[0]
plots = ["'< paste %s %s' using 1:($2/$4) title '%s/%s' lc %d" % (original, f, original.executable, f.executable,
color)
for f, color in zip(self.files[1:], cycle(self.COLORS))]
say("Generating plot: Speedup")
gnuplot([
'set ylabel "Speedup"',
'set xlabel "%s"' % self.xlabel,
'set key below',
"set term pdf color",
'set output "%s_speedup.pdf"' % self.name,
"plot %s" % ', '.join(plots)
])
def checkpoint(self):
self.make()
with say("Checkpoint %s..." % self.name):
try:
self._generate_stats(self._original)
finally:
self.clean()
def diff(self, input, candidate):
with say("Checking input %s with args %s..." % (input, input.args)):
out_original, _ = self._original.run(input, save_output=True)
out_candidate, _ = candidate.run(input, save_output=True)
if subprocess.call(['diff', '-u', out_original.filename, out_candidate.filename]):
raise RuntimeError("Differences detected!")
out_original.remove()
out_candidate.remove()
def full(self, check_diffs=True):
self.make()
with say("Benchmarking %s..." % self.name):
try:
if check_diffs:
self.check_differences()
stats = self.generate_stats()
stats.generate_plots()
finally:
self.clean()
def make(self):
if self.target is None:
raise RuntimeError("No target executable defined in benchmark: %s" % self.name)
self._original = Executable(self.target)
if self.candidates is None:
self.candidates = ["%s_fast%s" % (self._original.name, self._original.extension)]
try:
iter(self.candidates)
except TypeError:
self.candidates = [self.candidates]
self._candidates = [Executable(candidate) for candidate in self.candidates]
with say("Making executables..."):
self._original.make()
for candidate in self._candidates:
candidate.make()
return self._original, self._candidates