def make(self):
if self.target is None:
raise RuntimeError("No target executable defined in benchmark: %s" % self.name)
self._original = Executable(self.target)
if self.candidates is None:
self.candidates = ["%s_fast%s" % (self._original.name, self._original.extension)]
try:
iter(self.candidates)
except TypeError:
self.candidates = [self.candidates]
self._candidates = [Executable(candidate) for candidate in self.candidates]
with say("Making executables..."):
self._original.make()
for candidate in self._candidates:
candidate.make()
return self._original, self._candidates
class BenchmarkBase(object):
name = None
target = None
candidates = None
diff_script = None
instances = 20
executions = 1
xlabel = "Input"
stats_class = Stats
def __init__(self):
self._inputs = []
self._original = None
self._candidates = []
def make(self):
if self.target is None:
raise RuntimeError("No target executable defined in benchmark: %s" % self.name)
self._original = Executable(self.target)
if self.candidates is None:
self.candidates = ["%s_fast%s" % (self._original.name, self._original.extension)]
try:
iter(self.candidates)
except TypeError:
self.candidates = [self.candidates]
self._candidates = [Executable(candidate) for candidate in self.candidates]
with say("Making executables..."):
self._original.make()
for candidate in self._candidates:
candidate.make()
return self._original, self._candidates
def args(self, instance):
return []
def input(self, instance):
raise RuntimeError("input() not implemented in benchmark %s" % self.name)
def label(self, instance):
return instance
def generate_input(self, instance):
input = Input(benchmark=self.name, label=self.label(instance), args=self.args(instance))
with input.open('w') as f:
f.write(self.input(instance))
return input
def inputs(self):
for instance in range(1, self.instances+1):
try:
self._inputs[instance-1]
except IndexError:
self._inputs.append(self.generate_input(instance))
yield self._inputs[instance-1]
def checkpoint(self):
self.make()
with say("Checkpoint %s..." % self.name):
try:
self._generate_stats(self._original)
finally:
self.clean()
def full(self, check_diffs=True):
self.make()
with say("Benchmarking %s..." % self.name):
try:
if check_diffs:
self.check_differences()
stats = self.generate_stats()
stats.generate_plots()
finally:
self.clean()
def clean(self):
for input in self.inputs():
input.remove()
self._inputs = []
def check_differences(self):
if self.diff_script is None:
for candidate in self._candidates:
with say("Checking differences between %s and %s..." % (self._original, candidate)):
for input in self.inputs():
self.diff(input, candidate)
else:
with say("Executing differences script: %s" % self.diff_script):
if subprocess.call(["./%s" % self.diff_script]):
raise RuntimeError("Differences detected!")
def diff(self, input, candidate):
with say("Checking input %s with args %s..." % (input, input.args)):
out_original, _ = self._original.run(input, save_output=True)
out_candidate, _ = candidate.run(input, save_output=True)
if subprocess.call(['diff', '-u', out_original.filename, out_candidate.filename]):
raise RuntimeError("Differences detected!")
out_original.remove()
out_candidate.remove()
def generate_stats(self):
original_stat = Output(self._original, benchmark=self.name, label=self.instances, extension='.stats')
if not original_stat.exists():
raise RuntimeError("Original stats file for benchmark %s does not exist. Run checkpoint first." % self.name)
files = [original_stat]
files.extend([self._generate_stats(candidate) for candidate in self._candidates])
return self.stats_class(
name=self.name,
xlabel=self.xlabel,
files=files
)
def _generate_stats(self, executable):
stats_file = Output(executable=executable, benchmark=self.name, label=self.instances, extension='.stats')
with say("Generating stats for %s..." % executable), stats_file.open('w') as f:
say("%s%s" % (self.xlabel.ljust(20), "Time (s)"))
for input in self.inputs():
time = executable.average(input, self.executions)
f.write("%d %.4f\n" % (input.label, time))
say("%s%.4f" % (str(input.label).ljust(20), time))
return stats_file