def make(self): if self.target is None: raise RuntimeError("No target executable defined in benchmark: %s" % self.name) self._original = Executable(self.target) if self.candidates is None: self.candidates = ["%s_fast%s" % (self._original.name, self._original.extension)] try: iter(self.candidates) except TypeError: self.candidates = [self.candidates] self._candidates = [Executable(candidate) for candidate in self.candidates] with say("Making executables..."): self._original.make() for candidate in self._candidates: candidate.make() return self._original, self._candidates
class BenchmarkBase(object): name = None target = None candidates = None diff_script = None instances = 20 executions = 1 xlabel = "Input" stats_class = Stats def __init__(self): self._inputs = [] self._original = None self._candidates = [] def make(self): if self.target is None: raise RuntimeError("No target executable defined in benchmark: %s" % self.name) self._original = Executable(self.target) if self.candidates is None: self.candidates = ["%s_fast%s" % (self._original.name, self._original.extension)] try: iter(self.candidates) except TypeError: self.candidates = [self.candidates] self._candidates = [Executable(candidate) for candidate in self.candidates] with say("Making executables..."): self._original.make() for candidate in self._candidates: candidate.make() return self._original, self._candidates def args(self, instance): return [] def input(self, instance): raise RuntimeError("input() not implemented in benchmark %s" % self.name) def label(self, instance): return instance def generate_input(self, instance): input = Input(benchmark=self.name, label=self.label(instance), args=self.args(instance)) with input.open('w') as f: f.write(self.input(instance)) return input def inputs(self): for instance in range(1, self.instances+1): try: self._inputs[instance-1] except IndexError: self._inputs.append(self.generate_input(instance)) yield self._inputs[instance-1] def checkpoint(self): self.make() with say("Checkpoint %s..." % self.name): try: self._generate_stats(self._original) finally: self.clean() def full(self, check_diffs=True): self.make() with say("Benchmarking %s..." % self.name): try: if check_diffs: self.check_differences() stats = self.generate_stats() stats.generate_plots() finally: self.clean() def clean(self): for input in self.inputs(): input.remove() self._inputs = [] def check_differences(self): if self.diff_script is None: for candidate in self._candidates: with say("Checking differences between %s and %s..." % (self._original, candidate)): for input in self.inputs(): self.diff(input, candidate) else: with say("Executing differences script: %s" % self.diff_script): if subprocess.call(["./%s" % self.diff_script]): raise RuntimeError("Differences detected!") def diff(self, input, candidate): with say("Checking input %s with args %s..." % (input, input.args)): out_original, _ = self._original.run(input, save_output=True) out_candidate, _ = candidate.run(input, save_output=True) if subprocess.call(['diff', '-u', out_original.filename, out_candidate.filename]): raise RuntimeError("Differences detected!") out_original.remove() out_candidate.remove() def generate_stats(self): original_stat = Output(self._original, benchmark=self.name, label=self.instances, extension='.stats') if not original_stat.exists(): raise RuntimeError("Original stats file for benchmark %s does not exist. Run checkpoint first." % self.name) files = [original_stat] files.extend([self._generate_stats(candidate) for candidate in self._candidates]) return self.stats_class( name=self.name, xlabel=self.xlabel, files=files ) def _generate_stats(self, executable): stats_file = Output(executable=executable, benchmark=self.name, label=self.instances, extension='.stats') with say("Generating stats for %s..." % executable), stats_file.open('w') as f: say("%s%s" % (self.xlabel.ljust(20), "Time (s)")) for input in self.inputs(): time = executable.average(input, self.executions) f.write("%d %.4f\n" % (input.label, time)) say("%s%.4f" % (str(input.label).ljust(20), time)) return stats_file