def __init__(self, runs: t.List[dict] = None, append: bool = None, show_report: bool = None): """ Creates an instance and setup everything. :param runs: list of dictionaries that represent run program blocks if None Settings()["run/in"] is used :param append: append to the old benchmarks if there are any in the result file? :param show_report: show a short report after finishing the benchmarking? """ if runs is None: typecheck(Settings()["run/in"], ValidYamlFileName()) with open(Settings()["run/in"], "r") as f: runs = yaml.load(f) typecheck(runs, List(Dict({ "attributes": Dict(all_keys=False, key_type=Str()), "run_config": Dict(all_keys=False) }))) self.runs = runs # type: t.List[dict] """ List of dictionaries that represent run program blocks """ self.run_blocks = [] # type: t.List[RunProgramBlock] """ Run program blocks for each dictionary in ``runs```""" for (id, run) in enumerate(runs): self.run_blocks.append(RunProgramBlock.from_dict(id, copy.deepcopy(run))) self.append = Settings().default(append, "run/append") # type: bool """ Append to the old benchmarks if there are any in the result file? """ self.show_report = Settings().default(show_report, "run/show_report") # type: bool """ Show a short report after finishing the benchmarking? """ self.stats_helper = None # type: RunDataStatsHelper """ Used stats helper to help with measurements """ typecheck(Settings()["run/out"], FileName()) if self.append: run_data = [] try: if os.path.exists(Settings()["run/out"]): with open(Settings()["run/out"], "r") as f: run_data = yaml.load(f) self.stats_helper = RunDataStatsHelper.init_from_dicts(run_data, external=True) for run in runs: self.stats_helper.runs.append(RunData(attributes=run["attributes"])) except: self.teardown() raise else: self.stats_helper = RunDataStatsHelper.init_from_dicts(copy.deepcopy(runs)) #if Settings()["run/remote"]: # self.pool = RemoteRunWorkerPool(Settings()["run/remote"], Settings()["run/remote_port"]) if os.path.exists(Settings()["run/out"]): os.remove(Settings()["run/out"]) self.pool = None # type: AbstractRunWorkerPool """ Used run worker pool that abstracts the benchmarking """ if Settings()["run/cpuset/parallel"] == 0: self.pool = RunWorkerPool() else: self.pool = ParallelRunWorkerPool() self.run_block_size = Settings()["run/run_block_size"] # type: int """ Number of benchmarking runs that are done together """ self.discarded_runs = Settings()["run/discarded_runs"] # type: int """ First n runs that are discarded """ self.max_runs = Settings()["run/max_runs"] # type: int """ Maximum number of benchmarking runs """ self.min_runs = Settings()["run/min_runs"] # type: int """ Minimum number of benchmarking runs """ if self.min_runs > self.max_runs: logging.warning("min_runs ({}) is bigger than max_runs ({}), therefore they are swapped." .format(self.min_runs, self.max_runs)) tmp = self.min_runs self.min_runs = self.max_runs self.max_runs = tmp self.shuffle = Settings()["run/shuffle"] # type: bool """ Randomize the order in which the program blocks are benchmarked. """ self.fixed_runs = Settings()["run/runs"] != -1 # type: bool """ Do a fixed number of benchmarking runs? """ if self.fixed_runs: self.min_runs = self.max_runs = self.min_runs = Settings()["run/runs"] self.start_time = round(time.time()) # type: float """ Unix time stamp of the start of the benchmarking """ self.end_time = None # type: float """ Unix time stamp of the point in time that the benchmarking can at most reach """ try: self.end_time = self.start_time + pytimeparse.parse(Settings()["run/max_time"]) except: self.teardown() raise self.store_often = Settings()["run/store_often"] # type: bool """ Store the result file after each set of blocks is benchmarked """ self.block_run_count = 0 # type: int """ Number of benchmarked blocks """ self.erroneous_run_blocks = [] # type: t.List[t.Tuple[int, BenchmarkingResultBlock]] """ List of all failing run blocks (id and results till failing) """
class RunProcessor: """ This class handles the coordination of the whole benchmarking process. It is configured by setting the settings of the stats and run domain. Important note: the constructor also setups the cpu sets and plugins that can alter the system, e.g. confine most processes on only one core. Be sure to call the ``teardown()`` or the ``benchmark()`` method to make the system usable again. """ def __init__(self, runs: t.List[dict] = None, append: bool = None, show_report: bool = None): """ Creates an instance and setup everything. :param runs: list of dictionaries that represent run program blocks if None Settings()["run/in"] is used :param append: append to the old benchmarks if there are any in the result file? :param show_report: show a short report after finishing the benchmarking? """ if runs is None: typecheck(Settings()["run/in"], ValidYamlFileName()) with open(Settings()["run/in"], "r") as f: runs = yaml.load(f) typecheck(runs, List(Dict({ "attributes": Dict(all_keys=False, key_type=Str()), "run_config": Dict(all_keys=False) }))) self.runs = runs # type: t.List[dict] """ List of dictionaries that represent run program blocks """ self.run_blocks = [] # type: t.List[RunProgramBlock] """ Run program blocks for each dictionary in ``runs```""" for (id, run) in enumerate(runs): self.run_blocks.append(RunProgramBlock.from_dict(id, copy.deepcopy(run))) self.append = Settings().default(append, "run/append") # type: bool """ Append to the old benchmarks if there are any in the result file? """ self.show_report = Settings().default(show_report, "run/show_report") # type: bool """ Show a short report after finishing the benchmarking? """ self.stats_helper = None # type: RunDataStatsHelper """ Used stats helper to help with measurements """ typecheck(Settings()["run/out"], FileName()) if self.append: run_data = [] try: if os.path.exists(Settings()["run/out"]): with open(Settings()["run/out"], "r") as f: run_data = yaml.load(f) self.stats_helper = RunDataStatsHelper.init_from_dicts(run_data, external=True) for run in runs: self.stats_helper.runs.append(RunData(attributes=run["attributes"])) except: self.teardown() raise else: self.stats_helper = RunDataStatsHelper.init_from_dicts(copy.deepcopy(runs)) #if Settings()["run/remote"]: # self.pool = RemoteRunWorkerPool(Settings()["run/remote"], Settings()["run/remote_port"]) if os.path.exists(Settings()["run/out"]): os.remove(Settings()["run/out"]) self.pool = None # type: AbstractRunWorkerPool """ Used run worker pool that abstracts the benchmarking """ if Settings()["run/cpuset/parallel"] == 0: self.pool = RunWorkerPool() else: self.pool = ParallelRunWorkerPool() self.run_block_size = Settings()["run/run_block_size"] # type: int """ Number of benchmarking runs that are done together """ self.discarded_runs = Settings()["run/discarded_runs"] # type: int """ First n runs that are discarded """ self.max_runs = Settings()["run/max_runs"] # type: int """ Maximum number of benchmarking runs """ self.min_runs = Settings()["run/min_runs"] # type: int """ Minimum number of benchmarking runs """ if self.min_runs > self.max_runs: logging.warning("min_runs ({}) is bigger than max_runs ({}), therefore they are swapped." .format(self.min_runs, self.max_runs)) tmp = self.min_runs self.min_runs = self.max_runs self.max_runs = tmp self.shuffle = Settings()["run/shuffle"] # type: bool """ Randomize the order in which the program blocks are benchmarked. """ self.fixed_runs = Settings()["run/runs"] != -1 # type: bool """ Do a fixed number of benchmarking runs? """ if self.fixed_runs: self.min_runs = self.max_runs = self.min_runs = Settings()["run/runs"] self.start_time = round(time.time()) # type: float """ Unix time stamp of the start of the benchmarking """ self.end_time = None # type: float """ Unix time stamp of the point in time that the benchmarking can at most reach """ try: self.end_time = self.start_time + pytimeparse.parse(Settings()["run/max_time"]) except: self.teardown() raise self.store_often = Settings()["run/store_often"] # type: bool """ Store the result file after each set of blocks is benchmarked """ self.block_run_count = 0 # type: int """ Number of benchmarked blocks """ self.erroneous_run_blocks = [] # type: t.List[t.Tuple[int, BenchmarkingResultBlock]] """ List of all failing run blocks (id and results till failing) """ def _finished(self) -> bool: if self.fixed_runs: return self.block_run_count >= self.max_runs return (len(self.stats_helper.get_program_ids_to_bench()) == 0 \ or not self._can_run_next_block()) and self.min_runs <= self.block_run_count def _can_run_next_block(self) -> bool: if not in_standalone_mode: estimated_time = self.stats_helper.estimate_time_for_next_round(self.run_block_size, all=self.block_run_count < self.min_runs) to_bench_count = len(self.stats_helper.get_program_ids_to_bench()) if round(time.time() + estimated_time) > self.end_time: logging.warning("Ran to long ({}) and is therefore now aborted. " "{} program blocks should've been benchmarked again." .format(humanfriendly.format_timespan(time.time() + estimated_time - self.start_time), to_bench_count)) return False if self.block_run_count >= self.max_runs and self.block_run_count >= self.min_runs: #print("benchmarked too often, block run count ", self.block_run_count, self.block_run_count + self.run_block_size > self.min_runs) logging.warning("Benchmarked program blocks to often and aborted therefore now.") return False return True def benchmark(self): """ Benchmark and teardown. """ try: show_progress = Settings().has_log_level("info") and \ ("exec" != RunDriverRegistry.get_used() or "start_stop" not in ExecRunDriver.get_used()) showed_progress_before = False discard_label = "Make the {} discarded benchmarks".format(self.discarded_runs) if self.fixed_runs: label = "Benchmark {} times".format(self.max_runs) else: label = "Benchmark {} to {} times".format(self.min_runs, self.max_runs) start_label = discard_label if self.discarded_runs > 0 else label label_format = "{:32s}" if show_progress: with click.progressbar(range(0, self.max_runs + self.discarded_runs), label=label_format.format(start_label)) as runs: for run in runs: if run < self.discarded_runs: self._benchmarking_block_run(block_size=1, discard=True) else: if self._finished(): break self._benchmarking_block_run() if run == self.discarded_runs - 1: runs.label = label_format.format(label) else: time_per_run = self._make_discarded_runs() last_round_time = time.time() if time_per_run != None: last_round_time -= time_per_run * self.run_block_size while not self._finished(): self._benchmarking_block_run() except BaseException as ex: logging.error("Forced teardown of RunProcessor") self.store_and_teardown() if isinstance(ex, KeyboardInterrupt) and Settings()["log_level"] == "info" and self.block_run_count > 0\ and self.show_report: self.print_report() raise self.store_and_teardown() def _benchmarking_block_run(self, block_size: int = None, discard: bool = False, bench_all: bool = None): block_size = block_size or self.run_block_size if bench_all is None: bench_all = self.block_run_count < self.min_runs try: to_bench = list(enumerate(self.run_blocks)) if not bench_all and self.block_run_count < self.max_runs and not in_standalone_mode: to_bench = [(i, self.run_blocks[i]) for i in self.stats_helper.get_program_ids_to_bench()] to_bench = [(i, b) for (i, b) in to_bench if self.stats_helper.runs[i] is not None] if self.shuffle: random.shuffle(to_bench) if len(to_bench) == 0: return for (id, run_block) in to_bench: self.pool.submit(run_block, id, self.run_block_size) for (block, result, id) in self.pool.results(len(to_bench)): if result.error: self.erroneous_run_blocks.append((id, result)) self.stats_helper.disable_run_data(id) logging.error("Program block no. {} failed: {}".format(id, result.error)) self.store_erroneous() elif not discard: self.stats_helper.add_data_block(id, result.data) if not discard: self.block_run_count += block_size except BaseException as ex: self.store_and_teardown() logging.error("Forced teardown of RunProcessor") raise if not discard and self.store_often: self.store() def _make_discarded_runs(self) -> t.Optional[int]: if self.discarded_runs == 0: return None start_time = time.time() self._benchmarking_block_run(block_size=self.discarded_runs, discard=True, bench_all=True) return (time.time() - start_time) / self.discarded_runs def teardown(self): """ Teardown everything (make the system useable again) """ self.pool.teardown() def store_and_teardown(self): """ Teardown everything, store the result file, print a short report and send an email if configured to do so. """ if Settings().has_log_level("info") and self.show_report: self.print_report() self.teardown() self.store() if len(self.stats_helper.valid_runs()) > 0 \ and all(x.benchmarks() > 0 for x in self.stats_helper.valid_runs()): report = "" if not in_standalone_mode: report = ReporterRegistry.get_for_name("console", self.stats_helper)\ .report(with_tester_results=False, to_string = True) self.stats_helper.valid_runs()[0].description() subject = "Finished " + join_strs([repr(run.description()) for run in self.stats_helper.valid_runs()]) send_mail(Settings()["run/send_mail"], subject, report, [Settings()["run/out"]]) if len(self.erroneous_run_blocks) > 0: descrs = [] msgs = [] for (i, result) in self.erroneous_run_blocks: descr = repr(RunData(attributes=self.runs[i]["attributes"]).description()) descrs.append(descr) msg = descr + ":\n\t" + "\n\t".join(str(result.error).split("\n")) msgs.append(msg) subject = "Errors while benchmarking " + join_strs(descrs) send_mail(Settings()["run/send_mail"], subject, "\n\n".join(msgs), [Settings()["run/in"] + ".erroneous.yaml"]) def store(self): """ Store the result file """ try: self.stats_helper.add_property_descriptions(self.pool.run_driver.get_property_descriptions()) except (IOError, OSError) as ex: logging.error(ex) if len(self.stats_helper.valid_runs()) > 0 \ and all(x.benchmarks() > 0 for x in self.stats_helper.valid_runs()): with open(Settings()["run/out"], "w") as f: f.write(yaml.dump(self.stats_helper.serialize())) def store_erroneous(self): """ Store the failing program blocks in a file ending with ``.erroneous.yaml``. """ if len(self.erroneous_run_blocks) == 0: return file_name = Settings()["run/in"] + ".erroneous.yaml" try: blocks = [self.runs[x[0]] for x in self.erroneous_run_blocks] with open(file_name, "w") as f: f.write(yaml.dump(blocks)) except IOError as err: logging.error("Can't write erroneous program blocks to " + file_name) def print_report(self) -> str: if in_standalone_mode: return """ Print a short report if possible. """ try: if len(self.stats_helper.valid_runs()) > 0 and \ all(x.benchmarks() > 0 for x in self.stats_helper.valid_runs()): ReporterRegistry.get_for_name("console", self.stats_helper).report(with_tester_results=False) except: pass
class RunProcessor: """ This class handles the coordination of the whole benchmarking process. It is configured by setting the settings of the stats and run domain. Important note: the constructor also setups the cpu sets and plugins that can alter the system, e.g. confine most processes on only one core. Be sure to call the ``teardown()`` or the ``benchmark()`` method to make the system usable again. """ def __init__(self, runs: t.List[dict] = None, append: bool = None, show_report: bool = None): """ Creates an instance and setup everything. :param runs: list of dictionaries that represent run program blocks if None Settings()["run/in"] is used :param append: append to the old benchmarks if there are any in the result file? :param show_report: show a short report after finishing the benchmarking? """ if runs is None: typecheck(Settings()["run/in"], ValidYamlFileName()) with open(Settings()["run/in"], "r") as f: runs = yaml.load(f) typecheck( runs, List( Dict({ "attributes": Dict(all_keys=False, key_type=Str()), "run_config": Dict(all_keys=False) }))) self.runs = runs # type: t.List[dict] """ List of dictionaries that represent run program blocks """ self.run_blocks = [] # type: t.List[RunProgramBlock] """ Run program blocks for each dictionary in ``runs```""" for (id, run) in enumerate(runs): self.run_blocks.append( RunProgramBlock.from_dict(id, copy.deepcopy(run))) self.append = Settings().default(append, "run/append") # type: bool """ Append to the old benchmarks if there are any in the result file? """ self.show_report = Settings().default(show_report, "run/show_report") # type: bool """ Show a short report after finishing the benchmarking? """ self.stats_helper = None # type: RunDataStatsHelper """ Used stats helper to help with measurements """ typecheck(Settings()["run/out"], FileName()) if self.append: run_data = [] try: if os.path.exists(Settings()["run/out"]): with open(Settings()["run/out"], "r") as f: run_data = yaml.load(f) self.stats_helper = RunDataStatsHelper.init_from_dicts( run_data, external=True) for run in runs: self.stats_helper.runs.append( RunData(attributes=run["attributes"])) except: self.teardown() raise else: self.stats_helper = RunDataStatsHelper.init_from_dicts( copy.deepcopy(runs)) #if Settings()["run/remote"]: # self.pool = RemoteRunWorkerPool(Settings()["run/remote"], Settings()["run/remote_port"]) if os.path.exists(Settings()["run/out"]): os.remove(Settings()["run/out"]) self.pool = None # type: AbstractRunWorkerPool """ Used run worker pool that abstracts the benchmarking """ if Settings()["run/cpuset/parallel"] == 0: self.pool = RunWorkerPool() else: self.pool = ParallelRunWorkerPool() self.run_block_size = Settings()["run/run_block_size"] # type: int """ Number of benchmarking runs that are done together """ self.discarded_runs = Settings()["run/discarded_runs"] # type: int """ First n runs that are discarded """ self.max_runs = Settings()["run/max_runs"] # type: int """ Maximum number of benchmarking runs """ self.min_runs = Settings()["run/min_runs"] # type: int """ Minimum number of benchmarking runs """ if self.min_runs > self.max_runs: logging.warning( "min_runs ({}) is bigger than max_runs ({}), therefore they are swapped." .format(self.min_runs, self.max_runs)) tmp = self.min_runs self.min_runs = self.max_runs self.max_runs = tmp self.shuffle = Settings()["run/shuffle"] # type: bool """ Randomize the order in which the program blocks are benchmarked. """ self.fixed_runs = Settings()["run/runs"] != -1 # type: bool """ Do a fixed number of benchmarking runs? """ if self.fixed_runs: self.min_runs = self.max_runs = self.min_runs = Settings( )["run/runs"] self.start_time = round(time.time()) # type: float """ Unix time stamp of the start of the benchmarking """ self.end_time = None # type: float """ Unix time stamp of the point in time that the benchmarking can at most reach """ try: self.end_time = self.start_time + pytimeparse.parse( Settings()["run/max_time"]) except: self.teardown() raise self.store_often = Settings()["run/store_often"] # type: bool """ Store the result file after each set of blocks is benchmarked """ self.block_run_count = 0 # type: int """ Number of benchmarked blocks """ self.erroneous_run_blocks = [ ] # type: t.List[t.Tuple[int, BenchmarkingResultBlock]] """ List of all failing run blocks (id and results till failing) """ def _finished(self) -> bool: if self.fixed_runs: return self.block_run_count >= self.max_runs return (len(self.stats_helper.get_program_ids_to_bench()) == 0 \ or not self._can_run_next_block()) and self.min_runs <= self.block_run_count def _can_run_next_block(self) -> bool: if not in_standalone_mode: estimated_time = self.stats_helper.estimate_time_for_next_round( self.run_block_size, all=self.block_run_count < self.min_runs) to_bench_count = len(self.stats_helper.get_program_ids_to_bench()) if round(time.time() + estimated_time) > self.end_time: logging.warning( "Ran to long ({}) and is therefore now aborted. " "{} program blocks should've been benchmarked again.". format( humanfriendly.format_timespan(time.time() + estimated_time - self.start_time), to_bench_count)) return False if self.block_run_count >= self.max_runs and self.block_run_count >= self.min_runs: #print("benchmarked too often, block run count ", self.block_run_count, self.block_run_count + self.run_block_size > self.min_runs) logging.warning( "Benchmarked program blocks to often and aborted therefore now." ) return False return True def benchmark(self): """ Benchmark and teardown. """ try: show_progress = Settings().has_log_level("info") and \ ("exec" != RunDriverRegistry.get_used() or "start_stop" not in ExecRunDriver.get_used()) showed_progress_before = False discard_label = "Make the {} discarded benchmarks".format( self.discarded_runs) if self.fixed_runs: label = "Benchmark {} times".format(self.max_runs) else: label = "Benchmark {} to {} times".format( self.min_runs, self.max_runs) start_label = discard_label if self.discarded_runs > 0 else label label_format = "{:32s}" if show_progress: with click.progressbar( range(0, self.max_runs + self.discarded_runs), label=label_format.format(start_label)) as runs: for run in runs: if run < self.discarded_runs: self._benchmarking_block_run(block_size=1, discard=True) else: if self._finished(): break self._benchmarking_block_run() if run == self.discarded_runs - 1: runs.label = label_format.format(label) else: time_per_run = self._make_discarded_runs() last_round_time = time.time() if time_per_run != None: last_round_time -= time_per_run * self.run_block_size while not self._finished(): self._benchmarking_block_run() except BaseException as ex: logging.error("Forced teardown of RunProcessor") self.store_and_teardown() if isinstance(ex, KeyboardInterrupt) and Settings()["log_level"] == "info" and self.block_run_count > 0\ and self.show_report: self.print_report() raise self.store_and_teardown() def _benchmarking_block_run(self, block_size: int = None, discard: bool = False, bench_all: bool = None): block_size = block_size or self.run_block_size if bench_all is None: bench_all = self.block_run_count < self.min_runs try: to_bench = list(enumerate(self.run_blocks)) if not bench_all and self.block_run_count < self.max_runs and not in_standalone_mode: to_bench = [ (i, self.run_blocks[i]) for i in self.stats_helper.get_program_ids_to_bench() ] to_bench = [(i, b) for (i, b) in to_bench if self.stats_helper.runs[i] is not None] if self.shuffle: random.shuffle(to_bench) if len(to_bench) == 0: return for (id, run_block) in to_bench: self.pool.submit(run_block, id, self.run_block_size) for (block, result, id) in self.pool.results(len(to_bench)): if result.error: self.erroneous_run_blocks.append((id, result)) self.stats_helper.disable_run_data(id) logging.error("Program block no. {} failed: {}".format( id, result.error)) self.store_erroneous() elif not discard: self.stats_helper.add_data_block(id, result.data) if not discard: self.block_run_count += block_size except BaseException as ex: self.store_and_teardown() logging.error("Forced teardown of RunProcessor") raise if not discard and self.store_often: self.store() def _make_discarded_runs(self) -> t.Optional[int]: if self.discarded_runs == 0: return None start_time = time.time() self._benchmarking_block_run(block_size=self.discarded_runs, discard=True, bench_all=True) return (time.time() - start_time) / self.discarded_runs def teardown(self): """ Teardown everything (make the system useable again) """ self.pool.teardown() def store_and_teardown(self): """ Teardown everything, store the result file, print a short report and send an email if configured to do so. """ if Settings().has_log_level("info") and self.show_report: self.print_report() self.teardown() self.store() if len(self.stats_helper.valid_runs()) > 0 \ and all(x.benchmarks() > 0 for x in self.stats_helper.valid_runs()): report = "" if not in_standalone_mode: report = ReporterRegistry.get_for_name("console", self.stats_helper)\ .report(with_tester_results=False, to_string = True) self.stats_helper.valid_runs()[0].description() subject = "Finished " + join_strs([ repr(run.description()) for run in self.stats_helper.valid_runs() ]) send_mail(Settings()["run/send_mail"], subject, report, [Settings()["run/out"]]) if len(self.erroneous_run_blocks) > 0: descrs = [] msgs = [] for (i, result) in self.erroneous_run_blocks: descr = repr( RunData( attributes=self.runs[i]["attributes"]).description()) descrs.append(descr) msg = descr + ":\n\t" + "\n\t".join( str(result.error).split("\n")) msgs.append(msg) subject = "Errors while benchmarking " + join_strs(descrs) send_mail(Settings()["run/send_mail"], subject, "\n\n".join(msgs), [Settings()["run/in"] + ".erroneous.yaml"]) def store(self): """ Store the result file """ try: self.stats_helper.add_property_descriptions( self.pool.run_driver.get_property_descriptions()) except (IOError, OSError) as ex: logging.error(ex) if len(self.stats_helper.valid_runs()) > 0 \ and all(x.benchmarks() > 0 for x in self.stats_helper.valid_runs()): with open(Settings()["run/out"], "w") as f: f.write(yaml.dump(self.stats_helper.serialize())) def store_erroneous(self): """ Store the failing program blocks in a file ending with ``.erroneous.yaml``. """ if len(self.erroneous_run_blocks) == 0: return file_name = Settings()["run/in"] + ".erroneous.yaml" try: blocks = [self.runs[x[0]] for x in self.erroneous_run_blocks] with open(file_name, "w") as f: f.write(yaml.dump(blocks)) except IOError as err: logging.error("Can't write erroneous program blocks to " + file_name) def print_report(self) -> str: if in_standalone_mode: return """ Print a short report if possible. """ try: if len(self.stats_helper.valid_runs()) > 0 and \ all(x.benchmarks() > 0 for x in self.stats_helper.valid_runs()): ReporterRegistry.get_for_name( "console", self.stats_helper).report(with_tester_results=False) except: pass
def __init__(self, runs: t.List[dict] = None, append: bool = None, show_report: bool = None): """ Creates an instance and setup everything. :param runs: list of dictionaries that represent run program blocks if None Settings()["run/in"] is used :param append: append to the old benchmarks if there are any in the result file? :param show_report: show a short report after finishing the benchmarking? """ if runs is None: typecheck(Settings()["run/in"], ValidYamlFileName()) with open(Settings()["run/in"], "r") as f: runs = yaml.load(f) typecheck( runs, List( Dict({ "attributes": Dict(all_keys=False, key_type=Str()), "run_config": Dict(all_keys=False) }))) self.runs = runs # type: t.List[dict] """ List of dictionaries that represent run program blocks """ self.run_blocks = [] # type: t.List[RunProgramBlock] """ Run program blocks for each dictionary in ``runs```""" for (id, run) in enumerate(runs): self.run_blocks.append( RunProgramBlock.from_dict(id, copy.deepcopy(run))) self.append = Settings().default(append, "run/append") # type: bool """ Append to the old benchmarks if there are any in the result file? """ self.show_report = Settings().default(show_report, "run/show_report") # type: bool """ Show a short report after finishing the benchmarking? """ self.stats_helper = None # type: RunDataStatsHelper """ Used stats helper to help with measurements """ typecheck(Settings()["run/out"], FileName()) if self.append: run_data = [] try: if os.path.exists(Settings()["run/out"]): with open(Settings()["run/out"], "r") as f: run_data = yaml.load(f) self.stats_helper = RunDataStatsHelper.init_from_dicts( run_data, external=True) for run in runs: self.stats_helper.runs.append( RunData(attributes=run["attributes"])) except: self.teardown() raise else: self.stats_helper = RunDataStatsHelper.init_from_dicts( copy.deepcopy(runs)) #if Settings()["run/remote"]: # self.pool = RemoteRunWorkerPool(Settings()["run/remote"], Settings()["run/remote_port"]) if os.path.exists(Settings()["run/out"]): os.remove(Settings()["run/out"]) self.pool = None # type: AbstractRunWorkerPool """ Used run worker pool that abstracts the benchmarking """ if Settings()["run/cpuset/parallel"] == 0: self.pool = RunWorkerPool() else: self.pool = ParallelRunWorkerPool() self.run_block_size = Settings()["run/run_block_size"] # type: int """ Number of benchmarking runs that are done together """ self.discarded_runs = Settings()["run/discarded_runs"] # type: int """ First n runs that are discarded """ self.max_runs = Settings()["run/max_runs"] # type: int """ Maximum number of benchmarking runs """ self.min_runs = Settings()["run/min_runs"] # type: int """ Minimum number of benchmarking runs """ if self.min_runs > self.max_runs: logging.warning( "min_runs ({}) is bigger than max_runs ({}), therefore they are swapped." .format(self.min_runs, self.max_runs)) tmp = self.min_runs self.min_runs = self.max_runs self.max_runs = tmp self.shuffle = Settings()["run/shuffle"] # type: bool """ Randomize the order in which the program blocks are benchmarked. """ self.fixed_runs = Settings()["run/runs"] != -1 # type: bool """ Do a fixed number of benchmarking runs? """ if self.fixed_runs: self.min_runs = self.max_runs = self.min_runs = Settings( )["run/runs"] self.start_time = round(time.time()) # type: float """ Unix time stamp of the start of the benchmarking """ self.end_time = None # type: float """ Unix time stamp of the point in time that the benchmarking can at most reach """ try: self.end_time = self.start_time + pytimeparse.parse( Settings()["run/max_time"]) except: self.teardown() raise self.store_often = Settings()["run/store_often"] # type: bool """ Store the result file after each set of blocks is benchmarked """ self.block_run_count = 0 # type: int """ Number of benchmarked blocks """ self.erroneous_run_blocks = [ ] # type: t.List[t.Tuple[int, BenchmarkingResultBlock]] """ List of all failing run blocks (id and results till failing) """
class RunProcessor: """ This class handles the coordination of the whole benchmarking process. It is configured by setting the settings of the stats and run domain. Important note: the constructor also setups the cpu sets and plugins that can alter the system, e.g. confine most processes on only one core. Be sure to call the ``teardown()`` or the ``benchmark()`` method to make the system usable again. """ def __init__(self, runs: t.List[dict] = None, append: bool = None, show_report: bool = None): """ Creates an instance and setup everything. :param runs: list of dictionaries that represent run program blocks if None Settings()["run/in"] is used :param append: append to the old benchmarks if there are any in the result file? :param show_report: show a short report after finishing the benchmarking? """ if runs is None: typecheck(Settings()["run/in"], ValidYamlFileName(), value_name="run/in") with open(Settings()["run/in"], "r") as f: runs = yaml.safe_load(f) self.runs = runs # type: t.List[dict] """ List of dictionaries that represent run program blocks """ self.run_blocks = [] # type: t.List[RunProgramBlock] """ Run program blocks for each dictionary in ``runs```""" for (id, run) in enumerate(runs): self.run_blocks.append(RunProgramBlock.from_dict(id, copy.deepcopy(run))) old_blocks = self.run_blocks self.run_blocks = filter_runs(self.run_blocks, Settings()["run/included_blocks"]) self.runs = [runs[next(i for i, o in enumerate(old_blocks) if o == b)] for b in self.run_blocks] self.append = Settings().default(append, "run/append") # type: bool """ Append to the old benchmarks if there are any in the result file? """ self.show_report = Settings().default(show_report, "run/show_report") # type: bool """ Show a short report after finishing the benchmarking? """ self.stats_helper = None # type: RunDataStatsHelper """ Used stats helper to help with measurements """ typecheck(Settings()["run/out"], FileName()) if self.append: run_data = [] try: if os.path.exists(Settings()["run/out"]): with open(Settings()["run/out"], "r") as f: run_data = yaml.safe_load(f) self.stats_helper = RunDataStatsHelper.init_from_dicts(run_data, external=True) for run in runs: self.stats_helper.runs.append(RunData(attributes=run["attributes"])) except: self.teardown() raise else: self.stats_helper = RunDataStatsHelper.init_from_dicts(copy.deepcopy(runs), included_blocks=Settings()["run/included_blocks"]) #if Settings()["run/remote"]: # self.pool = RemoteRunWorkerPool(Settings()["run/remote"], Settings()["run/remote_port"]) if os.path.exists(Settings()["run/out"]): os.remove(Settings()["run/out"]) self.start_time = time.time() # type: float """ Unix time stamp of the start of the benchmarking """ self.end_time = -1 # type: float """ Unix time stamp of the point in time that the benchmarking can at most reach """ try: max_time = parse_timespan(Settings()["run/max_time"]) if max_time > -1: self.end_time = self.start_time + max_time except: self.teardown() raise self.pool = None # type: AbstractRunWorkerPool """ Used run worker pool that abstracts the benchmarking """ if Settings()["run/cpuset/parallel"] == 0: self.pool = RunWorkerPool(end_time=self.end_time) else: self.pool = ParallelRunWorkerPool(end_time=self.end_time) self.run_block_size = Settings()["run/run_block_size"] # type: int """ Number of benchmarking runs that are done together """ self.discarded_runs = Settings()["run/discarded_runs"] # type: int """ First n runs that are discarded """ self.max_runs = Settings()["run/max_runs"] # type: int """ Maximum number of benchmarking runs """ self.min_runs = Settings()["run/min_runs"] # type: int """ Minimum number of benchmarking runs """ if self.min_runs > self.max_runs: logging.warning("min_runs ({}) is bigger than max_runs ({}), therefore they are swapped." .format(self.min_runs, self.max_runs)) tmp = self.min_runs self.min_runs = self.max_runs self.max_runs = tmp self.shuffle = Settings()["run/shuffle"] # type: bool """ Randomize the order in which the program blocks are benchmarked. """ self.fixed_runs = Settings()["run/runs"] != -1 # type: bool """ Do a fixed number of benchmarking runs? """ if self.fixed_runs: self.min_runs = self.max_runs = self.min_runs = Settings()["run/runs"] self.store_often = Settings()["run/store_often"] # type: bool """ Store the result file after each set of blocks is benchmarked """ self.block_run_count = 0 # type: int """ Number of benchmarked blocks """ self.erroneous_run_blocks = [] # type: t.List[t.Tuple[int, BenchmarkingResultBlock]] """ List of all failing run blocks (id and results till failing) """ self.discard_all_data_for_block_on_error = Settings()["run/discard_all_data_for_block_on_error"] self.no_build = Settings()["run/no_build"] self.only_build = Settings()["run/only_build"] self.abort_after_build_error = Settings()["run/abort_after_build_error"] def _finished(self) -> bool: if not self.pool.has_time_left(): return True if self.fixed_runs: return self.block_run_count >= self.max_runs return (len(self.stats_helper.get_program_ids_to_bench()) == 0 \ or not self._can_run_next_block()) and self.maximum_of_min_runs() <= self.block_run_count def maximum_of_min_runs(self) -> int: return max(list(block.min_runs for block in self.run_blocks) + [self.min_runs]) def maximum_of_max_runs(self) -> int: return max(list(block.max_runs for block in self.run_blocks) + [self.max_runs]) def _can_run_next_block(self) -> bool: if not in_standalone_mode: estimated_time = self.stats_helper.estimate_time_for_next_round(self.run_block_size, all=self.block_run_count < self.min_runs) to_bench_count = len(self.stats_helper.get_program_ids_to_bench()) if -1 < self.end_time < round(time.time() + estimated_time): logging.warning("Ran too long ({}) and is therefore now aborted. " "{} program blocks should've been benchmarked again." .format(humanfriendly.format_timespan(time.time() + estimated_time - self.start_time), to_bench_count)) return False if self.block_run_count >= self.maximum_of_max_runs() and self.block_run_count >= self.maximum_of_min_runs(): logging.warning("Benchmarked program blocks too often and aborted therefore now.") return False return True def build(self): """ Build before benchmarking, essentially calls `temci build` where necessary and modifies the run configs """ if self.no_build: return to_build = [(i, conf) for i, conf in enumerate(self.runs) if "build_config" in conf] if len(to_build) == 0: return logging.info("Start building {} block(s)".format(len(to_build))) for i, block in to_build: if "working_dir" not in block["build_config"]: block["build_config"]["working_dir"] = self.run_blocks[i].data["cwd"] try: block = BuildProcessor.preprocess_build_blocks([block])[0] logging.info("Build {}".format(self.run_blocks[i].description())) block_builder = Builder(self.run_blocks[i].description(), block["build_config"]["working_dir"], block["build_config"]["cmd"], block["build_config"]["revision"], block["build_config"]["number"], block["build_config"]["base_dir"], block["build_config"]["branch"]) working_dirs = block_builder.build() block["cwds"] = working_dirs self.run_blocks[i].data["cwds"] = working_dirs except BuildError as err: self.stats_helper.add_error(i, err.error) self.erroneous_run_blocks.append((i, BenchmarkingResultBlock(data={}, error=err, recorded_error=err.error))) if self.abort_after_build_error: raise err def benchmark(self): """ Benchmark and teardown. """ if self.only_build: return try: show_progress = Settings().has_log_level("info") and \ ("exec" != RunDriverRegistry.get_used() or "start_stop" not in ExecRunDriver.get_used()) showed_progress_before = False discard_label = "Make the {} discarded benchmarks".format(self.discarded_runs) if self.fixed_runs: label = "Benchmark {} times".format(self.max_runs) else: label = "Benchmark {} to {} times".format(self.min_runs, self.max_runs) start_label = discard_label if self.discarded_runs > 0 else label label_format = "{:32s}" if show_progress: def bench(run: int) -> bool: if run < self.discarded_runs: self._benchmarking_block_run(block_size=1, discard=True) else: recorded_run = run - self.discarded_runs if self._finished() or all(b.max_runs < recorded_run for b in self.run_blocks): return False self._benchmarking_block_run(run=recorded_run) return True import click with click.progressbar(range(0, self.max_runs + self.discarded_runs), label=label_format.format(start_label), file=None if self.pool.run_driver.runs_benchmarks else open(os.devnull, 'w')) as runs: discard_label = "Discarded benchmark {{}} out of {}".format(self.discarded_runs) if self.fixed_runs: label = "Benchmark {{}} out of {}".format(self.max_runs) else: label = "Benchmark {{}} out of {} to {}".format(self.min_runs, self.max_runs) def alter_label(run: int): if run < self.discarded_runs: runs.label = label_format.format(discard_label.format(run + 1)) else: runs.label = label_format.format(label.format(run - self.discarded_runs + 1)) runs.short_limit = 0 every = Settings()["run/watch_every"] if Settings()["run/watch"]: with Screen(scroll=True, keep_first_lines=1) as f: import click._termui_impl click._termui_impl.BEFORE_BAR = "\r" click._termui_impl.AFTER_BAR = "\n" runs.file = f if self.pool.run_driver.runs_benchmarks else "-" runs._last_line = "" def render(): f.reset() runs._last_line = "" runs.render_progress() f.advance_line() print(ReporterRegistry.get_for_name("console", self.stats_helper).report( with_tester_results=False, to_string=True), file=f) for run in runs: alter_label(run) f.enable() render() if run % every == 0 or True: f.display() f.reset() if not bench(run): break f.disable() runs.finish() render() f.display() else: alter_label(0) for run in runs: alter_label(run) runs._last_line = "" runs.render_progress() if not bench(run): break runs.finish() runs.render_progress() else: time_per_run = self._make_discarded_runs() last_round_time = time.time() if time_per_run != None: last_round_time -= time_per_run * self.run_block_size run = 0 while not self._finished(): self._benchmarking_block_run(run) run += 1 except BaseException as ex: logging.error("Forced teardown of RunProcessor") self.store_and_teardown() if isinstance(ex, KeyboardInterrupt) and Settings()["log_level"] == "info" and self.block_run_count > 0\ and self.show_report: self.print_report() raise if self.show_report and not Settings()["run/watch"]: self.print_report() self.store_and_teardown() def _benchmarking_block_run(self, block_size: int = None, discard: bool = False, bench_all: bool = None, run: int = None): block_size = block_size or self.run_block_size if bench_all is None: bench_all = self.block_run_count < self.maximum_of_min_runs() try: to_bench = list((i, b) for (i, b) in enumerate(self.run_blocks) if self._should_run(b, run)) if not bench_all and self.block_run_count < self.max_runs and not in_standalone_mode: to_bench = [(i, self.run_blocks[i]) for i in self.stats_helper.get_program_ids_to_bench() if self._should_run(self.run_blocks[i], run)] to_bench = [(i, b) for (i, b) in to_bench if self.stats_helper.runs[i] is not None and not self.stats_helper.has_error(i)] if self.shuffle: random.shuffle(to_bench) if len(to_bench) == 0: return benched = 0 for (id, run_block) in to_bench: if self.pool.has_time_left() > 0: benched += 1 self.pool.submit(run_block, id, self.run_block_size) else: logging.warn("Ran into timeout") break for (block, result, id) in self.pool.results(benched): if result.error: self.erroneous_run_blocks.append((id, result)) if self.discard_all_data_for_block_on_error: self.stats_helper.discard_run_data(id) if result.recorded_error: if not self.discard_all_data_for_block_on_error: self.stats_helper.add_data_block(id, result.data) self.stats_helper.add_error(id, result.recorded_error) logging.error("Program block no. {} failed: {}".format(id, result.error)) log_program_error(result.recorded_error) logging.debug("".join(traceback.format_exception(None, result.error, result.error.__traceback__))) self.store_erroneous() if isinstance(result.error, KeyboardInterrupt): raise result.error elif not discard: self.stats_helper.add_data_block(id, result.data) if not discard: self.block_run_count += block_size except BaseException as ex: #self.store_and_teardown() #logging.error("Forced teardown of RunProcessor") raise if not discard and self.store_often: self.store() def _should_run(self, block: RunProgramBlock, run: int = None) -> bool: return run < block.max_runs if run is not None else not self.stats_helper.has_error(block.id) def _make_discarded_runs(self) -> t.Optional[int]: if self.discarded_runs == 0: return None start_time = time.time() self._benchmarking_block_run(block_size=self.discarded_runs, discard=True, bench_all=True) return (time.time() - start_time) / self.discarded_runs def recorded_error(self) -> bool: return len(self.erroneous_run_blocks) > 0 def teardown(self): """ Teardown everything (make the system useable again) """ self.pool.teardown() def store_and_teardown(self): """ Teardown everything, store the result file, print a short report and send an email if configured to do so. """ self.teardown() if not self.pool.run_driver.store_files: return self.store() if len(self.stats_helper.valid_runs()) > 0 \ and all(x.benchmarks() > 0 for x in self.stats_helper.valid_runs()): report = "" if not in_standalone_mode: report = ReporterRegistry.get_for_name("console", self.stats_helper)\ .report(with_tester_results=False, to_string=True) subject = "Finished " + join_strs([repr(run.description()) for run in self.stats_helper.valid_runs()]) send_mail(Settings()["run/send_mail"], subject, report, [Settings()["run/out"]]) if self.recorded_error(): descrs = [] msgs = [] for (i, result) in self.erroneous_run_blocks: descr = self.run_blocks[i].description() descrs.append(descr) msg = descr + ":\n\t" + "\n\t".join(str(result.error).split("\n")) msgs.append(msg) subject = "Errors while benchmarking " + join_strs(descrs) send_mail(Settings()["run/send_mail"], subject, "\n\n".join(msgs), [Settings()["run/in"] + ".erroneous.yaml"]) def store(self): """ Store the result file """ try: self.stats_helper.add_property_descriptions(self.pool.run_driver.get_property_descriptions()) except (IOError, OSError) as ex: logging.error(ex) if (len(self.stats_helper.valid_runs()) > 0 and all(x.benchmarks() > 0 for x in self.stats_helper.valid_runs())) \ or Settings()["run/record_errors_in_file"]: with open(Settings()["run/out"], "w") as f: self.stats_helper.update_env_info(), f.write(yaml.dump(self.stats_helper.serialize())) chown(f) def store_erroneous(self): """ Store the failing program blocks in a file ending with ``.erroneous.yaml``. """ if len(self.erroneous_run_blocks) == 0: return file_name = Settings()["run/in"] + ".erroneous.yaml" try: blocks = [self.runs[x[0]] for x in self.erroneous_run_blocks] with open(file_name, "w") as f: f.write(yaml.dump(blocks)) chown(f) except IOError as err: logging.error("Can't write erroneous program blocks to " + file_name) def print_report(self) -> str: if in_standalone_mode: return """ Print a short report if possible. """ try: ReporterRegistry.get_for_name("console", self.stats_helper).report(with_tester_results=False) except: pass