def store_and_teardown(self): """ Teardown everything, store the result file, print a short report and send an email if configured to do so. """ if Settings().has_log_level("info") and self.show_report: self.print_report() self.teardown() self.store() if len(self.stats_helper.valid_runs()) > 0 \ and all(x.benchmarks() > 0 for x in self.stats_helper.valid_runs()): report = "" if not in_standalone_mode: report = ReporterRegistry.get_for_name("console", self.stats_helper)\ .report(with_tester_results=False, to_string = True) self.stats_helper.valid_runs()[0].description() subject = "Finished " + join_strs([repr(run.description()) for run in self.stats_helper.valid_runs()]) send_mail(Settings()["run/send_mail"], subject, report, [Settings()["run/out"]]) if len(self.erroneous_run_blocks) > 0: descrs = [] msgs = [] for (i, result) in self.erroneous_run_blocks: descr = repr(RunData(attributes=self.runs[i]["attributes"]).description()) descrs.append(descr) msg = descr + ":\n\t" + "\n\t".join(str(result.error).split("\n")) msgs.append(msg) subject = "Errors while benchmarking " + join_strs(descrs) send_mail(Settings()["run/send_mail"], subject, "\n\n".join(msgs), [Settings()["run/in"] + ".erroneous.yaml"])
def store_and_teardown(self): """ Teardown everything, store the result file, print a short report and send an email if configured to do so. """ self.teardown() if not self.pool.run_driver.store_files: return self.store() if len(self.stats_helper.valid_runs()) > 0 \ and all(x.benchmarks() > 0 for x in self.stats_helper.valid_runs()): report = "" if not in_standalone_mode: report = ReporterRegistry.get_for_name("console", self.stats_helper)\ .report(with_tester_results=False, to_string=True) subject = "Finished " + join_strs([repr(run.description()) for run in self.stats_helper.valid_runs()]) send_mail(Settings()["run/send_mail"], subject, report, [Settings()["run/out"]]) if self.recorded_error(): descrs = [] msgs = [] for (i, result) in self.erroneous_run_blocks: descr = self.run_blocks[i].description() descrs.append(descr) msg = descr + ":\n\t" + "\n\t".join(str(result.error).split("\n")) msgs.append(msg) subject = "Errors while benchmarking " + join_strs(descrs) send_mail(Settings()["run/send_mail"], subject, "\n\n".join(msgs), [Settings()["run/in"] + ".erroneous.yaml"])
def __init__(self, misc_settings: dict = None): """ Creates an instance. Also calls the setup methods on all registered plugins. It calls the setup() method. :param misc_settings: further settings """ self.misc_settings = misc_settings """ Further settings """ self.used_plugins = [] # type: t.List[RunDriverPlugin] """ Used and active plugins """ miss_root_plugins = [] is_root = has_root_privileges() for used in self.get_used(): klass = self.get_class(used) if klass.needs_root_privileges and not is_root: miss_root_plugins.append(used) else: self.used_plugins.append(self.get_for_name(used)) if miss_root_plugins: logging.warning( "The following plugins are disabled because they need root privileges: " + join_strs(miss_root_plugins)) self.setup()
def _execute(self, db: Database): if self._unsupported_distribution(): expected = join_strs( ["{} {} ".format(*l) for l in self.possible_distributions], "or") self._fail( "Unsupported distribution {} and release {}, expected {}". format(get_distribution_name(), get_distribution_release(), expected))
def _match_return_code(self, cmd: str, exptected_codes: t.Union[t.List[int], int], return_code: int): if isinstance(exptected_codes, int): exptected_codes = [exptected_codes] if return_code not in exptected_codes: raise BenchmarkingError( "Unexpected return code {} of {!r}, expected {}".format( str(return_code), cmd, join_strs(list(map(str, exptected_codes)), "or")))
def __init__(self, misc_settings: dict = None): """ Creates an instance. Also calls the setup methods on all registered plugins. It calls the setup() method. :param misc_settings: further settings """ self.misc_settings = misc_settings """ Further settings """ self.used_plugins = [] # type: t.List[RunDriverPlugin] """ Used and active plugins """ miss_root_plugins = [] is_root = has_root_privileges() for used in self.get_used(): klass = self.get_class(used) if klass.needs_root_privileges and not is_root: miss_root_plugins.append(used) else: self.used_plugins.append(self.get_for_name(used)) if miss_root_plugins: logging.warning("The following plugins are disabled because they need root privileges: " + join_strs(miss_root_plugins)) self.setup()
def _execute(self, db: Database): if self._unsupported_distribution(): expected = join_strs(["{} {} ".format(*l) for l in self.possible_distributions], "or") self._fail("Unsupported distribution {} and release {}, expected {}".format(get_distribution_name(), get_distribution_release(), expected))
def _execute(self, db: Database): if self._unsupported_distribution(): self._fail("Unsupported distribution {}, expected {}".format(get_distribution_name(), join_strs(self.possible_distributions, "or")))
def register(cls, name: str, klass: type, misc_type: Type, activate_by_default: bool = None, deprecated: bool = False): """ Registers a new class. The constructor of the class gets as first argument the misc settings. :param name: common name of the registered class :param klass: actual class :param misc_type: type scheme of the {name}_misc settings :param misc_default: default value of the {name}_misc settings :param deprecated: is the registered class deprecated and should not be used? """ if deprecated: Settings().get_type_scheme( cls.settings_key_path).unknown_keys = True return def format_str_list(val: t.List[str], sep: str = "and") -> str: return join_strs(list(map(repr, val)), sep) misc_type_empty = misc_type == Dict() or misc_type == Dict({}) misc_default = misc_type.get_default() description = None use_key_path = "/".join([cls.settings_key_path, cls.use_key]) misc_key = "{}_misc".format("/".join([cls.settings_key_path, name])) if klass.__doc__ is not None: header = "" # "Description of {} (class {}):\n".format(name, klass.__qualname__) lines = str(klass.__doc__.strip()).split("\n") lines = map(lambda x: " " + x.strip(), lines) description = Description(header + "\n".join(lines)) klass.__description__ = description.description misc_type //= description else: klass.__description__ = "" logging.error("Class level documentation for {} is missing".format( klass.__name__)) Settings().modify_setting(misc_key, misc_type) if cls.use_list: if not Settings().validate_key_path(use_key_path.split("/")) \ or isinstance(Settings().get_type_scheme(use_key_path), Any): use_key_type = (StrList() | Exact(name)) use_key_type.typecheck_default = False Settings().modify_setting( use_key_path, use_key_type // Default(cls.default) if cls.default is not None else use_key_type) else: use_key_list = Settings().get_type_scheme(use_key_path) assert isinstance(use_key_list, StrList) use_key_list |= Exact(name) use_key_list = Settings().get_type_scheme(use_key_path) use_key_list // Description("Possible {} are {}".format( cls.plugin_synonym[1], format_str_list(use_key_list.allowed_values))) active_path = "{}_active".format("/".join( [cls.settings_key_path, name])) if not Settings().validate_key_path(active_path.split("/")): Settings().modify_setting( active_path, BoolOrNone() // Default(activate_by_default)) Settings().get_type_scheme(active_path) // Description( "Enable: " + klass.__description__) else: if not Settings().validate_key_path(use_key_path.split("/")) \ or not isinstance(Settings().get_type_scheme(use_key_path), ExactEither): use_key_type = ExactEither(name) use_key_type.typecheck_default = False Settings().modify_setting( use_key_path, use_key_type // Default(cls.default) if cls.default else use_key_type) else: Settings().modify_setting( use_key_path, Settings().get_type_scheme(use_key_path) | Exact(name)) use_key_type = Settings().get_type_scheme(use_key_path) use_key_type // Description("Possible {} are {}".format( cls.plugin_synonym[1], format_str_list( use_key_type.exp_values))) cls.registry[name] = klass if not sphinx_doc(): return use_text = "" cls_use_text = "" if cls.use_list: use_text = "To use this {plugin} add its name (`{name}`) to the list at settings key `{key}` or " \ "set `{active}` to true."\ .format(plugin=cls.plugin_synonym[0], name=name, key=use_key_path, active="{}_active".format("/".join([cls.settings_key_path, name]))) cls_use_text = "The used {plugins} can be configured by editing the list under the settings key `{key}`."\ .format(plugins=cls.plugin_synonym[1], key=use_key_path) else: use_text = "To use this {plugin} set the currently used {plugin} (at key `{key}`) to its name (`{name}`)."\ .format(plugin=cls.plugin_synonym[0], name=name, key=use_key_path) cls_use_text = "The used {plugin} can be configured by editing the settings key `{key}`."\ .format(plugin=cls.plugin_synonym[0], key=use_key_path) other_plugins_text = "" used_type = Settings().get_type_scheme(use_key_path) possible_plugins = used_type.allowed_values if cls.use_list else used_type.exp_values # type: t.List[str] possbible_p_wo_self = [x for x in possible_plugins if x != name] if len(possible_plugins) == 2: other_plugins_text = "Another usable {plugin} is `{p}`.".format( plugin=cls.plugin_synonym[0], p=possbible_p_wo_self[0]) if len(possible_plugins) > 2: other_plugins_text = "Other usable {plugins} are {p}.".format( plugins=cls.plugin_synonym[1], p=join_strs(["`{}`".format(x) for x in possbible_p_wo_self])) default_plugins_text = "" if cls.default: if cls.use_list and len(cls.default) > 1: p = join_strs(["`{}`".format(x) for x in cls.default]) default_plugins_text = "The default {plugins} are {p}.".format( plugins=cls.plugin_synonym[1], p=p) else: default = cls.default[0] if cls.use_list else cls.default default_plugins_text = "The default {plugin} is `{p}`.".format( plugin=cls.plugin_synonym[0], p=default) if not misc_type_empty: klass.__doc__ += """ Configuration format: .. code-block:: yaml {default_yaml} This {plugin} can be configured under the settings key `{misc_key}`. """.format(default_yaml="\n ".join( misc_type.get_default_yaml().split("\n")), misc_key=misc_key, plugin=cls.plugin_synonym[0]) klass.__doc__ += """ {use_text} {other} {default} """.format(use_text=use_text, other=other_plugins_text, default=default_plugins_text) if not hasattr(cls, "__old_documentation__"): cls.__old_documentation__ = cls.__doc__ or "" cls.__doc__ = cls.__old_documentation__ cls.__doc__ += """ {use_text} {possible} """.format(use_text=cls_use_text, possible=Settings().get_type_scheme(use_key_path).description)
def format_str_list(val: t.List[str], sep: str = "and") -> str: return join_strs(list(map(repr, val)), sep)
def _match_return_code(self, cmd: str, exptected_codes: t.Union[t.List[int], int], return_code: int): if isinstance(exptected_codes, int): exptected_codes = [exptected_codes] if return_code not in exptected_codes: raise BenchmarkingError("Unexpected return code {} of {!r}, expected {}" .format(str(return_code), cmd, join_strs(list(map(str, exptected_codes)), "or")))
def _execute(self, db: Database): if self._unsupported_distribution(): self._fail("Unsupported distribution {}, expected {}".format( get_distribution_name(), join_strs(self.possible_distributions, "or")))
def format_str_list(val: t.List[str], sep: str = "and") -> str: return join_strs(list(map(repr, val)), sep)
def register(cls, name: str, klass: type, misc_type: Type): """ Registers a new class. The constructor of the class gets as first argument the misc settings. :param name: common name of the registered class :param klass: actual class :param misc_type: type scheme of the {name}_misc settings :param misc_default: default value of the {name}_misc settings """ def format_str_list(val: t.List[str], sep: str = "and") -> str: return join_strs(list(map(repr, val)), sep) misc_type_empty = misc_type == Dict() or misc_type == Dict({}) misc_default = misc_type.get_default() description = None use_key_path = "/".join([cls.settings_key_path, cls.use_key]) misc_key = "{}_misc".format("/".join([cls.settings_key_path, name])) if klass.__doc__ is not None: header = ""# "Description of {} (class {}):\n".format(name, klass.__qualname__) lines = str(klass.__doc__.strip()).split("\n") lines = map(lambda x: " " + x.strip(), lines) description = Description(header + "\n".join(lines)) klass.__description__ = description.description misc_type //= description else: klass.__description__ = "" logging.error("Class level documentation for {} is missing".format(klass.__name__)) Settings().modify_setting(misc_key, misc_type) if cls.use_list: if not Settings().validate_key_path(use_key_path.split("/")) \ or isinstance(Settings().get_type_scheme(use_key_path), Any): use_key_type = (StrList() | Exact(name)) use_key_type.typecheck_default = False Settings().modify_setting(use_key_path, use_key_type // Default(cls.default) if cls.default else use_key_type) else: use_key_list = Settings().get_type_scheme(use_key_path) assert isinstance(use_key_list, StrList) use_key_list |= Exact(name) use_key_list = Settings().get_type_scheme(use_key_path) use_key_list // Description("Possible {} are {}".format(cls.plugin_synonym[1], format_str_list(use_key_list.allowed_values))) active_path = "{}_active".format("/".join([cls.settings_key_path, name])) if not Settings().validate_key_path(active_path.split("/")): Settings().modify_setting(active_path, BoolOrNone() // Default(None)) Settings().get_type_scheme(active_path) // Description("Enable: " + klass.__description__) else: if not Settings().validate_key_path(use_key_path.split("/")) \ or not isinstance(Settings().get_type_scheme(use_key_path), ExactEither): use_key_type = ExactEither(name) use_key_type.typecheck_default = False Settings().modify_setting(use_key_path, use_key_type // Default(cls.default) if cls.default else use_key_type) else: Settings().modify_setting(use_key_path, Settings().get_type_scheme(use_key_path) | Exact(name)) use_key_type = Settings().get_type_scheme(use_key_path) use_key_type // Description("Possible {} are {}".format(cls.plugin_synonym[1], format_str_list(use_key_type.exp_values))) cls.registry[name] = klass if not sphinx_doc(): return use_text = "" cls_use_text = "" if cls.use_list: use_text = "To use this {plugin} add it's name (`{name}`) to the list at settings key `{key}` or " \ "set `{active}` to true."\ .format(plugin=cls.plugin_synonym[0], name=name, key=use_key_path, active="{}_active".format("/".join([cls.settings_key_path, name]))) cls_use_text = "The used {plugins} can be configured by editing the list under the settings key `{key}`."\ .format(plugins=cls.plugin_synonym[1], key=use_key_path) else: use_text = "To use this {plugin} set the currently used {plugin} (at key `{key}`) to its name (`{name}`)."\ .format(plugin=cls.plugin_synonym[0], name=name, key=use_key_path) cls_use_text = "The used {plugin} can be configured by editing the settings key `{key}`."\ .format(plugin=cls.plugin_synonym[0], key=use_key_path) other_plugins_text = "" used_type = Settings().get_type_scheme(use_key_path) possible_plugins = used_type.allowed_values if cls.use_list else used_type.exp_values # type: t.List[str] possbible_p_wo_self = [x for x in possible_plugins if x != name] if len(possible_plugins) == 2: other_plugins_text = "Another usable {plugin} is `{p}`.".format(plugin=cls.plugin_synonym[0], p=possbible_p_wo_self[0]) if len(possible_plugins) > 2: other_plugins_text = "Other usable {plugins} are {p}.".format(plugins=cls.plugin_synonym[1], p=join_strs(["`{}`".format(x) for x in possbible_p_wo_self])) default_plugins_text = "" if cls.default: if cls.use_list and len(cls.default) > 1: p = join_strs(["`{}`".format(x) for x in cls.default]) default_plugins_text = "The default {plugins} are {p}.".format(plugins=cls.plugin_synonym[1], p=p) else: default = cls.default[0] if cls.use_list else cls.default default_plugins_text = "The default {plugin} is `{p}`.".format(plugin=cls.plugin_synonym[0], p=default) if not misc_type_empty: klass.__doc__ += """ Configuration format: .. code-block:: yaml {default_yaml} This {plugin} can be configured under the settings key `{misc_key}`. """.format(default_yaml="\n ".join(misc_type.get_default_yaml().split("\n")), misc_key=misc_key, plugin=cls.plugin_synonym[0]) klass.__doc__ += """ {use_text} {other} {default} """.format(use_text=use_text, other=other_plugins_text, default=default_plugins_text) if not hasattr(cls, "__old_documentation__"): cls.__old_documentation__ = cls.__doc__ or "" cls.__doc__ = cls.__old_documentation__ cls.__doc__ += """ {use_text} {possible} """.format( use_text=cls_use_text, possible=Settings().get_type_scheme(use_key_path).description )