def store_and_teardown(self):
"""
Teardown everything, store the result file, print a short report and send an email
if configured to do so.
"""
if Settings().has_log_level("info") and self.show_report:
self.print_report()
self.teardown()
self.store()
if len(self.stats_helper.valid_runs()) > 0 \
and all(x.benchmarks() > 0 for x in self.stats_helper.valid_runs()):
report = ""
if not in_standalone_mode:
report = ReporterRegistry.get_for_name("console", self.stats_helper)\
.report(with_tester_results=False, to_string = True)
self.stats_helper.valid_runs()[0].description()
subject = "Finished " + join_strs([repr(run.description()) for run in self.stats_helper.valid_runs()])
send_mail(Settings()["run/send_mail"], subject, report, [Settings()["run/out"]])
if len(self.erroneous_run_blocks) > 0:
descrs = []
msgs = []
for (i, result) in self.erroneous_run_blocks:
descr = repr(RunData(attributes=self.runs[i]["attributes"]).description())
descrs.append(descr)
msg = descr + ":\n\t" + "\n\t".join(str(result.error).split("\n"))
msgs.append(msg)
subject = "Errors while benchmarking " + join_strs(descrs)
send_mail(Settings()["run/send_mail"], subject, "\n\n".join(msgs), [Settings()["run/in"] + ".erroneous.yaml"])
def store_and_teardown(self):
"""
Teardown everything, store the result file, print a short report and send an email
if configured to do so.
"""
self.teardown()
if not self.pool.run_driver.store_files:
return
self.store()
if len(self.stats_helper.valid_runs()) > 0 \
and all(x.benchmarks() > 0 for x in self.stats_helper.valid_runs()):
report = ""
if not in_standalone_mode:
report = ReporterRegistry.get_for_name("console", self.stats_helper)\
.report(with_tester_results=False, to_string=True)
subject = "Finished " + join_strs([repr(run.description()) for run in self.stats_helper.valid_runs()])
send_mail(Settings()["run/send_mail"], subject, report, [Settings()["run/out"]])
if self.recorded_error():
descrs = []
msgs = []
for (i, result) in self.erroneous_run_blocks:
descr = self.run_blocks[i].description()
descrs.append(descr)
msg = descr + ":\n\t" + "\n\t".join(str(result.error).split("\n"))
msgs.append(msg)
subject = "Errors while benchmarking " + join_strs(descrs)
send_mail(Settings()["run/send_mail"], subject, "\n\n".join(msgs), [Settings()["run/in"] + ".erroneous.yaml"])
def __init__(self, misc_settings: dict = None):
"""
Creates an instance.
Also calls the setup methods on all registered plugins.
It calls the setup() method.
:param misc_settings: further settings
"""
self.misc_settings = misc_settings
""" Further settings """
self.used_plugins = [] # type: t.List[RunDriverPlugin]
""" Used and active plugins """
miss_root_plugins = []
is_root = has_root_privileges()
for used in self.get_used():
klass = self.get_class(used)
if klass.needs_root_privileges and not is_root:
miss_root_plugins.append(used)
else:
self.used_plugins.append(self.get_for_name(used))
if miss_root_plugins:
logging.warning(
"The following plugins are disabled because they need root privileges: "
+ join_strs(miss_root_plugins))
self.setup()
def _execute(self, db: Database):
if self._unsupported_distribution():
expected = join_strs(
["{} {} ".format(*l) for l in self.possible_distributions],
"or")
self._fail(
"Unsupported distribution {} and release {}, expected {}".
format(get_distribution_name(), get_distribution_release(),
expected))
def _match_return_code(self, cmd: str,
exptected_codes: t.Union[t.List[int],
int], return_code: int):
if isinstance(exptected_codes, int):
exptected_codes = [exptected_codes]
if return_code not in exptected_codes:
raise BenchmarkingError(
"Unexpected return code {} of {!r}, expected {}".format(
str(return_code), cmd,
join_strs(list(map(str, exptected_codes)), "or")))
def __init__(self, misc_settings: dict = None):
"""
Creates an instance.
Also calls the setup methods on all registered plugins.
It calls the setup() method.
:param misc_settings: further settings
"""
self.misc_settings = misc_settings
""" Further settings """
self.used_plugins = [] # type: t.List[RunDriverPlugin]
""" Used and active plugins """
miss_root_plugins = []
is_root = has_root_privileges()
for used in self.get_used():
klass = self.get_class(used)
if klass.needs_root_privileges and not is_root:
miss_root_plugins.append(used)
else:
self.used_plugins.append(self.get_for_name(used))
if miss_root_plugins:
logging.warning("The following plugins are disabled because they need root privileges: " +
join_strs(miss_root_plugins))
self.setup()
def _execute(self, db: Database):
if self._unsupported_distribution():
expected = join_strs(["{} {} ".format(*l) for l in self.possible_distributions], "or")
self._fail("Unsupported distribution {} and release {}, expected {}".format(get_distribution_name(),
get_distribution_release(),
expected))
def _execute(self, db: Database):
if self._unsupported_distribution():
self._fail("Unsupported distribution {}, expected {}".format(get_distribution_name(),
join_strs(self.possible_distributions, "or")))
def register(cls,
name: str,
klass: type,
misc_type: Type,
activate_by_default: bool = None,
deprecated: bool = False):
"""
Registers a new class.
The constructor of the class gets as first argument the misc settings.
:param name: common name of the registered class
:param klass: actual class
:param misc_type: type scheme of the {name}_misc settings
:param misc_default: default value of the {name}_misc settings
:param deprecated: is the registered class deprecated and should not be used?
"""
if deprecated:
Settings().get_type_scheme(
cls.settings_key_path).unknown_keys = True
return
def format_str_list(val: t.List[str], sep: str = "and") -> str:
return join_strs(list(map(repr, val)), sep)
misc_type_empty = misc_type == Dict() or misc_type == Dict({})
misc_default = misc_type.get_default()
description = None
use_key_path = "/".join([cls.settings_key_path, cls.use_key])
misc_key = "{}_misc".format("/".join([cls.settings_key_path, name]))
if klass.__doc__ is not None:
header = "" # "Description of {} (class {}):\n".format(name, klass.__qualname__)
lines = str(klass.__doc__.strip()).split("\n")
lines = map(lambda x: " " + x.strip(), lines)
description = Description(header + "\n".join(lines))
klass.__description__ = description.description
misc_type //= description
else:
klass.__description__ = ""
logging.error("Class level documentation for {} is missing".format(
klass.__name__))
Settings().modify_setting(misc_key, misc_type)
if cls.use_list:
if not Settings().validate_key_path(use_key_path.split("/")) \
or isinstance(Settings().get_type_scheme(use_key_path), Any):
use_key_type = (StrList() | Exact(name))
use_key_type.typecheck_default = False
Settings().modify_setting(
use_key_path, use_key_type // Default(cls.default)
if cls.default is not None else use_key_type)
else:
use_key_list = Settings().get_type_scheme(use_key_path)
assert isinstance(use_key_list, StrList)
use_key_list |= Exact(name)
use_key_list = Settings().get_type_scheme(use_key_path)
use_key_list // Description("Possible {} are {}".format(
cls.plugin_synonym[1],
format_str_list(use_key_list.allowed_values)))
active_path = "{}_active".format("/".join(
[cls.settings_key_path, name]))
if not Settings().validate_key_path(active_path.split("/")):
Settings().modify_setting(
active_path,
BoolOrNone() // Default(activate_by_default))
Settings().get_type_scheme(active_path) // Description(
"Enable: " + klass.__description__)
else:
if not Settings().validate_key_path(use_key_path.split("/")) \
or not isinstance(Settings().get_type_scheme(use_key_path), ExactEither):
use_key_type = ExactEither(name)
use_key_type.typecheck_default = False
Settings().modify_setting(
use_key_path, use_key_type //
Default(cls.default) if cls.default else use_key_type)
else:
Settings().modify_setting(
use_key_path,
Settings().get_type_scheme(use_key_path) | Exact(name))
use_key_type = Settings().get_type_scheme(use_key_path)
use_key_type // Description("Possible {} are {}".format(
cls.plugin_synonym[1], format_str_list(
use_key_type.exp_values)))
cls.registry[name] = klass
if not sphinx_doc():
return
use_text = ""
cls_use_text = ""
if cls.use_list:
use_text = "To use this {plugin} add its name (`{name}`) to the list at settings key `{key}` or " \
"set `{active}` to true."\
.format(plugin=cls.plugin_synonym[0], name=name, key=use_key_path,
active="{}_active".format("/".join([cls.settings_key_path, name])))
cls_use_text = "The used {plugins} can be configured by editing the list under the settings key `{key}`."\
.format(plugins=cls.plugin_synonym[1], key=use_key_path)
else:
use_text = "To use this {plugin} set the currently used {plugin} (at key `{key}`) to its name (`{name}`)."\
.format(plugin=cls.plugin_synonym[0], name=name, key=use_key_path)
cls_use_text = "The used {plugin} can be configured by editing the settings key `{key}`."\
.format(plugin=cls.plugin_synonym[0], key=use_key_path)
other_plugins_text = ""
used_type = Settings().get_type_scheme(use_key_path)
possible_plugins = used_type.allowed_values if cls.use_list else used_type.exp_values # type: t.List[str]
possbible_p_wo_self = [x for x in possible_plugins if x != name]
if len(possible_plugins) == 2:
other_plugins_text = "Another usable {plugin} is `{p}`.".format(
plugin=cls.plugin_synonym[0], p=possbible_p_wo_self[0])
if len(possible_plugins) > 2:
other_plugins_text = "Other usable {plugins} are {p}.".format(
plugins=cls.plugin_synonym[1],
p=join_strs(["`{}`".format(x) for x in possbible_p_wo_self]))
default_plugins_text = ""
if cls.default:
if cls.use_list and len(cls.default) > 1:
p = join_strs(["`{}`".format(x) for x in cls.default])
default_plugins_text = "The default {plugins} are {p}.".format(
plugins=cls.plugin_synonym[1], p=p)
else:
default = cls.default[0] if cls.use_list else cls.default
default_plugins_text = "The default {plugin} is `{p}`.".format(
plugin=cls.plugin_synonym[0], p=default)
if not misc_type_empty:
klass.__doc__ += """
Configuration format:
.. code-block:: yaml
{default_yaml}
This {plugin} can be configured under the settings key `{misc_key}`.
""".format(default_yaml="\n ".join(
misc_type.get_default_yaml().split("\n")),
misc_key=misc_key,
plugin=cls.plugin_synonym[0])
klass.__doc__ += """
{use_text}
{other}
{default}
""".format(use_text=use_text,
other=other_plugins_text,
default=default_plugins_text)
if not hasattr(cls, "__old_documentation__"):
cls.__old_documentation__ = cls.__doc__ or ""
cls.__doc__ = cls.__old_documentation__
cls.__doc__ += """
{use_text}
{possible}
""".format(use_text=cls_use_text,
possible=Settings().get_type_scheme(use_key_path).description)
def format_str_list(val: t.List[str], sep: str = "and") -> str:
return join_strs(list(map(repr, val)), sep)
def _match_return_code(self, cmd: str, exptected_codes: t.Union[t.List[int], int], return_code: int):
if isinstance(exptected_codes, int):
exptected_codes = [exptected_codes]
if return_code not in exptected_codes:
raise BenchmarkingError("Unexpected return code {} of {!r}, expected {}"
.format(str(return_code), cmd, join_strs(list(map(str, exptected_codes)), "or")))
def _execute(self, db: Database):
if self._unsupported_distribution():
self._fail("Unsupported distribution {}, expected {}".format(
get_distribution_name(),
join_strs(self.possible_distributions, "or")))
def format_str_list(val: t.List[str], sep: str = "and") -> str:
return join_strs(list(map(repr, val)), sep)
def register(cls, name: str, klass: type, misc_type: Type):
"""
Registers a new class.
The constructor of the class gets as first argument the misc settings.
:param name: common name of the registered class
:param klass: actual class
:param misc_type: type scheme of the {name}_misc settings
:param misc_default: default value of the {name}_misc settings
"""
def format_str_list(val: t.List[str], sep: str = "and") -> str:
return join_strs(list(map(repr, val)), sep)
misc_type_empty = misc_type == Dict() or misc_type == Dict({})
misc_default = misc_type.get_default()
description = None
use_key_path = "/".join([cls.settings_key_path, cls.use_key])
misc_key = "{}_misc".format("/".join([cls.settings_key_path, name]))
if klass.__doc__ is not None:
header = ""# "Description of {} (class {}):\n".format(name, klass.__qualname__)
lines = str(klass.__doc__.strip()).split("\n")
lines = map(lambda x: " " + x.strip(), lines)
description = Description(header + "\n".join(lines))
klass.__description__ = description.description
misc_type //= description
else:
klass.__description__ = ""
logging.error("Class level documentation for {} is missing".format(klass.__name__))
Settings().modify_setting(misc_key, misc_type)
if cls.use_list:
if not Settings().validate_key_path(use_key_path.split("/")) \
or isinstance(Settings().get_type_scheme(use_key_path), Any):
use_key_type = (StrList() | Exact(name))
use_key_type.typecheck_default = False
Settings().modify_setting(use_key_path,
use_key_type // Default(cls.default) if cls.default else use_key_type)
else:
use_key_list = Settings().get_type_scheme(use_key_path)
assert isinstance(use_key_list, StrList)
use_key_list |= Exact(name)
use_key_list = Settings().get_type_scheme(use_key_path)
use_key_list // Description("Possible {} are {}".format(cls.plugin_synonym[1],
format_str_list(use_key_list.allowed_values)))
active_path = "{}_active".format("/".join([cls.settings_key_path, name]))
if not Settings().validate_key_path(active_path.split("/")):
Settings().modify_setting(active_path, BoolOrNone() // Default(None))
Settings().get_type_scheme(active_path) // Description("Enable: " + klass.__description__)
else:
if not Settings().validate_key_path(use_key_path.split("/")) \
or not isinstance(Settings().get_type_scheme(use_key_path), ExactEither):
use_key_type = ExactEither(name)
use_key_type.typecheck_default = False
Settings().modify_setting(use_key_path,
use_key_type // Default(cls.default) if cls.default else use_key_type)
else:
Settings().modify_setting(use_key_path, Settings().get_type_scheme(use_key_path) | Exact(name))
use_key_type = Settings().get_type_scheme(use_key_path)
use_key_type // Description("Possible {} are {}".format(cls.plugin_synonym[1],
format_str_list(use_key_type.exp_values)))
cls.registry[name] = klass
if not sphinx_doc():
return
use_text = ""
cls_use_text = ""
if cls.use_list:
use_text = "To use this {plugin} add it's name (`{name}`) to the list at settings key `{key}` or " \
"set `{active}` to true."\
.format(plugin=cls.plugin_synonym[0], name=name, key=use_key_path,
active="{}_active".format("/".join([cls.settings_key_path, name])))
cls_use_text = "The used {plugins} can be configured by editing the list under the settings key `{key}`."\
.format(plugins=cls.plugin_synonym[1], key=use_key_path)
else:
use_text = "To use this {plugin} set the currently used {plugin} (at key `{key}`) to its name (`{name}`)."\
.format(plugin=cls.plugin_synonym[0], name=name, key=use_key_path)
cls_use_text = "The used {plugin} can be configured by editing the settings key `{key}`."\
.format(plugin=cls.plugin_synonym[0], key=use_key_path)
other_plugins_text = ""
used_type = Settings().get_type_scheme(use_key_path)
possible_plugins = used_type.allowed_values if cls.use_list else used_type.exp_values # type: t.List[str]
possbible_p_wo_self = [x for x in possible_plugins if x != name]
if len(possible_plugins) == 2:
other_plugins_text = "Another usable {plugin} is `{p}`.".format(plugin=cls.plugin_synonym[0],
p=possbible_p_wo_self[0])
if len(possible_plugins) > 2:
other_plugins_text = "Other usable {plugins} are {p}.".format(plugins=cls.plugin_synonym[1],
p=join_strs(["`{}`".format(x)
for x in possbible_p_wo_self]))
default_plugins_text = ""
if cls.default:
if cls.use_list and len(cls.default) > 1:
p = join_strs(["`{}`".format(x) for x in cls.default])
default_plugins_text = "The default {plugins} are {p}.".format(plugins=cls.plugin_synonym[1],
p=p)
else:
default = cls.default[0] if cls.use_list else cls.default
default_plugins_text = "The default {plugin} is `{p}`.".format(plugin=cls.plugin_synonym[0],
p=default)
if not misc_type_empty:
klass.__doc__ += """
Configuration format:
.. code-block:: yaml
{default_yaml}
This {plugin} can be configured under the settings key `{misc_key}`.
""".format(default_yaml="\n ".join(misc_type.get_default_yaml().split("\n")),
misc_key=misc_key, plugin=cls.plugin_synonym[0])
klass.__doc__ += """
{use_text}
{other}
{default}
""".format(use_text=use_text, other=other_plugins_text, default=default_plugins_text)
if not hasattr(cls, "__old_documentation__"):
cls.__old_documentation__ = cls.__doc__ or ""
cls.__doc__ = cls.__old_documentation__
cls.__doc__ += """
{use_text}
{possible}
""".format(
use_text=cls_use_text,
possible=Settings().get_type_scheme(use_key_path).description
)
