def _load_results(self):
"""Load the test results from the CSV file and/or apply the columns selector."""
_LOG.debug("summaries will be calculated for these columns: %s",
", ".join(self._smry_colnames))
_LOG.debug("additional colnames: %s", ", ".join(self._more_colnames))
for res in self.rsts:
_LOG.debug("hover colnames: %s", ", ".join(self._hov_colnames[res.reportid]))
colnames = []
for colname in self._hov_colnames[res.reportid] + self._more_colnames:
if colname in res.colnames_set:
colnames.append(colname)
csel = Trivial.list_dedup(self._smry_colnames + colnames)
res.set_csel(csel)
res.load_df()
# We'll be dropping columns and adding temporary columns, so we'll affect the original
# dataframe. This is more efficient than creating copies.
self._mangle_loaded_res(res)
# Some columns from the axes lists could have been dropped, update the lists.
self._drop_absent_colnames()
def _generate_metric_tabs(self):
"""Generate 'Metric Tabs' which contain the plots and summary tables for each metric."""
for res in self.rsts:
_LOG.debug("calculate summary functions for '%s'", res.reportid)
res.calc_smrys(regexs=self._smry_colnames, funcnames=self._smry_funcs)
plot_axes = [(x, y) for x, y in itertools.product(self.xaxes, self.yaxes) if x != y]
if self.exclude_xaxes and self.exclude_yaxes:
x_axes = self._refres.find_colnames([self.exclude_xaxes])
y_axes = self._refres.find_colnames([self.exclude_yaxes])
exclude_axes = list(itertools.product(x_axes, y_axes))
plot_axes = [axes for axes in plot_axes if axes not in exclude_axes]
tabs = []
tab_names = [y for _, y in plot_axes]
tab_names += self.chist + self.hist
tab_names = Trivial.list_dedup(tab_names)
for metric in tab_names:
_LOG.info("Generating %s tab.", metric)
tab_plots = []
smry_metrics = []
for axes in plot_axes:
if metric in axes:
# Only add plots which have the tab metric on one of the axes.
tab_plots.append(axes)
# Only add metrics shown in the diagrams to the summary table.
smry_metrics += axes
smry_metrics = Trivial.list_dedup(smry_metrics)
metric_tab = _MetricTab.MetricTabBuilder(metric, self.rsts, self.outdir)
metric_tab.add_smrytbl(smry_metrics, self._smry_funcs)
metric_tab.add_plots(tab_plots, self.hist, self.chist, self._hov_colnames)
tabs.append(metric_tab.get_tab())
return tabs
def add_smrytbl(self, smry_metrics, smry_funcs):
"""
Summaries table includes values like average and median values for a single metric (column).
It "summarizes" the metric. This function creates and dumps the summary table for this tab.
This means that it includes summaries of all the metrics referenced in this tab.
"""
smry_tbl = _SummaryTable.SummaryTable()
# Summaries are calculated only for numeric metrics. Tab metric name is represented by
# 'smrytblpath.name', this should come first.
metrics = [self.tabname] if self._refres.is_numeric(
self.tabname) else []
metrics += [
metric for metric in smry_metrics
if self._refres.is_numeric(metric)
]
# Dedupe the list so that each metric only appears once.
metrics = Trivial.list_dedup(metrics)
for metric in metrics:
# Create row in the summary table for each metric.
defs = self._refres.defs.info[metric]
fmt = "{:.2f}" if defs["type"] == "float" else None
smry_tbl.add_metric(metric, defs["short_unit"], defs["descr"], fmt)
# Select only those functions that are present in all test results. For example, 'std'
# will not be present if the result has only one datapoint. In this case, we need to
# exclude the 'std' function.
funcs = []
for funcname in smry_funcs:
if all(res.smrys[metric].get(funcname) for res in self._rsts):
funcs.append(funcname)
# Populate each row with summary functions for each result.
for res in self._rsts:
for funcname in funcs:
val = res.smrys[metric][funcname]
smry_tbl.add_smry_func(res.reportid, metric, funcname, val)
try:
smry_tbl.generate(self._smrytblpath)
except Error as err:
raise Error("Failed to generate summary table.") from err
def __init__(self, rsts, outdir, title_descr=None, xaxes=None, yaxes=None, hist=None,
chist=None, exclude_xaxes=None, exclude_yaxes=None):
"""
The class constructor. The arguments are as follows.
* rsts - list of 'RORawResult' objects representing the raw test results to generate the
HTML report for.
* outdir - the output directory path to store the HTML report at.
* title_descr - a string describing this report or a file path containing the description.
* The description will be put at the top part of the HTML report. It should
* describe the report in general (e.g., it compares platform A to platform
* B). By default no title description is added to the HTML report.
* xaxes - list of regular expressions matching datapoints CSV file column names to use for
the X axis of scatter plot diagrams. A scatter plot will be generated for each
combination of 'xaxes' and 'yaxes' column name pair (except for the pairs in
'exclude_xaxes'/'exclude_yaxes'). Default is the first column in the datapoints
CSV file.
* yaxes - list of regular expressions matching datapoints CSV file column names to use for
the Y axis of scatter plot diagrams. Default is the second column in the
datapoints CSV file.
* hist - list of regular expressions matching datapoints CSV file column names to create a
histogram for. Default is the first column in the datapoints CSV file. An empty
string can be used to disable histograms.
* chist - list of regular expressions matching datapoints CSV file column names to create
a cumulative histogram for. Default is he first column in the datapoints CSV
file. An empty string can be used to disable cumulative histograms.
* exclude_xaxes - by default all diagrams of X- vs Y-axes combinations will be created.
The 'exclude_xaxes' is a list regular expressions matching datapoints
CSV file column names. There will be no scatter plot for each
combinations of 'exclude_xaxes' and 'exclude_yaxes'. In other words,
this argument along with 'exclude_yaxes' allows for excluding some
diagrams from the 'xaxes' and 'yaxes' combinations.
* exclude_yaxes - same as 'exclude_xaxes', but for Y-axes.
"""
self.rsts = rsts
self.outdir = Path(outdir)
self.title_descr = title_descr
self.xaxes = xaxes
self.yaxes = yaxes
self.exclude_xaxes = exclude_xaxes
self.exclude_yaxes = exclude_yaxes
self.hist = hist
self.chist = chist
self._projname = "wult"
# Users can change this to 'True' to make the reports relocatable. In which case the raw
# results files will be copied from the test result directories to the output directory.
self.relocatable = False
# The first result is the 'reference' result.
self._refres = rsts[0]
# The raw reference result information.
self._refinfo = self._refres.info
# Names of columns in the datapoints CSV file to provide the summary function values for
# (e.g., median, 99th percentile). The summaries will show up in the summary tables (one
# table per metric).
self._smry_colnames = None
# List of functions to provide in the summary tables.
self._smry_funcs = ("nzcnt", "max", "99.999%", "99.99%", "99.9%", "99%", "med", "avg",
"min", "std")
# Per-test result list of column names to include into the hover text of the scatter plot.
# By default only the x and y axis values are included.
self._hov_colnames = {}
# Additional columns to load, if they exist in the CSV file.
self._more_colnames = []
self._validate_init_args()
self._init_colnames()
# We'll provide summaries for every column participating in at least one diagram.
smry_colnames = Trivial.list_dedup(self.yaxes + self.xaxes + self.hist + self.chist)
# Summary functions table includes all test results, but the results may have a bit
# different set of column names (e.g., they were collected with different wult versions # or
# using different methods, or on different systems). Therefore, include only common columns
# into it.
self._smry_colnames = []
for colname in smry_colnames:
for res in rsts:
if colname not in res.colnames_set:
break
else:
self._smry_colnames.append(colname)
self._init_assets()
if (self.exclude_xaxes and not self.exclude_yaxes) or \
(self.exclude_yaxes and not self.exclude_xaxes):
raise Error("'exclude_xaxes' and 'exclude_yaxes' must both be 'None' or both not be "
"'None'")