def _get_level(self, start, end, nums=None):
"""
Returns list of level 'end' values belonging to level 'start' for each ID in 'nums'. Returns
all values if 'nums' is None or "all". Offline CPUs are ignored.
"""
if start not in LEVELS or end not in LEVELS:
levels = ", ".join(LEVELS)
raise Error(f"bad levels '{start}','{end}', use: {levels}")
start_idx = LEVELS.index(start)
end_idx = LEVELS.index(end)
if start_idx > end_idx:
raise Error(f"bad level order, cannot get {end}s from level '{start}'")
items = {}
for line in self._get_lscpu():
if line.startswith("#"):
continue
# Each line has comma-separated integers for socket, node, core and cpu. For example:
# 1,1,9,61,Y. In case of offline CPU, the final element is going to be "N".
line = line.strip().split(",")
if line[-1] != "Y":
# Skip non-online CPUs.
continue
line = [int(val) for val in line[0:-1]]
if line[start_idx] in items.keys():
items[line[start_idx]].append(line[end_idx])
else:
items[line[start_idx]] = [line[end_idx]]
# So now 'items' is a dictionary with keys being the 'start' level elements and values being
# lists of the 'end' level elements.
# For example, suppose we are looking for CPUs in packages, and the system has 2 packages,
# each containing 8 CPUs. The 'items' dictionary will look like this:
# items[0] = {0, 2, 4, 6, 8, 10, 12, 14}
# items[1] = {1, 3, 6, 7, 9, 11, 13, 15}
# In this example, package 0 includes CPUs with even numbers, and package 1 includes CPUs
# with odd numbers.
if not nums or nums == "all":
nums = list(items.keys())
result = []
for num in nums:
if not Trivial.is_int(num):
raise Error(f"bad {start} number '{num}', should be an integer")
num = int(num)
if num not in items:
items_str = ", ".join(str(key) for key in items)
raise Error(f"{start} {num} does not exist{self._proc.hostmsg}, use: {items_str}")
result += items[num]
return Trivial.list_dedup(result)
def __init__(self,
rsts,
outdir,
title_descr=None,
xaxes=None,
yaxes=None,
hist=None,
chist=None):
"""The class constructor. The arguments are the same as in 'HTMLReportBase()'."""
args = {"xaxes": xaxes, "yaxes": yaxes, "hist": hist, "chist": chist}
for name, default in zip(
args,
(DEFAULT_XAXES, DEFAULT_YAXES, DEFAULT_HIST, DEFAULT_CHIST)):
if not args[name]:
args[name] = default.split(",")
super().__init__(rsts,
outdir,
title_descr=title_descr,
xaxes=args["xaxes"],
yaxes=args["yaxes"],
hist=args["hist"],
chist=args["chist"])
# Column names representing C-state residency.
self._cs_colnames = set()
for res in rsts:
for colname in res.defs.get_csres_colnames():
# Form the list of column names representing C-state residency. We'll need to load
# them in order to detect C-states with no residency.
self._cs_colnames.add(colname)
self._more_colnames.append(colname)
self._more_colnames = Trivial.list_dedup(self._more_colnames)
def _load_results(self):
"""Load the test results from the CSV file and/or apply the columns selector."""
_LOG.debug("stats colnames: %s", ", ".join(self._stats_colnames))
_LOG.debug("additional colnames: %s", ", ".join(self._more_colnames))
for res in self.rsts:
_LOG.debug("hover colnames: %s",
", ".join(self._hov_colnames[res.reportid]))
colnames = []
for colname in self._hov_colnames[
res.reportid] + self._more_colnames:
if colname in res.colnames_set:
colnames.append(colname)
csel = Trivial.list_dedup(self._stats_colnames + colnames)
res.clear_filts()
res.set_csel(csel)
res.load_df()
# We'll be dropping columns and adding temporary columns, so we'll affect the original
# dataframe. This is more effecient than creating copies.
self._mangle_loaded_res(res)
def __init__(self,
rsts,
outdir,
title_descr=None,
xaxes=None,
yaxes=None,
hist=None,
chist=None):
"""
The class constructor. The arguments are as follows.
* rsts - list of 'RORawResult' objects representing the raw test results to generate the
HTML report for.
* outdir - the output directory path to store the HTML report at.
* title_descr - a string describing this report or a file path containing the description.
* The description will be put at the top part of the HTML report. It should
* describe the report in general (e.g., it compares platform A to platform
* B). By default no title description is added to the HTML report.
* xaxes - list of datapoints CSV file column names to use on the X axis. Default is the
first column in the datapoints CSV file.
* yaxes - list of datapoints CSV file column names to use on the Y axis. Default is
'WakeLatency'. Default is the second column in the datapoints CSV file.
* hist - list of datapoints CSV file column names to create a histogram for. Default is
the first column in the datapoints CSV file. And empty string can be used to
disable histograms.
* chist - list of datapoints CSV file column names to create a cumulative histogram for.
Default is he first column in the datapoints CSV file. And empty string can be
used to disable cumulative histograms.
"""
self.rsts = rsts
self.outdir = Path(outdir)
self.title_descr = title_descr
self.xaxes = xaxes
self.yaxes = yaxes
self.hist = hist
self.chist = chist
# Users can change this to make the reports relocatable. In this case the statistics and
# other stuff will be copied from the test result directories to the output directory. By
# default symlinks are used.
self.relocatable = True
# The first result is the 'reference' result.
self._refres = rsts[0]
# The reference definitions - it contains helpful information about every CSV file column,
# for example the title, units, and so on.
self._refdefs = self._refres.defs
# The raw reference result information.
self._refinfo = self._refres.info
# Names of columns in the datapoints CSV file to provide the statistics for. The statistics
# will show up in the statistics table included into the report.
self._stats_colnames = None
# List of functions to provide in the statistics column.
self._stats_funcs = ("nzcnt", "max", "99.999%", "99.99%", "99.9%",
"99%", "med", "avg", "min", "std")
# The diagram/histogram transparency level. It is helpful to have some transparency in case
# there are several test results rendered on the same diagram.
self._opacity = 0.8 if len(self.rsts) > 1 else 1
# Plot information dictionaries. This is a dictionary of of lists, each list containing
# sub-dictionaries describing a single plot. The lists of sub-dictionaries are grouped by
# the X" and "Y" axis column names, because later plots with the same "Y" and "Y" axes will
# go to the same HTML page.
self._pinfos = OrderedDict()
# Per-test result List of column names to include into the hover text of the scatter plot.
# By default only the x and y axis values are included.
self._hov_colnames = {}
# Additional columns to load, if they exist in the CSV file.
self._more_colnames = []
self._init_colnames()
# We'll provide statistics for every column participating in at least one diagram.
stats_colnames = Trivial.list_dedup(self.yaxes + self.xaxes +
self.hist + self.chist)
# Since statistics table is global, include only common columns into it.
self._stats_colnames = []
for colname in stats_colnames:
for res in rsts:
if colname not in res.colnames_set:
break
else:
self._stats_colnames.append(colname)
self._validate_init_args()
def _generate_histograms(self):
"""Generate the scatter plots."""
def get_xbins(xcolname):
"""Returns the 'xbins' dictinary for plotly's 'Histrogram()' method."""
xmin, xmax = (float("inf"), -float("inf"))
for res in self.rsts:
xdata = self._base_unit(res.df, xcolname)
xmin = min(xmin, xdata.min())
xmax = max(xmax, xdata.max())
return {"size": (xmax - xmin) / 1000}
xcolnames = Trivial.list_dedup(self.hist + self.chist)
hist_set = set(self.hist)
chist_set = set(self.chist)
for xcolname in xcolnames:
if xcolname in hist_set:
ycolname = "Count"
pinfo = self._add_pinfo(xcolname, ycolname, is_hist=True)
_LOG.info("Generating histogram: %s vs %s.", xcolname,
ycolname)
gobjs = []
xbins = get_xbins(xcolname)
for res in self.rsts:
xdata = self._base_unit(res.df, xcolname)
try:
gobj = plotly.graph_objs.Histogram(
x=xdata,
name=res.reportid,
xbins=xbins,
opacity=self._opacity)
except Exception as err:
raise Error(f"failed to create histogram "
f"'{ycolname}-vs-{xcolname}':\n{err}")
gobjs.append(gobj)
self._create_diagram(gobjs, pinfo)
if xcolname in chist_set:
ycolname = "Percentile"
_LOG.info("Generating cumulative histogram: %s vs %s.",
xcolname, ycolname)
pinfo = self._add_pinfo(xcolname, ycolname, is_hist=True)
gobjs = []
if xcolname not in hist_set:
xbins = get_xbins(xcolname)
for res in self.rsts:
xdata = self._base_unit(res.df, xcolname)
try:
gobj = plotly.graph_objs.Histogram(
x=xdata,
name=res.reportid,
xbins=xbins,
cumulative=dict(enabled=True),
histnorm="percent",
opacity=self._opacity)
except Exception as err:
raise Error(f"failed to create cumulative histogram "
f"'{ycolname}-vs-{xcolname}':\n{err}")
gobjs.append(gobj)
self._create_diagram(gobjs, pinfo)