def validate_ldist(ldist):
"""
Validate and modify launch distanse. 'ldist' argument is string of single or two comma-separated
integers, and tells launch distance in microseconds. Return value is launch distance in
nanoseconds, as list of one or two integers.
"""
ldst = Trivial.split_csv_line(ldist)
for idx, val in enumerate(ldst):
ldst[idx] = Trivial.str_to_num(val, default=None)
if ldst[idx] is None:
raise Error(f"bad launch distance '{ldist}', should be an integer")
if ldst[idx] <= 0:
raise Error(
f"bad launch distance value '{ldst[idx]}', should be greater than zero"
)
ldst_str = ", ".join([str(val) for val in ldst])
if len(ldst) > 2:
raise Error(
f"bad launch distance range '{ldst_str}', it should include 2 numbers"
)
if len(ldst) == 2 and ldst[1] - ldst[0] < 0:
raise Error(
f"bad launch distance range '{ldst_str}', first number cannot be "
f"greater than the second number")
# Return launch distance as nanoseconds.
for idx, val in enumerate(ldst):
ldst[idx] = val * 1000
return ldst
def _set_launch_distance(self):
"""Set launch distance limits to driver."""
try:
limit_path = self._basedir / "ldist_max_nsec"
with self._proc.open(limit_path, "r") as fobj:
ldist_max = fobj.read().strip()
limit_path = self._basedir / "ldist_min_nsec"
with self._proc.open(limit_path, "r") as fobj:
ldist_min = fobj.read().strip()
except Error as err:
raise Error(
f"failed to read launch distance limit from '{limit_path}'"
f"{self._proc.hostmsg}:\n{err}")
ldist_min = Trivial.str_to_num(ldist_min)
ldist_max = Trivial.str_to_num(ldist_max)
from_path = self._basedir / "ldist_from_nsec"
to_path = self._basedir / "ldist_to_nsec"
for ldist, ldist_path in zip_longest(self._ldist, [from_path, to_path],
fillvalue=self._ldist[-1]):
if ldist < ldist_min or ldist > ldist_max:
raise Error(
f"launch distance '{ldist}' is out of range, it should be in range of "
f"[{ldist_min},{ldist_max}]")
if not FSHelpers.exists(ldist_path, proc=self._proc):
raise Error(
f"path i'{ldist_path}' does not exist{self._proc.hostmsg}")
# open()/write() doesn't work for this file when done over SSH.
self._proc.run_verify(f"echo {ldist} > {ldist_path}", shell=True)
def _get_level(self, start, end, nums=None):
"""
Returns list of level 'end' values belonging to level 'start' for each ID in 'nums'. Returns
all values if 'nums' is None or "all". Offline CPUs are ignored.
"""
if start not in LEVELS or end not in LEVELS:
levels = ", ".join(LEVELS)
raise Error(f"bad levels '{start}','{end}', use: {levels}")
start_idx = LEVELS.index(start)
end_idx = LEVELS.index(end)
if start_idx > end_idx:
raise Error(f"bad level order, cannot get {end}s from level '{start}'")
items = {}
for line in self._get_lscpu():
if line.startswith("#"):
continue
# Each line has comma-separated integers for socket, node, core and cpu. For example:
# 1,1,9,61,Y. In case of offline CPU, the final element is going to be "N".
line = line.strip().split(",")
if line[-1] != "Y":
# Skip non-online CPUs.
continue
line = [int(val) for val in line[0:-1]]
if line[start_idx] in items.keys():
items[line[start_idx]].append(line[end_idx])
else:
items[line[start_idx]] = [line[end_idx]]
# So now 'items' is a dictionary with keys being the 'start' level elements and values being
# lists of the 'end' level elements.
# For example, suppose we are looking for CPUs in packages, and the system has 2 packages,
# each containing 8 CPUs. The 'items' dictionary will look like this:
# items[0] = {0, 2, 4, 6, 8, 10, 12, 14}
# items[1] = {1, 3, 6, 7, 9, 11, 13, 15}
# In this example, package 0 includes CPUs with even numbers, and package 1 includes CPUs
# with odd numbers.
if not nums or nums == "all":
nums = list(items.keys())
result = []
for num in nums:
if not Trivial.is_int(num):
raise Error(f"bad {start} number '{num}', should be an integer")
num = int(num)
if num not in items:
items_str = ", ".join(str(key) for key in items)
raise Error(f"{start} {num} does not exist{self._proc.hostmsg}, use: {items_str}")
result += items[num]
return Trivial.list_dedup(result)
def validate_cpunum(cpunum, proc=None):
"""
Validate CPU number 'cpunum'. The optional 'cpugeom' argument is the CPU geometry dictionary
generated by 'CPUInfo.get_cpu_geometry()' function. If 'proc' is not provided, this function
just checks that 'cpunum' is a positive integer number, nothing else.
If 'proc' is provided, then this function discovers CPU count on the host associated with
'proc', and verifies that 'cpunum' does not exceed the host CPU count and the CPU is online.
Note, 'proc' shoulc be an 'SSH' or 'Proc' object.
"""
if not Trivial.is_int(cpunum) or int(cpunum) < 0:
raise Error(f"bad CPU number '{cpunum}', should be a positive integer")
cpunum = int(cpunum)
if proc:
with CPUInfo.CPUInfo(proc=proc) as cpuinfo:
cpugeom = cpuinfo.get_cpu_geometry()
if cpunum in cpugeom["offcpus"]:
raise Error(f"CPU '{cpunum}'{proc.hostmsg} is offline")
if cpunum not in cpugeom["cpus"]:
raise Error(f"CPU '{cpunum}' does not exist{proc.hostmsg}")
return cpunum
def stats_command(args):
"""Implements the 'stats' command for the 'wult' and 'ndl' tools."""
if args.list_funcs:
for name, descr in RORawResult.get_stat_funcs():
_LOG.info("%s: %s", name, descr)
return
if args.funcs:
funcnames = Trivial.split_csv_line(args.funcs)
all_funcs = True
else:
funcnames = None
all_funcs = False
res = RORawResult.RORawResult(args.respath)
apply_filters(args, res)
non_numeric = res.get_non_numeric_colnames()
if non_numeric and (args.csel or args.cfilt):
non_numeric = ", ".join(non_numeric)
_LOG.warning("skipping non-numeric column(s): %s", non_numeric)
res.calc_stats(funcnames=funcnames, all_funcs=all_funcs)
_LOG.info("Datapoints count: %d", len(res.df))
YAML.dump(res.cstats, sys.stdout, float_format="%.2f")
def is_root(proc=None):
"""
If 'proc' is 'None' or a 'Proc' object, return 'True' if current process' user name is 'root'
and 'False' if current process' user name is not 'root'. If 'proc' is an 'SSH' object, returns
'True' if the SSH user has 'root' permissions on the remote host, otherwise returns 'False'.
"""
if not proc or not proc.is_remote:
return Trivial.is_root()
stdout, _ = proc.run_verify("id -u")
stdout = stdout.strip()
if not Trivial.is_int(stdout):
raise Error("unexpected output from 'id -u' command, expected an integer, got:\n{stdout}")
return int(stdout) == 0
def find_processes(regex: str, proc=None):
"""
Find all processes which match the 'regex' regular expression on the host defined by 'proc'. The
regular expression is matched against the process executable name + command-line arguments.
By default this function operates on the local host, but the 'proc' argument can be used to pass
a connected 'SSH' object in which case this function will operate on the remote host.
Returns a list of tuples containing the PID and the command line.
"""
if not proc:
proc = Procs.Proc()
cmd = "ps axo pid,args"
stdout, stderr = proc.run_verify(cmd, join=False)
if len(stdout) < 2:
raise Error(f"no processes found at all{proc.hostmsg}\nExecuted this command:\n{cmd}\n"
f"stdout:\n{stdout}\nstderr:{stderr}\n")
procs = []
for line in stdout[1:]:
pid, comm = line.strip().split(" ", 1)
pid = int(pid)
if proc.hostname == "localhost" and pid == Trivial.get_pid():
continue
if re.search(regex, comm):
procs.append((int(pid), comm))
return procs
def report_command_open_raw_results(args):
"""
Opens the input raw test results for the 'report' command of the 'wult' and 'ndl' tools. Returns
the list of 'RORawResult' objects. At the same time, implements the '--list-columns' option by
printing the column names for each input raw result.
"""
if args.reportids:
reportids = Trivial.split_csv_line(args.reportids)
else:
reportids = []
if len(reportids) > len(args.respaths):
raise Error(f"there are {len(reportids)} report IDs to assign to {len(args.respaths)} "
f"input test results. Please, provide {len(args.respaths)} or less report IDs.")
# Append the required amount of 'None's to make the 'reportids' list be of the same length as
# the 'args.respaths' list.
reportids += [None] * (len(args.respaths) - len(reportids))
rsts = []
for respath, reportid in zip(args.respaths, reportids):
if reportid:
additional_chars = getattr(args, "reportid_additional_chars", None)
ReportID.validate_reportid(reportid, additional_chars=additional_chars)
res = RORawResult.RORawResult(respath, reportid=reportid)
rsts.append(res)
if args.list_columns:
_LOG.info("Column names in '%s':", respath)
for colname in res.colnames:
_LOG.info(" * %s: %s", colname, res.defs.info[colname]["title"])
return rsts
def _parse_ip_address_show(raw):
"""
Parse output of the 'ip address show <IFNAME>' command and return the resulting dictionary.
"""
info = {}
for line in raw.splitlines():
line = line.strip()
elts = Trivial.split_csv_line(line, sep=" ")
if re.match(r"^\d+:$", elts[0]):
info["ifname"] = elts[1][:-1]
elif elts[0] == "inet":
ipnet = ipaddress.IPv4Network(elts[1], strict=False)
info["ipv4"] = {}
info["ipv4"]["ip"] = ipnet.network_address
info["ipv4"]["mask"] = ipnet.netmask
info["ipv4"]["bcast"] = ipnet.broadcast_address
info["ipv4"]["ip_cidr"] = elts[1]
info["ipv4"]["cidr"] = str(ipnet)
elif elts[0] == "link/ether":
info["ether"] = {}
info["ether"]["mac"] = elts[1]
info["ether"]["bcast"] = elts[3]
return info
def _get_cstates_info(self, cpus, indexes, ordered):
"""Implements 'get_cstates_info()'."""
indexes_regex = cpus_regex = "[[:digit:]]+"
if cpus is not None:
cpus_regex = "|".join([str(cpu) for cpu in cpus])
if indexes is not None:
indexes_regex = "|".join([str(index) for index in indexes])
cmd = fr"find '{self._sysfs_base}' -type f -regextype posix-extended " \
fr"-regex '.*cpu({cpus_regex})/cpuidle/state({indexes_regex})/[^/]+' " \
fr"-exec printf '%s' {{}}: \; -exec grep . {{}} \;"
stdout, _ = self._proc.run_verify(cmd, join=False)
if not stdout:
raise Error(
f"failed to find C-states information in '{self._sysfs_base}'"
f"{self._proc.hostmsg}")
if ordered:
stdout = sorted(stdout)
regex = re.compile(
r".+/cpu([0-9]+)/cpuidle/state([0-9]+)/(.+):([^\n]+)")
info = {}
index = prev_index = cpu = prev_cpu = None
for line in stdout:
matchobj = re.match(regex, line)
if not matchobj:
raise Error(
f"failed to parse the follwoing line from file in '{self._sysfs_base}'"
f"{self._proc.hostmsg}:\n{line.strip()}")
cpu = int(matchobj.group(1))
index = int(matchobj.group(2))
key = matchobj.group(3)
val = matchobj.group(4)
if Trivial.is_int(val):
val = int(val)
if prev_cpu is None:
prev_cpu = cpu
if prev_index is None:
prev_index = index
if cpu != prev_cpu or index != prev_index:
info["cpu"] = prev_cpu
info["index"] = prev_index
yield info
prev_cpu = cpu
prev_index = index
info = {}
info[key] = val
info["cpu"] = prev_cpu
info["index"] = prev_index
yield info
def do_filter(res, ops):
"""Apply filter operations in 'ops' to wult test result 'res'."""
res.clear_filts()
for name, expr in ops.items():
if name.startswith("c"):
expr = Trivial.split_csv_line(expr)
getattr(res, f"set_{name}")(expr)
res.load_df()
def _normalize_cstates(self, cstates):
"""
Some methods accept the C-states to operate on as a string or a list. There may be C-state
names or indexes in the list. This method turns the user input into a list of C-state
indexes and returns this list.
"""
if isinstance(cstates, int):
cstates = str(cstates)
if cstates is not None:
if isinstance(cstates, str):
cstates = Trivial.split_csv_line(cstates, dedup=True)
indexes = []
for cstate in cstates:
if not Trivial.is_int(cstate):
cstate = self._name2idx(cstate)
indexes.append(int(cstate))
cstates = indexes
return cstates
def set_post_trigger(self, path, trange=None):
"""
Configure the post-trigger - a program that has to be executed after a datapoint is
collected. The arguments are as follows.
* path - path to the executable program to run. The program will be executed with the
'--latency <value>' option, where '<value>' is the observed wake latency value in
nanoseconds.
* trange - the post-trigger range. By default, the trigger program is executed on every
datapoint. But if the trigger range is provided, the trigger program will be executed
only when wake latency is in trigger range.
"""
if not FSHelpers.isexe(path, proc=self._proc):
raise Error(
f"post-trigger program '{path}' does not exist{self._proc.hostmsg} or it "
f"is not an executable file")
self._post_trigger = path
if trange is not None:
vals = Trivial.split_csv_line(trange)
for idx, val in enumerate(vals):
if not Trivial.is_int(val):
raise Error(
f"bad post-trigger range value '{val}', should be an integer "
f"amount of nanoseconds")
vals[idx] = Trivial.str_to_num(val, default=None)
if vals[idx] < 0:
raise Error(
f"bad post trigger range value '{vals[idx]}', should be greater or "
f"equal to zero")
if len(vals) != 2:
raise Error(
f"bad post trigger range '{trange}', it should include 2 numbers"
)
if vals[1] - vals[0] < 0:
raise Error(
f"bad post trigger range '{trange}', first number cannot be greater "
f"than the second number")
self._post_trigger_range = vals
def get_mtime(path: Path, proc=None):
"""Returns file or directory mtime."""
if not proc:
return path.stat().st_mtime
cmd = f"stat -c %Y -- {path}"
mtime = proc.run_verify(cmd)[0].strip()
if not Trivial.is_float(mtime):
raise Error(f"got erroneous mtime of '{path}'{proc.hostmsg}:\n{mtime}")
return float(mtime)
def _get_percentile(funcname):
"""
Parses and validates the percentile statistics function name (e.g., "99%") and returns the
percent value (99).
"""
percent = Trivial.str_to_num(funcname[:-1])
if percent <= 0 or percent >= 100:
raise Error(f"bad percentile number in '{funcname}', should be in range of "
f"(0, 100)")
return percent
def parse_int_list(nums, ints=False):
"""
Turn a string contaning a comma-separated list of numbers and ranges into a list of numbers and
return it. For example, a string like "0,1-3,7" would become ["0", "1", "2", "3", "7"]. The
'ints' argument controls whether the resulting list should contain strings or integers.
"""
if nums is None:
return None
if isinstance(nums, int):
nums = str(nums)
if isinstance(nums, str):
nums = Trivial.split_csv_line(nums)
nums_set = set()
for elts in nums:
elts = str(elts)
if "-" in elts:
elts = Trivial.split_csv_line(elts, sep="-")
if len(elts) != 2:
raise Error("bad range '%s', should be two integers separated by '-'" % elts)
else:
elts = [elts]
for elt in elts:
if not Trivial.is_int(elt):
raise Error("bad number '%s', should be an integer" % elt)
if len(elts) > 1:
if int(elts[0]) > int(elts[1]):
raise Error("bad range %s-%s, the first number should be smaller than thesecond"
% (elts[0], elts[1]))
nums_set.update([str(elt) for elt in range(int(elts[0]), int(elts[1]) + 1)])
else:
nums_set.add(elts[0])
result = sorted([int(num) for num in nums_set])
if not ints:
result = [str(num) for num in result]
return result
def _get_deployables(srcpath, proc):
"""
Returns the list of "deployables" (driver names or helper tool names) provided by tools or
drivers source code directory 'srcpath' on a host defined by 'proc'.
"""
cmd = f"make --silent -C '{srcpath}' list_deployables"
deployables, _ = proc.run_verify(cmd)
if deployables:
deployables = Trivial.split_csv_line(deployables, sep=" ")
return deployables
def get_resolution(self):
"""Returns resolution of the delayed event devices in nanoseconds."""
try:
path = self._basedir / "resolution_nsec"
with self._proc.open(path, "r") as fobj:
resolution = fobj.read().strip()
except Error as err:
raise Error(
f"failed to read the delayed event reslolution from '{path}'"
f"{self._proc.hostmsg}:\n{err}")
return Trivial.str_to_num(resolution)
def _get_latency(self, dp):
"""
Read the next latency data line from the 'ndlrunner' helper, parse it, and save the result
in the 'dp' dictionary.
"""
line = self._get_line(prefix="datapoint")
line = Trivial.split_csv_line(line)
if len(line) != 2:
msg = self._unexpected_line_error_prefix(line)
raise Error(
f"{msg}\nExpected 2 comma-separated integers, got {len(line)}")
for val in line:
if not Trivial.is_int(val):
msg = self._unexpected_line_error_prefix(line)
raise Error(
f"{msg}\n: Expected 2 comma-separated integers, got a non-integer "
f"'{val}'")
dp["RTD"] = int(line[0])
dp["LDist"] = int(line[1])
def _build_colmap(self):
"""Build and return the turbostat -> CSV column names map."""
self._colmap = {}
# Build the turbostat -> CSV column names map.
cmd = f"{self._ts_bin} -l"
stdout, _ = self._proc.run_verify(cmd)
for key in Trivial.split_csv_line(stdout):
if key == "Busy%":
self._colmap[key] = "CC0%"
elif key.startswith("CPU%c"):
self._colmap[key] = f"CC{key[5:]}%"
elif key.startswith("Pkg%pc"):
self._colmap[key] = f"PC{key[6:]}%"
def unmanage(self, ifnames):
"""
Mark network interfaces in 'ifnames' as 'unmanaged'. The managed state can later be restored
with the 'restore_managed()'.
"""
if not Trivial.is_iterable(ifnames):
ifnames = [ifnames]
for ifname in ifnames:
managed = self.is_managed(ifname)
if not managed:
continue
self._toggle_managed(ifname, False)
if ifname not in self._saved_managed:
self._saved_managed[ifname] = managed
def _get_cstate_indexes(self, cpu):
"""Yield tuples of of C-state indexes and sysfs paths for cpu number 'cpu'."""
basedir = self._sysfs_base / f"cpu{cpu}" / "cpuidle"
name = None
for name, path, typ in FSHelpers.lsdir(basedir, proc=self._proc):
errmsg = f"unexpected entry '{name}' in '{basedir}'{self._proc.hostmsg}"
if typ != "/" or not name.startswith("state"):
raise Error(errmsg)
index = name[len("state"):]
if not Trivial.is_int(index):
raise Error(errmsg)
yield int(index), Path(path)
if name is None:
raise Error(f"C-states are not supported{self._proc.hostmsg}")
def get_dpcnt(res, dpcnt):
"""
This helper function validates number of datapoints the user requested to collect ('dpcnt'). It
also looks at how many datapoints are already present in the 'res' object (represents a raw test
result) and returns the number datapoints to collect in order for 'rest' to end up with 'dpcnt'
datapoints.
"""
if not Trivial.is_int(dpcnt) or int(dpcnt) <= 0:
raise Error(
f"bad datapoints count '{dpcnt}', should be a positive integer")
dpcnt = int(dpcnt) - res.csv.initial_rows_cnt
if dpcnt <= 0:
_LOG.info("Raw test result at '%s' already includes %d datapoints",
res.dirpath, res.csv.initial_rows_cnt)
_LOG.info("Nothing to collect")
return 0
return dpcnt
def __init__(self,
rsts,
outdir,
title_descr=None,
xaxes=None,
yaxes=None,
hist=None,
chist=None):
"""The class constructor. The arguments are the same as in 'HTMLReportBase()'."""
args = {"xaxes": xaxes, "yaxes": yaxes, "hist": hist, "chist": chist}
for name, default in zip(
args,
(DEFAULT_XAXES, DEFAULT_YAXES, DEFAULT_HIST, DEFAULT_CHIST)):
if not args[name]:
args[name] = default.split(",")
super().__init__(rsts,
outdir,
title_descr=title_descr,
xaxes=args["xaxes"],
yaxes=args["yaxes"],
hist=args["hist"],
chist=args["chist"])
# Column names representing C-state residency.
self._cs_colnames = set()
for res in rsts:
for colname in res.defs.get_csres_colnames():
# Form the list of column names representing C-state residency. We'll need to load
# them in order to detect C-states with no residency.
self._cs_colnames.add(colname)
self._more_colnames.append(colname)
self._more_colnames = Trivial.list_dedup(self._more_colnames)
def set_post_trigger(self, path, thresh=None):
"""
Configure the post-trigger - a program that has to be executed after a datapoint is
collected. The arguments are as follows.
* path - path to the executable program to run. The program will be executed with the
'--latency <value>' option, where '<value>' is the observed latency value in
nanoseconds.
* thresh - the post-trigger threshold. By default, the trigger program is executed on evey
datapoint. But if a threshold is provided, the trigger program will be executed only
when latency exceeds the threshold.
"""
if not FSHelpers.isexe(path, proc=self._proc):
raise Error(
f"file '{path}' does not exist or it is not an executalbe file"
)
self._post_trigger = path
if thresh is not None:
if not Trivial.is_int(thresh):
raise Error(
f"bad post-trigger threshold value '{thresh}', should be an integer "
f"amount of nanoseconds")
self._post_trigger_thresh = int(thresh)
def _load_results(self):
"""Load the test results from the CSV file and/or apply the columns selector."""
_LOG.debug("stats colnames: %s", ", ".join(self._stats_colnames))
_LOG.debug("additional colnames: %s", ", ".join(self._more_colnames))
for res in self.rsts:
_LOG.debug("hover colnames: %s",
", ".join(self._hov_colnames[res.reportid]))
colnames = []
for colname in self._hov_colnames[
res.reportid] + self._more_colnames:
if colname in res.colnames_set:
colnames.append(colname)
csel = Trivial.list_dedup(self._stats_colnames + colnames)
res.clear_filts()
res.set_csel(csel)
res.load_df()
# We'll be dropping columns and adding temporary columns, so we'll affect the original
# dataframe. This is more effecient than creating copies.
self._mangle_loaded_res(res)
def prepare(self):
"""Prepare to start measurements."""
# Ensure the kernel is fresh enough.
kver = KernelVersion.get_kver(proc=self._proc)
if KernelVersion.kver_lt(kver, "5.1-rc1"):
raise Error(
f"version of the running kernel{self._proc.hostmsg} is {kver}, but it "
f"does not support the ETF qdisc.\nPlease, use kernel version 5.1 or "
f"newer")
try:
self._nmcli = _Nmcli.Nmcli(proc=self._proc)
except ErrorNotSupported:
pass
else:
# We have to configure the I210 network interface in a special way, but if it is managed
# by NetworkManager, the configuration may get reset at any point. Therefore, detach the
# network interface from NetworkManager.
_LOG.info("Detaching network interface '%s' from NetworkManager%s",
self._ifname, self._proc.hostmsg)
self._nmcli.unmanage(self._ifname)
# Ensure the interface exists and has carrier. It must be brought up before we can check the
# carrier status.
self._netif = _NetIface.NetIface(self._ifname, proc=self._proc)
self._netif.up()
self._netif.wait_for_carrier(10)
# Make sure the network interface has an IP address.
ipaddr = self._netif.get_ipv4_addr(default=None)
if ipaddr:
_LOG.debug("network interface '%s'%s has IP address '%s'",
self._ifname, self._proc.hostmsg, ipaddr)
else:
ipaddr = self._netif.get_unique_ipv4_addr()
ipaddr += "/16"
self._netif.set_ipv4_addr(ipaddr)
# Ensure the IP was set.
self._netif.get_ipv4_addr()
_LOG.info("Assigned IP address '%s' to interface '%s'%s", ipaddr,
self._ifname, self._proc.hostmsg)
self._drv.load(unload=True, opts=f"ifname={self._ifname}")
# We use the ETF qdisc for scheduling delayed network packets. Configure it and start the
# 'phc2sys' process in background in order to keep the host and NIC clocks in sync.
# Get the TAI offset first.
stdout, _ = self._proc.run_verify(
f"{self._ndlrunner_bin} --tai-offset")
tai_offset = self._get_line(prefix="TAI offset", line=stdout)
if not Trivial.is_int(tai_offset):
raise Error(
f"unexpected 'ndlrunner --tai-offset' output:\n{stdout}")
_LOG.info("Configuring the ETF qdisc%s", self._proc.hostmsg)
self._etfqdisc.configure()
_LOG.info("Starting NIC-to-system clock synchronization process%s",
self._proc.hostmsg)
self._etfqdisc.start_phc2sys(tai_offset=int(tai_offset))
def _next(self):
"""
Generator which yields a dictionary corresponding to one snapshot of turbostat output at a
time.
"""
cpus = OrderedDict()
table_started = False
nontable = {}
heading = totals = None
tbl_regex = re.compile(self._cols_regex)
for line in self._lines:
# Ignore empty and 'jitter' lines like "turbostat: cpu65 jitter 2574 5881".
if not line or line.startswith("turbostat: "):
continue
# Match the beginning of the turbostat table.
if not table_started and not re.match(tbl_regex, line):
_add_nontable_data(nontable, line)
continue
line = line.split()
if Trivial.is_float(line[0]):
# This is the continuation of the table we are currently parsing. It starts either
# with a floating-point 'Time_Of_Day_Seconds' an integer 'Core' value. Each line
# describes a single CPU.
cpu_data = _parse_turbostat_line(heading, line)
cpus[cpu_data["CPU"]] = cpu_data
else:
# This is the start of the new table.
if cpus or table_started:
if not cpus:
# This is the the special case for single-CPU systems. Turbostat does not
# print the totals because there is only one CPU and totals is the the same
# as the CPU information.
cpus[0] = totals
yield _construct_the_result(totals, cpus, nontable)
nontable = {}
cpus = OrderedDict()
heading = OrderedDict()
for key in line:
if "%" in key or "Watt" in key or key in ("Time_Of_Day_Seconds", ):
heading[key] = float
elif key in ("Package", "Core", "CPU"):
heading[key] = str
else:
heading[key] = int
# The next line is total statistics across all CPUs, exept if there is only one
# single CPU in the system.
# False pylint warning, see issue: https://github.com/PyCQA/pylint/issues/1830
line = next(self._lines).split() # pylint: disable=stop-iteration-return
# On systems with a single core turbostat does not include the "Core" colum. Similar
# to single CPU systems - the CPU column is excluded. Make sure we always have them.
for key in ("Core", "CPU"):
if key not in heading:
heading[key] = str
line.append("0")
totals = _parse_turbostat_line(heading, line)
table_started = True
yield _construct_the_result(totals, cpus, nontable)
def _deploy_prepare(args, toolname, minkver):
"""
Validate command-line arguments of the "deploy" command and prepare for builing the helpers and
drivers. The arguments are as follows.
o args - the command line arguments.
o toolname - name of the tool being deployed (e.g., 'ndl').
o minkver - the minimum required version number.
"""
args.tmpdir = None
args.kver = None
if not args.ihost:
args.ihost = "localhost"
if not args.bhost:
args.bhost = args.ihost
if args.ihost != args.bhost and not args.bhost == "localhost":
raise Error("build host (--build-host) must be the local host or the same as deploy host "
"(--host)")
if args.ihost == "localhost" and args.bhost == "localhost":
for attr in ("username", "privkey", "timeout"):
if getattr(args, attr) is not None:
_LOG.warning("ignoring the '--%s' option as it not useful for a local host", attr)
if not args.timeout:
args.timeout = 8
else:
args.timeout = Trivial.str_to_num(args.timeout)
if not args.username:
args.username = "******"
if args.privkey and not args.privkey.is_dir():
raise Error(f"path '{args.privkey}' does not exist or it is not a directory")
if hasattr(args, "drvsrc"):
if not args.drvsrc:
args.drvsrc = FSHelpers.search_for_app_data("wult", _DRV_SRC_SUBPATH/f"{toolname}",
pathdescr=f"{toolname} drivers sources")
if not args.drvsrc.is_dir():
raise Error(f"path '{args.drvsrc}' does not exist or it is not a directory")
if hasattr(args, "helpersrc"):
if not args.helpersrc:
args.helpersrc = FSHelpers.search_for_app_data("wult",
_HELPERS_SRC_SUBPATH/f"{toolname}",
pathdescr=f"{toolname} helper sources")
if not args.helpersrc.is_dir():
raise Error(f"path '{args.helpersrc}' does not exist or it is not a directory")
with contextlib.closing(get_proc(args, args.bhost)) as proc:
if not FSHelpers.which("make", default=None, proc=proc):
raise Error(f"please, install the 'make' tool{proc.hostmsg}")
if not args.ksrc:
args.kver = KernelVersion.get_kver(proc=proc)
if not args.ksrc:
args.ksrc = Path(f"/lib/modules/{args.kver}/build")
else:
args.ksrc = FSHelpers.abspath(args.ksrc, proc=proc)
if not FSHelpers.isdir(args.ksrc, proc=proc):
raise Error(f"kernel sources directory '{args.ksrc}' does not exist{proc.hostmsg}")
if not args.kver:
args.kver = KernelVersion.get_kver_ktree(args.ksrc, proc=proc)
_LOG.info("Kernel sources path: %s", args.ksrc)
_LOG.info("Kernel version: %s", args.kver)
if KernelVersion.kver_lt(args.kver, minkver):
raise Error(f"version of the kernel{proc.hostmsg} is {args.kver}, and it is not new "
f"enough.\nPlease, use kernel version {minkver} or newer.")
args.tmpdir = FSHelpers.mktemp(prefix=f"{toolname}-", proc=proc)
if hasattr(args, "drvsrc"):
_LOG.debug("copying the drivers to %s:\n '%s' -> '%s'",
proc.hostname, args.drvsrc, args.tmpdir)
proc.rsync(f"{args.drvsrc}/", args.tmpdir / "drivers", remotesrc=False, remotedst=True)
args.drvsrc = args.tmpdir / "drivers"
_LOG.info("Drivers will be compiled on host '%s'", proc.hostname)
if hasattr(args, "helpersrc"):
_LOG.debug("copying the helpers to %s:\n '%s' -> '%s'",
proc.hostname, args.helpersrc, args.tmpdir)
proc.rsync(f"{args.helpersrc}/", args.tmpdir / "helpers", remotesrc=False,
remotedst=True)
args.helpersrc = args.tmpdir / "helpers"
_LOG.info("Helpers will be compiled on host '%s'", proc.hostname)
with contextlib.closing(get_proc(args, args.ihost)) as proc:
if not args.kmodpath:
args.kmodpath = Path(f"/lib/modules/{args.kver}")
if not FSHelpers.isdir(args.kmodpath, proc=proc):
raise Error(f"kernel modules directory '{args.kmodpath}' does not exist{proc.hostmsg}")
_LOG.info("Drivers will be deployed to '%s'%s", args.kmodpath, proc.hostmsg)
_LOG.info("Kernel modules path%s: %s", proc.hostmsg, args.kmodpath)
if hasattr(args, "helpersrc"):
if not args.helpers_path:
args.helpers_path = get_helpers_deploy_path(proc, toolname)
_LOG.info("Helpers will be deployed to '%s'%s", args.helpers_path, proc.hostmsg)
def kill_pids(pids, sig: str = "SIGTERM", kill_children: bool = False, must_die: bool = False,
proc=None):
"""
This function kills or signals processes with PIDs in 'pids' on the host defined by 'procs'. The
'pids' argument can be a collection of PID numbers ('int' or 'str' types) or a single PID
number.
By default the processes are killed (SIGTERM), but you can specify any signal either by name or
by number.
The 'children' and 'must_die' arguments must only be used when killing processes (SIGTERM or
SIGKILL). The 'children' argument controls whether this function should also try killing the
children. If the 'must_die' argument is 'True', then this function also verifies that the
process(es) did actually die, and if any of them did not die, it raises an exception.
By default this function operates on the local host, but the 'proc' argument can be used to pass
a connected 'SSH' object in which case this function will operate on the remote host.
"""
def collect_zombies(proc):
"""In case of a local process we need to 'waitpid()' the children."""
if not proc.is_remote:
with contextlib.suppress(OSError):
os.waitpid(0, os.WNOHANG)
if not proc:
proc = Procs.Proc()
if not pids:
return
if not Trivial.is_iterable(pids):
pids = (pids, )
pids = [str(int(pid)) for pid in pids]
if sig is None:
sig = "SIGTERM"
else:
sig = str(sig)
killing = _is_sigterm(sig) or _is_sigkill(sig)
if (kill_children or must_die) and not killing:
raise Error(f"'children' and 'must_die' arguments cannot be used with '{sig}' signal")
if kill_children:
# Find all the children of the process.
for pid in pids:
children, _, exitcode = proc.run(f"pgrep -P {pid}", join=False)
if exitcode != 0:
break
pids += [child.strip() for child in children]
pids_spc = " ".join(pids)
pids_comma = ",".join(pids)
_LOG.debug("sending '%s' signal to the following process%s: %s",
sig, proc.hostmsg, pids_comma)
try:
proc.run_verify(f"kill -{sig} -- {pids_spc}")
except Error as err:
if not killing:
raise Error(f"failed to send signal '{sig}' to PIDs '{pids_comma}'{proc.hostmsg}:\n"
f"{err}")
# Some error happened on the first attempt. We've seen a couple of situations when this
# happens.
# 1. Most often, a PID does not exist anymore, the process exited already (race condition).
# 2 One of the processes in the list is owned by a different user (e.g., root). Let's call
# it process A. We have no permissions to kill process A, but we can kill other processes
# in the 'pids' list. But often killing other processes in the 'pids' list will make
# process A exit. This is why we do not error out just yet.
#
# So the strategy is to do the second signal sending round and often times it happens
# without errors, and all the processes that we want to kill just go away.
if not killing:
return
# Give the processes up to 4 seconds to die.
timeout = 4
start_time = time.time()
while time.time() - start_time <= timeout:
collect_zombies(proc)
_, _, exitcode = proc.run(f"kill -0 -- {pids_spc}")
if exitcode != 0:
return
time.sleep(0.2)
if _is_sigterm(sig):
# Something refused to die, try SIGKILL.
try:
proc.run_verify(f"kill -9 -- {pids_spc}")
except Error as err:
# It is fine if one of the processes exited meanwhile.
if "No such process" not in str(err):
raise
collect_zombies(proc)
if not must_die:
return
# Give the processes up to 4 seconds to die.
timeout = 4
start_time = time.time()
while time.time() - start_time <= timeout:
collect_zombies(proc)
_, _, exitcode = proc.run(f"kill -0 -- {pids_spc}")
if exitcode != 0:
return
time.sleep(0.2)
# Something refused to die, find out what.
msg, _, = proc.run_verify(f"ps -f {pids_spc}", join=False)
if len(msg) < 2:
msg = pids_comma
raise Error(f"one of the following processes{proc.hostmsg} did not die after 'SIGKILL': {msg}")
