def close(self):
"""Stop the measurements."""
if getattr(self, "_phc2sys_proc", None):
_LOG.debug("killing the the phc2sys process PID %d%s",
self._phc2sys_proc.pid, self._proc.hostmsg)
ProcHelpers.kill_pids(self._phc2sys_proc.pid,
kill_children=True,
must_die=False,
proc=self._proc)
self._phc2sys_proc = None
for attr in ("_tchk", ):
obj = getattr(self, attr, None)
if obj:
obj.close()
setattr(self, attr, None)
for attr in (
"_netif",
"_proc",
):
obj = getattr(self, attr, None)
if obj:
if getattr(self, f"_close{attr}", False):
getattr(obj, "close")()
setattr(self, attr, None)
def __init__(self, proc, timeout=30):
"""
Class constructor. The arguments are as follows.
* proc - the 'Proc' or 'SSH' object that defines the host to operate on. This object will
keep a 'proc' reference and use it in various methods.
* timeout - longest time in seconts to wait for data in the trace buffer.
"""
self._reader = None
self._proc = proc
self.timeout = timeout
self.raw_line = None
mntpoint = FSHelpers.mount_debugfs(proc=proc)
self.ftpath = mntpoint.joinpath("tracing/trace")
self.ftpipe_path = mntpoint.joinpath("tracing/trace_pipe")
for path in (self.ftpath, self.ftpipe_path):
if not FSHelpers.isfile(path, proc=proc):
raise ErrorNotSupported(
f"linux kernel function trace file was not found at "
f"'{path}'{proc.hostmsg}")
cmd = f"cat {self.ftpipe_path}"
name = "stale wult function trace reader process"
ProcHelpers.kill_processes(cmd, log=True, name=name, proc=self._proc)
self._clear()
self._reader = self._proc.run_async(cmd, shell=True)
def _start_stats_collect(self):
"""Helper function for 'configure()' that starts 'stats-collect'."""
self._init_paths()
# Kill a possibly running stale 'stats-collect' process.
msg = f"stale {self._sc_path} process{self._proc.hostmsg}"
ProcHelpers.kill_processes(self._sc_search,
log=True,
name=msg,
proc=self._proc)
if self._proc.is_remote:
# Kill a possibly running stale SSH tunnel process.
msg = f"stale stats-collect SSH tunnel process{self._proc.hostmsg}"
ProcHelpers.kill_processes(self._ssht_search,
log=True,
name=msg,
proc=self._proc)
# Format the command for executing 'stats-collect'.
self._cmd = f"{self._sc_path} --sut-name {self._sutname}"
if _LOG.getEffectiveLevel() == logging.DEBUG:
self._cmd = f"{self._cmd} -d"
self._logpath = self._logsdir / f"stats-collect-{self._proc.hostname}.log.txt"
self._cmd = f"{self._cmd} > '{self._logpath}' 2>&1"
# And format the 'stats-collect' command prefix.
cmd_prefix = ""
if self._unshare_path:
# Older version of 'unshare' did not support the '--kill-child' option. Note, unshare
# version structure is similar to kernel version, so we use 'KernelVersion' module.
ver = self._get_unshare_version()
if KernelVersion.kver_ge(ver, "2.32"):
opt_kc = " --kill-child"
else:
opt_kc = ""
cmd_prefix += f"{self._unshare_path} --pid --fork --mount-proc{opt_kc} -- "
if self._nice_path:
cmd_prefix += f"{self._nice_path} -n -20 -- "
if cmd_prefix:
self._cmd = f"{cmd_prefix} {self._cmd}"
self._sc = self._proc.run_async(self._cmd, shell=True)
self._fetch_stat_collect_socket_path()
if self._proc.is_remote:
# 'stats-collect' runs on the SUT and we cannot connect to the Unix socket file
# directly. Setup SSH forwarding.
self._setup_stats_collect_ssh_forwarding()
def _start_ndlrunner(self):
"""Start the 'ndlrunner' process on the measured system."""
regex = f"^.*{self._ndlrunner_bin} .*{self._ifname}.*$"
ProcHelpers.kill_processes(regex,
log=True,
name="stale 'ndlrunner' process",
proc=self._proc)
ldist_str = ",".join([str(val) for val in self._ldist])
cmd = f"{self._ndlrunner_bin} -l {ldist_str} "
cmd += f"{self._ifname}"
self._ndlrunner = self._proc.run_async(cmd)
def _stop_ndlrunner(self):
"""Make 'ndlrunner' process to terminate."""
ndlrunner = self._ndlrunner
self._ndlrunner = None
with contextlib.suppress(Error):
ndlrunner.stdin.write("q\n".encode("utf8"))
ndlrunner.stdin.flush()
_, _, exitcode = ndlrunner.wait_for_cmd(timeout=5)
if exitcode is None:
_LOG.warning(
"the 'ndlrunner' program PID %d%s failed to exit, killing it",
ndlrunner.pid, self._proc.hostmsg)
ProcHelpers.kill_pids(ndlrunner.pid,
kill_children=True,
must_die=False,
proc=self._proc)
def close(self):
"""Stop following the function trace buffer."""
if getattr(self, "_proc", None):
proc = self._proc
self._proc = None
else:
return
if getattr(self, "_reader", None) and getattr(self._reader, "pid",
None):
_LOG.debug("killing the function trace reader process PID %d%s",
self._reader.pid, proc.hostmsg)
ProcHelpers.kill_pids(self._reader.pid,
kill_children=True,
must_die=False,
proc=proc)
self._reader = None
def close(self):
"""Stop the measurements."""
if getattr(self, "_proc", None):
proc = self._proc
self._proc = None
else:
return
if getattr(self, "_phc2sys_proc", None):
_LOG.debug("killing the the phc2sys process PID %d%s",
self._phc2sys_proc.pid, proc.hostmsg)
ProcHelpers.kill_pids(self._phc2sys_proc.pid,
kill_children=True,
must_die=False,
proc=proc)
self._phc2sys_proc = None
if getattr(self, "_netif", None):
self._netif.close()
self._netif = None
def close(self):
"""Close the statistics collector."""
acquired = self._close_lock.acquire(timeout=1) # pylint: disable=consider-using-with
if not acquired:
return
try:
if getattr(self, "_sock", None):
if self._start_time:
with contextlib.suppress(Exception):
self._send_command("stop")
with contextlib.suppress(Exception):
self._send_command("exit")
with contextlib.suppress(Exception):
self._disconnect()
self._sock = None
if getattr(self, "_proc", None):
if self._ssht:
with contextlib.suppress(Exception):
ProcHelpers.kill_processes(self._ssht_search,
proc=self._proc)
self._ssht = None
if self._sc:
with contextlib.suppress(Exception):
ProcHelpers.kill_processes(self._sc_search,
proc=self._proc)
self._sc = None
# Remove the output directory if we created it.
if getattr(self, "_outdir_created", None):
with contextlib.suppress(Exception):
FSHelpers.rm_minus_rf(self.outdir, proc=self._proc)
self._outdir_created = None
self._proc = None
finally:
self._close_lock.release()
def start_phc2sys(self, sync_period=5, tai_offset=37):
"""
Start the 'phc2sys' process in order to synchronize the host and NIC time. The arguments are
as follows.
* sync_period - how often to synchronize (every 5 seconds).
* tai_offset - current TAI offset in seconds (TAI time - real time).
"""
# Kill a possibly stale 'phc2sys' process.
ProcHelpers.kill_processes(r"^phc2sys .*",
log=True,
name="stale 'phc2sys' processes",
proc=self._proc)
freq = 1.0 / sync_period
cmd = f"phc2sys -s CLOCK_REALTIME -c {self._ifname} -R {freq:.5} -O {tai_offset}"
self._phc2sys_proc = self._proc.run_async(cmd)
# Make sure the process did not exit immediately.
stdout, stderr, exitcode = self._phc2sys_proc.wait_for_cmd(timeout=1)
if exitcode is not None:
raise Error(
"can't synchronize the NIC and system clocks, 'phc2sys' exited:\n%s"
% self._phc2sys_proc.cmd_failed_msg(stdout, stderr, exitcode))
def _init_paths(self):
"""
Helper function for 'start_stats_collect()' that discovers and initializes various
paths.
"""
# Discover path to 'stats-collect'.
if not self._sc_path:
self._sc_path = FSHelpers.which("stats-collect", proc=self._proc)
is_root = ProcHelpers.is_root(proc=self._proc)
if not self._unshare_path and is_root:
# Unshare is used for running 'stats-collect' in a separate PID namespace. We do this
# because when the PID 1 process of the namespace is killed, all other processes get
# automatically killed. This helps to easily and reliably clean up processes upon exit.
# But creating a PID namespace requires 'root'.
self._unshare_path = FSHelpers.which("unshare",
default=None,
proc=self._proc)
if not self._unshare_path:
_LOG.warning(
"the 'unshare' tool is missing%s, it is recommended to have it "
"installed. This tool is part of the 'util-linux' project",
self._proc.hostmsg)
if not self._nice_path and is_root:
# We are trying to run 'stats-collect' with high priority, because we want the
# statistics to be collected at steady intervals. The 'nice' tool helps changing the
# priority of the process.
self._nice_path = FSHelpers.which("nice",
default=None,
proc=self._proc)
if not self._nice_path:
_LOG.warning(
"the 'nice' tool is missing%s, it is recommended to have it "
"installed. This tool is part of the 'coreutils' project",
self._proc.hostmsg)
def _fetch_stat_collect_socket_path(self):
"""
This is a helper for '_start_stats_collect()'. When 'stats-collect' starts, it prints unix
socket path it is listening for connections on. This functions parses 'stats-collect' output
and fetches the socket path.
"""
# Spend max. 5 secs waiting for 'stats-collect' to startup and print the socket file path.
attempts = 0
while not self._uspath and attempts < 5:
_, _, exitcode = self._sc.wait_for_cmd(timeout=1,
capture_output=False)
attempts += 1
logdata = logerr = None
try:
with self._proc.open(self._logpath, "r") as fobj:
logdata = fobj.read()
except Error as logerr:
pass
if exitcode is not None:
msg = self._proc.cmd_failed_msg(self._cmd, logdata, None,
exitcode)
if not logdata:
msg += f"\nCheck '{self._logpath}'{self._proc.hostmsg} for details"
raise Error(msg)
if not logdata:
# The log file has not been created yet or has no data yet.
continue
# Search for the socket file path in the log.
pfx = "Listening on Unix socket "
for line in logdata.splitlines():
if line.startswith(pfx):
self._uspath = line.strip()[len(pfx):]
break
if self._uspath:
_LOG.debug("stats-collect PID: %d, socket file path: %s",
self._sc.pid, self._uspath)
self._sc_id = f"{self._proc.hostname}:{self._uspath}"
msg = f"stats-collect (PID {self._sc.pid}) that reported it is listening on Unix " \
f"socket {self._uspath}{self._proc.hostmsg}"
try:
if FSHelpers.issocket(Path(self._uspath), proc=self._proc):
return
except Error as err:
msg = f"{msg}\nBut checking the file path failed: {err}"
else:
msg = f"{msg}\nBut this is not a Unix socket file"
else:
# Failed to extract socket file path.
if exitcode is None:
with contextlib.suppress(Error):
ProcHelpers.kill_pids([
self._sc.pid,
],
kill_children=True,
must_die=False,
proc=self._proc)
msg = f"failed to extract socket file path from 'stats-collect' log\n" \
f"The command was: {self._cmd}\n" \
f"The log is in '{self._logpath}'{self._proc.hostmsg}"
if logerr:
msg += f"\nFailed to read the log file: {logerr}"
elif logdata:
msg += f"\nLog file contents was:\n{logdata}"
raise Error(msg)