def test_cpu_usage(self):
"""
JIRA ID: DAOS-4826
Test Description: Test CPU usage of formatted and idle engine.
:avocado: tags=all,full_regression
:avocado: tags=server
:avocado: tags=cpu_usage
"""
# Get PID of daos_engine with ps.
ps_engine = r"ps -C daos_engine -o %\p"
pid_found = False
# At this point, daos_engine should be started, but do the repetetive
# calls just in case.
for _ in range(5):
task = run_task(hosts=self.hostlist_servers, command=ps_engine)
for output, _ in task.iter_buffers():
self.log.info("ps output = %s", output)
pid = str(output).splitlines()[-1]
self.log.info("PID = %s", pid)
if "PID" not in pid:
pid_found = True
if pid_found:
break
time.sleep(5)
if not pid_found:
self.fail("daos_engine PID couldn't be obtained!")
for _ in range(10):
# Get (instantaneous) CPU usage of the PID with top.
top_pid = "top -p {} -b -n 1".format(pid)
usage = -1
task = run_task(hosts=self.hostlist_servers, command=top_pid)
for output, _ in task.iter_buffers():
process_row = str(output).splitlines()[-1]
self.log.info("Process row = %s", process_row)
values = process_row.split()
self.log.info("Values = %s", values)
if len(values) < 9:
self.fail("{} returned invalid output!".format(top_pid))
usage = values[8]
self.log.info("CPU Usage = %s", usage)
if usage != -1 and float(usage) < 100:
break
time.sleep(2)
self.assertTrue(usage != -1,
"daos_engine CPU usage couldn't be obtained!")
self.assertTrue(
float(usage) < 100, "CPU usage is above 100%: {}%".format(usage))
def get_interface_ib_name(hosts, interface, verbose=True):
"""Get the InfiniBand name of this network interface on each host.
Args:
hosts (NodeSet): hosts on which to detect the InfiniBand name
interface (str): interface for which to obtain the InfiniBand name
verbose (bool, optional): display command details. Defaults to True.
Returns:
dict: a dictionary of InfiniBand name keys and NodeSet values on which they were detected
"""
net_path = os.path.join(os.path.sep, "sys", "class", "net")
command = f"ls -1 {os.path.join(net_path, interface, 'device', 'infiniband')}"
task = run_task(hosts, command, verbose=verbose)
if verbose:
display_task(task)
# Populate a dictionary of IB names with a NodSet of hosts on which it was detected
ib_names = {}
results = dict(task.iter_retcodes())
if 0 in results:
for output, nodelist in task.iter_buffers(results[0]):
ib_name_list = []
for line in output:
match = re.findall(r"([A-Za-z0-9;_+]+)", line.decode("utf-8"))
if len(match) == 1:
ib_name_list.append(match[0])
if ib_name_list:
ib_names[",".join(ib_name_list)] = NodeSet.fromlist(nodelist)
return ib_names
def service_running(self):
"""Determine if the job's service is active via the systemctl command.
The 'systemctl is-active <service>' command will return a string
indicating one of the following states:
active, inactive, activating, deactivating, failed, unknown
If the <service> is "active" or "activating" return True.
Returns:
bool: True id the service is running, False otherwise
"""
status = True
states = {}
valid_states = ["active", "activating"]
self._systemctl.unit_command.value = "is-active"
task = run_task(self._hosts, self.__str__(), self.timeout)
for output, nodelist in task.iter_buffers():
output = str(output)
nodeset = NodeSet.fromlist(nodelist)
status &= output in valid_states
if output not in states:
states[output] = NodeSet()
states[output].add(nodeset)
if self.timeout and task.num_timeout() > 0:
nodeset = NodeSet.fromlist(task.iter_keys_timeout())
states["timeout"] = nodeset
data = ["=".join([key, str(states[key])]) for key in sorted(states)]
self.log.info(" Detected %s states: %s",
self._systemctl.service.value, ", ".join(data))
return status
def get_nvme_readlink(self):
"""Get NVMe readlink from servers.
Returns:
dict: Dictionary of server readlink pci mapping with disk ID
'wolf-A': {'0000:da:00.0': 'nvme9n1'}.
Dictionary of server mapping with disk ID and size
'wolf-A': {'nvme2n1': '1600321314816'}.
"""
nvme_lsblk = self.get_nvme_lsblk()
nvme_readlink = {}
#Create the dictionary for NVMe readlink.
for server, items in nvme_lsblk.items():
tmp_dict = {}
for drive in items:
cmd = ('readlink /sys/block/{}/device/device'
.format(drive.split()[0]))
task = run_task([server], cmd)
for _rc_code, _node in task.iter_retcodes():
if _rc_code == 1:
print("Failed to readlink on {}".format(_node))
raise ValueError
#Get the drive size from each daos_io_servers
for buf, _node in task.iter_buffers():
output = str(buf).split('\n')
tmp_dict[output[0].split('/')[-1]] = drive.split()[0]
nvme_readlink[server] = tmp_dict
return nvme_lsblk, nvme_readlink
def get_nvme_lsblk(self):
"""Get NVMe size using lsblk from servers.
Returns:
dict: Dictionary of server mapping with disk ID and size
'wolf-A': {'nvme2n1': '1600321314816'}.
"""
nvme_data = {}
task = run_task(self.hostlist_servers, "lsblk -b /dev/nvme*n*")
for _rc_code, _node in task.iter_retcodes():
if _rc_code == 1:
print("Failed to lsblk on {}".format(_node))
raise ValueError
#Get the drive size from each daos_io_servers
for buf, nodelist in task.iter_buffers():
for node in nodelist:
disk_data = {}
output = str(buf).split('\n')
for _tmp in output[1:]:
if 'nvme' in _tmp:
disk_data[_tmp.split()[0]] = _tmp.split()[3]
nvme_data['{}'.format(node)] = disk_data
return nvme_data
def debug_numa_node(self, pci_addr_heads):
"""Debug numa_node file by searching it in /sys and call hwloc-ls.
Args:
pci_addr_heads (list): List of PCI address head.
"""
for pci_addr_head in pci_addr_heads:
self.log.debug("----- Search PCI Addr Head %s in /sys -----",
pci_addr_head)
task = run_task(
hosts=self.hostlist_servers,
command="find /sys -name \"{}\"".format(pci_addr_head))
for output, _ in task.iter_buffers():
self.log.debug(output)
# Another way to obtain the Socket ID is to use hwloc-ls --whole-io
# --verbose. It contains something like:
# Bridge Host->PCI L#9 (P#2 buses=0000:[80-81])
# Bridge PCI->PCI (P#524320 busid=0000:80:02.0 id=8086:2f04
# class=0604(PCI_B) buses=0000:[81-81])
# PCI 8086:2701 (P#528384 busid=0000:81:00.0 class=0108(NVMExp)
# PCISlot=801)
# In this case, the PCI address was 0000:81:00.0. We can figure out
# which NUMA node section these lines are in. This approach is clearly
# much more cumbersome than reading the numa_node, so it's called here
# for mainly debugging purpose.
self.log.debug("----- Show PCI Address in hwloc-ls -----")
pcmd(hosts=self.hostlist_servers,
command="hwloc-ls --whole-io --verbose")
def get_interface_numa_node(hosts, interface, verbose=True):
"""Get the NUMA node ID of this network interface on each host.
Args:
hosts (NodeSet): hosts on which to detect the NUMA node
interface (str): interface for which to obtain the NUMA node
verbose (bool, optional): display command details. Defaults to True.
Returns:
dict: a dictionary of NUMA node ID keys and NodeSet values on which they were detected
"""
net_path = os.path.join(os.path.sep, "sys", "class", "net")
command = f"cat {os.path.join(net_path, interface, 'device', 'numa_node')}"
task = run_task(hosts, command, verbose=verbose)
if verbose:
display_task(task)
# Populate a dictionary of numa node IDs with a NodSet of hosts on which it was detected
numa_nodes = {}
results = dict(task.iter_retcodes())
if 0 in results:
for output, nodelist in task.iter_buffers(results[0]):
output_lines = [line.decode("utf-8") for line in output]
nodeset = NodeSet.fromlist(nodelist)
for line in output_lines:
try:
numa_node = int(line.strip())
except ValueError:
continue
if numa_node not in numa_nodes:
numa_nodes[numa_node] = NodeSet()
numa_nodes[numa_node].update(nodeset)
return numa_nodes
def update_config_cmdlist(args):
"""Create the command lines to update slurmd.conf file.
Args:
args (Namespace): Commandline arguments
Returns:
cmd_list: list of cmdlines to update config file
"""
all_nodes = NodeSet("{},{}".format(str(args.control), str(args.nodes)))
if not args.sudo:
sudo = ""
else:
sudo = "sudo"
# Copy the slurm*example.conf files to /etc/slurm/
if execute_cluster_cmds(all_nodes, COPY_LIST, args.sudo) > 0:
exit(1)
cmd_list = [
"sed -i -e 's/ControlMachine=linux0/ControlMachine={}/g' {}".format(
args.control, SLURM_CONF),
"sed -i -e 's/ClusterName=linux/ClusterName=ci_cluster/g' {}".format(
SLURM_CONF),
"sed -i -e 's/SlurmUser=slurm/SlurmUser={}/g' {}".format(
args.user, SLURM_CONF),
"sed -i -e 's/NodeName/#NodeName/g' {}".format(SLURM_CONF),
]
# This info needs to be gathered from every node that can run a slurm job
command = r"lscpu | grep -E '(Socket|Core|Thread)\(s\)'"
task = run_task(all_nodes, command)
for output, nodes in task.iter_buffers():
info = {
data[0]: data[1]
for data in re.findall(r"(Socket|Core|Thread).*:\s+(\d+)",
str(output)) if len(data) > 1
}
if "Socket" not in info or "Core" not in info or "Thread" not in info:
# Did not find value for socket|core|thread so do not
# include in config file
pass
cmd_list.append("echo \"NodeName={0} Sockets={1} CoresPerSocket={2} "
"ThreadsPerCore={3}\" |{4} tee -a {5}".format(
NodeSet.fromlist(nodes), info["Socket"],
info["Core"], info["Thread"], sudo, SLURM_CONF))
#
cmd_list.append("echo \"PartitionName= {} Nodes={} Default=YES "
"MaxTime=INFINITE State=UP\" |{} tee -a {}".format(
args.partition, args.nodes, sudo, SLURM_CONF))
return execute_cluster_cmds(all_nodes, cmd_list, args.sudo)
def test_scan_ssd(self):
"""
JIRA ID: DAOS-3584
Test Description: Verify NVMe NUMA socket values.
:avocado: tags=all,small,full_regression,hw,control,ssd_socket
"""
# Call dmg storage scan --verbose and get the PCI addresses.
data = self.get_dmg_command().storage_scan(verbose=True)
pci_addrs = data[self.hostlist_servers[0]]["nvme"].keys()
self.log.info("Testing PCI addresses: %s", pci_addrs)
pci_addr_heads = []
errors = []
# For every PCI address, verify its Socket ID against its NUMA socket
# ID.
for pci_addr in pci_addrs:
# Get the PCI Address Head and construct the path to numa_node.
cmd_socket_id = data[self.hostlist_servers[0]]["nvme"][pci_addr]\
["socket"]
pci_addr_values = pci_addr.split(":")
pci_addr_head = "{}:{}".format(pci_addr_values[0],
pci_addr_values[1])
pci_addr_heads.append(pci_addr_head)
numa_node_path = "/sys/class/pci_bus/{}/device/numa_node".format(
pci_addr_head)
# Call cat on the server host, not necessarily the local test host.
task = run_task(hosts=[self.hostlist_servers[0]],
command="cat {}".format(numa_node_path))
# Obtain the numa_node content.
fs_socket_id = ""
for output, _ in task.iter_buffers():
fs_socket_id = str(output).splitlines()[-1]
# Test that the content is expected.
if fs_socket_id != cmd_socket_id:
errors.append("Unexpected socket ID! Cmd: {}; FS: {}".format(
cmd_socket_id, fs_socket_id))
if errors:
# Since we're dealing with system files and we don't have access to
# them in CI, we need some debugging info when the test fails to
# better understand the result.
self.debug_numa_node(pci_addr_heads)
self.fail("Error found!\n{}".format("\n".join(errors)))
def test_nvme_io_stats(self):
"""Jira ID: DAOS-4722.
Test Description:
Purpose of this test is to run IO test and check when NVME_IO_STATS
enabled in config, it generates the different statistics.
Use case:
Run ior and it will print the NVMe IO stats to control plane log
file.
:avocado: tags=all,hw,medium,nvme,ib2,nvme_io_stats,full_regression
"""
# run ior
self.run_ior_with_pool()
#Get the NVMe IO statistics from server control_log file.
cmd = 'cat {}'.format(get_log_file(self.control_log))
task = run_task(self.hostlist_servers, cmd)
for _rc_code, _node in task.iter_retcodes():
if _rc_code == 1:
self.fail("Failed to run cmd {} on {}".format(cmd, _node))
for buf, _nodes in task.iter_buffers():
output_list = str(buf).split('\n')
#Verify statistics are increasing for IO
target_stats = []
for _tmp in range(8):
target_stats.append(
[s for s in output_list if "tgt[{}]".format(_tmp) in s])
for stats in NVME_STATS:
for _tgt in range(len(target_stats)):
first_stats = re.findall(r'\d+', [
x for x in target_stats[_tgt][0].split()
if re.search(stats, x)
][0])[0]
last_stats = re.findall(r'\d+', [
x for x in target_stats[_tgt][-1].split()
if re.search(stats, x)
][0])[0]
#Last statistic should be higher from the initial statistics
if int(first_stats) >= int(last_stats):
self.fail(
'Failed: Stats {} for target {} did not increased'
' First_stat={} < Last_stat={}'.format(
stats, _tgt, first_stats, last_stats))
def get_interface_speeds(hosts, interface, verbose=True):
"""Get the speeds of this network interface on each host.
Args:
hosts (NodeSet): hosts on which to detect the interface speed
interface (str): interface for which to obtain the speed
verbose (bool, optional): display command details. Defaults to True.
Returns:
dict: a dictionary of interface speed keys and NodeSet values on which they were detected
"""
net_path = os.path.join(os.path.sep, "sys", "class", "net")
command = f"cat {os.path.join(net_path, interface, 'speed')}"
task = run_task(hosts, command, verbose=verbose)
if verbose:
display_task(task)
# Populate a dictionary of interface speeds with a NodSet of hosts on which it was detected
interface_speeds = {}
results = dict(task.iter_retcodes())
if 0 in results:
for output, nodelist in task.iter_buffers(results[0]):
output_lines = [line.decode("utf-8") for line in output]
nodeset = NodeSet.fromlist(nodelist)
for line in output_lines:
try:
speed = int(line.strip())
except IOError as io_error:
# KVM/Qemu/libvirt returns an EINVAL
if io_error.errno == errno.EINVAL:
speed = 1000
except ValueError:
# Any line not containing a speed (integer)
continue
if speed not in interface_speeds:
interface_speeds[speed] = NodeSet()
interface_speeds[speed].update(nodeset)
return interface_speeds
def test_cpu_usage(self):
# pylint: disable=pylint-bad-continuation
"""
JIRA ID: DAOS-4826
Test Description: Test CPU usage of formatted and idle daos_io_server.
:avocado: tags=all,hw,server,small,full_regression,cpu_usage
"""
ps_get_cpu = r"ps -C daos_io_server -o %\cpu"
prev_usage = 1
usage = 1
time.sleep(5)
for _ in range(10):
time.sleep(5)
task = run_task(hosts=self.hostlist_servers, command=ps_get_cpu)
# Sample output.
# %CPU
# 1798
for output, _ in task.iter_buffers():
usage = str(output).splitlines()[-1]
self.log.info("CPU usage = %s", usage)
# Check if daos_io_server has started.
if usage == "%CPU":
continue
usage = int(usage)
if usage == 0:
break
diff = usage - prev_usage
diff_p = (float(abs(diff)) / prev_usage) * 100
# Check if the CPU usage is stable; the change was less than 10%.
if diff_p <= float(10):
break
prev_usage = usage
self.assertTrue(
usage != "%CPU", "daos_io_server CPU usage couldn't be obtained!")
self.assertTrue(
usage < 100, "CPU usage is above 100%: {}%".format(usage))
def get_active_network_interfaces(hosts, verbose=True):
"""Get all active network interfaces on the hosts.
Args:
hosts (NodeSet): hosts on which to find active interfaces
verbose (bool, optional): display command details. Defaults to True.
Returns:
dict: a dictionary of interface keys and NodeSet values on which they were found
"""
net_path = os.path.join(os.path.sep, "sys", "class", "net")
operstate = os.path.join(net_path, "*", "operstate")
command = " | ".join([
f"grep -l 'up' {operstate}", "grep -Ev '/(lo|bonding_masters)/'",
"sort"
])
task = run_task(hosts, command, verbose=verbose)
if verbose:
display_task(task)
# Populate a dictionary of active interfaces with a NodSet of hosts on which it was found
active_interfaces = {}
for output, nodelist in task.iter_buffers():
output_lines = [line.decode("utf-8") for line in output]
nodeset = NodeSet.fromlist(nodelist)
for line in output_lines:
try:
interface = line.split("/")[-2]
if interface not in active_interfaces:
active_interfaces[interface] = NodeSet()
active_interfaces[interface].update(nodeset)
except IndexError:
pass
return active_interfaces
def get_ofi_info(hosts, supported=None, verbose=True):
"""Get the OFI provider information from the specified hosts.
Args:
hosts (NodeSet): hosts from which to gather the information
supported (list, optional): list of supported providers when if provided will limit the
inclusion to only those providers specified. Defaults to None.
verbose (bool, optional): display command details. Defaults to True.
Returns:
dict: a dictionary of interface keys with a dictionary value of a comma-separated string of
providers key with a NodeSet value where the providers where detected.
"""
task = run_task(hosts, "fi_info", verbose=verbose)
if verbose:
display_task(task)
# Populate a dictionary of interfaces with a list of provider lists and NodSet of hosts on which
# the providers were detected.
providers = {}
results = dict(task.iter_retcodes())
if 0 in results:
for output, nodelist in task.iter_buffers(results[0]):
output_lines = [
line.decode("utf-8").rstrip(os.linesep) for line in output
]
nodeset = NodeSet.fromlist(nodelist)
# Find all the provider and domain pairings. The fi_info output reports these on
# separate lines when processing the re matches ensure each domain is preceded by a
# provider.
interface_providers = {}
data = re.findall(r"(provider|domain):\s+([A-Za-z0-9;_+]+)",
"\n".join(output_lines))
while data:
provider = list(data.pop(0))
if provider[0] == "provider" and data[0][0] == "domain":
provider.pop(0)
domain = list(data.pop(0))
domain.pop(0)
# A provider and domain must be specified
if not provider or not domain:
continue
# Add 'ofi+' to the provider
provider = ["+".join(["ofi", item]) for item in provider]
# Only include supported providers if a supported list is provided
if supported and provider[0] not in supported:
continue
if domain[0] not in interface_providers:
interface_providers[domain[0]] = set()
interface_providers[domain[0]].update(provider)
for interface, provider_set in interface_providers.items():
if interface not in providers:
providers[interface] = {}
provider_key = ",".join(list(provider_set))
if provider_key not in providers[interface]:
providers[interface][provider_key] = NodeSet()
providers[interface][provider_key].update(nodeset)
return providers
def get_log_data(self, hosts, since, until=None, timeout=60):
"""Gather log output for the command running on each host.
Note (from journalctl man page):
Date specifications should be of the format "2012-10-30 18:17:16".
If the time part is omitted, "00:00:00" is assumed. If only the
seconds component is omitted, ":00" is assumed. If the date
component is omitted, the current day is assumed. Alternatively the
strings "yesterday", "today", "tomorrow" are understood, which refer
to 00:00:00 of the day before the current day, the current day, or
the day after the current day, respectively. "now" refers to the
current time. Finally, relative times may be specified, prefixed
with "-" or "+", referring to times before or after the current
time, respectively.
Args:
hosts (list): list of hosts from which to gather log data.
since (str): show log entries from this date.
until (str, optional): show log entries up to this date. Defaults
to None, in which case it is not utilized.
timeout (int, optional): timeout for issuing the command. Defaults
to 60 seconds.
Returns:
dict: log output per host
"""
# Setup the journalctl command to capture all unit activity from the
# specified start date to now or a specified end date
# --output=json?
command = [
"sudo",
"journalctl",
"--unit={}".format(self._systemctl.service.value),
"--since=\"{}\"".format(since),
]
if until:
command.append("--until=\"{}\"".format(until))
self.log.info("Gathering log data on %s: %s", str(hosts),
" ".join(command))
# Gather the log information per host
task = run_task(hosts, " ".join(command), timeout)
# Create a dictionary of hosts for each unique return code
results = {code: hosts for code, hosts in task.iter_retcodes()}
# Determine if the command completed successfully across all the hosts
status = len(results) == 1 and 0 in results
# Determine if any commands timed out
timed_out = [str(hosts) for hosts in task.iter_keys_timeout()]
if timed_out:
status = False
if not status:
self.log.info(" Errors detected running \"%s\":", command)
# List any hosts that timed out
if timed_out:
self.log.info(" %s: timeout detected after %s seconds",
str(NodeSet.fromlist(timed_out)), timeout)
# Display/return the command output
log_data = {}
for code in sorted(results):
# Get the command output from the hosts with this return code
output_data = list(task.iter_buffers(results[code]))
if not output_data:
output_data = [["<NONE>", results[code]]]
for output_buffer, output_hosts in output_data:
node_set = NodeSet.fromlist(output_hosts)
lines = str(output_buffer).splitlines()
if status:
# Add the successful output from each node to the dictionary
log_data[node_set] = lines
else:
# Display all of the results in the case of an error
if len(lines) > 1:
self.log.info(" %s: rc=%s, output:", node_set, code)
for line in lines:
self.log.info(" %s", line)
else:
self.log.info(" %s: rc=%s, output: %s", node_set,
code, output_buffer)
# Report any errors through an exception
if not status:
raise CommandFailure(
"Error(s) detected gathering {} log data on {}".format(
self._systemctl.service.value, NodeSet.fromlist(hosts)))
# Return the successful command output per set of hosts
return log_data
def get_host_data(self, command, text, error):
"""Get the data requested for each host using the specified command.
Args:
command (str): command used to obtain the data on each server
text (str): data identification string
error (str): data error string
Returns:
dict: a dictionary of data values for each NodeSet key
"""
# Find the data for each specified servers
self.log.info(" Obtaining %s data on %s", text, self.hosts)
task = run_task(self.hosts, command, self.timeout)
host_data = {}
# Create a list of NodeSets with the same return code
data = {code: hosts for code, hosts in task.iter_retcodes()}
# Multiple return codes or a single non-zero return code
# indicate at least one error obtaining the data
if len(data) > 1 or 0 not in data:
# Report the errors
messages = []
for code, hosts in data.items():
if code != 0:
output_data = list(task.iter_buffers(hosts))
if len(output_data) == 0:
messages.append("{}: rc={}, command=\"{}\"".format(
NodeSet.fromlist(hosts), code, command))
else:
for output, o_hosts in output_data:
lines = str(output).splitlines()
info = "rc={}{}".format(
code, ", {}".format(output) if len(lines) < 2
else "\n {}".format("\n ".join(lines)))
messages.append("{}: {}".format(
NodeSet.fromlist(o_hosts), info))
self.log.error(" %s on the following hosts:\n %s", error,
"\n ".join(messages))
# Return an error data set for all of the hosts
host_data = {NodeSet.fromlist(self.hosts): DATA_ERROR}
else:
# The command completed successfully on all servers.
for output, hosts in task.iter_buffers(data[0]):
# Find the maximum size of the all the devices reported by
# this group of hosts as only one needs to meet the minimum
nodes = NodeSet.fromlist(hosts)
try:
# The assumption here is that each line of command output
# will begin with a number and that for the purposes of
# checking this requirement the maximum of these numbers is
# needed
int_host_values = [
int(line.split()[0])
for line in str(output).splitlines()
]
host_data[nodes] = max(int_host_values)
except (IndexError, ValueError):
# Log the error
self.log.error(
" %s: Unable to obtain the maximum %s size due to "
"unexpected output:\n %s", nodes, text,
"\n ".join(str(output).splitlines()))
# Return an error data set for all of the hosts
host_data = {NodeSet.fromlist(hosts): DATA_ERROR}
break
return host_data
def get_ucx_info(hosts, supported=None, verbose=True):
"""Get the UCX provider information from the specified hosts.
Args:
hosts (NodeSet): hosts from which to gather the information
supported (list, optional): list of supported providers when if provided will limit the
inclusion to only those providers specified. Defaults to None.
verbose (bool, optional): display command details. Defaults to True.
Returns:
dict: a dictionary of interface keys with a dictionary value of a comma-separated string of
providers key with a NodeSet value where the providers where detected.
"""
task = run_task(hosts, "ucx_info -d", verbose=verbose)
if verbose:
display_task(task)
# Populate a dictionary of interfaces with a list of provider lists and NodSet of hosts on which
# the providers were detected.
providers = {}
results = dict(task.iter_retcodes())
if 0 in results:
for output, nodelist in task.iter_buffers(results[0]):
output_lines = [
line.decode("utf-8").rstrip(os.linesep) for line in output
]
nodeset = NodeSet.fromlist(nodelist)
# Find all the transport, device, and type pairings. The ucx_info output reports these
# on separate lines so when processing the re matches ensure each device is preceded by
# a provider.
interface_providers = {}
data = re.findall(r"(Transport|Device):\s+([A-Za-z0-9;_+]+)",
"\n".join(output_lines))
while data:
transport = list(data.pop(0))
if transport[0] == "Transport" and data[0][0] == "Device":
transport.pop(0)
device = list(data.pop(0))
device.pop(0)
# A transport and device must be specified
if not transport or not device:
continue
# Add 'ucx+' to the provider and replace 'mlx[0-9]' with 'x'
transport = [
"+".join(["ucx",
re.sub(r"mlx[0-9]+", "x", item)])
for item in transport
]
# Only include supported providers if a supported list is provided
if supported and transport[0] not in supported:
continue
if device[0] not in interface_providers:
interface_providers[device[0]] = set()
interface_providers[device[0]].update(transport)
for interface, provider_set in interface_providers.items():
if interface not in providers:
providers[interface] = {}
provider_key = ",".join(list(provider_set))
if provider_key not in providers[interface]:
providers[interface][provider_key] = NodeSet()
providers[interface][provider_key].update(nodeset)
return providers
