def _get_metrics(hosts, metrics, data=None):
"""Retrieve host metrics using SSH if necessary.
Note hosts will not appear in the returned results if:
* They are not contactable.
* There is an error in the command which returns the results.
Args:
hosts (list):
List of host fqdns.
metrics (list):
List in the form [(function, arg1, arg2, ...), ...]
data (dict):
Used for logging success/fail outcomes of the form {host: {}}
Examples:
Command failure:
>>> _get_metrics(['localhost'], [['elephant']])
({}, {'localhost': {'get_metrics': 'Command failed (exit: 1)'}})
Returns:
dict - {host: {(function, arg1, arg2, ...): result}}
"""
host_stats = {}
proc_map = {}
if not data:
data = {host: dict() for host in hosts}
# Start up commands on hosts
cmd = ['psutil']
kwargs = {'stdin_str': json.dumps(metrics), 'capture_process': True}
for host in hosts:
if is_remote_host(host):
proc_map[host] = remote_cylc_cmd(cmd, host=host, **kwargs)
else:
proc_map[host] = run_cmd(['cylc'] + cmd, **kwargs)
# Collect results from commands
while proc_map:
for host, proc in list(proc_map.copy().items()):
if proc.poll() is None:
continue
del proc_map[host]
out, err = (f.decode() for f in proc.communicate())
if proc.wait():
# Command failed in verbose/debug mode
LOG.warning('Could not evaluate "%s" (return code %d)\n%s',
host, proc.returncode, err)
data[host]['get_metrics'] = (
f'Command failed (exit: {proc.returncode})')
else:
host_stats[host] = dict(
zip(
metrics,
# convert JSON dicts -> namedtuples
_deserialise(metrics, parse_dirty_json(out))))
sleep(0.01)
return host_stats, data
def test_run_cmd_stdin_file(tmp_path):
"""Test passing stdin as a file."""
tmp_path = tmp_path / 'stdin'
with tmp_path.open('w+') as tmp_file:
tmp_file.write('1foo2')
tmp_file = tmp_path.open('rb')
proc = run_cmd(['sed', 's/foo/bar/'], stdin=tmp_file, capture_process=True)
assert [s.strip() for s in proc.communicate()] == [b'1bar2', b'']
def test_run_cmd_stdin_str():
"""Test passing stdin as a string."""
proc = run_cmd(
['sed', 's/foo/bar/'],
stdin_str='1foo2',
capture_process=True
)
assert [s.strip() for s in proc.communicate()] == [
b'1bar2',
b''
]
def _get_host_metrics(self):
"""Run "cylc get-host-metrics" commands on hosts.
Return (dict): {host: host-metrics-dict, ...}
"""
host_stats = {}
# Run "cylc get-host-metrics" commands on hosts
host_proc_map = {}
cmd = [self.CMD_BASE] + sorted(self._get_host_metrics_opts())
# Start up commands on hosts
for host in self.hosts:
if is_remote_host(host):
host_proc_map[host] = remote_cylc_cmd(cmd,
stdin=None,
host=host,
capture_process=True)
elif 'localhost' in host_proc_map:
continue # Don't duplicate localhost
else:
# 1st instance of localhost
host_proc_map['localhost'] = run_cmd(['cylc'] + cmd,
capture_process=True)
# Collect results from commands
while host_proc_map:
for host, proc in list(host_proc_map.copy().items()):
if proc.poll() is None:
continue
del host_proc_map[host]
out, err = (f.decode() for f in proc.communicate())
if proc.wait():
# Command failed in verbose/debug mode
LOG.warning(
"can't get host metric from '%s'" +
"%s # returncode=%d, err=%s\n", host, ' '.join(
(quote(item) for item in cmd)), proc.returncode,
err)
else:
# Command OK
# Users may have profile scripts that write to STDOUT.
# Drop all output lines until the the first character of a
# line is '{'. Hopefully this is enough to find us the
# first line that denotes the beginning of the expected
# JSON data structure.
out = ''.join(
dropwhile(lambda s: not s.startswith('{'),
out.splitlines(True)))
host_stats[host] = json.loads(out)
sleep(0.01)
return host_stats
def _get_host_metrics(self):
"""Run "cylc get-host-metrics" commands on hosts.
Return (dict): {host: host-metrics-dict, ...}
"""
host_stats = {}
# Run "cylc get-host-metrics" commands on hosts
host_proc_map = {}
cmd = [self.CMD_BASE] + sorted(self._get_host_metrics_opts())
# Start up commands on hosts
for host in self.hosts:
if is_remote_host(host):
host_proc_map[host] = remote_cylc_cmd(
cmd, stdin=None, host=host, capture_process=True)
elif 'localhost' in host_proc_map:
continue # Don't duplicate localhost
else:
# 1st instance of localhost
host_proc_map['localhost'] = run_cmd(
['cylc'] + cmd, capture_process=True)
# Collect results from commands
while host_proc_map:
for host, proc in list(host_proc_map.copy().items()):
if proc.poll() is None:
continue
del host_proc_map[host]
out, err = (f.decode() for f in proc.communicate())
if proc.wait():
# Command failed in verbose/debug mode
LOG.warning(
"can't get host metric from '%s'" +
"%s # returncode=%d, err=%s\n",
host, ' '.join((quote(item) for item in cmd)),
proc.returncode, err)
else:
# Command OK
# Users may have profile scripts that write to STDOUT.
# Drop all output lines until the the first character of a
# line is '{'. Hopefully this is enough to find us the
# first line that denotes the beginning of the expected
# JSON data structure.
out = ''.join(dropwhile(
lambda s: not s.startswith('{'), out.splitlines(True)))
host_stats[host] = json.loads(out)
sleep(0.01)
return host_stats
