def _distribute(host):
"""Re-invoke this command on a different host if requested."""
# Check whether a run host is explicitly specified, else select one.
if not host:
host = select_workflow_host()[0]
if is_remote_host(host):
# Prevent recursive host selection
cmd = sys.argv[1:]
cmd.append("--host=localhost")
_remote_cylc_cmd(cmd, host=host)
sys.exit(0)
async def async_request(self, command, args=None, timeout=None):
"""Send asynchronous request via SSH.
"""
timeout = timeout if timeout is not None else self.timeout
try:
async with ascyncto(timeout):
cmd, ssh_cmd, login_sh, cylc_path, msg = self.prepare_command(
command, args, timeout)
proc = _remote_cylc_cmd(cmd,
host=self.host,
stdin_str=msg,
ssh_cmd=ssh_cmd,
remote_cylc_path=cylc_path,
ssh_login_shell=login_sh,
capture_process=True)
while True:
if proc.poll() is not None:
break
await asyncio.sleep(self.SLEEP_INTERVAL)
out, err = (f.decode() for f in proc.communicate())
return_code = proc.wait()
if return_code:
raise ClientError(err, f"return-code={return_code}")
return json.loads(out)
except asyncio.TimeoutError:
raise ClientTimeout(f"Command exceeded the timeout {timeout}. "
f"This could be due to network problems. "
"Check the workflow log.")
def _get_metrics(hosts, metrics, data=None):
"""Retrieve host metrics using SSH if necessary.
Note hosts will not appear in the returned results if:
* They are not contactable.
* There is an error in the command which returns the results.
Args:
hosts (list):
List of host fqdns.
metrics (list):
List in the form [(function, arg1, arg2, ...), ...]
data (dict):
Used for logging success/fail outcomes of the form {host: {}}
Examples:
Command failure:
>>> _get_metrics(['localhost'], [['elephant']])
({}, {'localhost': {'get_metrics': 'Command failed (exit: 1)'}})
Returns:
dict - {host: {(function, arg1, arg2, ...): result}}
"""
host_stats = {}
proc_map = {}
if not data:
data = {host: {} for host in hosts}
# Start up commands on hosts
cmd = ['psutil']
kwargs = {'stdin_str': json.dumps(metrics), 'capture_process': True}
for host in hosts:
if is_remote_host(host):
proc_map[host] = _remote_cylc_cmd(cmd, host=host, **kwargs)
else:
proc_map[host] = run_cmd(['cylc'] + cmd, **kwargs)
# Collect results from commands
while proc_map:
for host, proc in list(proc_map.copy().items()):
if proc.poll() is None:
continue
del proc_map[host]
out, err = (f.decode() for f in proc.communicate())
if proc.wait():
# Command failed in verbose/debug mode
LOG.warning('Could not evaluate "%s" (return code %d)\n%s',
host, proc.returncode, err)
data[host]['get_metrics'] = (
f'Command failed (exit: {proc.returncode})')
else:
host_stats[host] = dict(
zip(
metrics,
# convert JSON dicts -> namedtuples
_deserialise(metrics, parse_dirty_json(out))))
sleep(0.01)
return host_stats, data
def send_request(self, command, args=None, timeout=None):
"""Send a request, using ssh.
Determines ssh_cmd, cylc_path and login_shell settings from the contact
file.
Converts message to JSON and sends this to stdin. Executes the Cylc
command, then deserialises the output.
Use ``__call__`` to call this method.
Args:
command (str): The name of the endpoint to call.
args (dict): Arguments to pass to the endpoint function.
timeout (float): Override the default timeout (seconds).
Raises:
ClientError: Coverall, on error from function call
Returns:
object: Deserialized output from function called.
"""
# Set environment variable to determine the communication for use on
# the scheduler
os.environ["CLIENT_COMMS_METH"] = CommsMeth.SSH.value
cmd = ["client"]
if timeout:
cmd += [f'comms_timeout={timeout}']
cmd += [self.workflow, command]
contact = load_contact_file(self.workflow)
ssh_cmd = contact[ContactFileFields.SCHEDULER_SSH_COMMAND]
login_shell = contact[ContactFileFields.SCHEDULER_USE_LOGIN_SHELL]
cylc_path = contact[ContactFileFields.SCHEDULER_CYLC_PATH]
cylc_path = None if cylc_path == 'None' else cylc_path
if not args:
args = {}
message = json.dumps(args)
proc = _remote_cylc_cmd(
cmd,
host=self.host,
stdin_str=message,
ssh_cmd=ssh_cmd,
remote_cylc_path=cylc_path,
ssh_login_shell=login_shell,
capture_process=True)
out, err = (f.decode() for f in proc.communicate())
return_code = proc.wait()
if return_code:
raise ClientError(err, f"return-code={return_code}")
return json.loads(out)
