def main(parser, options, suite, event_msg, event_id):
suite = os.path.normpath(suite)
LOG.info('Send to suite %s: "%s" (%s)', suite, event_msg, event_id)
pclient = get_client(suite, timeout=options.comms_timeout)
max_n_tries = int(options.max_n_tries)
retry_intvl_secs = float(options.retry_intvl_secs)
mutation_kwargs = {
'request_string': MUTATION,
'variables': {
'wFlows': [suite],
'eventMsg': event_msg,
'eventId': event_id,
}
}
for i_try in range(max_n_tries):
try:
pclient('graphql', mutation_kwargs)
except ClientError as exc:
LOG.exception(exc)
LOG.info(MSG_SEND_FAILED, i_try + 1, max_n_tries)
if i_try == max_n_tries - 1: # final attempt
raise CylcError('send failed')
LOG.info(MSG_SEND_RETRY, retry_intvl_secs, options.comms_timeout)
sleep(retry_intvl_secs)
else:
if i_try > 0:
LOG.info(MSG_SEND_SUCCEED, i_try + 1, max_n_tries)
break
async def scan_one(reg, host, port, timeout=None, methods=None):
if not methods:
methods = ['identify']
if is_remote_host(host):
try:
host = get_host_ip_by_name(host) # IP reduces DNS traffic
except socket.error as exc:
if cylc.flow.flags.debug:
raise
sys.stderr.write("ERROR: %s: %s\n" % (exc, host))
return (reg, host, port, None)
# NOTE: Connect to the suite by host:port, this was the
# SuiteRuntimeClient will not attempt to check the contact file
# which would be unnecessary as we have already done so.
# NOTE: This part of the scan *is* IO blocking.
client = SuiteRuntimeClient(reg, host=host, port=port, timeout=timeout)
result = {}
for method in methods:
# work our way up the chain of identity methods, extract as much
# information as we can before the suite rejects us
try:
msg = await client.async_request(method)
except ClientTimeout as exc:
LOG.exception(f"Timeout: name:{reg}, host:{host}, port:{port}")
return (reg, host, port, MSG_TIMEOUT)
except ClientError as exc:
LOG.exception("ClientError")
return (reg, host, port, result or None)
else:
result.update(msg)
return (reg, host, port, result)
def _receiver(self, message):
"""Wrap incoming messages and dispatch them to exposed methods.
Args:
message (dict): message contents
"""
# determine the server method to call
try:
method = getattr(self, message['command'])
args = message['args']
args.update({'user': message['user']})
if 'meta' in message:
args['meta'] = message['meta']
except KeyError:
# malformed message
return {'error': {
'message': 'Request missing required field(s).'}}
except AttributeError:
# no exposed method by that name
return {'error': {
'message': 'No method by the name "%s"' % message['command']}}
# generate response
try:
response = method(**args)
except Exception as exc:
# includes incorrect arguments (TypeError)
LOG.exception(exc) # note the error server side
import traceback
return {'error': {
'message': str(exc), 'traceback': traceback.format_exc()}}
return {'data': response}
async def _wrapper(fcn, scheduler, state, timings=None):
"""Wrapper for all plugin functions.
* Logs the function's execution.
* Times the function.
* Catches any exceptions which aren't subclasses of CylcError.
"""
sig = f'{fcn.__module__}:{fcn.__name__}'
LOG.debug(f'main_loop [run] {sig}')
start_time = time()
try:
await fcn(scheduler, state)
except CylcError as exc:
# allow CylcErrors through (e.g. SchedulerStop)
# NOTE: the `from None` bit gets rid of this gunk:
# > During handling of the above exception another exception
raise MainLoopPluginException(exc) from None
except Exception as exc:
LOG.error(f'Error in main loop plugin {sig}')
LOG.exception(exc)
duration = time() - start_time
LOG.debug(f'main_loop [end] {sig} ({duration:.3f}s)')
if timings is not None:
timings.append((start_time, duration))
def _run_command_init(cls,
ctx,
bad_hosts=None,
callback=None,
callback_args=None,
callback_255=None,
callback_255_args=None):
"""Prepare and launch shell command in ctx."""
try:
if ctx.cmd_kwargs.get('stdin_files'):
if len(ctx.cmd_kwargs['stdin_files']) > 1:
stdin_file = cls.get_temporary_file()
for file_ in ctx.cmd_kwargs['stdin_files']:
if hasattr(file_, 'read'):
stdin_file.write(file_.read())
else:
with open(file_, 'rb') as openfile:
stdin_file.write(openfile.read())
stdin_file.seek(0)
elif hasattr(ctx.cmd_kwargs['stdin_files'][0], 'read'):
stdin_file = ctx.cmd_kwargs['stdin_files'][0]
else:
stdin_file = open( # noqa: SIM115
# (nasty use of file handles, should avoid in future)
ctx.cmd_kwargs['stdin_files'][0],
'rb')
elif ctx.cmd_kwargs.get('stdin_str'):
stdin_file = cls.get_temporary_file()
stdin_file.write(ctx.cmd_kwargs.get('stdin_str').encode())
stdin_file.seek(0)
else:
stdin_file = DEVNULL
proc = procopen(
ctx.cmd,
stdin=stdin_file,
stdoutpipe=True,
stderrpipe=True,
# Execute command as a process group leader,
# so we can use "os.killpg" to kill the whole group.
preexec_fn=os.setpgrp,
env=ctx.cmd_kwargs.get('env'),
usesh=ctx.cmd_kwargs.get('shell'))
# calls to open a shell are aggregated in cylc_subproc.procopen()
# with logging for what is calling it and the commands given
except OSError as exc:
if exc.filename is None:
exc.filename = ctx.cmd[0]
LOG.exception(exc)
ctx.ret_code = 1
ctx.err = str(exc)
cls._run_command_exit(ctx,
bad_hosts=bad_hosts,
callback=callback,
callback_args=callback_args,
callback_255=callback_255,
callback_255_args=callback_255_args)
return None
else:
LOG.debug(ctx.cmd)
return proc
def main(parser, options, suite, event_msg, event_id):
LOG.info('Send to suite %s: "%s" (%s)', suite, event_msg, event_id)
pclient = SuiteRuntimeClient(suite, options.owner, options.host,
options.port)
max_n_tries = int(options.max_n_tries)
retry_intvl_secs = float(options.retry_intvl_secs)
for i_try in range(max_n_tries):
try:
pclient('put_ext_trigger', {
'message': event_msg,
'id': event_id
},
timeout=options.comms_timeout)
except ClientError as exc:
LOG.exception(exc)
LOG.info(MSG_SEND_FAILED, i_try + 1, max_n_tries)
if i_try == max_n_tries - 1: # final attempt
raise CylcError('send failed')
else:
LOG.info(MSG_SEND_RETRY, retry_intvl_secs,
options.comms_timeout)
sleep(retry_intvl_secs)
else:
if i_try > 0:
LOG.info(MSG_SEND_SUCCEED, i_try + 1, max_n_tries)
break
def _receiver(self, message):
"""Wrap incoming messages and dispatch them to exposed methods.
Args:
message (dict): message contents
"""
# determine the server method to call
try:
method = getattr(self, message['command'])
args = message['args']
args.update({'user': message['user']})
if 'meta' in message:
args['meta'] = message['meta']
except KeyError:
# malformed message
return {'error': {
'message': 'Request missing required field(s).'}}
except AttributeError:
# no exposed method by that name
return {'error': {
'message': 'No method by the name "%s"' % message['command']}}
# generate response
try:
response = method(**args)
except Exception as exc:
# includes incorrect arguments (TypeError)
LOG.exception(exc) # note the error server side
import traceback
return {'error': {
'message': str(exc), 'traceback': traceback.format_exc()}}
return {'data': response}
def ensure_keypair_exists(auth_parent_dir, auth_child_dir, tag):
"""Check if a set of public/private keys exist and if not, create them.
Args:
auth_parent_dir (str):
Parent containing public/private key directories (auth_child_dir).
auth_child_dir (str):
Child containing public/private keys.
tag (str): Filename/Basename of key.
Returns:
bool: True if the keypair (now) exists.
"""
public_key_location, private_key_location = return_key_locations(
auth_child_dir) # where the keys should be, else where to create them
if (os.path.exists(public_key_location)
and os.path.exists(private_key_location)):
return True
# Ensure parent dir exists (child dir will be created if it does not)
if not os.path.exists(auth_parent_dir):
os.mkdir(auth_parent_dir)
try:
generate_key_store(auth_parent_dir, tag)
return True
except Exception:
# Catch anything so we can otherwise be sure the key store exists.
LOG.exception("Failed to create %s authentication keys." % tag)
return False
def main(parser: COP, options: 'Values', workflow: str, event_msg: str,
event_id: str) -> None:
workflow, _ = parse_reg(workflow)
LOG.info('Send to workflow %s: "%s" (%s)', workflow, event_msg, event_id)
pclient = get_client(workflow, timeout=options.comms_timeout)
max_n_tries = int(options.max_n_tries)
retry_intvl_secs = float(options.retry_intvl_secs)
mutation_kwargs = {
'request_string': MUTATION,
'variables': {
'wFlows': [workflow],
'eventMsg': event_msg,
'eventId': event_id,
}
}
for i_try in range(max_n_tries):
try:
pclient('graphql', mutation_kwargs)
except ClientError as exc:
LOG.exception(exc)
LOG.info(MSG_SEND_FAILED, i_try + 1, max_n_tries)
if i_try == max_n_tries - 1: # final attempt
raise CylcError('send failed')
LOG.info(MSG_SEND_RETRY, retry_intvl_secs, options.comms_timeout)
sleep(retry_intvl_secs)
else:
if i_try > 0:
LOG.info(MSG_SEND_SUCCEED, i_try + 1, max_n_tries)
break
def create_directory(dir_, name):
"""Create directory. Raise GlobalConfigError on error."""
try:
os.makedirs(dir_, exist_ok=True)
except OSError as exc:
LOG.exception(exc)
raise GlobalConfigError(
'Failed to create directory "' + name + '"')
def insert_db_job(self, row_idx, row):
"""Load job element from DB post restart."""
if row_idx == 0:
LOG.info("LOADING job data")
(point_string, name, status, submit_num, time_submit, time_run,
time_run_exit, batch_sys_name, batch_sys_job_id, platform_name) = row
if status not in JOB_STATUS_SET:
return
t_id = f'{self.workflow_id}{ID_DELIM}{point_string}{ID_DELIM}{name}'
j_id = f'{t_id}{ID_DELIM}{submit_num}'
try:
tdef = self.schd.config.get_taskdef(name)
j_owner = self.schd.owner
if platform_name:
j_host = get_host_from_platform(get_platform(platform_name))
else:
j_host = self.schd.host
j_buf = PbJob(
stamp=f'{j_id}@{time()}',
id=j_id,
submit_num=submit_num,
state=status,
task_proxy=t_id,
submitted_time=time_submit,
started_time=time_run,
finished_time=time_run_exit,
batch_sys_name=batch_sys_name,
batch_sys_job_id=batch_sys_job_id,
host=j_host,
owner=j_owner,
name=name,
cycle_point=point_string,
)
# Add in log files.
j_buf.job_log_dir = get_task_job_log(self.schd.suite, point_string,
name, submit_num)
overrides = self.schd.task_events_mgr.broadcast_mgr.get_broadcast(
TaskID.get(name, point_string))
if overrides:
rtconfig = pdeepcopy(tdef.rtconfig)
poverride(rtconfig, overrides, prepend=True)
else:
rtconfig = tdef.rtconfig
j_buf.extra_logs.extend([
os.path.expanduser(os.path.expandvars(log_file))
for log_file in rtconfig['extra log files']
])
except SuiteConfigError:
LOG.exception(
('ignoring job %s from the suite run database\n'
'(its task definition has probably been deleted).') % j_id)
except Exception:
LOG.exception('could not load job %s' % j_id)
else:
self.added[j_id] = j_buf
self.task_jobs.setdefault(t_id, set()).add(j_id)
self.updates_pending = True
def _manip_task_jobs_callback(self,
ctx,
suite,
itasks,
summary_callback,
more_callbacks=None):
"""Callback when submit/poll/kill tasks command exits."""
if ctx.ret_code:
LOG.error(ctx)
else:
LOG.debug(ctx)
# A dict for easy reference of (CYCLE, NAME, SUBMIT_NUM) -> TaskProxy
#
# Note for "reload": A TaskProxy instance may be replaced on reload, so
# the "itasks" list may not reference the TaskProxy objects that
# replace the old ones. The .reload_successor attribute provides the
# link(s) for us to get to the latest replacement.
#
# Note for "kill": It is possible for a job to trigger its trap and
# report back to the suite back this logic is called. If so, the task
# will no longer be TASK_STATUS_SUBMITTED or TASK_STATUS_RUNNING, and
# its output line will be ignored here.
tasks = {}
for itask in itasks:
while itask.reload_successor is not None:
itask = itask.reload_successor
if itask.point is not None and itask.submit_num:
submit_num = "%02d" % (itask.submit_num)
tasks[(str(itask.point), itask.tdef.name, submit_num)] = itask
handlers = [(self.batch_sys_mgr.OUT_PREFIX_SUMMARY, summary_callback)]
if more_callbacks:
for prefix, callback in more_callbacks.items():
handlers.append((prefix, callback))
out = ctx.out
if not out:
out = ""
bad_tasks = dict(tasks)
for line in out.splitlines(True):
for prefix, callback in handlers:
if line.startswith(prefix):
line = line[len(prefix):].strip()
try:
path = line.split("|", 2)[1] # timestamp, path, status
point, name, submit_num = path.split(os.sep, 2)
if prefix == self.batch_sys_mgr.OUT_PREFIX_SUMMARY:
del bad_tasks[(point, name, submit_num)]
itask = tasks[(point, name, submit_num)]
callback(suite, itask, ctx, line)
except (LookupError, ValueError, KeyError) as exc:
LOG.warning('Unhandled %s output: %s', ctx.cmd_key,
line)
LOG.exception(exc)
# Task jobs that are in the original command but did not get a status
# in the output. Handle as failures.
for key, itask in sorted(bad_tasks.items()):
line = ("|".join([ctx.timestamp, os.sep.join(key), "1"]) + "\n")
summary_callback(suite, itask, ctx, line)
def _manip_task_jobs_callback(
self, ctx, suite, itasks, summary_callback, more_callbacks=None):
"""Callback when submit/poll/kill tasks command exits."""
if ctx.ret_code:
LOG.error(ctx)
else:
LOG.debug(ctx)
# A dict for easy reference of (CYCLE, NAME, SUBMIT_NUM) -> TaskProxy
#
# Note for "reload": A TaskProxy instance may be replaced on reload, so
# the "itasks" list may not reference the TaskProxy objects that
# replace the old ones. The .reload_successor attribute provides the
# link(s) for us to get to the latest replacement.
#
# Note for "kill": It is possible for a job to trigger its trap and
# report back to the suite back this logic is called. If so, the task
# will no longer be TASK_STATUS_SUBMITTED or TASK_STATUS_RUNNING, and
# its output line will be ignored here.
tasks = {}
for itask in itasks:
while itask.reload_successor is not None:
itask = itask.reload_successor
if itask.point is not None and itask.submit_num:
submit_num = "%02d" % (itask.submit_num)
tasks[(str(itask.point), itask.tdef.name, submit_num)] = itask
handlers = [(self.batch_sys_mgr.OUT_PREFIX_SUMMARY, summary_callback)]
if more_callbacks:
for prefix, callback in more_callbacks.items():
handlers.append((prefix, callback))
out = ctx.out
if not out:
out = ""
bad_tasks = dict(tasks)
for line in out.splitlines(True):
for prefix, callback in handlers:
if line.startswith(prefix):
line = line[len(prefix):].strip()
try:
path = line.split("|", 2)[1] # timestamp, path, status
point, name, submit_num = path.split(os.sep, 2)
if prefix == self.batch_sys_mgr.OUT_PREFIX_SUMMARY:
del bad_tasks[(point, name, submit_num)]
itask = tasks[(point, name, submit_num)]
callback(suite, itask, ctx, line)
except (LookupError, ValueError, KeyError) as exc:
LOG.warning(
'Unhandled %s output: %s', ctx.cmd_key, line)
LOG.exception(exc)
# Task jobs that are in the original command but did not get a status
# in the output. Handle as failures.
for key, itask in sorted(bad_tasks.items()):
line = (
"|".join([ctx.timestamp, os.sep.join(key), "1"]) + "\n")
summary_callback(suite, itask, ctx, line)
async def scan_one(reg, host, port, pub_port, api, timeout=None, methods=None):
"""Connect to and identify workflow server if possible.
Args:
reg (str): Registered name of workflow.
host (str): Workflow host.
port (int): Workflow server port.
pub_port (int): Workflow publisher port.
api (str): Workflow API version.
timeout (float, optional): Client socket receiver timeout.
methods (list): List of methods/endpoints to request.
Returns:
tuple: (reg, host, port, pub_port, result)
"""
if not methods:
methods = ['identify']
if is_remote_host(host):
try:
host = get_host_ip_by_name(host) # IP reduces DNS traffic
except socket.error as exc:
if cylc.flow.flags.debug:
raise
sys.stderr.write("ERROR: %s: %s\n" % (exc, host))
return (reg, host, port, pub_port, api, None)
# NOTE: Connect to the suite by host:port, this was the
# SuiteRuntimeClient will not attempt to check the contact file
# which would be unnecessary as we have already done so.
# NOTE: This part of the scan *is* IO blocking.
client = SuiteRuntimeClient(reg, host=host, port=port, timeout=timeout)
result = {}
for method in methods:
# work our way up the chain of identity methods, extract as much
# information as we can before the suite rejects us
try:
msg = await client.async_request(method)
except ClientTimeout as exc:
LOG.exception(
"Timeout: name:%s, host:%s, port:%s", reg, host, port)
return (reg, host, port, pub_port, api, MSG_TIMEOUT)
except ClientError as exc:
LOG.exception("ClientError")
return (reg, host, port, pub_port, api, result or None)
else:
result.update(msg)
return (reg, host, port, pub_port, api, result)
def run_reftest(config, ctx):
"""Run reference test at shutdown."""
reffilename = config.get_ref_log_name()
curfilename = get_workflow_test_log_name(ctx.workflow)
ref = _load_reflog(reffilename)
cur = _load_reflog(curfilename)
if ref == cur:
LOG.info('WORKFLOW REFERENCE TEST PASSED')
else:
exc = WorkflowEventError(
'WORKFLOW REFERENCE TEST FAILED\n'
'triggering is NOT consistent with the reference log:\n%s\n' %
'\n'.join(unified_diff(ref, cur, 'reference', 'this run')))
LOG.exception(exc)
raise exc
def _event_email_callback(self, proc_ctx, schd_ctx):
"""Call back when email notification command exits."""
for id_key in proc_ctx.cmd_kwargs["id_keys"]:
key1, point, name, submit_num = id_key
try:
if proc_ctx.ret_code == 0:
del self.event_timers[id_key]
log_ctx = SubProcContext((key1, submit_num), None)
log_ctx.ret_code = 0
log_task_job_activity(log_ctx, schd_ctx.suite, point, name,
submit_num)
else:
self.event_timers[id_key].unset_waiting()
except KeyError as exc:
LOG.exception(exc)
def _listener(self):
"""The server main loop, listen for and serve requests."""
while True:
# process any commands passed to the listener by its parent process
if self.queue.qsize():
command = self.queue.get()
if command == 'STOP':
break
raise ValueError('Unknown command "%s"' % command)
try:
# wait RECV_TIMEOUT for a message
msg = self.socket.recv_string()
except zmq.error.Again:
# timeout, continue with the loop, this allows the listener
# thread to stop
continue
except zmq.error.ZMQError as exc:
LOG.exception('unexpected error: %s', exc)
continue
# attempt to decode the message, authenticating the user in the
# process
try:
message = decode_(msg)
except Exception as exc: # purposefully catch generic exception
# failed to decode message, possibly resulting from failed
# authentication
LOG.exception('failed to decode message: "%s"', exc)
else:
# success case - serve the request
res = self._receiver(message)
if message['command'] in PB_METHOD_MAP:
response = res['data']
else:
response = encode_(res).encode()
# send back the string to bytes response
self.socket.send(response)
# Note: we are using CurveZMQ to secure the messages (see
# self.curve_auth, self.socket.curve_...key etc.). We have set up
# public-key cryptography on the ZMQ messaging and sockets, so
# there is no need to encrypt messages ourselves before sending.
sleep(0) # yield control to other threads
def _run_command_init(cls, ctx, callback=None, callback_args=None):
"""Prepare and launch shell command in ctx."""
try:
if ctx.cmd_kwargs.get('stdin_files'):
if len(ctx.cmd_kwargs['stdin_files']) > 1:
stdin_file = cls.get_temporary_file()
for file_ in ctx.cmd_kwargs['stdin_files']:
if hasattr(file_, 'read'):
stdin_file.write(file_.read())
else:
stdin_file.write(open(file_, 'rb').read())
stdin_file.seek(0)
elif hasattr(ctx.cmd_kwargs['stdin_files'][0], 'read'):
stdin_file = ctx.cmd_kwargs['stdin_files'][0]
else:
stdin_file = open(
ctx.cmd_kwargs['stdin_files'][0], 'rb')
elif ctx.cmd_kwargs.get('stdin_str'):
stdin_file = cls.get_temporary_file()
stdin_file.write(ctx.cmd_kwargs.get('stdin_str').encode())
stdin_file.seek(0)
else:
stdin_file = open(os.devnull)
proc = procopen(
ctx.cmd, stdin=stdin_file, stdoutpipe=True, stderrpipe=True,
# Execute command as a process group leader,
# so we can use "os.killpg" to kill the whole group.
preexec_fn=os.setpgrp,
env=ctx.cmd_kwargs.get('env'),
usesh=ctx.cmd_kwargs.get('shell'))
# calls to open a shell are aggregated in cylc_subproc.procopen()
# with logging for what is calling it and the commands given
except (IOError, OSError) as exc:
if exc.filename is None:
exc.filename = ctx.cmd[0]
LOG.exception(exc)
ctx.ret_code = 1
ctx.err = str(exc)
cls._run_command_exit(ctx, callback, callback_args)
return None
else:
LOG.debug(ctx.cmd)
return proc
def run_reftest(schd: 'Scheduler') -> None:
"""Run reference test at shutdown."""
reffilename = schd.config.get_ref_log_name()
curfilename = get_workflow_test_log_name(schd.workflow)
ref = _load_reflog(reffilename)
cur = _load_reflog(curfilename)
if not ref:
raise WorkflowEventError("No triggering events in reference log.")
if not cur:
raise WorkflowEventError("No triggering events in test log.")
if ref == cur:
LOG.info('WORKFLOW REFERENCE TEST PASSED')
else:
exc = WorkflowEventError(
'WORKFLOW REFERENCE TEST FAILED\n'
'triggering is NOT consistent with the reference log:\n%s\n'
% '\n'.join(unified_diff(ref, cur, 'reference', 'this run'))
)
LOG.exception(exc)
raise exc
def log_task_job_activity(ctx, suite, point, name, submit_num=None):
"""Log an activity for a task job."""
ctx_str = str(ctx)
if not ctx_str:
return
if isinstance(ctx.cmd_key, tuple): # An event handler
submit_num = ctx.cmd_key[-1]
job_activity_log = get_task_job_activity_log(suite, point, name,
submit_num)
try:
with open(job_activity_log, "ab") as handle:
handle.write((ctx_str + '\n').encode())
except IOError as exc:
# This happens when there is no job directory, e.g. if job host
# selection command causes an submission failure, there will be no job
# directory. In this case, just send the information to the suite log.
LOG.exception(exc)
LOG.info(ctx_str)
if ctx.cmd and ctx.ret_code:
LOG.error(ctx_str)
elif ctx.cmd:
LOG.debug(ctx_str)
def _listener(self):
"""The server main loop, listen for and serve requests."""
while True:
# process any commands passed to the listener by its parent process
if self.queue.qsize():
command = self.queue.get()
if command == 'STOP':
break
else:
raise ValueError('Unknown command "%s"' % command)
try:
# wait RECV_TIMEOUT for a message
msg = self.socket.recv_string()
except zmq.error.Again:
# timeout, continue with the loop, this allows the listener
# thread to stop
continue
# attempt to decode the message, authenticating the user in the
# process
try:
message = self.decode(msg, self.secret())
except Exception as exc: # purposefully catch generic exception
# failed to decode message, possibly resulting from failed
# authentication
LOG.exception(f'failed to decode message: {str(exc)}')
else:
# success case - serve the request
res = self._receiver(message)
if message['command'] in PB_METHOD_MAP:
response = res['data']
else:
response = self.encode(res, self.secret()).encode()
# send back the string to bytes response
self.socket.send(response)
sleep(0) # yield control to other threads
def _job_logs_retrieval_callback(self, proc_ctx, schd_ctx):
"""Call back when log job retrieval completes."""
if proc_ctx.ret_code:
LOG.error(proc_ctx)
else:
LOG.debug(proc_ctx)
for id_key in proc_ctx.cmd_kwargs["id_keys"]:
key1, point, name, submit_num = id_key
try:
# All completed jobs are expected to have a "job.out".
fnames = [JOB_LOG_OUT]
try:
if key1[1] not in 'succeeded':
fnames.append(JOB_LOG_ERR)
except TypeError:
pass
fname_oks = {}
for fname in fnames:
fname_oks[fname] = os.path.exists(
get_task_job_log(schd_ctx.suite, point, name,
submit_num, fname))
# All expected paths must exist to record a good attempt
log_ctx = SubProcContext((key1, submit_num), None)
if all(fname_oks.values()):
log_ctx.ret_code = 0
del self.event_timers[id_key]
else:
log_ctx.ret_code = 1
log_ctx.err = "File(s) not retrieved:"
for fname, exist_ok in sorted(fname_oks.items()):
if not exist_ok:
log_ctx.err += " %s" % fname
self.event_timers[id_key].unset_waiting()
log_task_job_activity(log_ctx, schd_ctx.suite, point, name,
submit_num)
except KeyError as exc:
LOG.exception(exc)
async def _run(parser, options, reg, is_restart, scheduler):
"""Run the workflow and handle exceptions."""
# run cylc run
ret = 0
try:
await scheduler.run()
# stop cylc stop
except SchedulerError:
ret = 1
except KeyboardInterrupt as exc:
try:
await scheduler.shutdown(exc)
except Exception as exc2:
# In case of exceptions in the shutdown method itself.
LOG.exception(exc2)
raise exc2 from None
ret = 2
except Exception:
ret = 3
# kthxbye
finally:
return ret
async def graphql_query(flow, fields, filters=None):
"""Obtain information from a GraphQL request to the flow.
Requires:
* is_active(True)
* contact_info
Args:
flow (dict):
Flow information dictionary, provided by scan through the pipe.
fields (iterable):
Iterable containing the fields to request e.g::
['id', 'name']
One level of nesting is supported e.g::
{'name': None, 'meta': ['title']}
filters (list):
Filter by the data returned from the query.
List in the form ``[(key, ...), value]``, e.g::
# state must be running
[('state',), 'running']
# state must be running or paused
[('state',), ('running', 'paused')]
"""
query = f'query {{ workflows(ids: ["{flow["name"]}"]) {{ {fields} }} }}'
try:
client = WorkflowRuntimeClient(
flow['name'],
# use contact_info data if present for efficiency
host=flow.get('CYLC_WORKFLOW_HOST'),
port=flow.get('CYLC_WORKFLOW_PORT'))
except WorkflowStopped:
LOG.warning(f'Workflow not running: {flow["name"]}')
return False
try:
ret = await client.async_request('graphql', {
'request_string': query,
'variables': {}
})
except ClientTimeout:
LOG.exception(f'Timeout: name: {flow["name"]}, '
f'host: {client.host}, '
f'port: {client.port}')
return False
except ClientError as exc:
LOG.exception(exc)
return False
else:
# stick the result into the flow object
for item in ret:
if 'error' in item:
LOG.exception(item['error']['message'])
return False
for workflow in ret.get('workflows', []):
flow.update(workflow)
# process filters
for field, value in filters or []:
for field_ in field:
value_ = flow[field_]
if isinstance(value, Iterable):
if value_ not in value:
return False
else:
if value_ != value:
return False
return flow
