def __init__(self, **kw):
super(HostRunner, self).__init__(**kw)
self._spawned_pids = set()
if self._config.reuse:
log.warning('Reuse not supported for HostRunner.')
def stop_running_tasks(self):
"""Stop containers started by Popper."""
for c in self._spawned_containers:
log.info(f"Stopping container {c}")
_, ecode, _ = HostRunner._exec_cmd(["podman", "stop", c], logging=False)
if ecode != 0:
log.warning(f"Failed to stop the {c} container")
def check_for_unreachable_actions(self, skip=None):
"""Validates a workflow by checking for unreachable nodes / gaps
in the workflow.
Args:
skip (list) : The list actions to skip if applicable.
"""
def _traverse(entrypoint, reachable, actions):
for node in entrypoint:
reachable.add(node)
_traverse(actions[node].get(
'next', []), reachable, actions)
reachable = set()
skipped = set(self.props.get('skip_list', []))
actions = set(map(lambda a: a[0], self.action.items()))
_traverse(self.root, reachable, self.action)
unreachable = actions - reachable
if unreachable - skipped:
if skip:
log.fail('Actions {} are unreachable.'.format(
', '.join(unreachable - skipped))
)
else:
log.warning('Actions {} are unreachable.'.format(
', '.join(unreachable))
)
for a in unreachable:
self.action.pop(a)
def _stop_out_stream(self):
_out_stream_pid = list(self._out_stream_pid)[0]
try:
os.kill(_out_stream_pid, 0)
os.kill(_out_stream_pid, signal.SIGKILL)
except ProcessLookupError:
log.warning('Tail process was killed by some other process.')
self._out_stream_thread.join()
def _stop_out_stream(self):
if len(self._out_stream_pid) != 1:
log.fail("Cannot find PID for tail process")
_out_stream_pid = list(self._out_stream_pid)[0]
try:
os.kill(_out_stream_pid, signal.SIGKILL)
except ProcessLookupError:
log.warning("Tail process was stopped by some other process.")
self._out_stream_thread.join()
def run_workflow(**kwargs):
kwargs['wfile'] = pu.find_default_wfile(kwargs['wfile'])
log.info('Found and running workflow at ' + kwargs['wfile'])
# Initialize a Worklow. During initialization all the validation
# takes place automatically.
wf = Workflow(kwargs['wfile'])
wf_runner = WorkflowRunner(wf)
# Check for injected actions
pre_wfile = os.environ.get('POPPER_PRE_WORKFLOW_PATH')
post_wfile = os.environ.get('POPPER_POST_WORKFLOW_PATH')
# Saving workflow instance for signal handling
popper.cli.interrupt_params['parallel'] = kwargs['parallel']
if kwargs['parallel']:
if sys.version_info[0] < 3:
log.fail('--parallel is only supported on Python3')
log.warning("Using --parallel may result in interleaved output. "
"You may use --quiet flag to avoid confusion.")
if kwargs['with_dependencies'] and (not kwargs['action']):
log.fail('`--with-dependencies` can be used only with '
'action argument.')
if kwargs['skip'] and kwargs['action']:
log.fail('`--skip` can\'t be used when action argument ' 'is passed.')
on_failure = kwargs.pop('on_failure')
wfile = kwargs.pop('wfile')
try:
if pre_wfile:
pre_wf = Workflow(pre_wfile)
pre_wf_runner = WorkflowRunner(pre_wf)
pre_wf_runner.run(**kwargs)
wf_runner.run(**kwargs)
if post_wfile:
post_wf = Workflow(post_wfile)
pre_wf_runner = WorkflowRunner(post_wf)
pre_wf_runner.run(**kwargs)
except SystemExit as e:
if (e.code != 0) and on_failure:
kwargs['skip'] = list()
kwargs['action'] = on_failure
wf_runner.run(**kwargs)
else:
raise
if kwargs['action']:
log.info('Action "{}" finished successfully.'.format(kwargs['action']))
else:
log.info('Workflow "{}" finished successfully.'.format(wfile))
def check_for_unreachable_steps(self, skip=None):
"""Validates a workflow by checking for unreachable nodes / gaps in the
workflow.
Args:
skip(list, optional): The list of steps to skip if applicable.
(Default value = None)
Returns:
None
"""
if not skip or self.wf_fmt == 'yml':
# noop
return
def _traverse(entrypoint, reachable, steps):
"""
Args:
entrypoint(set): Set containing the entry point of part of the
workflow.
reachable(set): Set containing all the reachable parts of
workflow.
steps(dict): Dictionary containing the identifier of the
workflow and its description.
Returns:
None
"""
for node in entrypoint:
reachable.add(node)
_traverse(steps[node].get('next', []), reachable, steps)
reachable = set()
skipped = set(self.props.get('skip_list', []))
steps = set(map(lambda a: a[0], self.steps.items()))
_traverse(self.root, reachable, self.steps)
unreachable = steps - reachable
if unreachable - skipped:
if skip:
log.fail(
f'Unreachable step(s): {", ".join(unreachable-skipped)}.')
else:
log.warning(f'Unreachable step(s): {", ".join(unreachable)}.')
for a in unreachable:
self.steps.pop(a)
def cli(ctx, action, wfile, skip_clone, skip_pull, skip, workspace, reuse,
recursive, quiet, debug, dry_run, parallel, log_file,
with_dependencies, on_failure):
"""Executes one or more pipelines and reports on their status.
"""
popper.scm.get_git_root_folder()
level = 'ACTION_INFO'
if quiet:
level = 'INFO'
if debug:
level = 'DEBUG'
log.setLevel(level)
if log_file:
logging.add_log(log, log_file)
if os.environ.get('CI') == 'true':
log.info("Running in CI environment.")
if recursive:
log.warning('When CI variable is set, --recursive is ignored.')
wfile_list = pu.find_recursive_wfile()
wfile_list = workflows_from_commit_message(wfile_list)
else:
if recursive:
if action:
log.fail(
"An 'action' argument and the --recursive flag cannot be "
"both given.")
wfile_list = pu.find_recursive_wfile()
else:
wfile_list = [wfile]
if not wfile_list:
log.fail("No workflow to execute.")
for wfile in wfile_list:
wfile = pu.find_default_wfile(wfile)
log.info("Found and running workflow at " + wfile)
run_pipeline(action, wfile, skip_clone, skip_pull, skip, workspace,
reuse, dry_run, parallel, with_dependencies, on_failure)
def stop_running_tasks(self):
for job_name in self._spawned_jobs:
log.info(f"Cancelling job {job_name}")
_, ecode, _ = HostRunner._exec_cmd(["scancel", "--name", job_name])
if ecode != 0:
log.warning(f"Failed to cancel the job {job_name}.")
def stop_running_tasks(self):
for job_name in self._spawned_jobs:
log.info(f'Cancelling job {job_name}')
_, ecode, _ = HostRunner._exec_cmd(['scancel', '--name', job_name])
if ecode != 0:
log.warning(f'Failed to cancel the job {job_name}.')
