def cli(ctx, wfile, skip, colors):
"""Creates a graph in the .dot format representing the workflow.
"""
# Args:
# ctx(Popper.cli.context): For process inter-command communication
# context is used.For reference visit
# https://click.palletsprojects.com/en/7.x/commands
# wfile(str): Name of the file containing definition of workflow.
# skip(tuple): List of steps that are to be skipped.
# colors(bool): Flag for colors.
# Returns:
# None
def add_to_graph(dot_str, wf, parent, children, node_attrs, stage_edges):
"""Recursively goes over the children ("next" attribute) of the given
parent, adding an edge from parent to children
Args:
dot_str(str): The intermediate string to which further nodes are
to be added.
wf(popper.parser.workflow): Instance of the workflow class.
parent(str): Step Identifier.
children(list/set): The node that is to be attached as a children.
node_attrs(str): These are the attributes of the node of the graph.
stage_edges(set): Intermediate sets containing the nodes and edges.
Returns:
str: The string containing nodes and their description.
"""
for n in children:
edge = f' "{parent}" -> "{n}";\n'
if edge in stage_edges:
continue
dot_str += edge + f' "{n}" [{node_attrs}];\n'
stage_edges.add(edge)
for M in wf.steps[n].get('next', []):
dot_str = add_to_graph(dot_str, wf, n, [M], node_attrs,
stage_edges)
return dot_str
wf = Workflow.new(wfile)
wf.parse()
wf = Workflow.skip_steps(wf, skip)
wf.check_for_unreachable_steps()
node_attrs = ('shape=box, style="filled{}", fillcolor=transparent{}')
wf_attr = node_attrs.format(',rounded', ',color=red' if colors else '')
act_attr = node_attrs.format('', ',color=cyan' if colors else '')
dot_str = add_to_graph("", wf, wf.name, wf.root, act_attr, set())
dot_str += f' "{wf.name}" [{wf_attr}];\n'
log.info("digraph G { graph [bgcolor=transparent];\n" + dot_str + "}\n")
def test_scaffold(self):
wf_dir = tempfile.mkdtemp()
runner = CliRunner()
file_loc = f'{wf_dir}/wf.yml'
result = runner.invoke(scaffold.cli, ['-f', file_loc])
self.assertEqual(result.exit_code, 0)
self.assertTrue(os.path.isfile(file_loc))
wf = Workflow.new(file_loc)
self.assertDictEqual(
wf.steps, {
'1': {
'uses': 'popperized/bin/sh@master',
'args': ['ls'],
'name': '1',
'next': {'2'}
},
'2': {
'uses': 'docker://alpine:3.11',
'args': ['ls'],
'name': '2',
'needs': ['1']
}
})
with self.assertLogs('popper') as test_logger:
result = runner.invoke(run.cli, ['-f', file_loc])
self.assertEqual(result.exit_code, 0)
self.assertTrue(len(test_logger.output))
self.assertTrue("INFO:popper:Step '1' ran successfully !" in
test_logger.output)
self.assertTrue("INFO:popper:Step '2' ran successfully !" in
test_logger.output)
def cli(ctx, step, wfile, debug, dry_run, log_file, quiet, reuse, engine,
resource_manager, skip, skip_pull, skip_clone, substitution,
allow_loose, with_dependencies, workspace, conf):
"""Runs a Popper workflow. Only executes STEP if given.
To specify a container engine to use other than docker, use the --engine/-e
flag. For executing on a resource manager such as SLURM or Kubernetes, use
the --resource-manager/-r flag. Alternatively, a configuration file can be
given (--conf flag) that can specify container options, resource manager
options, or both (see "Workflow Syntax and Execution Runtime" section of
the Popper documentation for more).
If the container engine (-e) or resource manager (-r) are specified with a
flag and a configuration file is given as well, the values passed via the
flags are given preference over those contained in the configuration file.
"""
# set the logging levels.
level = 'STEP_INFO'
if quiet:
level = 'INFO'
if debug:
level = 'DEBUG'
log.setLevel(level)
if dry_run:
logging.msg_prefix = "DRYRUN: "
if log_file:
# also log to a file
logging.add_log(log, log_file)
# check conflicting flags and fail if needed
if with_dependencies and not step:
log.fail('`--with-dependencies` can only be used when '
'STEP argument is given.')
if skip and step:
log.fail('`--skip` can not be used when STEP argument is passed.')
# invoke wf factory; handles formats, validations, filtering
wf = Workflow.new(wfile,
step=step,
skipped_steps=skip,
substitutions=substitution,
allow_loose=allow_loose,
include_step_dependencies=with_dependencies)
config = PopperConfig(engine_name=engine,
resman_name=resource_manager,
config_file=conf,
reuse=reuse,
dry_run=dry_run,
skip_pull=skip_pull,
skip_clone=skip_clone,
workspace_dir=workspace)
runner = WorkflowRunner(config)
try:
runner.run(wf)
except Exception as e:
log.debug(traceback.format_exc())
log.fail(e)
