def test_workflow(runner, project):
"""Test workflow command."""
result = runner.invoke(cli.cli, ['run', 'touch', 'data.csv'])
assert 0 == result.exit_code
with open('counted.txt', 'w') as stdout:
with contextlib.redirect_stdout(stdout):
try:
cli.cli.main(
args=('run', 'wc', 'data.csv'),
prog_name=runner.get_default_prog_name(cli.cli),
)
except SystemExit as e:
assert e.code in {None, 0}
result = runner.invoke(
cli.cli,
['workflow', 'create', 'counted.txt', '-o', 'workflow.cwl'],
catch_exceptions=False,
)
assert 0 == result.exit_code
with open('workflow.cwl', 'r') as f:
workflow = Workflow.from_cwl(yaml.safe_load(f))
assert workflow.steps[0].run.startswith('.renku/workflow/')
# Compare default log and log for a specific file.
result_default = runner.invoke(cli.cli, ['log'])
result_arg = runner.invoke(cli.cli, ['log', 'counted.txt'])
assert 0 == result_default.exit_code
assert 0 == result_arg.exit_code
assert result_default.output == result_arg.output
def test_workflow(runner):
"""Test workflow command."""
result = runner.invoke(cli.cli, ['run', 'touch', 'data.csv'])
assert result.exit_code == 0
with open('counted.txt', 'w') as stdout:
with contextlib.redirect_stdout(stdout):
try:
cli.cli.main(
args=('run', 'wc', 'data.csv'),
prog_name=runner.get_default_prog_name(cli.cli),
)
except SystemExit as e:
assert e.code in {None, 0}
result = runner.invoke(
cli.cli, ['workflow', 'create', 'counted.txt', '-o', 'workflow.cwl'])
assert result.exit_code == 0
with open('workflow.cwl', 'r') as f:
workflow = Workflow.from_cwl(yaml.load(f))
assert workflow.steps[0].run.startswith('.renku/workflow/')
def add_tool(self,
commit,
path,
file_key=None,
expand_workflow=True,
is_step=False):
"""Add a tool and its dependencies to the graph."""
data = (commit.tree / path).data_stream.read()
cwl = yaml.load(data)
try:
tool = CommandLineTool.from_cwl(cwl)
except TypeError:
if expand_workflow:
return self.add_workflow(commit,
path,
file_key=file_key,
cwl=cwl)
tool = Workflow.from_cwl(cwl)
tool_key = self.add_node(commit, path, tool=tool)
if is_step:
return tool_key
for input_path, input_id in self.iter_file_inputs(
tool, os.path.dirname(path)):
input_key = self.add_file(input_path,
revision='{0}^'.format(commit))
#: Edge from an input to the tool.
self.G.add_edge(input_key, tool_key, id=input_id)
if file_key:
_, path = file_key
output_id = tool.get_output_id(path)
if output_id:
self.G.add_edge(tool_key, file_key, id=output_id)
return tool_key
def add_tool(self,
commit,
path,
file_key=None,
expand_workflow=True,
is_step=False):
"""Add a tool and its dependencies to the graph."""
data = (commit.tree / path).data_stream.read()
cwl = yaml.load(data)
try:
tool = CommandLineTool.from_cwl(cwl)
except TypeError:
if expand_workflow:
return self.add_workflow(commit,
path,
file_key=file_key,
cwl=cwl)
tool = Workflow.from_cwl(cwl)
tool_key = self.add_node(commit, path, tool=tool)
if is_step:
return tool_key
for input_id, input_path in self.iter_input_files(
tool, os.path.dirname(path)):
input_key = self.add_file(input_path,
revision='{0}^'.format(commit))
#: Edge from an input to the tool.
self.G.add_edge(input_key, tool_key, id=input_id)
# Find ALL siblings that MUST be generated in the same commit.
for output_id, path in self.iter_output_files(tool):
self.G.add_edge(tool_key, (str(commit), path), id=output_id)
return tool_key
def add_workflow(self, commit, path, cwl=None, file_key=None):
"""Add a workflow and its dependencies to the graph."""
if cwl is None:
data = (commit.tree / path).data_stream.read()
cwl = yaml.load(data)
workflow = Workflow.from_cwl(cwl)
basedir = os.path.dirname(path)
# Keep track of node identifiers for steps, inputs and outputs:
step_map = {}
input_map = {}
output_map = {}
#: First find workflow inputs, but don't connect them yet.
for input_id, input_path in self.iter_input_files(workflow, basedir):
input_key = self.add_file(input_path,
revision='{0}^'.format(commit))
input_map[input_id] = input_key
for step in workflow.steps:
tool_key = self.add_tool(
commit,
os.path.join(basedir, step.run),
file_key=file_key,
is_step=True,
)
step_tool = self.G.nodes[tool_key]['tool']
for input_id, input_path in self.iter_input_files(
step_tool, basedir):
if input_path in commit.stats.files:
#: Check intermediate committed files
input_key = self.add_node(commit, input_path)
#: Edge from an input to the tool.
self.G.add_edge(input_key, tool_key, id=input_id)
else:
#: Global workflow input
source = step.in_[input_id]
self.G.add_edge(input_map[source], tool_key, id=input_id)
# Find ALL siblings that MUST be generated in the same commit.
for output_id, output_path in self.iter_output_files(step_tool):
self.G.add_edge(tool_key, (str(commit), output_path),
id=output_id)
output_map.update({
step.id + '/' + name: target
for target, _, name in self.G.in_edges(tool_key, data='id')
})
step_map[step.id] = tool_key
self.G.nodes[tool_key]['workflow'] = workflow
self.G.nodes[tool_key][
'workflow_path'] = path + '#steps/' + step.id
for step in workflow.steps:
for alias, source in step.in_.items():
name = step.id + '/' + alias
if name in output_map and '/' in source:
other_step, id_ = source.split('/')
other_key = step_map[other_step]
self.G.add_edge(other_key, output_map[name], id=id_)
return workflow
