def test_workflow(runner, project): """Test workflow command.""" result = runner.invoke(cli.cli, ['run', 'touch', 'data.csv']) assert 0 == result.exit_code with open('counted.txt', 'w') as stdout: with contextlib.redirect_stdout(stdout): try: cli.cli.main( args=('run', 'wc', 'data.csv'), prog_name=runner.get_default_prog_name(cli.cli), ) except SystemExit as e: assert e.code in {None, 0} result = runner.invoke( cli.cli, ['workflow', 'create', 'counted.txt', '-o', 'workflow.cwl'], catch_exceptions=False, ) assert 0 == result.exit_code with open('workflow.cwl', 'r') as f: workflow = Workflow.from_cwl(yaml.safe_load(f)) assert workflow.steps[0].run.startswith('.renku/workflow/') # Compare default log and log for a specific file. result_default = runner.invoke(cli.cli, ['log']) result_arg = runner.invoke(cli.cli, ['log', 'counted.txt']) assert 0 == result_default.exit_code assert 0 == result_arg.exit_code assert result_default.output == result_arg.output
def test_workflow(runner): """Test workflow command.""" result = runner.invoke(cli.cli, ['run', 'touch', 'data.csv']) assert result.exit_code == 0 with open('counted.txt', 'w') as stdout: with contextlib.redirect_stdout(stdout): try: cli.cli.main( args=('run', 'wc', 'data.csv'), prog_name=runner.get_default_prog_name(cli.cli), ) except SystemExit as e: assert e.code in {None, 0} result = runner.invoke( cli.cli, ['workflow', 'create', 'counted.txt', '-o', 'workflow.cwl']) assert result.exit_code == 0 with open('workflow.cwl', 'r') as f: workflow = Workflow.from_cwl(yaml.load(f)) assert workflow.steps[0].run.startswith('.renku/workflow/')
def add_tool(self, commit, path, file_key=None, expand_workflow=True, is_step=False): """Add a tool and its dependencies to the graph.""" data = (commit.tree / path).data_stream.read() cwl = yaml.load(data) try: tool = CommandLineTool.from_cwl(cwl) except TypeError: if expand_workflow: return self.add_workflow(commit, path, file_key=file_key, cwl=cwl) tool = Workflow.from_cwl(cwl) tool_key = self.add_node(commit, path, tool=tool) if is_step: return tool_key for input_path, input_id in self.iter_file_inputs( tool, os.path.dirname(path)): input_key = self.add_file(input_path, revision='{0}^'.format(commit)) #: Edge from an input to the tool. self.G.add_edge(input_key, tool_key, id=input_id) if file_key: _, path = file_key output_id = tool.get_output_id(path) if output_id: self.G.add_edge(tool_key, file_key, id=output_id) return tool_key
def add_tool(self, commit, path, file_key=None, expand_workflow=True, is_step=False): """Add a tool and its dependencies to the graph.""" data = (commit.tree / path).data_stream.read() cwl = yaml.load(data) try: tool = CommandLineTool.from_cwl(cwl) except TypeError: if expand_workflow: return self.add_workflow(commit, path, file_key=file_key, cwl=cwl) tool = Workflow.from_cwl(cwl) tool_key = self.add_node(commit, path, tool=tool) if is_step: return tool_key for input_id, input_path in self.iter_input_files( tool, os.path.dirname(path)): input_key = self.add_file(input_path, revision='{0}^'.format(commit)) #: Edge from an input to the tool. self.G.add_edge(input_key, tool_key, id=input_id) # Find ALL siblings that MUST be generated in the same commit. for output_id, path in self.iter_output_files(tool): self.G.add_edge(tool_key, (str(commit), path), id=output_id) return tool_key
def add_workflow(self, commit, path, cwl=None, file_key=None): """Add a workflow and its dependencies to the graph.""" if cwl is None: data = (commit.tree / path).data_stream.read() cwl = yaml.load(data) workflow = Workflow.from_cwl(cwl) basedir = os.path.dirname(path) # Keep track of node identifiers for steps, inputs and outputs: step_map = {} input_map = {} output_map = {} #: First find workflow inputs, but don't connect them yet. for input_id, input_path in self.iter_input_files(workflow, basedir): input_key = self.add_file(input_path, revision='{0}^'.format(commit)) input_map[input_id] = input_key for step in workflow.steps: tool_key = self.add_tool( commit, os.path.join(basedir, step.run), file_key=file_key, is_step=True, ) step_tool = self.G.nodes[tool_key]['tool'] for input_id, input_path in self.iter_input_files( step_tool, basedir): if input_path in commit.stats.files: #: Check intermediate committed files input_key = self.add_node(commit, input_path) #: Edge from an input to the tool. self.G.add_edge(input_key, tool_key, id=input_id) else: #: Global workflow input source = step.in_[input_id] self.G.add_edge(input_map[source], tool_key, id=input_id) # Find ALL siblings that MUST be generated in the same commit. for output_id, output_path in self.iter_output_files(step_tool): self.G.add_edge(tool_key, (str(commit), output_path), id=output_id) output_map.update({ step.id + '/' + name: target for target, _, name in self.G.in_edges(tool_key, data='id') }) step_map[step.id] = tool_key self.G.nodes[tool_key]['workflow'] = workflow self.G.nodes[tool_key][ 'workflow_path'] = path + '#steps/' + step.id for step in workflow.steps: for alias, source in step.in_.items(): name = step.id + '/' + alias if name in output_map and '/' in source: other_step, id_ = source.split('/') other_key = step_map[other_step] self.G.add_edge(other_key, output_map[name], id=id_) return workflow