def update(client, revision, no_output, siblings, paths): """Update existing files by rerunning their outdated workflow.""" graph = Graph(client) outputs = graph.build(revision=revision, can_be_cwl=no_output, paths=paths) outputs = {node for node in outputs if graph.need_update(node)} if not outputs: click.secho("All files were generated from the latest inputs.", fg="green") sys.exit(0) # Check or extend siblings of outputs. outputs = siblings(graph, outputs) output_paths = {node.path for node in outputs if _safe_path(node.path)} # Get all clean nodes. input_paths = {node.path for node in graph.nodes} - output_paths # Store the generated workflow used for updating paths. workflow = graph.as_workflow(input_paths=input_paths, output_paths=output_paths, outputs=outputs,) wf, path = CWLConverter.convert(workflow, client) # Don't compute paths if storage is disabled. if client.check_external_storage(): # Make sure all inputs are pulled from a storage. paths_ = (i.consumes.path for i in workflow.inputs) client.pull_paths_from_storage(*paths_) execute(client, path, output_paths=output_paths) paths = [o.produces.path for o in workflow.outputs] client.repo.git.add(*paths) if client.repo.is_dirty(): commit_msg = "renku update: committing {} newly added files".format(len(paths)) committer = Actor("renku {0}".format(__version__), version_url) client.repo.index.commit( commit_msg, committer=committer, skip_hooks=True, ) workflow_name = "{0}_update.yaml".format(uuid.uuid4().hex) path = client.workflow_path / workflow_name workflow.update_id_and_label_from_commit_path(client, client.repo.head.commit, path) with with_reference(path): cls = WorkflowRun if workflow.subprocesses else ProcessRun run = cls.from_run(run=workflow, client=client, path=path, update_commits=True) run.to_yaml() client.add_to_activity_index(run)
def create(client, output_file, revision, paths): """Create a workflow description for a file.""" graph = Graph(client) outputs = graph.build(paths=paths, revision=revision) workflow = graph.as_workflow(outputs=outputs, ) if output_file: output_file = Path(output_file) wf, path = CWLConverter.convert(workflow, client, path=output_file) if not output_file: click.echo(wf.export_string())
def rerun(client, revision, roots, siblings, inputs, paths): """Recreate files generated by a sequence of ``run`` commands.""" graph = Graph(client) outputs = graph.build(paths=paths, revision=revision) # Check or extend siblings of outputs. outputs = siblings(graph, outputs) output_paths = {node.path for node in outputs} # Normalize and check all starting paths. roots = {graph.normalize_path(root) for root in roots} output_paths -= roots outputs = [o for o in outputs if o.path not in roots] # Generate workflow and check inputs. # NOTE The workflow creation is done before opening a new file. workflow = inputs( client, graph.as_workflow( input_paths=roots, output_paths=output_paths, outputs=outputs, ) ) wf, path = CWLConverter.convert(workflow, client) # Don't compute paths if storage is disabled. if client.check_external_storage(): # Make sure all inputs are pulled from a storage. paths_ = (i.consumes.path for i in workflow.inputs) client.pull_paths_from_storage(*paths_) # Execute the workflow and relocate all output files. # FIXME get new output paths for edited tools # output_paths = {path for _, path in workflow.iter_output_files()} execute( client, path, output_paths=output_paths, ) paths = [o.produces.path for o in workflow.outputs] client.repo.git.add(*paths) if client.repo.is_dirty(): commit_msg = ('renku rerun: ' 'committing {} newly added files').format(len(paths)) committer = Actor('renku {0}'.format(__version__), version_url) client.repo.index.commit( commit_msg, committer=committer, skip_hooks=True, ) workflow_name = '{0}_rerun.yaml'.format(uuid.uuid4().hex) path = client.workflow_path / workflow_name workflow.update_id_and_label_from_commit_path( client, client.repo.head.commit, path ) with with_reference(path): run = WorkflowRun.from_run(workflow, client, path) run.to_yaml() client.add_to_activity_index(run)