def update(client, revision, no_output, siblings, paths): """Update existing files by rerunning their outdated workflow.""" graph = Graph(client) outputs = graph.build(revision=revision, can_be_cwl=no_output, paths=paths) outputs = {node for node in outputs if graph.need_update(node)} if not outputs: click.secho("All files were generated from the latest inputs.", fg="green") sys.exit(0) # Check or extend siblings of outputs. outputs = siblings(graph, outputs) output_paths = {node.path for node in outputs if _safe_path(node.path)} # Get all clean nodes. input_paths = {node.path for node in graph.nodes} - output_paths # Store the generated workflow used for updating paths. workflow = graph.as_workflow(input_paths=input_paths, output_paths=output_paths, outputs=outputs,) wf, path = CWLConverter.convert(workflow, client) # Don't compute paths if storage is disabled. if client.check_external_storage(): # Make sure all inputs are pulled from a storage. paths_ = (i.consumes.path for i in workflow.inputs) client.pull_paths_from_storage(*paths_) execute(client, path, output_paths=output_paths) paths = [o.produces.path for o in workflow.outputs] client.repo.git.add(*paths) if client.repo.is_dirty(): commit_msg = "renku update: committing {} newly added files".format(len(paths)) committer = Actor("renku {0}".format(__version__), version_url) client.repo.index.commit( commit_msg, committer=committer, skip_hooks=True, ) workflow_name = "{0}_update.yaml".format(uuid.uuid4().hex) path = client.workflow_path / workflow_name workflow.update_id_and_label_from_commit_path(client, client.repo.head.commit, path) with with_reference(path): cls = WorkflowRun if workflow.subprocesses else ProcessRun run = cls.from_run(run=workflow, client=client, path=path, update_commits=True) run.to_yaml() client.add_to_activity_index(run)
def update(client, revision, no_output, siblings, paths): """Update existing files by rerunning their outdated workflow.""" graph = Graph(client) outputs = graph.build(revision=revision, can_be_cwl=no_output, paths=paths) outputs = {node for node in outputs if graph.need_update(node)} if not outputs: click.secho('All files were generated from the latest inputs.', fg='green') sys.exit(0) # Check or extend siblings of outputs. outputs = siblings(graph, outputs) output_paths = {node.path for node in outputs if _safe_path(node.path)} # Get all clean nodes. input_paths = {node.path for node in graph.nodes} - output_paths # Store the generated workflow used for updating paths. import yaml output_file = client.workflow_path / '{0}.cwl'.format(uuid.uuid4().hex) workflow = graph.ascwl( input_paths=input_paths, output_paths=output_paths, outputs=outputs, ) # Don't compute paths if storage is disabled. if client.has_external_storage: # Make sure all inputs are pulled from a storage. paths_ = ( path for _, path in workflow.iter_input_files(client.workflow_path)) client.pull_paths_from_storage(*paths_) with output_file.open('w') as f: f.write( yaml.dump(ascwl( workflow, filter=lambda _, x: x is not None, basedir=client.workflow_path, ), default_flow_style=False)) execute(client, output_file, output_paths=output_paths)