def rerun(client, revision, roots, siblings, inputs, paths):
"""Recreate files generated by a sequence of ``run`` commands."""
graph = Graph(client)
outputs = graph.build(paths=paths, revision=revision)
# Check or extend siblings of outputs.
outputs = siblings(graph, outputs)
output_paths = {node.path for node in outputs}
# Normalize and check all starting paths.
roots = {graph.normalize_path(root) for root in roots}
assert not roots & output_paths, '--from colides with output paths'
# Generate workflow and check inputs.
# NOTE The workflow creation is done before opening a new file.
workflow = inputs(
client,
graph.ascwl(
input_paths=roots,
output_paths=output_paths,
outputs=outputs,
))
# Don't compute paths if storage is disabled.
if client.has_external_storage:
# Make sure all inputs are pulled from a storage.
paths_ = (
path
for _, path in workflow.iter_input_files(client.workflow_path))
client.pull_paths_from_storage(*paths_)
# Store the generated workflow used for updating paths.
import yaml
output_file = client.workflow_path / '{0}.cwl'.format(uuid.uuid4().hex)
with output_file.open('w') as f:
f.write(
yaml.dump(ascwl(
workflow,
filter=lambda _, x: x is not None,
basedir=client.workflow_path,
),
default_flow_style=False))
# Execute the workflow and relocate all output files.
# FIXME get new output paths for edited tools
# output_paths = {path for _, path in workflow.iter_output_files()}
execute(
client,
output_file,
output_paths=output_paths,
)
def rerun(client, revision, roots, siblings, inputs, paths):
"""Recreate files generated by a sequence of ``run`` commands."""
graph = Graph(client)
outputs = graph.build(paths=paths, revision=revision)
# Check or extend siblings of outputs.
outputs = siblings(graph, outputs)
output_paths = {node.path for node in outputs}
# Normalize and check all starting paths.
roots = {graph.normalize_path(root) for root in roots}
output_paths -= roots
outputs = [o for o in outputs if o.path not in roots]
# Generate workflow and check inputs.
# NOTE The workflow creation is done before opening a new file.
workflow = inputs(
client,
graph.as_workflow(
input_paths=roots,
output_paths=output_paths,
outputs=outputs,
)
)
wf, path = CWLConverter.convert(workflow, client)
# Don't compute paths if storage is disabled.
if client.check_external_storage():
# Make sure all inputs are pulled from a storage.
paths_ = (i.consumes.path for i in workflow.inputs)
client.pull_paths_from_storage(*paths_)
# Execute the workflow and relocate all output files.
# FIXME get new output paths for edited tools
# output_paths = {path for _, path in workflow.iter_output_files()}
execute(
client,
path,
output_paths=output_paths,
)
paths = [o.produces.path for o in workflow.outputs]
client.repo.git.add(*paths)
if client.repo.is_dirty():
commit_msg = ('renku rerun: '
'committing {} newly added files').format(len(paths))
committer = Actor('renku {0}'.format(__version__), version_url)
client.repo.index.commit(
commit_msg,
committer=committer,
skip_hooks=True,
)
workflow_name = '{0}_rerun.yaml'.format(uuid.uuid4().hex)
path = client.workflow_path / workflow_name
workflow.update_id_and_label_from_commit_path(
client, client.repo.head.commit, path
)
with with_reference(path):
run = WorkflowRun.from_run(workflow, client, path)
run.to_yaml()
client.add_to_activity_index(run)