def test_existing_output_directory(client, runner, project):
"""Test creation of InitialWorkDirRequirement for output."""
from renku.core.models.workflow.converters.cwl import CWLConverter
client.path = client.path
output = client.path / 'output'
argv = ['script', 'output']
factory = CommandLineToolFactory(
argv,
directory=client.path,
working_dir=client.path,
)
with factory.watch(client, no_output=True) as tool:
# Script creates the directory.
output.mkdir(parents=True)
run = factory.generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
cwl, _ = CWLConverter.convert(run.association.plan, client)
assert 0 == len([r for r in cwl.requirements if hasattr(r, 'listing')])
output.mkdir(parents=True, exist_ok=True)
with factory.watch(client) as tool:
# The directory already exists.
(output / 'result.txt').touch()
assert 1 == len(tool.inputs)
run = tool.generate_process_run(client=client,
commit=client.repo.head.commit,
path='dummy.yaml')
cwl, _ = CWLConverter.convert(run.association.plan, client)
reqs = [r for r in cwl.requirements if hasattr(r, 'listing')]
assert 1 == len(reqs)
assert output.name == reqs[0].listing[0].entryname
assert 1 == len(tool.outputs)
def update(client, revision, no_output, siblings, paths):
"""Update existing files by rerunning their outdated workflow."""
graph = Graph(client)
outputs = graph.build(revision=revision, can_be_cwl=no_output, paths=paths)
outputs = {node for node in outputs if graph.need_update(node)}
if not outputs:
click.secho("All files were generated from the latest inputs.", fg="green")
sys.exit(0)
# Check or extend siblings of outputs.
outputs = siblings(graph, outputs)
output_paths = {node.path for node in outputs if _safe_path(node.path)}
# Get all clean nodes.
input_paths = {node.path for node in graph.nodes} - output_paths
# Store the generated workflow used for updating paths.
workflow = graph.as_workflow(input_paths=input_paths, output_paths=output_paths, outputs=outputs,)
wf, path = CWLConverter.convert(workflow, client)
# Don't compute paths if storage is disabled.
if client.check_external_storage():
# Make sure all inputs are pulled from a storage.
paths_ = (i.consumes.path for i in workflow.inputs)
client.pull_paths_from_storage(*paths_)
execute(client, path, output_paths=output_paths)
paths = [o.produces.path for o in workflow.outputs]
client.repo.git.add(*paths)
if client.repo.is_dirty():
commit_msg = "renku update: committing {} newly added files".format(len(paths))
committer = Actor("renku {0}".format(__version__), version_url)
client.repo.index.commit(
commit_msg, committer=committer, skip_hooks=True,
)
workflow_name = "{0}_update.yaml".format(uuid.uuid4().hex)
path = client.workflow_path / workflow_name
workflow.update_id_and_label_from_commit_path(client, client.repo.head.commit, path)
with with_reference(path):
cls = WorkflowRun if workflow.subprocesses else ProcessRun
run = cls.from_run(run=workflow, client=client, path=path, update_commits=True)
run.to_yaml()
client.add_to_activity_index(run)
def create(client, output_file, revision, paths):
"""Create a workflow description for a file."""
graph = Graph(client)
outputs = graph.build(paths=paths, revision=revision)
workflow = graph.as_workflow(outputs=outputs, )
if output_file:
output_file = Path(output_file)
wf, path = CWLConverter.convert(workflow, client, path=output_file)
if not output_file:
click.echo(wf.export_string())
def rerun(client, revision, roots, siblings, inputs, paths):
"""Recreate files generated by a sequence of ``run`` commands."""
graph = Graph(client)
outputs = graph.build(paths=paths, revision=revision)
# Check or extend siblings of outputs.
outputs = siblings(graph, outputs)
output_paths = {node.path for node in outputs}
# Normalize and check all starting paths.
roots = {graph.normalize_path(root) for root in roots}
output_paths -= roots
outputs = [o for o in outputs if o.path not in roots]
# Generate workflow and check inputs.
# NOTE The workflow creation is done before opening a new file.
workflow = inputs(
client,
graph.as_workflow(
input_paths=roots,
output_paths=output_paths,
outputs=outputs,
)
)
wf, path = CWLConverter.convert(workflow, client)
# Don't compute paths if storage is disabled.
if client.check_external_storage():
# Make sure all inputs are pulled from a storage.
paths_ = (i.consumes.path for i in workflow.inputs)
client.pull_paths_from_storage(*paths_)
# Execute the workflow and relocate all output files.
# FIXME get new output paths for edited tools
# output_paths = {path for _, path in workflow.iter_output_files()}
execute(
client,
path,
output_paths=output_paths,
)
paths = [o.produces.path for o in workflow.outputs]
client.repo.git.add(*paths)
if client.repo.is_dirty():
commit_msg = ('renku rerun: '
'committing {} newly added files').format(len(paths))
committer = Actor('renku {0}'.format(__version__), version_url)
client.repo.index.commit(
commit_msg,
committer=committer,
skip_hooks=True,
)
workflow_name = '{0}_rerun.yaml'.format(uuid.uuid4().hex)
path = client.workflow_path / workflow_name
workflow.update_id_and_label_from_commit_path(
client, client.repo.head.commit, path
)
with with_reference(path):
run = WorkflowRun.from_run(workflow, client, path)
run.to_yaml()
client.add_to_activity_index(run)