def create(client, output_file, revision, paths):
"""Create a workflow description for a file."""
graph = Graph(client)
outputs = graph.build(paths=paths, revision=revision)
output_file.write(
yaml.dump(ascwl(
graph.ascwl(outputs=outputs),
filter=lambda _, x: x is not None and x != [],
basedir=os.path.dirname(getattr(output_file, 'name', '.')) or '.',
),
default_flow_style=False))
def rerun(client, revision, roots, siblings, inputs, paths):
"""Recreate files generated by a sequence of ``run`` commands."""
graph = Graph(client)
outputs = graph.build(paths=paths, revision=revision)
# Check or extend siblings of outputs.
outputs = siblings(graph, outputs)
output_paths = {node.path for node in outputs}
# Normalize and check all starting paths.
roots = {graph.normalize_path(root) for root in roots}
assert not roots & output_paths, '--from colides with output paths'
# Generate workflow and check inputs.
# NOTE The workflow creation is done before opening a new file.
workflow = inputs(
client,
graph.ascwl(
input_paths=roots,
output_paths=output_paths,
outputs=outputs,
))
# Don't compute paths if storage is disabled.
if client.has_external_storage:
# Make sure all inputs are pulled from a storage.
paths_ = (
path
for _, path in workflow.iter_input_files(client.workflow_path))
client.pull_paths_from_storage(*paths_)
# Store the generated workflow used for updating paths.
import yaml
output_file = client.workflow_path / '{0}.cwl'.format(uuid.uuid4().hex)
with output_file.open('w') as f:
f.write(
yaml.dump(ascwl(
workflow,
filter=lambda _, x: x is not None,
basedir=client.workflow_path,
),
default_flow_style=False))
# Execute the workflow and relocate all output files.
# FIXME get new output paths for edited tools
# output_paths = {path for _, path in workflow.iter_output_files()}
execute(
client,
output_file,
output_paths=output_paths,
)
def update(client, revision, no_output, siblings, paths):
"""Update existing files by rerunning their outdated workflow."""
graph = Graph(client)
outputs = graph.build(revision=revision, can_be_cwl=no_output, paths=paths)
outputs = {node for node in outputs if graph.need_update(node)}
if not outputs:
click.secho('All files were generated from the latest inputs.',
fg='green')
sys.exit(0)
# Check or extend siblings of outputs.
outputs = siblings(graph, outputs)
output_paths = {node.path for node in outputs if _safe_path(node.path)}
# Get all clean nodes.
input_paths = {node.path for node in graph.nodes} - output_paths
# Store the generated workflow used for updating paths.
import yaml
output_file = client.workflow_path / '{0}.cwl'.format(uuid.uuid4().hex)
workflow = graph.ascwl(
input_paths=input_paths,
output_paths=output_paths,
outputs=outputs,
)
# Don't compute paths if storage is disabled.
if client.has_external_storage:
# Make sure all inputs are pulled from a storage.
paths_ = (
path
for _, path in workflow.iter_input_files(client.workflow_path))
client.pull_paths_from_storage(*paths_)
with output_file.open('w') as f:
f.write(
yaml.dump(ascwl(
workflow,
filter=lambda _, x: x is not None,
basedir=client.workflow_path,
),
default_flow_style=False))
execute(client, output_file, output_paths=output_paths)