def menu(node: pipelines.Node):
if isinstance(node, pipelines.Pipeline):
code, choice = d.menu(
text='Pipeline ' +
'.'.join(node.path()) if node.parent else 'Root pipeline',
choices=[('▶ ', 'Run'), ('>> ', 'Run selected')] +
[(child.id,
'→' if isinstance(child, pipelines.Pipeline) else 'Run')
for child in node.nodes.values()])
if code == d.CANCEL:
return
if choice == '▶ ':
if not run_pipeline(node):
sys.exit(-1)
elif choice == '>> ':
code, node_ids = d.checklist(
'Select sub-nodes to run. If you want to run all, then select none.',
choices=[(node_id, '', False)
for node_id in node.nodes.keys()])
if code == d.OK:
if not run_pipeline(node,
{node.nodes[id]
for id in node_ids}, False):
sys.exit(-1)
else:
menu(node.nodes[choice])
return
else:
if not run_pipeline(pipeline=node.parent, nodes=[node]):
sys.exit(-1)
def action_buttons(node: pipelines.Node):
"""The action buttons to be displayed on a node page"""
path = node.path()
return [
response.ActionButton(
action=flask.url_for('data_integration.run_page', path='/'.join(path[:-1]),
with_upstreams=True, ids=path[-1]),
label='Run with upstreams', icon='play',
title=f'Run the task and all its upstreams in the pipeline "{node.parent.id}"'),
response.ActionButton(
action=flask.url_for('data_integration.run_page', path='/'.join(path[:-1]),
with_upstreams=False, ids=path[-1]),
label='Run', icon='play',
title=f'Run only this task, without upstreams')]
def compute_cost(node: pipelines.Node, node_durations_and_run_times: {tuple: [float, float]}) -> float:
"""
Computes the cost of a node as maximum cumulative run time of a node and all its downstreams.
Stores the result in `node` and also returns it
Args:
node: The node for which to compute the cost
node_durations_and_run_times: Duration and run time information as computed by
the `node_durations_and_run_times` function
"""
path = tuple(node.path())
if node.cost is None:
node.cost = (
max([compute_cost(downstream, node_durations_and_run_times)
for downstream in node.downstreams] or [0])
+ (node_durations_and_run_times.get(path, [0, 0])[1] or 0))
return node.cost