def menu(node: pipelines.Node): if isinstance(node, pipelines.Pipeline): code, choice = d.menu( text='Pipeline ' + '.'.join(node.path()) if node.parent else 'Root pipeline', choices=[('▶ ', 'Run'), ('>> ', 'Run selected')] + [(child.id, '→' if isinstance(child, pipelines.Pipeline) else 'Run') for child in node.nodes.values()]) if code == d.CANCEL: return if choice == '▶ ': if not run_pipeline(node): sys.exit(-1) elif choice == '>> ': code, node_ids = d.checklist( 'Select sub-nodes to run. If you want to run all, then select none.', choices=[(node_id, '', False) for node_id in node.nodes.keys()]) if code == d.OK: if not run_pipeline(node, {node.nodes[id] for id in node_ids}, False): sys.exit(-1) else: menu(node.nodes[choice]) return else: if not run_pipeline(pipeline=node.parent, nodes=[node]): sys.exit(-1)
def action_buttons(node: pipelines.Node): """The action buttons to be displayed on a node page""" path = node.path() return [ response.ActionButton( action=flask.url_for('data_integration.run_page', path='/'.join(path[:-1]), with_upstreams=True, ids=path[-1]), label='Run with upstreams', icon='play', title=f'Run the task and all its upstreams in the pipeline "{node.parent.id}"'), response.ActionButton( action=flask.url_for('data_integration.run_page', path='/'.join(path[:-1]), with_upstreams=False, ids=path[-1]), label='Run', icon='play', title=f'Run only this task, without upstreams')]
def compute_cost(node: pipelines.Node, node_durations_and_run_times: {tuple: [float, float]}) -> float: """ Computes the cost of a node as maximum cumulative run time of a node and all its downstreams. Stores the result in `node` and also returns it Args: node: The node for which to compute the cost node_durations_and_run_times: Duration and run time information as computed by the `node_durations_and_run_times` function """ path = tuple(node.path()) if node.cost is None: node.cost = ( max([compute_cost(downstream, node_durations_and_run_times) for downstream in node.downstreams] or [0]) + (node_durations_and_run_times.get(path, [0, 0])[1] or 0)) return node.cost