def card(node: pipelines.Node) -> str:
"""A card that shows the system stats, the time line and output for the last runs or a node"""
return bootstrap.card(
id='last-runs-card',
header_left=[
'Last runs ',
_.div(style='display:inline-block;margin-left:20px;')[
html.asynchronous_content(
flask.url_for('data_integration.last_runs_selector',
path=node.url_path()))]
],
body=[
html.spinner_js_function(),
html.asynchronous_content(url=flask.url_for(
'data_integration.system_stats',
path=node.url_path(),
run_id=None),
div_id='system-stats'),
html.asynchronous_content(url=flask.url_for(
'data_integration.timeline_chart',
path=node.url_path(),
run_id=None),
div_id='timeline-chart'),
html.asynchronous_content(url=flask.url_for(
'data_integration.run_output',
path=node.url_path(),
run_id=None,
limit=True),
div_id='run-output')
])
def menu(node: pipelines.Node):
if isinstance(node, pipelines.Pipeline):
code, choice = d.menu(
text='Pipeline ' +
'.'.join(node.path()) if node.parent else 'Root pipeline',
choices=[('▶ ', 'Run'), ('>> ', 'Run selected')] +
[(child.id,
'→' if isinstance(child, pipelines.Pipeline) else 'Run')
for child in node.nodes.values()])
if code == d.CANCEL:
return
if choice == '▶ ':
if not run_pipeline(node):
sys.exit(-1)
elif choice == '>> ':
code, node_ids = d.checklist(
'Select sub-nodes to run. If you want to run all, then select none.',
choices=[(node_id, '', False)
for node_id in node.nodes.keys()])
if code == d.OK:
if not run_pipeline(node,
{node.nodes[id]
for id in node_ids}, False):
sys.exit(-1)
else:
menu(node.nodes[choice])
return
else:
if not run_pipeline(pipeline=node.parent, nodes=[node]):
sys.exit(-1)
def compute_cost(node: pipelines.Node, node_durations_and_run_times: {tuple: [float, float]}) -> float:
"""
Computes the cost of a node as maximum cumulative run time of a node and all its downstreams.
Stores the result in `node` and also returns it
Args:
node: The node for which to compute the cost
node_durations_and_run_times: Duration and run time information as computed by
the `node_durations_and_run_times` function
"""
path = tuple(node.path())
if node.cost is None:
node.cost = (
max([compute_cost(downstream, node_durations_and_run_times)
for downstream in node.downstreams] or [0])
+ (node_durations_and_run_times.get(path, [0, 0])[1] or 0))
return node.cost
def action_buttons(node: pipelines.Node):
"""The action buttons to be displayed on a node page"""
path = node.path()
return [
response.ActionButton(
action=flask.url_for('data_integration.run_page', path='/'.join(path[:-1]),
with_upstreams=True, ids=path[-1]),
label='Run with upstreams', icon='play',
title=f'Run the task and all its upstreams in the pipeline "{node.parent.id}"'),
response.ActionButton(
action=flask.url_for('data_integration.run_page', path='/'.join(path[:-1]),
with_upstreams=False, ids=path[-1]),
label='Run', icon='play',
title=f'Run only this task, without upstreams')]
def node_url(node: pipelines.Node) -> str:
"""The url of the page that documents a node"""
return flask.url_for('data_integration.node_page', path=node.url_path())
def card(node: pipelines.Node):
"""A card that shows the duration of the node and its top children over time"""
return bootstrap.card(header_left='Run times',
body=html.asynchronous_content(
flask.url_for('data_integration.run_time_chart',
path=node.url_path())))