Esempio n. 1
0
def node_page(path: str):
    """Creates a node visualization page including title, action buttons, etc."""
    node, found = pipelines.find_node(path.split('/'))
    if not found and node:
        return flask.redirect(views.node_url(node), 302)
    elif not node:
        flask.abort(404, f'Node "{path}" not found')

    title = [node.__class__.__name__, ' ',
             [[_.a(href=views.node_url(parent))[parent.id], ' / '] for parent in node.parents()[1:-1]],
             node.id] if node.parent else 'Data Integration'
    return response.Response(
        title=title,
        action_buttons=action_buttons(node) if config.allow_run_from_web_ui() else [],
        html=[_.script['''
var nodePage = null;
document.addEventListener('DOMContentLoaded', function() {
     nodePage = NodePage("''' + flask.url_for('data_integration.node_page', path='') + '''");
});'''],
              dependency_graph.card(node),
              run_time_chart.card(node),
              node_content(node),
              last_runs.card(node)],
        js_files=['https://www.gstatic.com/charts/loader.js',
                  flask.url_for('data_integration.static', filename='node-page.js'),
                  flask.url_for('data_integration.static', filename='utils.js'),
                  flask.url_for('data_integration.static', filename='run-time-chart.js'),
                  flask.url_for('data_integration.static', filename='system-stats-chart.js'),
                  flask.url_for('data_integration.static', filename='timeline-chart.js'),
                  flask.url_for('data_integration.static', filename='kolorwheel.js')],
        css_files=[flask.url_for('data_integration.static', filename='common.css'),
                   flask.url_for('data_integration.static', filename='node-page.css'),
                   flask.url_for('data_integration.static', filename='timeline-chart.css')])
Esempio n. 2
0
def pipeline_children_table(path: str):
    """Creates a table that documents all child nodes of a table"""
    pipeline, __ = pipelines.find_node(path.split('/'))
    assert (isinstance(pipeline, pipelines.Pipeline))

    node_durations_and_run_times = node_cost.node_durations_and_run_times(pipeline.path())

    rows = []
    for node in pipeline.nodes.values():
        [avg_duration, avg_run_time] = node_durations_and_run_times.get(tuple(node.path()), ['', ''])

        rows.append(
            _.tr[_.td[_.a(href=views.node_url(node))[node.id.replace('_', '_<wbr>')]],
                 _.td[node.description],
                 _.td[views.format_labels(node)],
                 _.td[node_cost.format_duration(avg_duration)],
                 _.td(style='color:#bbb' if avg_duration == avg_run_time else '')[
                     node_cost.format_duration(avg_run_time)],
                 _.td[node_cost.format_duration(
                     node_cost.compute_cost(node, node_durations_and_run_times))],
                 _.td[(_.input(class_='pipeline-node-checkbox', type='checkbox',
                               value=node.id, name='ids[]', onchange='runButtons.update()')
                 if config.allow_run_from_web_ui() else '')]])

    return \
        str(_.script['var runButtons = new PipelineRunButtons();']) \
        + str(bootstrap.table(['ID', 'Description', '', 'Avg duration', 'Avg run time', 'Cost', ''], rows)) \
        + str(_.script['floatMaraTableHeaders();'])
Esempio n. 3
0
def dependency_graph(nodes: {str: pipelines.Node},
                     current_node: pipelines.Node = None) -> str:
    """
    Draws a list of pipeline nodes and the dependencies between them using graphviz

    Args:
        nodes: The nodes to render
        current_node: If not null, then this node is highlighted

    Returns:
        An svg representation of the graph
    """
    graph = graphviz.Digraph(graph_attr={'rankdir': 'TD', 'ranksep': '0.25', 'nodesep': '0.1'})
    for node in nodes.values():

        node_attributes = {'fontname': ' ',  # use website default
                           'fontsize': '10.5px'  # fontsize unfortunately must be set
                           }

        if node != current_node:
            node_attributes.update(
                {'href': views.node_url(node), 'fontcolor': '#0275d8',
                 'tooltip': node.description, 'color': 'transparent'})
        else:
            node_attributes.update({'color': '#888888', 'style': 'dotted'})

        if isinstance(node, pipelines.Pipeline):
            node_attributes.update({'shape': 'rectangle', 'style': 'dotted', 'color': '#888888'})
        elif isinstance(node, pipelines.ParallelTask):
            node_attributes.update({'shape': 'ellipse', 'style': 'dotted', 'color': '#888888'})
        else:
            node_attributes['shape'] = 'rectangle'

        graph.node(name=node.id, label=node.id.replace('_', '\n'), _attributes=node_attributes)

        for upstream in node.upstreams:
            if upstream.id in nodes:
                graph.edge(upstream.id, node.id, _attributes={'color': '#888888', 'arrowsize': '0.7'})
            elif (not current_node) or node in current_node.upstreams:
                graph.node(name=f'{upstream.id}_{node.id}',
                           _attributes={'style': 'invis', 'label': '', 'height': '0.1', 'fixedsize': 'true'})
                graph.edge(f'{upstream.id}_{node.id}', node.id,
                           _attributes={'color': '#888888', 'arrowsize': '0.7',
                                        'edgetooltip': upstream.id, 'style': 'dotted'})

        for downstream in node.downstreams:
            if downstream.id not in nodes and (not current_node or node in current_node.downstreams):
                graph.node(name=f'{downstream.id}_{node.id}',
                           _attributes={'style': 'invis', 'label': '', 'height': '0.1', 'fixedsize': 'true'})
                graph.edge(node.id, f'{downstream.id}_{node.id}',
                           _attributes={'color': '#888888', 'arrowsize': '0.7',
                                        'edgetooltip': downstream.id, 'style': 'dotted'})

    return graph.pipe('svg').decode('utf-8')
Esempio n. 4
0
def run_page(path: str, with_upstreams: bool, ids: str):
    if not config.allow_run_from_web_ui():
        flask.abort(
            403,
            'Running piplelines from web ui is disabled for this instance')

    # the pipeline to run
    pipeline, found = pipelines.find_node(path.split('/'))
    if not found:
        flask.abort(404, f'Pipeline "{path}" not found')
    assert (isinstance(pipeline, pipelines.Pipeline))

    # a list of nodes to run selectively in the pipeline
    nodes = []
    for id in (ids.split('/') if ids else []):
        node = pipeline.nodes.get(id)
        if not node:
            flask.abort(404, f'Node "{id}" not found in pipeline "{path}"')
        else:
            nodes.append(node)

    stream_url = flask.url_for('data_integration.do_run',
                               path=path,
                               with_upstreams=with_upstreams,
                               ids=ids)

    title = [
        'Run ', 'with upstreams ' if with_upstreams else '', ' / '.join([
            str(_.a(href=views.node_url(parent))[parent.id])
            for parent in pipeline.parents()[1:]
        ])
    ]
    if nodes:
        title += [
            ' / [', ', '.join([
                str(_.a(href=views.node_url(node))[node.id]) for node in nodes
            ]), ']'
        ]

    return response.Response(
        html=[
            _.script['''
document.addEventListener('DOMContentLoaded', function() {
     processRunEvents(''' + json.dumps(
                flask.url_for('data_integration.node_page', path='')) + ', ' +
                     json.dumps(stream_url) + ', ' +
                     json.dumps(pipeline.path()) + ''');
});'''],
            _.style[
                'span.action-buttons > * {display:none}'],  # hide reload button until run finishes
            _.div(class_='row')
            [_.div(class_='col-lg-7')[bootstrap.card(body=_.div(
                id='main-output-area', class_='run-output')[''])],
             _.div(class_='col-lg-5 scroll-container')[
                 bootstrap.
                 card(header_left='Timeline',
                      body=[
                          _.div(id='system-stats-chart', class_='google-chart'
                                )[' '],
                          _.div(id='timeline-chart')[' ']
                      ]),
                 _.div(id='failed-tasks-container')[''],
                 _.div(id='running-tasks-container')[''],
                 _.div(id='succeeded-tasks-container')[''],
                 bootstrap.card(id='card-template',
                                header_left=' ',
                                header_right=' ',
                                body=[_.div(class_='run-output')['']])]]
        ],
        js_files=[
            'https://www.gstatic.com/charts/loader.js',
            flask.url_for('data_integration.static',
                          filename='timeline-chart.js'),
            flask.url_for('data_integration.static',
                          filename='system-stats-chart.js'),
            flask.url_for('data_integration.static', filename='utils.js'),
            flask.url_for('data_integration.static', filename='run-page.js')
        ],
        css_files=[
            flask.url_for('data_integration.static',
                          filename='timeline-chart.css'),
            flask.url_for('data_integration.static', filename='run-page.css'),
            flask.url_for('data_integration.static', filename='common.css')
        ],
        action_buttons=[
            response.ActionButton(
                action='javascript:location.reload()',
                label='Run again',
                icon='play',
                title='Run pipeline again with same parameters as before')
        ],
        title=title,
    )