def action_buttons(node: pipelines.Node):
"""The action buttons to be displayed on a node page"""
path = node.path()
return [
response.ActionButton(
action=flask.url_for('data_integration.run_page', path='/'.join(path[:-1]),
with_upstreams=True, ids=path[-1]),
label='Run with upstreams', icon='play',
title=f'Run the task and all its upstreams in the pipeline "{node.parent.id}"'),
response.ActionButton(
action=flask.url_for('data_integration.run_page', path='/'.join(path[:-1]),
with_upstreams=False, ids=path[-1]),
label='Run', icon='play',
title=f'Run only this task, without upstreams')]
def __(pipeline: pipelines.Pipeline):
return [
response.ActionButton(action=flask.url_for('data_integration.run_page',
path=pipeline.url_path(),
with_upstreams=False),
label='Run',
icon='play',
title='Run the pipeline')
]
def acl_page():
roles = {}
with mara_db.postgresql.postgres_cursor_context(
'mara') as cursor: # type: psycopg2.extensions.cursor
cursor.execute(f'SELECT email, role FROM acl_user ORDER BY role')
for email, role in cursor.fetchall():
rolekey = keys.user_key(role)
if not rolekey in roles:
roles[rolekey] = {'name': role, 'users': {}}
roles[keys.user_key(role)]['users'][keys.user_key(role,
email)] = email
def resource_tree(name, key, children):
result = {'name': name, 'key': key, 'children': []}
for resource in children: # type: acl.AclResource
child_key = key + '__' + keys.escape_key(resource.name)
result['children'].append(
resource_tree(resource.name, child_key, resource.children))
return result
resources = resource_tree('All', 'resource__All', config.resources())
return response.Response(
flask.render_template('acl.html',
roles=roles,
permissions=permissions.all_permissions(),
resources=resources,
acl_base_url=flask.url_for('mara_acl.acl_page'),
bootstrap_card=bootstrap.card),
title='Users, Roles & Permissions',
js_files=[flask.url_for('mara_acl.static', filename='acl.js')],
css_files=[flask.url_for('mara_acl.static', filename='acl.css')],
action_buttons=[
response.ActionButton('javascript:savePermissions()', 'Save',
'Save permissions', 'save'),
response.ActionButton('javascript:inviteNewUser()', 'Invite',
'Invite new user', 'plus')
])
def index_page(db_alias: str):
"""A page that visiualizes the schemas of a database"""
if db_alias not in config.databases():
flask.abort(404, f'unkown database {db_alias}')
return response.Response(
title=f'Schema of database {db_alias}',
html=[bootstrap.card(sections=[
html.asynchronous_content(flask.url_for('mara_db.schema_selection', db_alias=db_alias)),
[_.div(id='schema-container')]]),
html.spinner_js_function()],
js_files=[flask.url_for('mara_db.static', filename='schema-page.js')],
action_buttons=[response.ActionButton(
action='javascript:schemaPage.downloadSvg()', label='SVG',
title='Save current chart as SVG file', icon='download')]
)
def data_set_page(data_set_id, query_id):
from .data_set import find_data_set
ds = find_data_set(data_set_id)
if not ds:
flask.flash(f'Data set "{data_set_id}" does not exist anymore', category='danger')
return flask.redirect(flask.url_for('mara_data_explorer.index_page'))
action_buttons = []
action_buttons.append(response.ActionButton(action='javascript:dataSetPage.downloadCSV()',
icon='download',
label='CSV', title='Download as CSV'))
if config.google_sheet_oauth2_client_config():
action_buttons.append(response.ActionButton(action='javascript:dataSetPage.exportToGoogleSheet()',
icon='cloud-upload',
label='Google sheet', title='Export to a Google sheet'))
action_buttons.append(response.ActionButton(action='javascript:dataSetPage.load()',
icon='folder-open',
label='Load', title='Load previously saved query'))
action_buttons.append(response.ActionButton(action='javascript:dataSetPage.save()',
icon='save',
label='Save', title='Save query'))
action_buttons.append(response.ActionButton(action='javascript:dataSetPage.displayQuery()',
icon='eye',
label='SQL', title='Display query'))
if query_id:
action_buttons.insert(1, response.ActionButton(
action=flask.url_for('mara_data_explorer._delete_query', data_set_id=data_set_id, query_id=query_id),
icon='trash', label='Delete', title='Delete query'))
return response.Response(
title=f'Query "{query_id}" on "{ds.name}"' if query_id else f'New query on "{ds.name}"',
html=[_.div(class_='row')[
_.div(class_='col-md-3')[
bootstrap.card(header_left='Query', body=_.div(id='query-details')[html.spinner()]),
bootstrap.card(header_left='Columns',
header_right=_.a(id='select-all', href='#')[' Select all'],
body=[_.div(class_="form-group")[
_.input(type="search", class_="columns-search form-control",
value="", placeholder="Filter")],
_.div(id='columns-list')[html.spinner()]])],
_.div(class_='col-md-9')[
bootstrap.card(
id='filter-card',
header_left=[_.div(class_="dropdown")[
_.a(**{'class': 'dropdown-toggle', 'data-toggle': 'dropdown', 'href': '#'})[
_.span(class_='fa fa-plus')[' '], ' Add filter'],
_.div(class_="dropdown-menu", id='filter-menu')[
_.div(class_="dropdown-item")[
_.input(type="text", class_="columns-search form-control", value="",
placeholder="Filter")]]]],
fixed_header_height=False,
body=_.div(id='filters')[html.spinner()]),
bootstrap.card(header_left=_.div(id='row-counts')[html.spinner()],
header_right=_.div(id='pagination')[html.spinner()],
body=_.div(id='preview')[html.spinner()]),
_.div(class_='row', id='distribution-charts')['']
]], _.script[f"""
var dataSetPage = null;
document.addEventListener('DOMContentLoaded', function() {{
dataSetPage = DataSetPage('{flask.url_for('mara_data_explorer.index_page')}',
{json.dumps(
{'data_set_id': data_set_id, 'query_id': query_id, 'query': flask.request.get_json()})},
15, '{config.charts_color()}');
}});
"""],
html.spinner_js_function(),
_.div(class_='col-xl-4 col-lg-6', id='distribution-chart-template', style='display: none')[
bootstrap.card(header_left=html.spinner(), body=_.div(class_='chart-container google-chart')[
html.spinner()])],
_.div(class_='modal fade', id='load-query-dialog', tabindex="-1")[
_.div(class_='modal-dialog', role='document')[
_.div(class_='modal-content')[
_.div(class_='modal-header')[
_.h5(class_='modal-title')['Load query'],
_.button(**{'type': "button", 'class': "close", 'data-dismiss': "modal",
'aria-label': "Close"})[
_.span(**{'aria-hidden': 'true'})['×']]],
_.div(class_='modal-body', id='query-list')['']
]
]
],
_.div(class_='modal fade', id='display-query-dialog', tabindex="-1")[
_.div(class_='modal-dialog', role='document')[
_.div(class_='modal-content')[
_.div(class_='modal-header')[
_.h5(class_='modal-title')['Query statement'],
_.button(**{'type': "button", 'class': "close", 'data-dismiss': "modal",
'aria-label': "Close"})[
_.span(**{'aria-hidden': 'true'})['×']]],
_.div(class_='modal-body', id='query-display')['']
]
]
],
_.form(action=flask.url_for('mara_data_explorer.download_csv', data_set_id=data_set_id), method='post')[
_.div(class_="modal fade", id="download-csv-dialog", tabindex="-1")[
_.div(class_="modal-dialog", role='document')[
_.div(class_="modal-content")[
_.div(class_="modal-header")[
_.h5(class_='modal-title')['Download as CSV'],
_.button(**{'type': "button", 'class': "close", 'data-dismiss': "modal",
'aria-label': "Close"})[
_.span(**{'aria-hidden': 'true'})['×']]],
_.div(class_="modal-body")[
'Delimiter:  ',
_.input(type="radio", value="\t", name="delimiter",
checked="checked"), ' tab   ',
_.input(type="radio", value=";", name="delimiter"), ' semicolon   ',
_.input(type="radio", value=",", name="delimiter"), ' comma   ',
_.hr,
'Number format:  ',
_.input(type="radio", value=".", name="decimal-mark",
checked="checked"), ' 42.7   ',
_.input(type="radio", value=",", name="decimal-mark"), ' 42,7   ',
_.input(type="hidden", name="query")],
_.div(class_="modal-footer")[
_.button(id="csv-download-button", type="submit", class_="btn btn-primary")[
'Download']]]]]],
_.form(action=flask.url_for('mara_data_explorer.oauth2_export_to_google_sheet', data_set_id=data_set_id),
method='post',
target="_blank")[
_.div(class_="modal fade", id="google-sheet-export-dialog", tabindex="-1")[
_.div(class_="modal-dialog", role='document')[
_.div(class_="modal-content")[
_.div(class_="modal-header")[
_.h5(class_='modal-title')['Google sheet export'],
_.button(**{'type': "button", 'class': "close", 'data-dismiss': "modal",
'aria-label': "Close"})[
_.span(**{'aria-hidden': 'true'})['×']]],
_.div(class_="modal-body")[
'Number format:  ',
_.input(type="radio", value=".", name="decimal-mark",
checked="checked"), ' 42.7   ',
_.input(type="radio", value=",", name="decimal-mark"), ' 42,7   ',
_.hr,
'Array format:  ',
_.input(type="radio", value="curly", name="array-format",
checked="checked"), ' {"a", "b"}   ',
_.input(type="radio", value="normal", name="array-format"), ' ["a", "b"]   ',
_.input(type="radio", value="tuple", name="array-format"), ' ("a", "b")   ',
_.hr,
'By clicking Export below:',
_.br,
_.ul[
_.li['Google authentication will be required.'],
_.li['A maximum limit of 100.000 rows will be applied.'],
_.li['A maximum limit of 50.000 characters per cell will be applied.'],
_.li['A Google sheet with the selected data will be available in a new tab.']
],
_.input(type="hidden", name="query")
],
_.div(class_="modal-footer")[
_.button(id="export-to-google-sheet", type="submit", class_="btn btn-primary")[
'Export']]]]]]
],
action_buttons=action_buttons,
js_files=['https://www.gstatic.com/charts/loader.js',
flask.url_for('mara_data_explorer.static', filename='tagsinput.js'),
flask.url_for('mara_data_explorer.static', filename='typeahead.js'),
flask.url_for('mara_data_explorer.static', filename='data-sets.js')],
css_files=[flask.url_for('mara_data_explorer.static', filename='tagsinput.css'),
flask.url_for('mara_data_explorer.static', filename='data-sets.css')])
def run_page(path: str, with_upstreams: bool, ids: str):
if not config.allow_run_from_web_ui():
flask.abort(
403,
'Running piplelines from web ui is disabled for this instance')
# the pipeline to run
pipeline, found = pipelines.find_node(path.split('/'))
if not found:
flask.abort(404, f'Pipeline "{path}" not found')
assert (isinstance(pipeline, pipelines.Pipeline))
# a list of nodes to run selectively in the pipeline
nodes = []
for id in (ids.split('/') if ids else []):
node = pipeline.nodes.get(id)
if not node:
flask.abort(404, f'Node "{id}" not found in pipeline "{path}"')
else:
nodes.append(node)
stream_url = flask.url_for('mara_pipelines.do_run',
path=path,
with_upstreams=with_upstreams,
ids=ids)
title = [
'Run ', 'with upstreams ' if with_upstreams else '', ' / '.join([
str(_.a(href=views.node_url(parent))[parent.id])
for parent in pipeline.parents()[1:]
])
]
if nodes:
title += [
' / [', ', '.join([
str(_.a(href=views.node_url(node))[node.id]) for node in nodes
]), ']'
]
return response.Response(
html=[
_.script['''
document.addEventListener('DOMContentLoaded', function() {
processRunEvents(''' + json.dumps(
flask.url_for('mara_pipelines.node_page', path='')) + ', ' +
json.dumps(stream_url) + ', ' +
json.dumps(pipeline.path()) + ''');
});'''],
_.style[
'span.action-buttons > * {display:none}'], # hide reload button until run finishes
_.div(class_='row')
[_.div(class_='col-lg-7')[bootstrap.card(body=_.div(
id='main-output-area', class_='run-output')[''])],
_.div(class_='col-lg-5 scroll-container')[
bootstrap.
card(header_left='Timeline',
body=[
_.div(id='system-stats-chart', class_='google-chart'
)[' '],
_.div(id='timeline-chart')[' ']
]),
_.div(id='failed-tasks-container')[''],
_.div(id='running-tasks-container')[''],
_.div(id='succeeded-tasks-container')[''],
bootstrap.card(id='card-template',
header_left=' ',
header_right=' ',
body=[_.div(class_='run-output')['']])]]
],
js_files=[
'https://www.gstatic.com/charts/loader.js',
flask.url_for('mara_pipelines.static',
filename='timeline-chart.js'),
flask.url_for('mara_pipelines.static',
filename='system-stats-chart.js'),
flask.url_for('mara_pipelines.static', filename='utils.js'),
flask.url_for('mara_pipelines.static', filename='run-page.js')
],
css_files=[
flask.url_for('mara_pipelines.static',
filename='timeline-chart.css'),
flask.url_for('mara_pipelines.static', filename='run-page.css'),
flask.url_for('mara_pipelines.static', filename='common.css')
],
action_buttons=[
response.ActionButton(
action='javascript:location.reload()',
label='Run again',
icon='play',
title='Run pipeline again with same parameters as before')
],
title=title,
)
def data_set_page(data_set_id, query_id):
ds = find_data_set(data_set_id)
if not ds:
flask.flash(f'Data set "{data_set_id}" does not exist anymore',
category='danger')
return flask.redirect(flask.url_for('data_sets.index_page'))
action_buttons = [
response.ActionButton(action='javascript:dataSetPage.downloadCSV()',
icon='download',
label='CSV',
title='Download as CSV'),
response.ActionButton(action='javascript:dataSetPage.load()',
icon='folder-open',
label='Load',
title='Load previously saved query'),
response.ActionButton(action='javascript:dataSetPage.save()',
icon='save',
label='Save',
title='Save query')
]
if query_id:
action_buttons.insert(
1,
response.ActionButton(action=flask.url_for(
'data_sets._delete_query',
data_set_id=data_set_id,
query_id=query_id),
icon='trash',
label='Delete',
title='Delete query'))
return response.Response(
title=f'Query "{query_id}" on "{ds.name}"'
if query_id else f'New query on "{ds.name}"',
html=[
_.div(
class_='row')[_.div(
class_='col-md-3'
)[bootstrap.card(header_left='Query',
body=_.div(
id='query-details')[html.spinner()]),
bootstrap.
card(header_left='Columns',
body=[
_.div(
class_="form-group"
)[_.input(type="search",
class_="columns-search form-control",
value="",
placeholder="Filter")],
_.div(id='columns-list')[html.spinner()]
])],
_.div(class_='col-md-9')[bootstrap.card(
id='filter-card',
header_left=[
_.div(class_="dropdown")[_.a(
**{
'class': 'dropdown-toggle',
'data-toggle': 'dropdown',
'href': '#'
}
)[_.span(
class_='fa fa-plus')[' '],
' Add filter'],
_.
div(class_=
"dropdown-menu",
id=
'filter-menu'
)
[_.div(
class_
="dropdown-item"
)[_.input(
type
="text",
class_=
"columns-search form-control",
value="",
placeholder=
"Filter")]]]
],
fixed_header_height=False,
body=_.div(id='filters')[html.spinner()]),
bootstrap.
card(header_left=_.div(
id='row-counts'
)[html.spinner()],
header_right=_.div(
id='pagination'
)[html.spinner()],
body=_.div(
id='preview'
)[html.spinner()]),
_.
div(class_='row',
id=
'distribution-charts'
)['']]],
_.script[f"""
var dataSetPage = null;
document.addEventListener('DOMContentLoaded', function() {{
dataSetPage = DataSetPage('{flask.url_for('data_sets.index_page')}',
{json.dumps({'data_set_id': data_set_id, 'query_id': query_id, 'query': flask.request.get_json()})},
15, '{config.charts_color()}');
}});
"""],
html.spinner_js_function(),
_.div(class_='col-xl-4 col-lg-6',
id='distribution-chart-template',
style='display: none')[bootstrap.card(
header_left=html.spinner(),
body=_.div(class_='chart-container google-chart')[
html.spinner()])],
_.div(
class_='modal fade', id='load-query-dialog',
tabindex="-1")[_.div(
class_='modal-dialog', role='document')[_.div(
class_='modal-content')[_.div(class_='modal-header')[
_.h5(class_='modal-title')['Load query'],
_.button(
**{
'type': "button",
'class': "close",
'data-dismiss': "modal",
'aria-label': "Close"
})[_.span(
**{'aria-hidden': 'true'})['×']]],
_.div(class_='modal-body',
id='query-list')['']]]],
_.form(
action=flask.url_for('data_sets.download_csv'),
method='post')[_.div(
class_="modal fade",
id="download-csv-dialog",
tabindex="-1"
)[_.div(class_="modal-dialog", role='document')[_.div(
class_="modal-content"
)[_.div(class_="modal-header")
[_.h5(class_='modal-title')['Download as CSV'],
_.button(
**{
'type': "button",
'class': "close",
'data-dismiss': "modal",
'aria-label': "Close"
})[_.span(
**{'aria-hidden': 'true'})['×']]],
_.div(class_="modal-body")[
'Delimiter:  ',
_.input(type="radio",
value="\t",
name="delimiter",
checked="checked"),
' tab   ',
_.input(type="radio", value=";", name="delimiter"),
' semicolon   ',
_.input(type="radio", value=",", name="delimiter"),
' comma   ', _.hr, 'Number format:  ',
_.input(type="radio",
value=".",
name="decimal-mark",
checked="checked"), ' 42.7   ',
_.input(type="radio", value=",", name="decimal-mark"),
' 42,7   ',
_.input(type="hidden", name="query")],
_.div(class_="modal-footer"
)[_.button(id="csv-download-button",
type="submit",
class_="btn btn-primary")['Download']]]]]]
],
action_buttons=action_buttons,
js_files=[
'https://www.gstatic.com/charts/loader.js',
flask.url_for('data_sets.static', filename='tagsinput.js'),
flask.url_for('data_sets.static', filename='typeahead.js'),
flask.url_for('data_sets.static', filename='data-sets.js')
],
css_files=[
flask.url_for('data_sets.static', filename='tagsinput.css'),
flask.url_for('data_sets.static', filename='data-sets.css')
])
