def render_entry(entry: navigation.NavigationEntry, level: int = 1):
attrs = {}
if entry.children:
attrs.update({'onClick': 'toggleNavigationEntry(this)'})
else:
attrs.update({
'onClick':
'highlightNavigationEntry(\'' + entry.uri_fn() +
'\');collapseNavigation()',
'href':
entry.uri_fn()
})
if entry.description:
attrs.update({
'title': entry.description,
'data-toggle': 'tooltip',
'data-container': 'body',
'data-placement': 'right'
})
return _.div(
_class='mara-nav-entry level-' + str(level),
style='display:none' if level > 1 else
'')[_.a(**attrs)[_.span(_class='fa fa-fw fa-' + entry.icon + (
' fa-lg' if level == 1 else ''))[''] if entry.icon else '',
entry.label.replace('_', '_<wbr>'),
_.div(_class='mara-caret fa fa-caret-down'
)[''] if entry.children else ''],
render_entries(entry.children, level + 1)]
def schema_selection(db_alias: str):
"""Asynchronously computes the list of schemas with foreign key constraints"""
schemas_with_fk_constraints = schemas_with_foreign_key_constraints(db_alias)
if not schemas_with_fk_constraints or len(schemas_with_fk_constraints) == 0:
return str(_.i['No schemas with foreign key constraints found'])
return ''.join(xml.render([
[_.div(class_='form-check form-check-inline')[
_.label(class_='form-check-label')[
_.input(class_="form-check-input schema-checkbox", type="checkbox", value=schema_name)[
''], ' ', schema_name]]
for schema_name in sorted(schemas_with_fk_constraints)],
'   ',
_.div(class_='form-check form-check-inline')[
_.label(class_='form-check-label')[
_.input(class_="form-check-input", id='hide-columns-checkbox', type="checkbox")[
''], ' ', 'hide columns']],
'   ',
_.div(class_='form-check form-check-inline')[
_.label(class_='form-check-label')[
'graphviz engine ',
_.select(id='engine', style='border:none;background-color:white;')[
[_.option(value=engine)[engine] for engine in ['neato', 'dot', 'twopi', 'fdp']]
]]],
_.script['''
var schemaPage = SchemaPage("''' + flask.url_for('mara_db.index_page', db_alias=db_alias) + '''", "''' + db_alias + '''");
''']]))
def render_entry(entry: navigation.NavigationEntry, level: int = 1):
attrs = {}
if entry.children:
attrs['onClick'] = 'toggleNavigationEntry(this)'
else:
attrs['href'] = entry.uri_fn()
if entry.description:
attrs.update({
'title': entry.description,
'data-toggle': 'tooltip',
'data-container': 'body',
'data-placement': 'right'
})
return _.div(
class_='mara-nav-entry level-' + str(level),
style='display:none' if level > 1 else ''
)[_.a(**attrs)[_.div(
class_='mara-nav-entry-icon fa fa-fw fa-' + entry.icon +
(' fa-lg' if level == 1 else ''))[''] if entry.icon else '',
_.div(class_='mara-nav-entry-text'
)[entry.label.replace('_', '_<wbr>')],
_.div(class_='mara-caret fa fa-caret-down'
)[''] if entry.children else ''],
render_entries(entry.children, level + 1)]
def card(title_left='',
title_right='',
fixed_title_height: bool = False,
body=[],
sections=[]):
"""
Renders a bootstrap card `bootstrap_card`_
Args:
title_left: A title that is displayed at the top left of the card
title_right: A title that is displayed at the top right of the card
fixed_title_height: When true, then the title is restricted to 1 line
body: Elements to be shown on the card
sections: Parts of the card that are separated by an horizontal line
Returns:
The rendered card
.. _bootstrap_card:
https://v4-alpha.getbootstrap.com/components/card/
"""
return _.div(_class="card")[_.div(_class='card-block')[(
_.div(_class='card-title' +
(' fixed-title-height' if fixed_title_height else ''))[
_.div(_class='card-title-left')[title_left],
_.div(_class='card-title-right')[title_right]] if
title_left != '' or title_right != '' else ''), body], (_.ul(
_class='list-group list-group-flush'
)[[_.li(_class='list-group-item')[section]
for section in sections]] if sections else '')]
def document(doc_id, folder_id=""):
docs = all_docs()
if folder_id:
full_doc_id = folder_id + '/' + doc_id
else:
full_doc_id = doc_id
if full_doc_id not in docs:
raise flask.abort(404, f"Documentation {doc_id} is not known.")
doc = docs[full_doc_id]
if not doc.path.exists():
raise flask.abort(404, f"Documentation {doc_id} is not found ({doc.path}).")
with doc.path.open() as f:
md_content = f.read()
md_escaped = flask.escape(md_content)
return response.Response(
title=f'Doc "{doc.full_name}"',
html=[bootstrap.card(
body=[
_.div(style="display:none")[_.pre(id_='markdown-source')[md_escaped]],
_.div(id_='markdown-rendered-content')[
_.span(class_='fa fa-spinner fa-spin')[' ']
],
]),
_.script(type="text/javascript")[__render_code],
],
js_files=[
'https://cdnjs.cloudflare.com/ajax/libs/mermaid/8.7.0/mermaid.min.js',
'https://cdnjs.cloudflare.com/ajax/libs/markdown-it/11.0.0/markdown-it.min.js',
'https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.1.2/highlight.min.js',
flask.url_for('docs.static', filename='markdown-it-naive-mermaid.js'),
],
css_files=[
'https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.1.2/styles/default.min.css',
]
)
def card(header_left='',
header_right='',
fixed_header_height: bool = True,
body=[],
sections=[],
id: str = None):
"""
Renders a bootstrap card `bootstrap_card`_
Args:
header_left: A header that is displayed at the top left of the card
header_right: A header that is displayed at the top right of the card
fixed_header_height: When true, then the header is restricted to 1 line
body: Elements to be shown on the card
sections: Parts of the card that are separated by an horizontal line
id: An optional id for the outer dom element of the card
Returns:
The rendered card
.. _bootstrap_card:
https://v4-alpha.getbootstrap.com/components/card/
"""
return _.div(id=id or uuid.uuid1(), class_="card mara-card")[(_.div(
class_='card-header' +
(' fixed-header-height' if fixed_header_height else '')
)[(_.div(class_='card-header-left')[header_left] if header_left else ''),
(_.div(class_='card-header-right')[header_right] if header_right else ''
)] if header_left != '' or header_right != '' else ''), (_.div(
class_='card-block')[_.div(
class_='card-block-content')[body]] if body else ''), [
_.div(class_='card-block card-section')[_.div(
class_='card-block-content')[section]]
for section in sections
] or '']
def run_time_chart(path: str):
node, found = pipelines.find_node(path.split('/'))
if not found:
flask.abort(404, f'Node "{path}" not found')
query = (pathlib.Path(__file__).parent / 'run_time_chart.sql').read_text()
with mara_db.postgresql.postgres_cursor_context(
'mara') as cursor: # type: psycopg2.extensions.cursor
cursor.execute(query)
cursor.execute(
f'SELECT row_to_json(t) FROM pg_temp.node_run_times({"%s"}) t',
(node.path(), ))
rows = [row[0] for row in cursor.fetchall()]
if rows and len(rows) > 1:
number_of_child_runs = len(
rows[0]['child_runs']) if rows[0]['child_runs'] else 0
return str(
_.div[_.div(id='run-time-chart',
class_='google-chart',
style=f'height:{100 + 15 * number_of_child_runs}px'
)[' '], _.script[f'''
drawRunTimeChart('run-time-chart', '{path}', {json.dumps(rows)});
''']])
else:
return str(_.i(style='color:#888')['Not enough data'])
def card(node: pipelines.Node) -> str:
"""A card that shows the system stats, the time line and output for the last runs or a node"""
return bootstrap.card(
id='last-runs-card',
header_left=[
'Last runs ',
_.div(style='display:inline-block;margin-left:20px;')[
html.asynchronous_content(
flask.url_for('mara_pipelines.last_runs_selector',
path=node.url_path()))]
],
body=[
html.spinner_js_function(),
html.asynchronous_content(url=flask.url_for(
'mara_pipelines.system_stats',
path=node.url_path(),
run_id=None),
div_id='system-stats'),
html.asynchronous_content(url=flask.url_for(
'mara_pipelines.timeline_chart',
path=node.url_path(),
run_id=None),
div_id='timeline-chart'),
html.asynchronous_content(url=flask.url_for(
'mara_pipelines.run_output',
path=node.url_path(),
run_id=None,
limit=True),
div_id='run-output')
])
def schema_selection(db_alias: str):
"""Asynchronously computes the list of schemas with foreign key constraints"""
schemas_with_fk_constraints = []
with mara_db.postgresql.postgres_cursor_context(db_alias) as cursor:
cursor.execute('''
SELECT
array_cat(array_agg(DISTINCT constrained_table_schema.nspname), array_agg(DISTINCT referenced_table_schema.nspname))
FROM pg_constraint
JOIN pg_class constrained_table ON constrained_table.oid = pg_constraint.conrelid
JOIN pg_namespace constrained_table_schema ON constrained_table.relnamespace = constrained_table_schema.oid
JOIN pg_class referenced_table ON referenced_table.oid = pg_constraint.confrelid
JOIN pg_namespace referenced_table_schema ON referenced_table.relnamespace = referenced_table_schema.oid'''
)
result = cursor.fetchone()
if result != (None, ):
schemas_with_fk_constraints = sorted(list(set(result[0])))
if len(schemas_with_fk_constraints) == 0:
return str(_.i['No schemas with foreign key constraints found'])
return ''.join(
xml.render(
[[
_.div(class_='form-check form-check-inline')[_.label(
class_='form-check-label')[
_.input(class_="form-check-input schema-checkbox",
type="checkbox",
value=schema_name)[''], ' ', schema_name]]
for schema_name in sorted(schemas_with_fk_constraints)
], '   ',
_.div(class_='form-check form-check-inline')[_.label(
class_='form-check-label')[_.input(class_="form-check-input",
id='hide-columns-checkbox',
type="checkbox")[''], ' ',
'hide columns']], '   ',
_.div(class_='form-check form-check-inline')[_.label(
class_='form-check-label'
)['graphviz engine ',
_.select(id='engine',
style='border:none;background-color:white;')[[
_.option(value=engine)[engine]
for engine in ['neato', 'dot', 'twopi', 'fdp']
]]]], _.script['''
var schemaPage = SchemaPage("''' + flask.url_for('mara_db.index_page',
db_alias=db_alias) +
'''", "''' + db_alias + '''");
''']]))
def page_header(response: mara_page.response.Response):
"""Renders the fixed top part of the page"""
return _.nav(id='mara-page-header', class_='navbar fixed-top')[
_.a(class_='navigation-toggle-button fa fa-lg fa-reorder',
onclick='toggleNavigation()')[' '],
_.img(src=config.logo_url() + '?' + _current_git_commit()),
_.h1[response.title],
_.div(class_='action-buttons')[map(action_button, response.
action_buttons)], ]
def render_function(function_name, function):
return _.tr[
_.td(style='max-width:15%;')[_.div(style='display:block;overflow:hidden;text-overflow:ellipsis')[
function_name.replace('_', '_<wbr/>')]],
_.td(style='width:30%')[_.em[function['doc']]],
_.td(style='width:55%;')[
_.pre(style='margin:0px;padding-top:3px;overflow:hidden;text-overflow:ellipsis;')[
html.escape(pprint.pformat(function['value']))]],
]
def flash_messages(response: mara_page.response.Response) -> xml.XMLElement:
"""Displays flask flash messages"""
return [
_.div(id='alerts'),
_.script(type='text/javascript')[map(
lambda m: 'showAlert("' + m[1].replace('"', '"') + '","' +
(m[0] if m[0] != 'message' else 'info') + '");',
flask.get_flashed_messages(True))]
]
def index_page():
"""Overview page of mara_db"""
return response.Response(
title=f'Database schemas',
html=bootstrap.card(body=[
_.div(
style=
'display:inline-block; margin-top:15px; margin-bottom:15px; margin-right:50px;'
)[_.a(href=flask.url_for('mara_db.schema_page', db_alias=db_alias)
)[_.span(class_='fa fa-database')[''], ' ', db_alias], _.br,
_.span(style='color:#888')[escape(str(type(db).__name__))]]
for db_alias, db in config.databases().items()
]),
js_files=[flask.url_for('mara_db.static', filename='schema-page.js')])
def index_page(db_alias: str):
"""A page that visiualizes the schemas of a database"""
if db_alias not in config.databases():
flask.abort(404, f'unkown database {db_alias}')
return response.Response(
title=f'Schema of database {db_alias}',
html=[bootstrap.card(sections=[
html.asynchronous_content(flask.url_for('mara_db.schema_selection', db_alias=db_alias)),
[_.div(id='schema-container')]]),
html.spinner_js_function()],
js_files=[flask.url_for('mara_db.static', filename='schema-page.js')],
action_buttons=[response.ActionButton(
action='javascript:schemaPage.downloadSvg()', label='SVG',
title='Save current chart as SVG file', icon='download')]
)
def asynchronous_content(url: str, div_id: str = None) -> [str]:
"""
Creates a div whose content will be asynchronously replaced with the content retrieved from `url`.
Requires the implementation of the javascript function `loadContentAsynchronously` that takes four arguments
- the container div
- the url to load
- a localStorage key for storing the final height of the div
- an optional javascript snippet that is called once the content is loaded
Args:
url: The url from which to retrieve the content
div_id: The id of the container div
Returns:
Html markup of the container div
"""
div_id = div_id or str(uuid.uuid1())
return _.div(id=div_id)[spinner(),
_.script["""
(function() {
// immediately (even before the DOM is completely loaded) set the height of the div
// to the last content height (stored in local storage) to avoid height flickering
var divHeightKey = 'div-height--' + window.location.pathname + '--' + '"""
+ url + """';
var divHeight = localStorage.getItem(divHeightKey);
if (divHeight) {
document.getElementById('""" + div_id +
"""').style.height = divHeight + 'px';
}
document.addEventListener('DOMContentLoaded', function() {
if (typeof loadContentAsynchronously == 'undefined') {
console.error('Please implement function "loadContentAsynchronously"');
} else {
loadContentAsynchronously('""" + div_id + """', '""" + url +
"""', divHeightKey);
}
});
})();
"""]]
def system_stats(path: str, run_id: int):
node, __ = pipelines.find_node(path.split('/'))
run_id = run_id or _latest_run_id(node.path())
if not run_id:
return ''
with mara_db.postgresql.postgres_cursor_context(
'mara') as cursor: # type: psycopg2.extensions.cursor
cursor.execute(
f'''
SELECT
-- needs to be spelled out to be able to rely on the order in the postprocessing of the row
-- run_id is not needed in the frontend...
stats.timestamp,
stats.disc_read,
stats.disc_write,
stats.net_recv,
stats.net_sent,
stats.cpu_usage,
stats.mem_usage,
stats.swap_usage,
stats.iowait
FROM data_integration_node_run nr
JOIN data_integration_system_statistics stats ON stats.timestamp BETWEEN nr.start_time AND nr.end_time
-- -1 is fallback for old cases where we didn't have a node ID -> can be removed after 2021-01-01 or so
AND (stats.run_id = nr.run_id OR stats.run_id = -1)
WHERE nr.run_id = {"%s"} AND nr.node_path = {"%s"};''', (run_id, node.path()))
data = [[row[0].isoformat()] + list(row[1:])
for row in cursor.fetchall()]
if len(data) >= 15:
return str(_.div(id='system-stats-chart', class_='google-chart')[' ']) \
+ str(_.script[f'nodePage.showSystemStats({json.dumps(data)});'])
else:
return ''
def system_stats(path: str, run_id: int):
node, __ = pipelines.find_node(path.split('/'))
run_id = run_id or _latest_run_id(node.path())
if not run_id:
return ''
with mara_db.postgresql.postgres_cursor_context(
'mara') as cursor: # type: psycopg2.extensions.cursor
cursor.execute(
f'''
SELECT data_integration_system_statistics.*
FROM data_integration_system_statistics
JOIN data_integration_node_run ON timestamp BETWEEN start_time AND end_time
WHERE run_id = {"%s"} AND node_path = {"%s"};''', (run_id, node.path()))
data = [[row[0].isoformat()] + list(row[1:])
for row in cursor.fetchall()]
if len(data) >= 15:
return str(_.div(id='system-stats-chart', class_='google-chart')[' ']) \
+ str(_.script[f'nodePage.showSystemStats({json.dumps(data)});'])
else:
return ''
def start_page():
import mara_pipelines.config
from mara_data_explorer.data_set import find_data_set
data_set_for_preview = find_data_set('order_items')
assert (data_set_for_preview)
return response.Response(
title='MyCompany BI',
html=_.div(class_='row')[_.div(
class_='col-lg-6'
)[bootstrap.
card(header_left=_.b['Welcome'],
body=[
_.
p['This is the first thing that users of your data warehouse will see. ',
'Please add links to relevant documentation, tutorials & other ',
'data tools in your organization.'], _.p[
_.a(href='https://github.com/mara/mara-example-project-1/blob/master/app/ui/start_page.py'
)[
'Here'],
' is the source code for this page, and here is a picture of a ',
_.a(href='https://en.wikipedia.org/wiki/Mara_(mammal)'
)[
'mara'], ':'],
_.
img(src=flask.url_for('ui.static', filename='mara.jpg'),
style
='width:40%; margin-left: auto; margin-right:auto; display:block;'
)
]),
bootstrap.card(header_left=[
_.b[_.a(href=flask.url_for('mara_metabase.metabase'))[_.span(
class_='fa fa-bar-chart')[''], ' Metabase']], ' & ', _.
b[_.a(href=flask.url_for('mara_mondrian.saiku'))[_.span(
class_='fa fa-bar-chart')[''], ' Saiku']],
': Company wide dashboards, pivoting & ad hoc analysis'
],
body=[
_.
p['Metabase tutorial: ',
_.
a(href=
'https://www.metabase.com/docs/latest/getting-started.html'
)
['https://www.metabase.com/docs/latest/getting-started.html']],
_.
p['Saiku introduction: ',
_.
a(href=
'https://saiku-documentation.readthedocs.io/en/latest/'
)
['https://saiku-documentation.readthedocs.io/en/latest/']]
]),
bootstrap.card(header_left=[
_.b[_.a(href=flask.url_for('mara_data_explorer.index_page'))[
_.span(class_='fa fa-table')[''],
' Explore']], ': Raw data access & segmentation'
],
body=[
_.p[_.a(
href=flask.
url_for('mara_data_explorer.data_set_page',
data_set_id=data_set_for_preview.id
))[data_set_for_preview.name], ':',
html.asynchronous_content(
flask.url_for(
'mara_data_explorer.data_set_preview',
data_set_id=data_set_for_preview.id)
)], _.
p['Other data sets: ', ', '.join([
str(
_.a(href=flask.url_for(
'mara_data_explorer.data_set_page',
data_set_id=ds.id))[ds.name])
for ds in mara_data_explorer.config.data_sets(
) if ds.id != data_set_for_preview.id
])]
])],
_.div(class_='col-lg-6')
[bootstrap.
card(header_left=[
_.b[_.a(href=flask.url_for(
'mara_schema.index_page'))[_.span(
class_='fa fa-book'
)[''], ' Data sets']],
': Documentation of attributes and metrics of all data sets'
],
body=html.asynchronous_content(
url=flask.
url_for('mara_schema.overview_graph'
))),
bootstrap.card(header_left=[
_.b[_.a(
href=flask.
url_for('mara_pipelines.node_page')
)[_.span(class_='fa fa-wrench')[''],
' Pipelines']],
': The data integration pipelines that create the DWH'
],
body=html.
asynchronous_content(
flask.url_for(
'mara_pipelines.dependency_graph',
path='/'))),
bootstrap.card(header_left=[
_.b[_.a(href=flask.
url_for('mara_db.index_page')
)[_.span(
class_='fa fa-database')[''],
' Database Schemas']],
': Schemas of all databases connections'
],
body=[
html.asynchronous_content(
flask.url_for(
'mara_db.draw_schema',
db_alias=
mara_pipelines.
config.
default_db_alias(
),
schemas='ec_dim'
) +
'?hide-columns=True')
])]])
def data_set_page(id: str) -> response.Response:
"""Renders the pages for individual data sets"""
from ..config import data_sets
data_set = next(
(data_set for data_set in data_sets() if data_set.id() == id), None)
if not data_set:
flask.flash(f'Could not find data set "{id}"', category='warning')
return flask.redirect(flask.url_for('mara_schema.index_page'))
base_url = flask.url_for('mara_schema.data_set_sql_query',
id=data_set.id())
def attribute_rows(data_set: DataSet) -> []:
rows = []
for path, attributes in data_set.connected_attributes().items():
if path:
rows.append(_.tr[_.td(
colspan=3, style='border-top:none; padding-top: 20px;'
)[[[
'→ ',
_.a(
href=data_set_url(entity.data_set)
)[link_title] if entity.data_set else link_title, ' '
] for entity, link_title in [(
entity_link.target_entity,
entity_link.prefix or entity_link.target_entity.name)
for entity_link in path]],
[' ', _.
i[path[-1].description]] if path[-1].description else '']])
for prefixed_name, attribute in attributes.items():
rows.append(_.tr[_.td[escape(prefixed_name)], _.td[_.i[escape(
attribute.description
)]], _.td[_.tt[escape(
f'{path[-1].target_entity.table_name + "." if path else ""}{attribute.column_name}'
)]]])
return rows
return response.Response(
html=[
bootstrap.card(
header_left=_.i[escape(data_set.entity.description)],
body=[
_.p['Entity table: ', _.code[escape(
f'{data_set.entity.schema_name}.{data_set.entity.table_name}'
)]],
html.asynchronous_content(
flask.url_for('mara_schema.data_set_graph',
id=data_set.id())),
]),
bootstrap.card(
header_left='Metrics',
body=[
html.asynchronous_content(
flask.url_for('mara_schema.metrics_graph',
id=data_set.id())),
bootstrap.table(['Name', 'Description', 'Computation'], [[
_.tr[_.td[escape(metric.name)],
_.td[_.i[escape(metric.description)]],
_.td[_.code[escape(metric.display_formula())]]]
for metric in data_set.metrics.values()
]]),
]),
bootstrap.card(header_left='Attributes',
body=bootstrap.table(
["Name", "Description", "Column name"],
attribute_rows(data_set))),
bootstrap.card(
header_left=[
'Data set sql query: ',
[
_.div(class_='form-check form-check-inline')
[" ",
_.label(class_='form-check-label')[
_.input(class_="form-check-input param-checkbox",
type="checkbox",
value=param)[''], ' ', param]]
for param in [
'human readable columns',
'pre-computed metrics',
'star schema',
'personal data',
'high cardinality attributes',
]
]
],
body=[
_.div(id='sql-container')[html.asynchronous_content(
base_url, 'sql-container')], _.script['''
document.addEventListener('DOMContentLoaded', function() {
DataSetSqlQuery("''' + base_url + '''");
});
''']
])
],
title=f'Data set "{data_set.name}"',
js_files=[
flask.url_for('mara_schema.static',
filename='data-set-sql-query.js')
],
)
def content_area(response: mara_page.response.Response) -> xml.XMLElement:
"""Renders the main content area"""
return _.div(id='mara-main',
_class='container-fluid')[response.get_data(as_text=True)]
def data_set_page(data_set_id, query_id):
ds = find_data_set(data_set_id)
if not ds:
flask.flash(f'Data set "{data_set_id}" does not exist anymore',
category='danger')
return flask.redirect(flask.url_for('data_sets.index_page'))
action_buttons = [
response.ActionButton(action='javascript:dataSetPage.downloadCSV()',
icon='download',
label='CSV',
title='Download as CSV'),
response.ActionButton(action='javascript:dataSetPage.load()',
icon='folder-open',
label='Load',
title='Load previously saved query'),
response.ActionButton(action='javascript:dataSetPage.save()',
icon='save',
label='Save',
title='Save query')
]
if query_id:
action_buttons.insert(
1,
response.ActionButton(action=flask.url_for(
'data_sets._delete_query',
data_set_id=data_set_id,
query_id=query_id),
icon='trash',
label='Delete',
title='Delete query'))
return response.Response(
title=f'Query "{query_id}" on "{ds.name}"'
if query_id else f'New query on "{ds.name}"',
html=[
_.div(
class_='row')[_.div(
class_='col-md-3'
)[bootstrap.card(header_left='Query',
body=_.div(
id='query-details')[html.spinner()]),
bootstrap.
card(header_left='Columns',
body=[
_.div(
class_="form-group"
)[_.input(type="search",
class_="columns-search form-control",
value="",
placeholder="Filter")],
_.div(id='columns-list')[html.spinner()]
])],
_.div(class_='col-md-9')[bootstrap.card(
id='filter-card',
header_left=[
_.div(class_="dropdown")[_.a(
**{
'class': 'dropdown-toggle',
'data-toggle': 'dropdown',
'href': '#'
}
)[_.span(
class_='fa fa-plus')[' '],
' Add filter'],
_.
div(class_=
"dropdown-menu",
id=
'filter-menu'
)
[_.div(
class_
="dropdown-item"
)[_.input(
type
="text",
class_=
"columns-search form-control",
value="",
placeholder=
"Filter")]]]
],
fixed_header_height=False,
body=_.div(id='filters')[html.spinner()]),
bootstrap.
card(header_left=_.div(
id='row-counts'
)[html.spinner()],
header_right=_.div(
id='pagination'
)[html.spinner()],
body=_.div(
id='preview'
)[html.spinner()]),
_.
div(class_='row',
id=
'distribution-charts'
)['']]],
_.script[f"""
var dataSetPage = null;
document.addEventListener('DOMContentLoaded', function() {{
dataSetPage = DataSetPage('{flask.url_for('data_sets.index_page')}',
{json.dumps({'data_set_id': data_set_id, 'query_id': query_id, 'query': flask.request.get_json()})},
15, '{config.charts_color()}');
}});
"""],
html.spinner_js_function(),
_.div(class_='col-xl-4 col-lg-6',
id='distribution-chart-template',
style='display: none')[bootstrap.card(
header_left=html.spinner(),
body=_.div(class_='chart-container google-chart')[
html.spinner()])],
_.div(
class_='modal fade', id='load-query-dialog',
tabindex="-1")[_.div(
class_='modal-dialog', role='document')[_.div(
class_='modal-content')[_.div(class_='modal-header')[
_.h5(class_='modal-title')['Load query'],
_.button(
**{
'type': "button",
'class': "close",
'data-dismiss': "modal",
'aria-label': "Close"
})[_.span(
**{'aria-hidden': 'true'})['×']]],
_.div(class_='modal-body',
id='query-list')['']]]],
_.form(
action=flask.url_for('data_sets.download_csv'),
method='post')[_.div(
class_="modal fade",
id="download-csv-dialog",
tabindex="-1"
)[_.div(class_="modal-dialog", role='document')[_.div(
class_="modal-content"
)[_.div(class_="modal-header")
[_.h5(class_='modal-title')['Download as CSV'],
_.button(
**{
'type': "button",
'class': "close",
'data-dismiss': "modal",
'aria-label': "Close"
})[_.span(
**{'aria-hidden': 'true'})['×']]],
_.div(class_="modal-body")[
'Delimiter:  ',
_.input(type="radio",
value="\t",
name="delimiter",
checked="checked"),
' tab   ',
_.input(type="radio", value=";", name="delimiter"),
' semicolon   ',
_.input(type="radio", value=",", name="delimiter"),
' comma   ', _.hr, 'Number format:  ',
_.input(type="radio",
value=".",
name="decimal-mark",
checked="checked"), ' 42.7   ',
_.input(type="radio", value=",", name="decimal-mark"),
' 42,7   ',
_.input(type="hidden", name="query")],
_.div(class_="modal-footer"
)[_.button(id="csv-download-button",
type="submit",
class_="btn btn-primary")['Download']]]]]]
],
action_buttons=action_buttons,
js_files=[
'https://www.gstatic.com/charts/loader.js',
flask.url_for('data_sets.static', filename='tagsinput.js'),
flask.url_for('data_sets.static', filename='typeahead.js'),
flask.url_for('data_sets.static', filename='data-sets.js')
],
css_files=[
flask.url_for('data_sets.static', filename='tagsinput.css'),
flask.url_for('data_sets.static', filename='data-sets.css')
])
def view_plots():
return response.Response(
html=[
_.div(id='forecast-container')[_.div(class_="")[[
[
_.div(class_='row')[_.div(class_='col-xl-12')[_.div(
class_='section-header')[forecast.metric_name]]],
_.div(class_='row')[_.div(class_='col-xl-12')[
_.p()['Number of days: ' +
str(forecast.number_of_days),
_.br(), 'Time-series query: ',
html.highlight_syntax(forecast.time_series_query,
language='postgresql')],
bootstrap.card(
header_left='Forecast plot of "' +
forecast.metric_name + '"',
header_right=_.div()[
# config.forecast_table_name
# _.a(class_='query-control', style='margin-right: 10px',
# href='#')[
# _.span(class_='fa fa-download')[' ']],
_.
a(class_='query-control',
href=f"javascript:showQueryDetails('"
f"{flask.url_for('forecasts.query_details', name=forecast.metric_name)}')"
)[_.span(class_='fa fa-eye')[' ']]],
body=[
html.asynchronous_content(
flask.url_for('forecasts.get_plot_image',
name=forecast.metric_name.
lower().replace(' ', '_').
replace('-', '_'),
components='False')),
_.br(),
_.br(),
_.div(class_='modal fade',
id='query-details-dialog',
tabindex="-1"
)[_.div(class_='modal-dialog modal-lg',
role='document')[_.div(
class_='modal-content'
)[_.div(
class_='modal-header'
)[_.h5(class_='modal-title'
)['Time-series query'],
_.button(
**{
'type': "button",
'class': "close",
'data-dismiss': "modal",
'aria-label': "Close"
}
)[_.span(
**{'aria-hidden': 'true'}
)['×']]],
_.div(class_='modal-body',
id='query-details'
)['']]]],
])],
_.
div(class_='col-xl-12')[bootstrap.card(
header_left=
'Forecast components (trend, holidays, seasonality) of "{}"'
.format(forecast.metric_name),
header_right='',
body=[
html.asynchronous_content(
flask.url_for(
'forecasts.get_plot_image',
name=forecast.
metric_name.lower(
).replace(' ', '_').
replace('-', '_'),
components='True')),
])],
_.hr()]
] for forecast in config.forecasts()
]]],
_.script['''''']
],
title='Forecasts',
css_files=[
'https://fonts.googleapis.com/css?family=Open+Sans:300,400,700',
flask.url_for('forecasts.static', filename='forecast.css')
],
js_files=[
flask.url_for('forecasts.static', filename='forecast.js'),
'https://www.gstatic.com/charts/loader.js'
])
def run_page(path: str, with_upstreams: bool, ids: str):
if not config.allow_run_from_web_ui():
flask.abort(
403,
'Running piplelines from web ui is disabled for this instance')
# the pipeline to run
pipeline, found = pipelines.find_node(path.split('/'))
if not found:
flask.abort(404, f'Pipeline "{path}" not found')
assert (isinstance(pipeline, pipelines.Pipeline))
# a list of nodes to run selectively in the pipeline
nodes = []
for id in (ids.split('/') if ids else []):
node = pipeline.nodes.get(id)
if not node:
flask.abort(404, f'Node "{id}" not found in pipeline "{path}"')
else:
nodes.append(node)
stream_url = flask.url_for('mara_pipelines.do_run',
path=path,
with_upstreams=with_upstreams,
ids=ids)
title = [
'Run ', 'with upstreams ' if with_upstreams else '', ' / '.join([
str(_.a(href=views.node_url(parent))[parent.id])
for parent in pipeline.parents()[1:]
])
]
if nodes:
title += [
' / [', ', '.join([
str(_.a(href=views.node_url(node))[node.id]) for node in nodes
]), ']'
]
return response.Response(
html=[
_.script['''
document.addEventListener('DOMContentLoaded', function() {
processRunEvents(''' + json.dumps(
flask.url_for('mara_pipelines.node_page', path='')) + ', ' +
json.dumps(stream_url) + ', ' +
json.dumps(pipeline.path()) + ''');
});'''],
_.style[
'span.action-buttons > * {display:none}'], # hide reload button until run finishes
_.div(class_='row')
[_.div(class_='col-lg-7')[bootstrap.card(body=_.div(
id='main-output-area', class_='run-output')[''])],
_.div(class_='col-lg-5 scroll-container')[
bootstrap.
card(header_left='Timeline',
body=[
_.div(id='system-stats-chart', class_='google-chart'
)[' '],
_.div(id='timeline-chart')[' ']
]),
_.div(id='failed-tasks-container')[''],
_.div(id='running-tasks-container')[''],
_.div(id='succeeded-tasks-container')[''],
bootstrap.card(id='card-template',
header_left=' ',
header_right=' ',
body=[_.div(class_='run-output')['']])]]
],
js_files=[
'https://www.gstatic.com/charts/loader.js',
flask.url_for('mara_pipelines.static',
filename='timeline-chart.js'),
flask.url_for('mara_pipelines.static',
filename='system-stats-chart.js'),
flask.url_for('mara_pipelines.static', filename='utils.js'),
flask.url_for('mara_pipelines.static', filename='run-page.js')
],
css_files=[
flask.url_for('mara_pipelines.static',
filename='timeline-chart.css'),
flask.url_for('mara_pipelines.static', filename='run-page.css'),
flask.url_for('mara_pipelines.static', filename='common.css')
],
action_buttons=[
response.ActionButton(
action='javascript:location.reload()',
label='Run again',
icon='play',
title='Run pipeline again with same parameters as before')
],
title=title,
)
def data_set_page(data_set_id, query_id):
from .data_set import find_data_set
ds = find_data_set(data_set_id)
if not ds:
flask.flash(f'Data set "{data_set_id}" does not exist anymore', category='danger')
return flask.redirect(flask.url_for('mara_data_explorer.index_page'))
action_buttons = []
action_buttons.append(response.ActionButton(action='javascript:dataSetPage.downloadCSV()',
icon='download',
label='CSV', title='Download as CSV'))
if config.google_sheet_oauth2_client_config():
action_buttons.append(response.ActionButton(action='javascript:dataSetPage.exportToGoogleSheet()',
icon='cloud-upload',
label='Google sheet', title='Export to a Google sheet'))
action_buttons.append(response.ActionButton(action='javascript:dataSetPage.load()',
icon='folder-open',
label='Load', title='Load previously saved query'))
action_buttons.append(response.ActionButton(action='javascript:dataSetPage.save()',
icon='save',
label='Save', title='Save query'))
action_buttons.append(response.ActionButton(action='javascript:dataSetPage.displayQuery()',
icon='eye',
label='SQL', title='Display query'))
if query_id:
action_buttons.insert(1, response.ActionButton(
action=flask.url_for('mara_data_explorer._delete_query', data_set_id=data_set_id, query_id=query_id),
icon='trash', label='Delete', title='Delete query'))
return response.Response(
title=f'Query "{query_id}" on "{ds.name}"' if query_id else f'New query on "{ds.name}"',
html=[_.div(class_='row')[
_.div(class_='col-md-3')[
bootstrap.card(header_left='Query', body=_.div(id='query-details')[html.spinner()]),
bootstrap.card(header_left='Columns',
header_right=_.a(id='select-all', href='#')[' Select all'],
body=[_.div(class_="form-group")[
_.input(type="search", class_="columns-search form-control",
value="", placeholder="Filter")],
_.div(id='columns-list')[html.spinner()]])],
_.div(class_='col-md-9')[
bootstrap.card(
id='filter-card',
header_left=[_.div(class_="dropdown")[
_.a(**{'class': 'dropdown-toggle', 'data-toggle': 'dropdown', 'href': '#'})[
_.span(class_='fa fa-plus')[' '], ' Add filter'],
_.div(class_="dropdown-menu", id='filter-menu')[
_.div(class_="dropdown-item")[
_.input(type="text", class_="columns-search form-control", value="",
placeholder="Filter")]]]],
fixed_header_height=False,
body=_.div(id='filters')[html.spinner()]),
bootstrap.card(header_left=_.div(id='row-counts')[html.spinner()],
header_right=_.div(id='pagination')[html.spinner()],
body=_.div(id='preview')[html.spinner()]),
_.div(class_='row', id='distribution-charts')['']
]], _.script[f"""
var dataSetPage = null;
document.addEventListener('DOMContentLoaded', function() {{
dataSetPage = DataSetPage('{flask.url_for('mara_data_explorer.index_page')}',
{json.dumps(
{'data_set_id': data_set_id, 'query_id': query_id, 'query': flask.request.get_json()})},
15, '{config.charts_color()}');
}});
"""],
html.spinner_js_function(),
_.div(class_='col-xl-4 col-lg-6', id='distribution-chart-template', style='display: none')[
bootstrap.card(header_left=html.spinner(), body=_.div(class_='chart-container google-chart')[
html.spinner()])],
_.div(class_='modal fade', id='load-query-dialog', tabindex="-1")[
_.div(class_='modal-dialog', role='document')[
_.div(class_='modal-content')[
_.div(class_='modal-header')[
_.h5(class_='modal-title')['Load query'],
_.button(**{'type': "button", 'class': "close", 'data-dismiss': "modal",
'aria-label': "Close"})[
_.span(**{'aria-hidden': 'true'})['×']]],
_.div(class_='modal-body', id='query-list')['']
]
]
],
_.div(class_='modal fade', id='display-query-dialog', tabindex="-1")[
_.div(class_='modal-dialog', role='document')[
_.div(class_='modal-content')[
_.div(class_='modal-header')[
_.h5(class_='modal-title')['Query statement'],
_.button(**{'type': "button", 'class': "close", 'data-dismiss': "modal",
'aria-label': "Close"})[
_.span(**{'aria-hidden': 'true'})['×']]],
_.div(class_='modal-body', id='query-display')['']
]
]
],
_.form(action=flask.url_for('mara_data_explorer.download_csv', data_set_id=data_set_id), method='post')[
_.div(class_="modal fade", id="download-csv-dialog", tabindex="-1")[
_.div(class_="modal-dialog", role='document')[
_.div(class_="modal-content")[
_.div(class_="modal-header")[
_.h5(class_='modal-title')['Download as CSV'],
_.button(**{'type': "button", 'class': "close", 'data-dismiss': "modal",
'aria-label': "Close"})[
_.span(**{'aria-hidden': 'true'})['×']]],
_.div(class_="modal-body")[
'Delimiter:  ',
_.input(type="radio", value="\t", name="delimiter",
checked="checked"), ' tab   ',
_.input(type="radio", value=";", name="delimiter"), ' semicolon   ',
_.input(type="radio", value=",", name="delimiter"), ' comma   ',
_.hr,
'Number format:  ',
_.input(type="radio", value=".", name="decimal-mark",
checked="checked"), ' 42.7   ',
_.input(type="radio", value=",", name="decimal-mark"), ' 42,7   ',
_.input(type="hidden", name="query")],
_.div(class_="modal-footer")[
_.button(id="csv-download-button", type="submit", class_="btn btn-primary")[
'Download']]]]]],
_.form(action=flask.url_for('mara_data_explorer.oauth2_export_to_google_sheet', data_set_id=data_set_id),
method='post',
target="_blank")[
_.div(class_="modal fade", id="google-sheet-export-dialog", tabindex="-1")[
_.div(class_="modal-dialog", role='document')[
_.div(class_="modal-content")[
_.div(class_="modal-header")[
_.h5(class_='modal-title')['Google sheet export'],
_.button(**{'type': "button", 'class': "close", 'data-dismiss': "modal",
'aria-label': "Close"})[
_.span(**{'aria-hidden': 'true'})['×']]],
_.div(class_="modal-body")[
'Number format:  ',
_.input(type="radio", value=".", name="decimal-mark",
checked="checked"), ' 42.7   ',
_.input(type="radio", value=",", name="decimal-mark"), ' 42,7   ',
_.hr,
'Array format:  ',
_.input(type="radio", value="curly", name="array-format",
checked="checked"), ' {"a", "b"}   ',
_.input(type="radio", value="normal", name="array-format"), ' ["a", "b"]   ',
_.input(type="radio", value="tuple", name="array-format"), ' ("a", "b")   ',
_.hr,
'By clicking Export below:',
_.br,
_.ul[
_.li['Google authentication will be required.'],
_.li['A maximum limit of 100.000 rows will be applied.'],
_.li['A maximum limit of 50.000 characters per cell will be applied.'],
_.li['A Google sheet with the selected data will be available in a new tab.']
],
_.input(type="hidden", name="query")
],
_.div(class_="modal-footer")[
_.button(id="export-to-google-sheet", type="submit", class_="btn btn-primary")[
'Export']]]]]]
],
action_buttons=action_buttons,
js_files=['https://www.gstatic.com/charts/loader.js',
flask.url_for('mara_data_explorer.static', filename='tagsinput.js'),
flask.url_for('mara_data_explorer.static', filename='typeahead.js'),
flask.url_for('mara_data_explorer.static', filename='data-sets.js')],
css_files=[flask.url_for('mara_data_explorer.static', filename='tagsinput.css'),
flask.url_for('mara_data_explorer.static', filename='data-sets.css')])
def content_area(response: mara_page.response.Response) -> xml.XMLElement:
"""Renders the main content area"""
return _.div(id='mara-main', class_='container-fluid')[response.response]
