def index_page() -> response.Response:
"""Renders the overview page"""
from ..config import data_sets
return response.Response(html=[
bootstrap.card(header_left='Entities & their relations',
body=html.asynchronous_content(
flask.url_for('mara_schema.overview_graph'))),
bootstrap.card(
header_left='Data sets',
body=bootstrap.table(['Name', 'Description'], [
_.tr[_.td[_.a(
href=flask.
url_for('mara_schema.data_set_page', id=data_set.id()
))[escape(data_set.name)]],
_.td[_.i[escape(data_set.entity.description)]], ]
for data_set in data_sets()
]),
)
],
title='Data sets documentation',
css_files=[
flask.url_for('mara_schema.static',
filename='schema.css')
])
def preview():
from .query import Query
from .data_set import Column
query = Query.from_dict(flask.request.json['query'])
def header(column: Column):
if column.sortable():
if query.sort_column_name == column.column_name and query.sort_order == 'ASC':
icon = _.span(class_=('fa fa-sort-amount-asc'))['']
elif query.sort_column_name == column.column_name and query.sort_order == 'DESC':
icon = _.span(class_=('fa fa-sort-amount-desc'))['']
else:
icon = ''
return _.a(href="#", name=flask.escape(column.column_name))[
icon, ' ',
flask.escape(column.column_name)]
else:
return flask.escape(column.column_name)
if current_user_has_permission(query):
rows = [_render_preview_row(query, row)
for row in query.run(limit=flask.request.json['limit'], offset=flask.request.json['offset'],
include_personal_data=acl.current_user_has_permission(personal_data_acl_resource))]
else:
rows = _.tr[_.td(colspan=len(query.column_names))[acl.inline_permission_denied_message()]]
if rows:
return str(bootstrap.table(headers=[header(query.data_set.columns[c]) for c in query.column_names], rows=rows))
else:
return '∅'
def pipeline_children_table(path: str):
"""Creates a table that documents all child nodes of a table"""
pipeline, __ = pipelines.find_node(path.split('/'))
assert (isinstance(pipeline, pipelines.Pipeline))
node_durations_and_run_times = node_cost.node_durations_and_run_times(pipeline.path())
rows = []
for node in pipeline.nodes.values():
[avg_duration, avg_run_time] = node_durations_and_run_times.get(tuple(node.path()), ['', ''])
rows.append(
_.tr[_.td[_.a(href=views.node_url(node))[node.id.replace('_', '_<wbr>')]],
_.td[node.description],
_.td[views.format_labels(node)],
_.td[node_cost.format_duration(avg_duration)],
_.td(style='color:#bbb' if avg_duration == avg_run_time else '')[
node_cost.format_duration(avg_run_time)],
_.td[node_cost.format_duration(
node_cost.compute_cost(node, node_durations_and_run_times))],
_.td[(_.input(class_='pipeline-node-checkbox', type='checkbox',
value=node.id, name='ids[]', onchange='runButtons.update()')
if config.allow_run_from_web_ui() else '')]])
return \
str(_.script['var runButtons = new PipelineRunButtons();']) \
+ str(bootstrap.table(['ID', 'Description', '', 'Avg duration', 'Avg run time', 'Cost', ''], rows)) \
+ str(_.script['floatMaraTableHeaders();'])
def _render_command(command: pipelines.Command):
"""Creates a html documentation for a command"""
from mara_page.xml import render
def __mask_passwords(content):
masks = config.password_masks()
if masks:
content = ''.join(render(content))
for mask in masks:
content = content.replace(mask, "***")
return content
try:
doc = bootstrap.table([], [
_.tr[_.td[_.div[section]],
_.td(style='width:90%')[__mask_passwords(content)]]
for section, content in command.html_doc_items()
])
except Exception as e:
import traceback
doc = [
_.p(style='color:red')[
_.i['Error in rendering command documentation']],
_.pre(style='color:red')[traceback.format_exc()]
]
return [_.p[_.b[command.__class__.__name__]], doc]
def configuration_page():
config_modules = {}
for configs in get_get_current_config():
api_name, func = configs
module_name = func.__module__
module = sys.modules[module_name]
if module_name not in config_modules:
config_modules[module_name] = {
'doc': module.__doc__,
'functions': {}
}
try:
value = func()
except Exception:
value = 'error calling function'
config_modules[module_name]['functions'][api_name] \
= {'doc': func.__doc__ or '', 'value': value}
return response.Response(html=[(bootstrap.card(
title_left=_.b[html.escape(module_name)],
body=[
_.p[html.escape(str(config['doc']))],
bootstrap.table([], [
_.tr[_.td[function_name.replace('_', '_<wbr/>')],
_.td[_.em[function['doc']]], _.
td[_.pre[html.escape(pprint.pformat(function['value']))]]]
for function_name, function in config['functions'].items()
])
]) if config['functions'] else '') for module_name, config in sorted(
config_modules.items())],
title='Mara Configuration')
def __(task: pipelines.ParallelTask):
if not acl.current_user_has_permission(views.acl_resource):
return bootstrap.card(
header_left=acl.inline_permission_denied_message())
else:
return [
bootstrap.card(header_left='Commands before',
fixed_header_height=True,
sections=[
_render_command(command)
for command in task.commands_before
]) if task.commands_before else '',
bootstrap.card(header_left='Sub task creation',
body=bootstrap.table([], [
_.tr[_.td[_.div[section]],
_.td(style='width:90%')[content]]
for section, content in task.html_doc_items()
])),
bootstrap.card(header_left='Commands after',
fixed_header_height=True,
sections=[
_render_command(command)
for command in task.commands_after
]) if task.commands_after else ''
]
def configuration_page():
config = get_config_for_display()
current_user_has_permission = acl.current_user_has_permission(acl_resource)
return response.Response(html=[(bootstrap.card(
id=module_name,
header_left=html.escape(module_name),
body=[
_.p[_.em[html.escape(str(module_content.doc))]],
bootstrap.table([], [
_.tr[_.td[
_.tt[html.escape(config_name).replace('_', '_<wbr/>')], [
_.br, ' ⟵ ', _.
tt[html.escape(config_function.func_desc).
replace('.', '<wbr/>.').replace('_', '_<wbr/>')]
] if config_function.set_func else ''],
_.td[_.em[html.escape(config_function.doc)]], _.
td[_.pre[html.
escape(str(config_function)
)] if current_user_has_permission else acl
.inline_permission_denied_message()]]
for config_name, config_function in module_content.items()
])
]) if module_content.config_functions else '') for module_name,
module_content in config.items()],
title='Mara Configuration')
def configuration_page():
import pprint
from . import app
# gather all config functions by package
current_user_has_permission = acl.current_user_has_permission(acl_resource)
return response.Response(html=[(bootstrap.card(
id=module_name,
header_left=html.escape(module_name),
body=[
_.p[_.em[html.escape(str(config['doc']))]],
bootstrap.table([], [
_.tr[_.td[
_.tt[html.escape(function_name).replace('_', '_<wbr/>')], [
_.br, ' ⟵ ', _.
tt[html.escape(function['new_function']).
replace('.', '<wbr/>.').replace('_', '_<wbr/>')]
] if function['new_function'] else ''],
_.td[_.em[html.escape(function['doc'])]], _.
td[_.pre[html.
escape(pprint.pformat(function['value'])
)] if current_user_has_permission else acl
.inline_permission_denied_message()]]
for function_name, function in config['functions'].items()
])
]) if config['functions'] else '') for module_name, config in sorted(
_config_modules().items())],
title='Mara Configuration')
def _render_command(command: pipelines.Command):
"""Creates a html documentation for a command"""
return [
_.p[_.b[command.__class__.__name__]],
bootstrap.table([], [
_.tr[_.td[_.div[section]],
_.td(style='width:90%')[content]]
for section, content in command.html_doc_items()
])
]
def _render_command(command: pipelines.Command):
"""Creates a html documentation for a command"""
try:
doc = bootstrap.table([], [_.tr[_.td[_.div[section]], _.td(style='width:90%')[content]]
for section, content in command.html_doc_items()])
except Exception as e:
import traceback
doc = [_.p(style='color:red')[_.i['Error in rendering command documentation']],
_.pre(style='color:red')[traceback.format_exc()]]
return [_.p[_.b[command.__class__.__name__]], doc]
def query_list(data_set_id):
from .query import list_queries
queries = list_queries(data_set_id)
if queries:
return str(bootstrap.table(
headers=['Query', 'Last changed', 'By'],
rows=[_.tr[_.td[
_.a(href=flask.url_for('mara_data_explorer.data_set_page', data_set_id=data_set_id, query_id=row[0]))[
row[0]]],
_.td[row[1].strftime('%Y-%m-%d')],
_.td[row[2]]] for row in queries]))
else:
return 'No queries saved yet'
def query_details(name):
query = ''
for forecast in config.forecasts():
if forecast.metric_name == name:
query = forecast.time_series_query
days = forecast.number_of_days
break
return str(
bootstrap.table(
headers=[],
rows=[
_.tr[_.td[html.highlight_syntax(query,
language='postgresql')]],
_.tr[_.td['Number of days forecasted: ' + str(days)]]
]))
def data_set_preview(data_set_id):
from .query import Query
query = Query(data_set_id=data_set_id)
if query.column_names:
if current_user_has_permission(query):
rows = [_render_preview_row(query, row) for row
in query.run(limit=7, offset=0,
include_personal_data=acl.current_user_has_permission(personal_data_acl_resource))]
else:
rows = _.tr[_.td(colspan=len(query.column_names))[acl.inline_permission_denied_message()]]
return str(
bootstrap.table(headers=[flask.escape(column_name) for column_name in query.column_names], rows=rows))
else:
return '∅'
def data_set_page(id: str) -> response.Response:
"""Renders the pages for individual data sets"""
from ..config import data_sets
data_set = next(
(data_set for data_set in data_sets() if data_set.id() == id), None)
if not data_set:
flask.flash(f'Could not find data set "{id}"', category='warning')
return flask.redirect(flask.url_for('mara_schema.index_page'))
base_url = flask.url_for('mara_schema.data_set_sql_query',
id=data_set.id())
def attribute_rows(data_set: DataSet) -> []:
rows = []
for path, attributes in data_set.connected_attributes().items():
if path:
rows.append(_.tr[_.td(
colspan=3, style='border-top:none; padding-top: 20px;'
)[[[
'→ ',
_.a(
href=data_set_url(entity.data_set)
)[link_title] if entity.data_set else link_title, ' '
] for entity, link_title in [(
entity_link.target_entity,
entity_link.prefix or entity_link.target_entity.name)
for entity_link in path]],
[' ', _.
i[path[-1].description]] if path[-1].description else '']])
for prefixed_name, attribute in attributes.items():
rows.append(_.tr[_.td[escape(prefixed_name)], _.td[_.i[escape(
attribute.description
)]], _.td[_.tt[escape(
f'{path[-1].target_entity.table_name + "." if path else ""}{attribute.column_name}'
)]]])
return rows
return response.Response(
html=[
bootstrap.card(
header_left=_.i[escape(data_set.entity.description)],
body=[
_.p['Entity table: ', _.code[escape(
f'{data_set.entity.schema_name}.{data_set.entity.table_name}'
)]],
html.asynchronous_content(
flask.url_for('mara_schema.data_set_graph',
id=data_set.id())),
]),
bootstrap.card(
header_left='Metrics',
body=[
html.asynchronous_content(
flask.url_for('mara_schema.metrics_graph',
id=data_set.id())),
bootstrap.table(['Name', 'Description', 'Computation'], [[
_.tr[_.td[escape(metric.name)],
_.td[_.i[escape(metric.description)]],
_.td[_.code[escape(metric.display_formula())]]]
for metric in data_set.metrics.values()
]]),
]),
bootstrap.card(header_left='Attributes',
body=bootstrap.table(
["Name", "Description", "Column name"],
attribute_rows(data_set))),
bootstrap.card(
header_left=[
'Data set sql query: ',
[
_.div(class_='form-check form-check-inline')
[" ",
_.label(class_='form-check-label')[
_.input(class_="form-check-input param-checkbox",
type="checkbox",
value=param)[''], ' ', param]]
for param in [
'human readable columns',
'pre-computed metrics',
'star schema',
'personal data',
'high cardinality attributes',
]
]
],
body=[
_.div(id='sql-container')[html.asynchronous_content(
base_url, 'sql-container')], _.script['''
document.addEventListener('DOMContentLoaded', function() {
DataSetSqlQuery("''' + base_url + '''");
});
''']
])
],
title=f'Data set "{data_set.name}"',
js_files=[
flask.url_for('mara_schema.static',
filename='data-set-sql-query.js')
],
)
