def render_entry(entry: navigation.NavigationEntry, level: int = 1): attrs = {} if entry.children: attrs.update({'onClick': 'toggleNavigationEntry(this)'}) else: attrs.update({ 'onClick': 'highlightNavigationEntry(\'' + entry.uri_fn() + '\');collapseNavigation()', 'href': entry.uri_fn() }) if entry.description: attrs.update({ 'title': entry.description, 'data-toggle': 'tooltip', 'data-container': 'body', 'data-placement': 'right' }) return _.div( _class='mara-nav-entry level-' + str(level), style='display:none' if level > 1 else '')[_.a(**attrs)[_.span(_class='fa fa-fw fa-' + entry.icon + ( ' fa-lg' if level == 1 else ''))[''] if entry.icon else '', entry.label.replace('_', '_<wbr>'), _.div(_class='mara-caret fa fa-caret-down' )[''] if entry.children else ''], render_entries(entry.children, level + 1)]
def schema_selection(db_alias: str): """Asynchronously computes the list of schemas with foreign key constraints""" schemas_with_fk_constraints = schemas_with_foreign_key_constraints(db_alias) if not schemas_with_fk_constraints or len(schemas_with_fk_constraints) == 0: return str(_.i['No schemas with foreign key constraints found']) return ''.join(xml.render([ [_.div(class_='form-check form-check-inline')[ _.label(class_='form-check-label')[ _.input(class_="form-check-input schema-checkbox", type="checkbox", value=schema_name)[ ''], ' ', schema_name]] for schema_name in sorted(schemas_with_fk_constraints)], '   ', _.div(class_='form-check form-check-inline')[ _.label(class_='form-check-label')[ _.input(class_="form-check-input", id='hide-columns-checkbox', type="checkbox")[ ''], ' ', 'hide columns']], '   ', _.div(class_='form-check form-check-inline')[ _.label(class_='form-check-label')[ 'graphviz engine ', _.select(id='engine', style='border:none;background-color:white;')[ [_.option(value=engine)[engine] for engine in ['neato', 'dot', 'twopi', 'fdp']] ]]], _.script[''' var schemaPage = SchemaPage("''' + flask.url_for('mara_db.index_page', db_alias=db_alias) + '''", "''' + db_alias + '''"); ''']]))
def render_entry(entry: navigation.NavigationEntry, level: int = 1): attrs = {} if entry.children: attrs['onClick'] = 'toggleNavigationEntry(this)' else: attrs['href'] = entry.uri_fn() if entry.description: attrs.update({ 'title': entry.description, 'data-toggle': 'tooltip', 'data-container': 'body', 'data-placement': 'right' }) return _.div( class_='mara-nav-entry level-' + str(level), style='display:none' if level > 1 else '' )[_.a(**attrs)[_.div( class_='mara-nav-entry-icon fa fa-fw fa-' + entry.icon + (' fa-lg' if level == 1 else ''))[''] if entry.icon else '', _.div(class_='mara-nav-entry-text' )[entry.label.replace('_', '_<wbr>')], _.div(class_='mara-caret fa fa-caret-down' )[''] if entry.children else ''], render_entries(entry.children, level + 1)]
def card(title_left='', title_right='', fixed_title_height: bool = False, body=[], sections=[]): """ Renders a bootstrap card `bootstrap_card`_ Args: title_left: A title that is displayed at the top left of the card title_right: A title that is displayed at the top right of the card fixed_title_height: When true, then the title is restricted to 1 line body: Elements to be shown on the card sections: Parts of the card that are separated by an horizontal line Returns: The rendered card .. _bootstrap_card: https://v4-alpha.getbootstrap.com/components/card/ """ return _.div(_class="card")[_.div(_class='card-block')[( _.div(_class='card-title' + (' fixed-title-height' if fixed_title_height else ''))[ _.div(_class='card-title-left')[title_left], _.div(_class='card-title-right')[title_right]] if title_left != '' or title_right != '' else ''), body], (_.ul( _class='list-group list-group-flush' )[[_.li(_class='list-group-item')[section] for section in sections]] if sections else '')]
def document(doc_id, folder_id=""): docs = all_docs() if folder_id: full_doc_id = folder_id + '/' + doc_id else: full_doc_id = doc_id if full_doc_id not in docs: raise flask.abort(404, f"Documentation {doc_id} is not known.") doc = docs[full_doc_id] if not doc.path.exists(): raise flask.abort(404, f"Documentation {doc_id} is not found ({doc.path}).") with doc.path.open() as f: md_content = f.read() md_escaped = flask.escape(md_content) return response.Response( title=f'Doc "{doc.full_name}"', html=[bootstrap.card( body=[ _.div(style="display:none")[_.pre(id_='markdown-source')[md_escaped]], _.div(id_='markdown-rendered-content')[ _.span(class_='fa fa-spinner fa-spin')[' '] ], ]), _.script(type="text/javascript")[__render_code], ], js_files=[ 'https://cdnjs.cloudflare.com/ajax/libs/mermaid/8.7.0/mermaid.min.js', 'https://cdnjs.cloudflare.com/ajax/libs/markdown-it/11.0.0/markdown-it.min.js', 'https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.1.2/highlight.min.js', flask.url_for('docs.static', filename='markdown-it-naive-mermaid.js'), ], css_files=[ 'https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.1.2/styles/default.min.css', ] )
def card(header_left='', header_right='', fixed_header_height: bool = True, body=[], sections=[], id: str = None): """ Renders a bootstrap card `bootstrap_card`_ Args: header_left: A header that is displayed at the top left of the card header_right: A header that is displayed at the top right of the card fixed_header_height: When true, then the header is restricted to 1 line body: Elements to be shown on the card sections: Parts of the card that are separated by an horizontal line id: An optional id for the outer dom element of the card Returns: The rendered card .. _bootstrap_card: https://v4-alpha.getbootstrap.com/components/card/ """ return _.div(id=id or uuid.uuid1(), class_="card mara-card")[(_.div( class_='card-header' + (' fixed-header-height' if fixed_header_height else '') )[(_.div(class_='card-header-left')[header_left] if header_left else ''), (_.div(class_='card-header-right')[header_right] if header_right else '' )] if header_left != '' or header_right != '' else ''), (_.div( class_='card-block')[_.div( class_='card-block-content')[body]] if body else ''), [ _.div(class_='card-block card-section')[_.div( class_='card-block-content')[section]] for section in sections ] or '']
def run_time_chart(path: str): node, found = pipelines.find_node(path.split('/')) if not found: flask.abort(404, f'Node "{path}" not found') query = (pathlib.Path(__file__).parent / 'run_time_chart.sql').read_text() with mara_db.postgresql.postgres_cursor_context( 'mara') as cursor: # type: psycopg2.extensions.cursor cursor.execute(query) cursor.execute( f'SELECT row_to_json(t) FROM pg_temp.node_run_times({"%s"}) t', (node.path(), )) rows = [row[0] for row in cursor.fetchall()] if rows and len(rows) > 1: number_of_child_runs = len( rows[0]['child_runs']) if rows[0]['child_runs'] else 0 return str( _.div[_.div(id='run-time-chart', class_='google-chart', style=f'height:{100 + 15 * number_of_child_runs}px' )[' '], _.script[f''' drawRunTimeChart('run-time-chart', '{path}', {json.dumps(rows)}); ''']]) else: return str(_.i(style='color:#888')['Not enough data'])
def card(node: pipelines.Node) -> str: """A card that shows the system stats, the time line and output for the last runs or a node""" return bootstrap.card( id='last-runs-card', header_left=[ 'Last runs ', _.div(style='display:inline-block;margin-left:20px;')[ html.asynchronous_content( flask.url_for('mara_pipelines.last_runs_selector', path=node.url_path()))] ], body=[ html.spinner_js_function(), html.asynchronous_content(url=flask.url_for( 'mara_pipelines.system_stats', path=node.url_path(), run_id=None), div_id='system-stats'), html.asynchronous_content(url=flask.url_for( 'mara_pipelines.timeline_chart', path=node.url_path(), run_id=None), div_id='timeline-chart'), html.asynchronous_content(url=flask.url_for( 'mara_pipelines.run_output', path=node.url_path(), run_id=None, limit=True), div_id='run-output') ])
def schema_selection(db_alias: str): """Asynchronously computes the list of schemas with foreign key constraints""" schemas_with_fk_constraints = [] with mara_db.postgresql.postgres_cursor_context(db_alias) as cursor: cursor.execute(''' SELECT array_cat(array_agg(DISTINCT constrained_table_schema.nspname), array_agg(DISTINCT referenced_table_schema.nspname)) FROM pg_constraint JOIN pg_class constrained_table ON constrained_table.oid = pg_constraint.conrelid JOIN pg_namespace constrained_table_schema ON constrained_table.relnamespace = constrained_table_schema.oid JOIN pg_class referenced_table ON referenced_table.oid = pg_constraint.confrelid JOIN pg_namespace referenced_table_schema ON referenced_table.relnamespace = referenced_table_schema.oid''' ) result = cursor.fetchone() if result != (None, ): schemas_with_fk_constraints = sorted(list(set(result[0]))) if len(schemas_with_fk_constraints) == 0: return str(_.i['No schemas with foreign key constraints found']) return ''.join( xml.render( [[ _.div(class_='form-check form-check-inline')[_.label( class_='form-check-label')[ _.input(class_="form-check-input schema-checkbox", type="checkbox", value=schema_name)[''], ' ', schema_name]] for schema_name in sorted(schemas_with_fk_constraints) ], '   ', _.div(class_='form-check form-check-inline')[_.label( class_='form-check-label')[_.input(class_="form-check-input", id='hide-columns-checkbox', type="checkbox")[''], ' ', 'hide columns']], '   ', _.div(class_='form-check form-check-inline')[_.label( class_='form-check-label' )['graphviz engine ', _.select(id='engine', style='border:none;background-color:white;')[[ _.option(value=engine)[engine] for engine in ['neato', 'dot', 'twopi', 'fdp'] ]]]], _.script[''' var schemaPage = SchemaPage("''' + flask.url_for('mara_db.index_page', db_alias=db_alias) + '''", "''' + db_alias + '''"); ''']]))
def page_header(response: mara_page.response.Response): """Renders the fixed top part of the page""" return _.nav(id='mara-page-header', class_='navbar fixed-top')[ _.a(class_='navigation-toggle-button fa fa-lg fa-reorder', onclick='toggleNavigation()')[' '], _.img(src=config.logo_url() + '?' + _current_git_commit()), _.h1[response.title], _.div(class_='action-buttons')[map(action_button, response. action_buttons)], ]
def render_function(function_name, function): return _.tr[ _.td(style='max-width:15%;')[_.div(style='display:block;overflow:hidden;text-overflow:ellipsis')[ function_name.replace('_', '_<wbr/>')]], _.td(style='width:30%')[_.em[function['doc']]], _.td(style='width:55%;')[ _.pre(style='margin:0px;padding-top:3px;overflow:hidden;text-overflow:ellipsis;')[ html.escape(pprint.pformat(function['value']))]], ]
def flash_messages(response: mara_page.response.Response) -> xml.XMLElement: """Displays flask flash messages""" return [ _.div(id='alerts'), _.script(type='text/javascript')[map( lambda m: 'showAlert("' + m[1].replace('"', '"') + '","' + (m[0] if m[0] != 'message' else 'info') + '");', flask.get_flashed_messages(True))] ]
def index_page(): """Overview page of mara_db""" return response.Response( title=f'Database schemas', html=bootstrap.card(body=[ _.div( style= 'display:inline-block; margin-top:15px; margin-bottom:15px; margin-right:50px;' )[_.a(href=flask.url_for('mara_db.schema_page', db_alias=db_alias) )[_.span(class_='fa fa-database')[''], ' ', db_alias], _.br, _.span(style='color:#888')[escape(str(type(db).__name__))]] for db_alias, db in config.databases().items() ]), js_files=[flask.url_for('mara_db.static', filename='schema-page.js')])
def index_page(db_alias: str): """A page that visiualizes the schemas of a database""" if db_alias not in config.databases(): flask.abort(404, f'unkown database {db_alias}') return response.Response( title=f'Schema of database {db_alias}', html=[bootstrap.card(sections=[ html.asynchronous_content(flask.url_for('mara_db.schema_selection', db_alias=db_alias)), [_.div(id='schema-container')]]), html.spinner_js_function()], js_files=[flask.url_for('mara_db.static', filename='schema-page.js')], action_buttons=[response.ActionButton( action='javascript:schemaPage.downloadSvg()', label='SVG', title='Save current chart as SVG file', icon='download')] )
def asynchronous_content(url: str, div_id: str = None) -> [str]: """ Creates a div whose content will be asynchronously replaced with the content retrieved from `url`. Requires the implementation of the javascript function `loadContentAsynchronously` that takes four arguments - the container div - the url to load - a localStorage key for storing the final height of the div - an optional javascript snippet that is called once the content is loaded Args: url: The url from which to retrieve the content div_id: The id of the container div Returns: Html markup of the container div """ div_id = div_id or str(uuid.uuid1()) return _.div(id=div_id)[spinner(), _.script[""" (function() { // immediately (even before the DOM is completely loaded) set the height of the div // to the last content height (stored in local storage) to avoid height flickering var divHeightKey = 'div-height--' + window.location.pathname + '--' + '""" + url + """'; var divHeight = localStorage.getItem(divHeightKey); if (divHeight) { document.getElementById('""" + div_id + """').style.height = divHeight + 'px'; } document.addEventListener('DOMContentLoaded', function() { if (typeof loadContentAsynchronously == 'undefined') { console.error('Please implement function "loadContentAsynchronously"'); } else { loadContentAsynchronously('""" + div_id + """', '""" + url + """', divHeightKey); } }); })(); """]]
def system_stats(path: str, run_id: int): node, __ = pipelines.find_node(path.split('/')) run_id = run_id or _latest_run_id(node.path()) if not run_id: return '' with mara_db.postgresql.postgres_cursor_context( 'mara') as cursor: # type: psycopg2.extensions.cursor cursor.execute( f''' SELECT -- needs to be spelled out to be able to rely on the order in the postprocessing of the row -- run_id is not needed in the frontend... stats.timestamp, stats.disc_read, stats.disc_write, stats.net_recv, stats.net_sent, stats.cpu_usage, stats.mem_usage, stats.swap_usage, stats.iowait FROM data_integration_node_run nr JOIN data_integration_system_statistics stats ON stats.timestamp BETWEEN nr.start_time AND nr.end_time -- -1 is fallback for old cases where we didn't have a node ID -> can be removed after 2021-01-01 or so AND (stats.run_id = nr.run_id OR stats.run_id = -1) WHERE nr.run_id = {"%s"} AND nr.node_path = {"%s"};''', (run_id, node.path())) data = [[row[0].isoformat()] + list(row[1:]) for row in cursor.fetchall()] if len(data) >= 15: return str(_.div(id='system-stats-chart', class_='google-chart')[' ']) \ + str(_.script[f'nodePage.showSystemStats({json.dumps(data)});']) else: return ''
def system_stats(path: str, run_id: int): node, __ = pipelines.find_node(path.split('/')) run_id = run_id or _latest_run_id(node.path()) if not run_id: return '' with mara_db.postgresql.postgres_cursor_context( 'mara') as cursor: # type: psycopg2.extensions.cursor cursor.execute( f''' SELECT data_integration_system_statistics.* FROM data_integration_system_statistics JOIN data_integration_node_run ON timestamp BETWEEN start_time AND end_time WHERE run_id = {"%s"} AND node_path = {"%s"};''', (run_id, node.path())) data = [[row[0].isoformat()] + list(row[1:]) for row in cursor.fetchall()] if len(data) >= 15: return str(_.div(id='system-stats-chart', class_='google-chart')[' ']) \ + str(_.script[f'nodePage.showSystemStats({json.dumps(data)});']) else: return ''
def start_page(): import mara_pipelines.config from mara_data_explorer.data_set import find_data_set data_set_for_preview = find_data_set('order_items') assert (data_set_for_preview) return response.Response( title='MyCompany BI', html=_.div(class_='row')[_.div( class_='col-lg-6' )[bootstrap. card(header_left=_.b['Welcome'], body=[ _. p['This is the first thing that users of your data warehouse will see. ', 'Please add links to relevant documentation, tutorials & other ', 'data tools in your organization.'], _.p[ _.a(href='https://github.com/mara/mara-example-project-1/blob/master/app/ui/start_page.py' )[ 'Here'], ' is the source code for this page, and here is a picture of a ', _.a(href='https://en.wikipedia.org/wiki/Mara_(mammal)' )[ 'mara'], ':'], _. img(src=flask.url_for('ui.static', filename='mara.jpg'), style ='width:40%; margin-left: auto; margin-right:auto; display:block;' ) ]), bootstrap.card(header_left=[ _.b[_.a(href=flask.url_for('mara_metabase.metabase'))[_.span( class_='fa fa-bar-chart')[''], ' Metabase']], ' & ', _. b[_.a(href=flask.url_for('mara_mondrian.saiku'))[_.span( class_='fa fa-bar-chart')[''], ' Saiku']], ': Company wide dashboards, pivoting & ad hoc analysis' ], body=[ _. p['Metabase tutorial: ', _. a(href= 'https://www.metabase.com/docs/latest/getting-started.html' ) ['https://www.metabase.com/docs/latest/getting-started.html']], _. p['Saiku introduction: ', _. a(href= 'https://saiku-documentation.readthedocs.io/en/latest/' ) ['https://saiku-documentation.readthedocs.io/en/latest/']] ]), bootstrap.card(header_left=[ _.b[_.a(href=flask.url_for('mara_data_explorer.index_page'))[ _.span(class_='fa fa-table')[''], ' Explore']], ': Raw data access & segmentation' ], body=[ _.p[_.a( href=flask. url_for('mara_data_explorer.data_set_page', data_set_id=data_set_for_preview.id ))[data_set_for_preview.name], ':', html.asynchronous_content( flask.url_for( 'mara_data_explorer.data_set_preview', data_set_id=data_set_for_preview.id) )], _. p['Other data sets: ', ', '.join([ str( _.a(href=flask.url_for( 'mara_data_explorer.data_set_page', data_set_id=ds.id))[ds.name]) for ds in mara_data_explorer.config.data_sets( ) if ds.id != data_set_for_preview.id ])] ])], _.div(class_='col-lg-6') [bootstrap. card(header_left=[ _.b[_.a(href=flask.url_for( 'mara_schema.index_page'))[_.span( class_='fa fa-book' )[''], ' Data sets']], ': Documentation of attributes and metrics of all data sets' ], body=html.asynchronous_content( url=flask. url_for('mara_schema.overview_graph' ))), bootstrap.card(header_left=[ _.b[_.a( href=flask. url_for('mara_pipelines.node_page') )[_.span(class_='fa fa-wrench')[''], ' Pipelines']], ': The data integration pipelines that create the DWH' ], body=html. asynchronous_content( flask.url_for( 'mara_pipelines.dependency_graph', path='/'))), bootstrap.card(header_left=[ _.b[_.a(href=flask. url_for('mara_db.index_page') )[_.span( class_='fa fa-database')[''], ' Database Schemas']], ': Schemas of all databases connections' ], body=[ html.asynchronous_content( flask.url_for( 'mara_db.draw_schema', db_alias= mara_pipelines. config. default_db_alias( ), schemas='ec_dim' ) + '?hide-columns=True') ])]])
def data_set_page(id: str) -> response.Response: """Renders the pages for individual data sets""" from ..config import data_sets data_set = next( (data_set for data_set in data_sets() if data_set.id() == id), None) if not data_set: flask.flash(f'Could not find data set "{id}"', category='warning') return flask.redirect(flask.url_for('mara_schema.index_page')) base_url = flask.url_for('mara_schema.data_set_sql_query', id=data_set.id()) def attribute_rows(data_set: DataSet) -> []: rows = [] for path, attributes in data_set.connected_attributes().items(): if path: rows.append(_.tr[_.td( colspan=3, style='border-top:none; padding-top: 20px;' )[[[ '→ ', _.a( href=data_set_url(entity.data_set) )[link_title] if entity.data_set else link_title, ' ' ] for entity, link_title in [( entity_link.target_entity, entity_link.prefix or entity_link.target_entity.name) for entity_link in path]], [' ', _. i[path[-1].description]] if path[-1].description else '']]) for prefixed_name, attribute in attributes.items(): rows.append(_.tr[_.td[escape(prefixed_name)], _.td[_.i[escape( attribute.description )]], _.td[_.tt[escape( f'{path[-1].target_entity.table_name + "." if path else ""}{attribute.column_name}' )]]]) return rows return response.Response( html=[ bootstrap.card( header_left=_.i[escape(data_set.entity.description)], body=[ _.p['Entity table: ', _.code[escape( f'{data_set.entity.schema_name}.{data_set.entity.table_name}' )]], html.asynchronous_content( flask.url_for('mara_schema.data_set_graph', id=data_set.id())), ]), bootstrap.card( header_left='Metrics', body=[ html.asynchronous_content( flask.url_for('mara_schema.metrics_graph', id=data_set.id())), bootstrap.table(['Name', 'Description', 'Computation'], [[ _.tr[_.td[escape(metric.name)], _.td[_.i[escape(metric.description)]], _.td[_.code[escape(metric.display_formula())]]] for metric in data_set.metrics.values() ]]), ]), bootstrap.card(header_left='Attributes', body=bootstrap.table( ["Name", "Description", "Column name"], attribute_rows(data_set))), bootstrap.card( header_left=[ 'Data set sql query: ', [ _.div(class_='form-check form-check-inline') [" ", _.label(class_='form-check-label')[ _.input(class_="form-check-input param-checkbox", type="checkbox", value=param)[''], ' ', param]] for param in [ 'human readable columns', 'pre-computed metrics', 'star schema', 'personal data', 'high cardinality attributes', ] ] ], body=[ _.div(id='sql-container')[html.asynchronous_content( base_url, 'sql-container')], _.script[''' document.addEventListener('DOMContentLoaded', function() { DataSetSqlQuery("''' + base_url + '''"); }); '''] ]) ], title=f'Data set "{data_set.name}"', js_files=[ flask.url_for('mara_schema.static', filename='data-set-sql-query.js') ], )
def content_area(response: mara_page.response.Response) -> xml.XMLElement: """Renders the main content area""" return _.div(id='mara-main', _class='container-fluid')[response.get_data(as_text=True)]
def data_set_page(data_set_id, query_id): ds = find_data_set(data_set_id) if not ds: flask.flash(f'Data set "{data_set_id}" does not exist anymore', category='danger') return flask.redirect(flask.url_for('data_sets.index_page')) action_buttons = [ response.ActionButton(action='javascript:dataSetPage.downloadCSV()', icon='download', label='CSV', title='Download as CSV'), response.ActionButton(action='javascript:dataSetPage.load()', icon='folder-open', label='Load', title='Load previously saved query'), response.ActionButton(action='javascript:dataSetPage.save()', icon='save', label='Save', title='Save query') ] if query_id: action_buttons.insert( 1, response.ActionButton(action=flask.url_for( 'data_sets._delete_query', data_set_id=data_set_id, query_id=query_id), icon='trash', label='Delete', title='Delete query')) return response.Response( title=f'Query "{query_id}" on "{ds.name}"' if query_id else f'New query on "{ds.name}"', html=[ _.div( class_='row')[_.div( class_='col-md-3' )[bootstrap.card(header_left='Query', body=_.div( id='query-details')[html.spinner()]), bootstrap. card(header_left='Columns', body=[ _.div( class_="form-group" )[_.input(type="search", class_="columns-search form-control", value="", placeholder="Filter")], _.div(id='columns-list')[html.spinner()] ])], _.div(class_='col-md-9')[bootstrap.card( id='filter-card', header_left=[ _.div(class_="dropdown")[_.a( **{ 'class': 'dropdown-toggle', 'data-toggle': 'dropdown', 'href': '#' } )[_.span( class_='fa fa-plus')[' '], ' Add filter'], _. div(class_= "dropdown-menu", id= 'filter-menu' ) [_.div( class_ ="dropdown-item" )[_.input( type ="text", class_= "columns-search form-control", value="", placeholder= "Filter")]]] ], fixed_header_height=False, body=_.div(id='filters')[html.spinner()]), bootstrap. card(header_left=_.div( id='row-counts' )[html.spinner()], header_right=_.div( id='pagination' )[html.spinner()], body=_.div( id='preview' )[html.spinner()]), _. div(class_='row', id= 'distribution-charts' )['']]], _.script[f""" var dataSetPage = null; document.addEventListener('DOMContentLoaded', function() {{ dataSetPage = DataSetPage('{flask.url_for('data_sets.index_page')}', {json.dumps({'data_set_id': data_set_id, 'query_id': query_id, 'query': flask.request.get_json()})}, 15, '{config.charts_color()}'); }}); """], html.spinner_js_function(), _.div(class_='col-xl-4 col-lg-6', id='distribution-chart-template', style='display: none')[bootstrap.card( header_left=html.spinner(), body=_.div(class_='chart-container google-chart')[ html.spinner()])], _.div( class_='modal fade', id='load-query-dialog', tabindex="-1")[_.div( class_='modal-dialog', role='document')[_.div( class_='modal-content')[_.div(class_='modal-header')[ _.h5(class_='modal-title')['Load query'], _.button( **{ 'type': "button", 'class': "close", 'data-dismiss': "modal", 'aria-label': "Close" })[_.span( **{'aria-hidden': 'true'})['×']]], _.div(class_='modal-body', id='query-list')['']]]], _.form( action=flask.url_for('data_sets.download_csv'), method='post')[_.div( class_="modal fade", id="download-csv-dialog", tabindex="-1" )[_.div(class_="modal-dialog", role='document')[_.div( class_="modal-content" )[_.div(class_="modal-header") [_.h5(class_='modal-title')['Download as CSV'], _.button( **{ 'type': "button", 'class': "close", 'data-dismiss': "modal", 'aria-label': "Close" })[_.span( **{'aria-hidden': 'true'})['×']]], _.div(class_="modal-body")[ 'Delimiter:  ', _.input(type="radio", value="\t", name="delimiter", checked="checked"), ' tab   ', _.input(type="radio", value=";", name="delimiter"), ' semicolon   ', _.input(type="radio", value=",", name="delimiter"), ' comma   ', _.hr, 'Number format:  ', _.input(type="radio", value=".", name="decimal-mark", checked="checked"), ' 42.7   ', _.input(type="radio", value=",", name="decimal-mark"), ' 42,7   ', _.input(type="hidden", name="query")], _.div(class_="modal-footer" )[_.button(id="csv-download-button", type="submit", class_="btn btn-primary")['Download']]]]]] ], action_buttons=action_buttons, js_files=[ 'https://www.gstatic.com/charts/loader.js', flask.url_for('data_sets.static', filename='tagsinput.js'), flask.url_for('data_sets.static', filename='typeahead.js'), flask.url_for('data_sets.static', filename='data-sets.js') ], css_files=[ flask.url_for('data_sets.static', filename='tagsinput.css'), flask.url_for('data_sets.static', filename='data-sets.css') ])
def view_plots(): return response.Response( html=[ _.div(id='forecast-container')[_.div(class_="")[[ [ _.div(class_='row')[_.div(class_='col-xl-12')[_.div( class_='section-header')[forecast.metric_name]]], _.div(class_='row')[_.div(class_='col-xl-12')[ _.p()['Number of days: ' + str(forecast.number_of_days), _.br(), 'Time-series query: ', html.highlight_syntax(forecast.time_series_query, language='postgresql')], bootstrap.card( header_left='Forecast plot of "' + forecast.metric_name + '"', header_right=_.div()[ # config.forecast_table_name # _.a(class_='query-control', style='margin-right: 10px', # href='#')[ # _.span(class_='fa fa-download')[' ']], _. a(class_='query-control', href=f"javascript:showQueryDetails('" f"{flask.url_for('forecasts.query_details', name=forecast.metric_name)}')" )[_.span(class_='fa fa-eye')[' ']]], body=[ html.asynchronous_content( flask.url_for('forecasts.get_plot_image', name=forecast.metric_name. lower().replace(' ', '_'). replace('-', '_'), components='False')), _.br(), _.br(), _.div(class_='modal fade', id='query-details-dialog', tabindex="-1" )[_.div(class_='modal-dialog modal-lg', role='document')[_.div( class_='modal-content' )[_.div( class_='modal-header' )[_.h5(class_='modal-title' )['Time-series query'], _.button( **{ 'type': "button", 'class': "close", 'data-dismiss': "modal", 'aria-label': "Close" } )[_.span( **{'aria-hidden': 'true'} )['×']]], _.div(class_='modal-body', id='query-details' )['']]]], ])], _. div(class_='col-xl-12')[bootstrap.card( header_left= 'Forecast components (trend, holidays, seasonality) of "{}"' .format(forecast.metric_name), header_right='', body=[ html.asynchronous_content( flask.url_for( 'forecasts.get_plot_image', name=forecast. metric_name.lower( ).replace(' ', '_'). replace('-', '_'), components='True')), ])], _.hr()] ] for forecast in config.forecasts() ]]], _.script[''''''] ], title='Forecasts', css_files=[ 'https://fonts.googleapis.com/css?family=Open+Sans:300,400,700', flask.url_for('forecasts.static', filename='forecast.css') ], js_files=[ flask.url_for('forecasts.static', filename='forecast.js'), 'https://www.gstatic.com/charts/loader.js' ])
def run_page(path: str, with_upstreams: bool, ids: str): if not config.allow_run_from_web_ui(): flask.abort( 403, 'Running piplelines from web ui is disabled for this instance') # the pipeline to run pipeline, found = pipelines.find_node(path.split('/')) if not found: flask.abort(404, f'Pipeline "{path}" not found') assert (isinstance(pipeline, pipelines.Pipeline)) # a list of nodes to run selectively in the pipeline nodes = [] for id in (ids.split('/') if ids else []): node = pipeline.nodes.get(id) if not node: flask.abort(404, f'Node "{id}" not found in pipeline "{path}"') else: nodes.append(node) stream_url = flask.url_for('mara_pipelines.do_run', path=path, with_upstreams=with_upstreams, ids=ids) title = [ 'Run ', 'with upstreams ' if with_upstreams else '', ' / '.join([ str(_.a(href=views.node_url(parent))[parent.id]) for parent in pipeline.parents()[1:] ]) ] if nodes: title += [ ' / [', ', '.join([ str(_.a(href=views.node_url(node))[node.id]) for node in nodes ]), ']' ] return response.Response( html=[ _.script[''' document.addEventListener('DOMContentLoaded', function() { processRunEvents(''' + json.dumps( flask.url_for('mara_pipelines.node_page', path='')) + ', ' + json.dumps(stream_url) + ', ' + json.dumps(pipeline.path()) + '''); });'''], _.style[ 'span.action-buttons > * {display:none}'], # hide reload button until run finishes _.div(class_='row') [_.div(class_='col-lg-7')[bootstrap.card(body=_.div( id='main-output-area', class_='run-output')[''])], _.div(class_='col-lg-5 scroll-container')[ bootstrap. card(header_left='Timeline', body=[ _.div(id='system-stats-chart', class_='google-chart' )[' '], _.div(id='timeline-chart')[' '] ]), _.div(id='failed-tasks-container')[''], _.div(id='running-tasks-container')[''], _.div(id='succeeded-tasks-container')[''], bootstrap.card(id='card-template', header_left=' ', header_right=' ', body=[_.div(class_='run-output')['']])]] ], js_files=[ 'https://www.gstatic.com/charts/loader.js', flask.url_for('mara_pipelines.static', filename='timeline-chart.js'), flask.url_for('mara_pipelines.static', filename='system-stats-chart.js'), flask.url_for('mara_pipelines.static', filename='utils.js'), flask.url_for('mara_pipelines.static', filename='run-page.js') ], css_files=[ flask.url_for('mara_pipelines.static', filename='timeline-chart.css'), flask.url_for('mara_pipelines.static', filename='run-page.css'), flask.url_for('mara_pipelines.static', filename='common.css') ], action_buttons=[ response.ActionButton( action='javascript:location.reload()', label='Run again', icon='play', title='Run pipeline again with same parameters as before') ], title=title, )
def data_set_page(data_set_id, query_id): from .data_set import find_data_set ds = find_data_set(data_set_id) if not ds: flask.flash(f'Data set "{data_set_id}" does not exist anymore', category='danger') return flask.redirect(flask.url_for('mara_data_explorer.index_page')) action_buttons = [] action_buttons.append(response.ActionButton(action='javascript:dataSetPage.downloadCSV()', icon='download', label='CSV', title='Download as CSV')) if config.google_sheet_oauth2_client_config(): action_buttons.append(response.ActionButton(action='javascript:dataSetPage.exportToGoogleSheet()', icon='cloud-upload', label='Google sheet', title='Export to a Google sheet')) action_buttons.append(response.ActionButton(action='javascript:dataSetPage.load()', icon='folder-open', label='Load', title='Load previously saved query')) action_buttons.append(response.ActionButton(action='javascript:dataSetPage.save()', icon='save', label='Save', title='Save query')) action_buttons.append(response.ActionButton(action='javascript:dataSetPage.displayQuery()', icon='eye', label='SQL', title='Display query')) if query_id: action_buttons.insert(1, response.ActionButton( action=flask.url_for('mara_data_explorer._delete_query', data_set_id=data_set_id, query_id=query_id), icon='trash', label='Delete', title='Delete query')) return response.Response( title=f'Query "{query_id}" on "{ds.name}"' if query_id else f'New query on "{ds.name}"', html=[_.div(class_='row')[ _.div(class_='col-md-3')[ bootstrap.card(header_left='Query', body=_.div(id='query-details')[html.spinner()]), bootstrap.card(header_left='Columns', header_right=_.a(id='select-all', href='#')[' Select all'], body=[_.div(class_="form-group")[ _.input(type="search", class_="columns-search form-control", value="", placeholder="Filter")], _.div(id='columns-list')[html.spinner()]])], _.div(class_='col-md-9')[ bootstrap.card( id='filter-card', header_left=[_.div(class_="dropdown")[ _.a(**{'class': 'dropdown-toggle', 'data-toggle': 'dropdown', 'href': '#'})[ _.span(class_='fa fa-plus')[' '], ' Add filter'], _.div(class_="dropdown-menu", id='filter-menu')[ _.div(class_="dropdown-item")[ _.input(type="text", class_="columns-search form-control", value="", placeholder="Filter")]]]], fixed_header_height=False, body=_.div(id='filters')[html.spinner()]), bootstrap.card(header_left=_.div(id='row-counts')[html.spinner()], header_right=_.div(id='pagination')[html.spinner()], body=_.div(id='preview')[html.spinner()]), _.div(class_='row', id='distribution-charts')[''] ]], _.script[f""" var dataSetPage = null; document.addEventListener('DOMContentLoaded', function() {{ dataSetPage = DataSetPage('{flask.url_for('mara_data_explorer.index_page')}', {json.dumps( {'data_set_id': data_set_id, 'query_id': query_id, 'query': flask.request.get_json()})}, 15, '{config.charts_color()}'); }}); """], html.spinner_js_function(), _.div(class_='col-xl-4 col-lg-6', id='distribution-chart-template', style='display: none')[ bootstrap.card(header_left=html.spinner(), body=_.div(class_='chart-container google-chart')[ html.spinner()])], _.div(class_='modal fade', id='load-query-dialog', tabindex="-1")[ _.div(class_='modal-dialog', role='document')[ _.div(class_='modal-content')[ _.div(class_='modal-header')[ _.h5(class_='modal-title')['Load query'], _.button(**{'type': "button", 'class': "close", 'data-dismiss': "modal", 'aria-label': "Close"})[ _.span(**{'aria-hidden': 'true'})['×']]], _.div(class_='modal-body', id='query-list')[''] ] ] ], _.div(class_='modal fade', id='display-query-dialog', tabindex="-1")[ _.div(class_='modal-dialog', role='document')[ _.div(class_='modal-content')[ _.div(class_='modal-header')[ _.h5(class_='modal-title')['Query statement'], _.button(**{'type': "button", 'class': "close", 'data-dismiss': "modal", 'aria-label': "Close"})[ _.span(**{'aria-hidden': 'true'})['×']]], _.div(class_='modal-body', id='query-display')[''] ] ] ], _.form(action=flask.url_for('mara_data_explorer.download_csv', data_set_id=data_set_id), method='post')[ _.div(class_="modal fade", id="download-csv-dialog", tabindex="-1")[ _.div(class_="modal-dialog", role='document')[ _.div(class_="modal-content")[ _.div(class_="modal-header")[ _.h5(class_='modal-title')['Download as CSV'], _.button(**{'type': "button", 'class': "close", 'data-dismiss': "modal", 'aria-label': "Close"})[ _.span(**{'aria-hidden': 'true'})['×']]], _.div(class_="modal-body")[ 'Delimiter:  ', _.input(type="radio", value="\t", name="delimiter", checked="checked"), ' tab   ', _.input(type="radio", value=";", name="delimiter"), ' semicolon   ', _.input(type="radio", value=",", name="delimiter"), ' comma   ', _.hr, 'Number format:  ', _.input(type="radio", value=".", name="decimal-mark", checked="checked"), ' 42.7   ', _.input(type="radio", value=",", name="decimal-mark"), ' 42,7   ', _.input(type="hidden", name="query")], _.div(class_="modal-footer")[ _.button(id="csv-download-button", type="submit", class_="btn btn-primary")[ 'Download']]]]]], _.form(action=flask.url_for('mara_data_explorer.oauth2_export_to_google_sheet', data_set_id=data_set_id), method='post', target="_blank")[ _.div(class_="modal fade", id="google-sheet-export-dialog", tabindex="-1")[ _.div(class_="modal-dialog", role='document')[ _.div(class_="modal-content")[ _.div(class_="modal-header")[ _.h5(class_='modal-title')['Google sheet export'], _.button(**{'type': "button", 'class': "close", 'data-dismiss': "modal", 'aria-label': "Close"})[ _.span(**{'aria-hidden': 'true'})['×']]], _.div(class_="modal-body")[ 'Number format:  ', _.input(type="radio", value=".", name="decimal-mark", checked="checked"), ' 42.7   ', _.input(type="radio", value=",", name="decimal-mark"), ' 42,7   ', _.hr, 'Array format:  ', _.input(type="radio", value="curly", name="array-format", checked="checked"), ' {"a", "b"}   ', _.input(type="radio", value="normal", name="array-format"), ' ["a", "b"]   ', _.input(type="radio", value="tuple", name="array-format"), ' ("a", "b")   ', _.hr, 'By clicking Export below:', _.br, _.ul[ _.li['Google authentication will be required.'], _.li['A maximum limit of 100.000 rows will be applied.'], _.li['A maximum limit of 50.000 characters per cell will be applied.'], _.li['A Google sheet with the selected data will be available in a new tab.'] ], _.input(type="hidden", name="query") ], _.div(class_="modal-footer")[ _.button(id="export-to-google-sheet", type="submit", class_="btn btn-primary")[ 'Export']]]]]] ], action_buttons=action_buttons, js_files=['https://www.gstatic.com/charts/loader.js', flask.url_for('mara_data_explorer.static', filename='tagsinput.js'), flask.url_for('mara_data_explorer.static', filename='typeahead.js'), flask.url_for('mara_data_explorer.static', filename='data-sets.js')], css_files=[flask.url_for('mara_data_explorer.static', filename='tagsinput.css'), flask.url_for('mara_data_explorer.static', filename='data-sets.css')])
def content_area(response: mara_page.response.Response) -> xml.XMLElement: """Renders the main content area""" return _.div(id='mara-main', class_='container-fluid')[response.response]