def auto_complete(): from .data_set import find_data_set ds = find_data_set(flask.request.args['data-set-id']) column_name = flask.request.args['column-name'] if not acl.current_user_has_permission(data_set_acl_resources[ds.id]): return flask.jsonify([]) elif not acl.current_user_has_permission( personal_data_acl_resource) and column_name in ds.personal_data_column_names: return flask.jsonify([]) else: return flask.jsonify(ds.autocomplete_text_column(column_name, flask.request.args['term']))
def distribution_chart(pos: int): from .query import Query query = Query.from_dict(flask.request.json) column = list(query.data_set.columns.values())[pos] if not current_user_has_permission(query): return flask.make_response(acl.inline_permission_denied_message(), 403) elif column.column_name in query.data_set.personal_data_column_names \ and not acl.current_user_has_permission(personal_data_acl_resource): return flask.make_response( acl.inline_permission_denied_message('Restricted personal data'), 403) else: if column.type == 'number': data = query.number_distribution(column.column_name) elif column.type == 'text': data = query.text_distribution(column.column_name) elif column.type == 'text[]': data = query.text_array_distribution(column.column_name) elif column.type == 'date': data = query.date_distribution(column.column_name) else: data = [] return flask.jsonify({'column': column.to_dict(), 'data': data})
def preview(): from .query import Query from .data_set import Column query = Query.from_dict(flask.request.json['query']) def header(column: Column): if column.sortable(): if query.sort_column_name == column.column_name and query.sort_order == 'ASC': icon = _.span(class_=('fa fa-sort-amount-asc'))[''] elif query.sort_column_name == column.column_name and query.sort_order == 'DESC': icon = _.span(class_=('fa fa-sort-amount-desc'))[''] else: icon = '' return _.a(href="#", name=flask.escape(column.column_name))[ icon, ' ', flask.escape(column.column_name)] else: return flask.escape(column.column_name) if current_user_has_permission(query): rows = [_render_preview_row(query, row) for row in query.run(limit=flask.request.json['limit'], offset=flask.request.json['offset'], include_personal_data=acl.current_user_has_permission(personal_data_acl_resource))] else: rows = _.tr[_.td(colspan=len(query.column_names))[acl.inline_permission_denied_message()]] if rows: return str(bootstrap.table(headers=[header(query.data_set.columns[c]) for c in query.column_names], rows=rows)) else: return '∅'
def __(task: pipelines.ParallelTask): if not acl.current_user_has_permission(views.acl_resource): return bootstrap.card( header_left=acl.inline_permission_denied_message()) else: return [ bootstrap.card(header_left='Commands before', fixed_header_height=True, sections=[ _render_command(command) for command in task.commands_before ]) if task.commands_before else '', bootstrap.card(header_left='Sub task creation', body=bootstrap.table([], [ _.tr[_.td[_.div[section]], _.td(style='width:90%')[content]] for section, content in task.html_doc_items() ])), bootstrap.card(header_left='Commands after', fixed_header_height=True, sections=[ _render_command(command) for command in task.commands_after ]) if task.commands_after else '' ]
def configuration_page(): config = get_config_for_display() current_user_has_permission = acl.current_user_has_permission(acl_resource) return response.Response(html=[(bootstrap.card( id=module_name, header_left=html.escape(module_name), body=[ _.p[_.em[html.escape(str(module_content.doc))]], bootstrap.table([], [ _.tr[_.td[ _.tt[html.escape(config_name).replace('_', '_<wbr/>')], [ _.br, ' ⟵ ', _. tt[html.escape(config_function.func_desc). replace('.', '<wbr/>.').replace('_', '_<wbr/>')] ] if config_function.set_func else ''], _.td[_.em[html.escape(config_function.doc)]], _. td[_.pre[html. escape(str(config_function) )] if current_user_has_permission else acl .inline_permission_denied_message()]] for config_name, config_function in module_content.items() ]) ]) if module_content.config_functions else '') for module_name, module_content in config.items()], title='Mara Configuration')
def configuration_page(): import pprint from . import app # gather all config functions by package current_user_has_permission = acl.current_user_has_permission(acl_resource) return response.Response(html=[(bootstrap.card( id=module_name, header_left=html.escape(module_name), body=[ _.p[_.em[html.escape(str(config['doc']))]], bootstrap.table([], [ _.tr[_.td[ _.tt[html.escape(function_name).replace('_', '_<wbr/>')], [ _.br, ' ⟵ ', _. tt[html.escape(function['new_function']). replace('.', '<wbr/>.').replace('_', '_<wbr/>')] ] if function['new_function'] else ''], _.td[_.em[html.escape(function['doc'])]], _. td[_.pre[html. escape(pprint.pformat(function['value']) )] if current_user_has_permission else acl .inline_permission_denied_message()]] for function_name, function in config['functions'].items() ]) ]) if config['functions'] else '') for module_name, config in sorted( _config_modules().items())], title='Mara Configuration')
def __(task: pipelines.Task): if not acl.current_user_has_permission(views.acl_resource): return bootstrap.card(header_left='Commands', body=acl.inline_permission_denied_message()) else: return bootstrap.card( header_left='Commands', fixed_header_height=True, sections=[_render_command(command) for command in task.commands])
def _delete_query(data_set_id, query_id): from .query import delete_query if not acl.current_user_has_permission(data_set_acl_resources[data_set_id]): flask.flash(f'Not enough permissions to delete queries from "{data_set_id}"', 'danger') else: delete_query(data_set_id, query_id) flask.flash(f'Deleted query "{query_id}" from "{data_set_id}"', 'success') return flask.redirect(flask.url_for('mara_data_explorer.data_set_page', data_set_id=data_set_id))
def download_csv(data_set_id): from .query import Query query = Query.from_dict(json.loads(flask.request.form['query'])) if not current_user_has_permission(query): return flask.abort(403, 'Not enough permissions to download this data set') else: file_name = query.data_set_id + ('-' + query.query_id if query.query_id else '') \ + '-' + datetime.date.today().isoformat() + '.csv' response = flask.make_response(query.as_csv(flask.request.form['delimiter'], flask.request.form['decimal-mark'], acl.current_user_has_permission(personal_data_acl_resource))) response.headers['Content-type'] = 'text/csv; charset = utf-8' response.headers['Content-disposition'] = f'attachment; filename="{file_name}"' return response
def data_set_preview(data_set_id): from .query import Query query = Query(data_set_id=data_set_id) if query.column_names: if current_user_has_permission(query): rows = [_render_preview_row(query, row) for row in query.run(limit=7, offset=0, include_personal_data=acl.current_user_has_permission(personal_data_acl_resource))] else: rows = _.tr[_.td(colspan=len(query.column_names))[acl.inline_permission_denied_message()]] return str( bootstrap.table(headers=[flask.escape(column_name) for column_name in query.column_names], rows=rows)) else: return '∅'
def current_user_has_permission(query: 'Query') -> bool: """Checks whether the current user has permissions to query the data set""" return acl.current_user_has_permission(data_set_acl_resources[query.data_set.id])
def google_sheet_oauth2callback(): try: import google_auth_oauthlib.flow import googleapiclient.discovery except ImportError: return flask.make_response( str(_.tt(style='color:red')[ "Please install the packages 'google_auth_oauthlib' and 'google-api-python-client'"]), 500) from .query import Query query = Query.from_dict(flask.session['query_for_google_sheet_callback']) # Specify the state when creating the flow in the callback so that it can # verified in the authorization server response. state = flask.session['state'] flow = google_auth_oauthlib.flow.Flow.from_client_config( config.google_sheet_oauth2_client_config(), scopes=SCOPES, state=state) flow.redirect_uri = flask.url_for('mara_data_explorer.google_sheet_oauth2callback', _external=True) # Use the authorization server's response to fetch the OAuth 2.0 tokens. authorization_response = flask.request.url flow.fetch_token(authorization_response=authorization_response) spreadsheet_title = query.data_set_id + ('-' + query.query_id if query.query_id else '') \ + '-' + datetime.date.today().isoformat() decimal_mark = flask.session.pop('decimal_mark') array_format = flask.session.pop('array_format') credentials = flow.credentials service = googleapiclient.discovery.build('sheets', 'v4', credentials=credentials) spreadsheet_body = { 'properties': { 'title': spreadsheet_title, # Determine decimal-mark through the Google sheet locale 'locale': 'de_DE' if decimal_mark == ',' else 'en_US' } } spreadsheet = service.spreadsheets().create(body=spreadsheet_body, fields='spreadsheetId') api_response = spreadsheet.execute() spreadsheet_id = api_response.get('spreadsheetId') # Upload data from generator in batches of 10K rows each. Each batch is a request # Api limits: https://developers.google.com/sheets/api/limits data_batch = [] google_sheet_rows = query.as_rows_for_google_sheet( array_format=array_format, limit=100000, include_personal_data=acl.current_user_has_permission(personal_data_acl_resource)) row_count = 0 batch_length = 10000 for row in google_sheet_rows: row_count += 1 data_batch.append(row) if row_count % batch_length == 0: body = { "data": [ { "values": data_batch, "range": "A" + str(row_count - batch_length + 1), "majorDimension": "ROWS" } ], # USER_ENTERED: The values will be parsed as if the user typed them into the UI. # Numbers will stay as numbers, but strings may be converted to numbers, dates, etc. # RAW: The values the user has entered will not be parsed and will be stored as-is. "valueInputOption": "USER_ENTERED" } # Request of the current batch service.spreadsheets().values().batchUpdate(spreadsheetId=spreadsheet_id, body=body).execute() # Initialize next batch data_batch = [] # Append remaining or less batch_length data if any if row_count < batch_length: body = {"data": [{"values": data_batch, "range": "A1", "majorDimension": "ROWS"}], "valueInputOption": "USER_ENTERED"} service.spreadsheets().values().batchUpdate(spreadsheetId=spreadsheet_id, body=body).execute() elif row_count % batch_length > 0: body = {"data": [{"values": data_batch, "range": "A" + str(row_count - len(data_batch) + 1), "majorDimension": "ROWS"}], "valueInputOption": "USER_ENTERED"} service.spreadsheets().values().batchUpdate(spreadsheetId=spreadsheet_id, body=body).execute() return flask.redirect('https://docs.google.com/spreadsheets/d/' + str(spreadsheet_id))