def auto_complete():
from .data_set import find_data_set
ds = find_data_set(flask.request.args['data-set-id'])
column_name = flask.request.args['column-name']
if not acl.current_user_has_permission(data_set_acl_resources[ds.id]):
return flask.jsonify([])
elif not acl.current_user_has_permission(
personal_data_acl_resource) and column_name in ds.personal_data_column_names:
return flask.jsonify([])
else:
return flask.jsonify(ds.autocomplete_text_column(column_name, flask.request.args['term']))
def distribution_chart(pos: int):
from .query import Query
query = Query.from_dict(flask.request.json)
column = list(query.data_set.columns.values())[pos]
if not current_user_has_permission(query):
return flask.make_response(acl.inline_permission_denied_message(), 403)
elif column.column_name in query.data_set.personal_data_column_names \
and not acl.current_user_has_permission(personal_data_acl_resource):
return flask.make_response(
acl.inline_permission_denied_message('Restricted personal data'), 403)
else:
if column.type == 'number':
data = query.number_distribution(column.column_name)
elif column.type == 'text':
data = query.text_distribution(column.column_name)
elif column.type == 'text[]':
data = query.text_array_distribution(column.column_name)
elif column.type == 'date':
data = query.date_distribution(column.column_name)
else:
data = []
return flask.jsonify({'column': column.to_dict(), 'data': data})
def preview():
from .query import Query
from .data_set import Column
query = Query.from_dict(flask.request.json['query'])
def header(column: Column):
if column.sortable():
if query.sort_column_name == column.column_name and query.sort_order == 'ASC':
icon = _.span(class_=('fa fa-sort-amount-asc'))['']
elif query.sort_column_name == column.column_name and query.sort_order == 'DESC':
icon = _.span(class_=('fa fa-sort-amount-desc'))['']
else:
icon = ''
return _.a(href="#", name=flask.escape(column.column_name))[
icon, ' ',
flask.escape(column.column_name)]
else:
return flask.escape(column.column_name)
if current_user_has_permission(query):
rows = [_render_preview_row(query, row)
for row in query.run(limit=flask.request.json['limit'], offset=flask.request.json['offset'],
include_personal_data=acl.current_user_has_permission(personal_data_acl_resource))]
else:
rows = _.tr[_.td(colspan=len(query.column_names))[acl.inline_permission_denied_message()]]
if rows:
return str(bootstrap.table(headers=[header(query.data_set.columns[c]) for c in query.column_names], rows=rows))
else:
return '∅'
def __(task: pipelines.ParallelTask):
if not acl.current_user_has_permission(views.acl_resource):
return bootstrap.card(
header_left=acl.inline_permission_denied_message())
else:
return [
bootstrap.card(header_left='Commands before',
fixed_header_height=True,
sections=[
_render_command(command)
for command in task.commands_before
]) if task.commands_before else '',
bootstrap.card(header_left='Sub task creation',
body=bootstrap.table([], [
_.tr[_.td[_.div[section]],
_.td(style='width:90%')[content]]
for section, content in task.html_doc_items()
])),
bootstrap.card(header_left='Commands after',
fixed_header_height=True,
sections=[
_render_command(command)
for command in task.commands_after
]) if task.commands_after else ''
]
def configuration_page():
config = get_config_for_display()
current_user_has_permission = acl.current_user_has_permission(acl_resource)
return response.Response(html=[(bootstrap.card(
id=module_name,
header_left=html.escape(module_name),
body=[
_.p[_.em[html.escape(str(module_content.doc))]],
bootstrap.table([], [
_.tr[_.td[
_.tt[html.escape(config_name).replace('_', '_<wbr/>')], [
_.br, ' ⟵ ', _.
tt[html.escape(config_function.func_desc).
replace('.', '<wbr/>.').replace('_', '_<wbr/>')]
] if config_function.set_func else ''],
_.td[_.em[html.escape(config_function.doc)]], _.
td[_.pre[html.
escape(str(config_function)
)] if current_user_has_permission else acl
.inline_permission_denied_message()]]
for config_name, config_function in module_content.items()
])
]) if module_content.config_functions else '') for module_name,
module_content in config.items()],
title='Mara Configuration')
def configuration_page():
import pprint
from . import app
# gather all config functions by package
current_user_has_permission = acl.current_user_has_permission(acl_resource)
return response.Response(html=[(bootstrap.card(
id=module_name,
header_left=html.escape(module_name),
body=[
_.p[_.em[html.escape(str(config['doc']))]],
bootstrap.table([], [
_.tr[_.td[
_.tt[html.escape(function_name).replace('_', '_<wbr/>')], [
_.br, ' ⟵ ', _.
tt[html.escape(function['new_function']).
replace('.', '<wbr/>.').replace('_', '_<wbr/>')]
] if function['new_function'] else ''],
_.td[_.em[html.escape(function['doc'])]], _.
td[_.pre[html.
escape(pprint.pformat(function['value'])
)] if current_user_has_permission else acl
.inline_permission_denied_message()]]
for function_name, function in config['functions'].items()
])
]) if config['functions'] else '') for module_name, config in sorted(
_config_modules().items())],
title='Mara Configuration')
def __(task: pipelines.Task):
if not acl.current_user_has_permission(views.acl_resource):
return bootstrap.card(header_left='Commands', body=acl.inline_permission_denied_message())
else:
return bootstrap.card(
header_left='Commands',
fixed_header_height=True,
sections=[_render_command(command) for command in task.commands])
def _delete_query(data_set_id, query_id):
from .query import delete_query
if not acl.current_user_has_permission(data_set_acl_resources[data_set_id]):
flask.flash(f'Not enough permissions to delete queries from "{data_set_id}"', 'danger')
else:
delete_query(data_set_id, query_id)
flask.flash(f'Deleted query "{query_id}" from "{data_set_id}"', 'success')
return flask.redirect(flask.url_for('mara_data_explorer.data_set_page', data_set_id=data_set_id))
def download_csv(data_set_id):
from .query import Query
query = Query.from_dict(json.loads(flask.request.form['query']))
if not current_user_has_permission(query):
return flask.abort(403, 'Not enough permissions to download this data set')
else:
file_name = query.data_set_id + ('-' + query.query_id if query.query_id else '') \
+ '-' + datetime.date.today().isoformat() + '.csv'
response = flask.make_response(query.as_csv(flask.request.form['delimiter'], flask.request.form['decimal-mark'],
acl.current_user_has_permission(personal_data_acl_resource)))
response.headers['Content-type'] = 'text/csv; charset = utf-8'
response.headers['Content-disposition'] = f'attachment; filename="{file_name}"'
return response
def data_set_preview(data_set_id):
from .query import Query
query = Query(data_set_id=data_set_id)
if query.column_names:
if current_user_has_permission(query):
rows = [_render_preview_row(query, row) for row
in query.run(limit=7, offset=0,
include_personal_data=acl.current_user_has_permission(personal_data_acl_resource))]
else:
rows = _.tr[_.td(colspan=len(query.column_names))[acl.inline_permission_denied_message()]]
return str(
bootstrap.table(headers=[flask.escape(column_name) for column_name in query.column_names], rows=rows))
else:
return '∅'
def current_user_has_permission(query: 'Query') -> bool:
"""Checks whether the current user has permissions to query the data set"""
return acl.current_user_has_permission(data_set_acl_resources[query.data_set.id])
def google_sheet_oauth2callback():
try:
import google_auth_oauthlib.flow
import googleapiclient.discovery
except ImportError:
return flask.make_response(
str(_.tt(style='color:red')[
"Please install the packages 'google_auth_oauthlib' and 'google-api-python-client'"]),
500)
from .query import Query
query = Query.from_dict(flask.session['query_for_google_sheet_callback'])
# Specify the state when creating the flow in the callback so that it can
# verified in the authorization server response.
state = flask.session['state']
flow = google_auth_oauthlib.flow.Flow.from_client_config(
config.google_sheet_oauth2_client_config(),
scopes=SCOPES,
state=state)
flow.redirect_uri = flask.url_for('mara_data_explorer.google_sheet_oauth2callback', _external=True)
# Use the authorization server's response to fetch the OAuth 2.0 tokens.
authorization_response = flask.request.url
flow.fetch_token(authorization_response=authorization_response)
spreadsheet_title = query.data_set_id + ('-' + query.query_id if query.query_id else '') \
+ '-' + datetime.date.today().isoformat()
decimal_mark = flask.session.pop('decimal_mark')
array_format = flask.session.pop('array_format')
credentials = flow.credentials
service = googleapiclient.discovery.build('sheets', 'v4', credentials=credentials)
spreadsheet_body = {
'properties': {
'title': spreadsheet_title,
# Determine decimal-mark through the Google sheet locale
'locale': 'de_DE' if decimal_mark == ',' else 'en_US'
}
}
spreadsheet = service.spreadsheets().create(body=spreadsheet_body, fields='spreadsheetId')
api_response = spreadsheet.execute()
spreadsheet_id = api_response.get('spreadsheetId')
# Upload data from generator in batches of 10K rows each. Each batch is a request
# Api limits: https://developers.google.com/sheets/api/limits
data_batch = []
google_sheet_rows = query.as_rows_for_google_sheet(
array_format=array_format,
limit=100000,
include_personal_data=acl.current_user_has_permission(personal_data_acl_resource))
row_count = 0
batch_length = 10000
for row in google_sheet_rows:
row_count += 1
data_batch.append(row)
if row_count % batch_length == 0:
body = {
"data": [
{
"values": data_batch,
"range": "A" + str(row_count - batch_length + 1),
"majorDimension": "ROWS"
}
],
# USER_ENTERED: The values will be parsed as if the user typed them into the UI.
# Numbers will stay as numbers, but strings may be converted to numbers, dates, etc.
# RAW: The values the user has entered will not be parsed and will be stored as-is.
"valueInputOption": "USER_ENTERED"
}
# Request of the current batch
service.spreadsheets().values().batchUpdate(spreadsheetId=spreadsheet_id, body=body).execute()
# Initialize next batch
data_batch = []
# Append remaining or less batch_length data if any
if row_count < batch_length:
body = {"data": [{"values": data_batch, "range": "A1", "majorDimension": "ROWS"}],
"valueInputOption": "USER_ENTERED"}
service.spreadsheets().values().batchUpdate(spreadsheetId=spreadsheet_id, body=body).execute()
elif row_count % batch_length > 0:
body = {"data": [{"values": data_batch, "range": "A" + str(row_count - len(data_batch) + 1),
"majorDimension": "ROWS"}],
"valueInputOption": "USER_ENTERED"}
service.spreadsheets().values().batchUpdate(spreadsheetId=spreadsheet_id, body=body).execute()
return flask.redirect('https://docs.google.com/spreadsheets/d/' + str(spreadsheet_id))
