def common_bootsrap_payload(self):
"""Common data always sent to the client"""
messages = get_flashed_messages(with_categories=True)
locale = str(get_locale())
return {
'flash_messages': messages,
'conf': {k: conf.get(k) for k in FRONTEND_CONF_KEYS},
'locale': locale,
'language_pack': get_language_pack(locale),
'feature_flags': conf.get('FEATURE_FLAGS'),
}
def sync_role_definitions():
"""Inits the Superset application with security roles and such"""
logging.info("Syncing role definition")
get_or_create_main_db()
create_custom_permissions()
pvms = db.session.query(ab_models.PermissionView).all()
pvms = [p for p in pvms if p.permission and p.view_menu]
# cleanup
pvms_to_delete = [p for p in pvms if not (p.permission and p.view_menu)]
for pvm_to_delete in pvms_to_delete:
sm.get_session.delete(pvm_to_delete)
# Creating default roles
set_role('Admin', pvms, is_admin_pvm)
set_role('Alpha', pvms, is_alpha_pvm)
set_role('Gamma', pvms, is_gamma_pvm)
set_role('granter', pvms, is_granter_pvm)
set_role('sql_lab', pvms, is_sql_lab_pvm)
if conf.get('PUBLIC_ROLE_LIKE_GAMMA', False):
set_role('Public', pvms, is_gamma_pvm)
view_menu_set = db.session.query(models.SqlaTable).all()
create_missing_datasource_perms(view_menu_set)
create_missing_database_perms(view_menu_set)
create_missing_metrics_perm(view_menu_set)
# commit role and view menu updates
sm.get_session.commit()
def get_or_create_main_db():
logging.info("Creating database reference")
dbobj = (
db.session.query(models.Database)
.filter_by(database_name='main')
.first()
)
if not dbobj:
dbobj = models.Database(database_name="main")
logging.info(conf.get("SQLALCHEMY_DATABASE_URI"))
dbobj.set_sqlalchemy_uri(conf.get("SQLALCHEMY_DATABASE_URI"))
dbobj.expose_in_sqllab = True
dbobj.allow_run_sync = True
db.session.add(dbobj)
db.session.commit()
return dbobj
def get_error_msg():
if conf.get("SHOW_STACKTRACE"):
error_msg = traceback.format_exc()
else:
error_msg = "FATAL ERROR \n"
error_msg += (
"Stacktrace is hidden. Change the SHOW_STACKTRACE "
"configuration setting to enable it")
return error_msg
def refresh_datasources(self, datasource_name=None, merge_flag=False):
"""Refresh metadata of all datasources in the cluster
If ``datasource_name`` is specified, only that datasource is updated
"""
self.druid_version = self.get_druid_version()
for datasource in self.get_datasources():
if datasource not in conf.get('DRUID_DATA_SOURCE_BLACKLIST', []):
if not datasource_name or datasource_name == datasource:
DruidDatasource.sync_to_db(datasource, self, merge_flag)
def get_error_msg():
if conf.get('SHOW_STACKTRACE'):
error_msg = traceback.format_exc()
else:
error_msg = 'FATAL ERROR \n'
error_msg += (
'Stacktrace is hidden. Change the SHOW_STACKTRACE '
'configuration setting to enable it')
return error_msg
def latest_metadata(self):
"""Returns segment metadata from the latest segment"""
client = self.cluster.get_pydruid_client()
results = client.time_boundary(datasource=self.datasource_name)
if not results:
return
max_time = results[0]['result']['maxTime']
max_time = dparse(max_time)
# Query segmentMetadata for 7 days back. However, due to a bug,
# we need to set this interval to more than 1 day ago to exclude
# realtime segments, which triggered a bug (fixed in druid 0.8.2).
# https://groups.google.com/forum/#!topic/druid-user/gVCqqspHqOQ
lbound = (max_time - timedelta(days=7)).isoformat()
rbound = max_time.isoformat()
if not self.version_higher(self.cluster.druid_version, '0.8.2'):
rbound = (max_time - timedelta(1)).isoformat()
segment_metadata = None
try:
segment_metadata = client.segment_metadata(
datasource=self.datasource_name,
intervals=lbound + '/' + rbound,
merge=self.merge_flag,
analysisTypes=conf.get('DRUID_ANALYSIS_TYPES'))
except Exception as e:
logging.warning("Failed first attempt to get latest segment")
logging.exception(e)
if not segment_metadata:
# if no segments in the past 7 days, look at all segments
lbound = datetime(1901, 1, 1).isoformat()[:10]
rbound = datetime(2050, 1, 1).isoformat()[:10]
if not self.version_higher(self.cluster.druid_version, '0.8.2'):
rbound = datetime.now().isoformat()[:10]
try:
segment_metadata = client.segment_metadata(
datasource=self.datasource_name,
intervals=lbound + '/' + rbound,
merge=self.merge_flag,
analysisTypes=conf.get('DRUID_ANALYSIS_TYPES'))
except Exception as e:
logging.warning("Failed 2nd attempt to get latest segment")
logging.exception(e)
if segment_metadata:
return segment_metadata[-1]['columns']
def get_or_create_main_db():
from superset import conf, db
from superset.models import core as models
logging.info('Creating database reference')
dbobj = get_main_database(db.session)
if not dbobj:
dbobj = models.Database(database_name='main')
dbobj.set_sqlalchemy_uri(conf.get('SQLALCHEMY_DATABASE_URI'))
dbobj.expose_in_sqllab = True
dbobj.allow_csv_upload = True
db.session.add(dbobj)
db.session.commit()
return dbobj
def refresh_datasources(
self,
datasource_name=None,
merge_flag=True,
refreshAll=True):
"""Refresh metadata of all datasources in the cluster
If ``datasource_name`` is specified, only that datasource is updated
"""
self.druid_version = self.get_druid_version()
ds_list = self.get_datasources()
blacklist = conf.get('DRUID_DATA_SOURCE_BLACKLIST', [])
ds_refresh = []
if not datasource_name:
ds_refresh = list(filter(lambda ds: ds not in blacklist, ds_list))
elif datasource_name not in blacklist and datasource_name in ds_list:
ds_refresh.append(datasource_name)
else:
return
self.refresh_async(ds_refresh, merge_flag, refreshAll)
def sync_role_definitions():
"""Inits the Superset application with security roles and such"""
logging.info("Syncing role definition")
get_or_create_main_db()
create_custom_permissions()
# Creating default roles
set_role('Admin', is_admin_pvm)
set_role('Alpha', is_alpha_pvm)
set_role('Gamma', is_gamma_pvm)
set_role('granter', is_granter_pvm)
set_role('sql_lab', is_sql_lab_pvm)
if conf.get('PUBLIC_ROLE_LIKE_GAMMA', False):
set_role('Public', is_gamma_pvm)
create_missing_perms()
# commit role and view menu updates
sm.get_session.commit()
def sync_role_definitions(self):
"""Inits the Superset application with security roles and such"""
from superset import conf
logging.info('Syncing role definition')
self.create_custom_permissions()
# Creating default roles
self.set_role('Admin', self.is_admin_pvm)
self.set_role('Alpha', self.is_alpha_pvm)
self.set_role('Gamma', self.is_gamma_pvm)
self.set_role('granter', self.is_granter_pvm)
self.set_role('sql_lab', self.is_sql_lab_pvm)
if conf.get('PUBLIC_ROLE_LIKE_GAMMA', False):
self.set_role('Public', self.is_gamma_pvm)
self.create_missing_perms()
# commit role and view menu updates
self.get_session.commit()
self.clean_perms()
from flask import g
from flask_babel import lazy_gettext as _
import pandas
from sqlalchemy import select
from sqlalchemy.engine import create_engine
from sqlalchemy.engine.url import make_url
from sqlalchemy.sql import text
import sqlparse
from werkzeug.utils import secure_filename
from superset import app, cache_util, conf, db, utils
from superset.utils import QueryStatus, SupersetTemplateException
config = app.config
tracking_url_trans = conf.get('TRACKING_URL_TRANSFORMER')
Grain = namedtuple('Grain', 'name label function')
class LimitMethod(object):
"""Enum the ways that limits can be applied"""
FETCH_MANY = 'fetch_many'
WRAP_SQL = 'wrap_sql'
class BaseEngineSpec(object):
"""Abstract class for database engine specific configurations"""
engine = 'base' # str as defined in sqlalchemy.engine.engine
cursor_execute_kwargs = {}
def sync_role_definitions():
"""Inits the Superset application with security roles and such"""
logging.info("Syncing role definition")
# Creating default roles
alpha = sm.add_role("Alpha")
admin = sm.add_role("Admin")
gamma = sm.add_role("Gamma")
public = sm.add_role("Public")
sql_lab = sm.add_role("sql_lab")
granter = sm.add_role("granter")
dashboard_access = sm.add_role("dashboard_access")
dashboard_edit = sm.add_role("dashboard_edit")
slice_access = sm.add_role("slice_access")
slice_edit = sm.add_role("slice_edit")
datasource_access = sm.add_role("datasource_access")
datasource_edit = sm.add_role("datasource_edit")
manage_edit = sm.add_role("manage_edit")
user_role_edit = sm.add_role("user_role_edit")
get_or_create_main_db()
# Global perms
merge_perm(sm, 'all_datasource_access', 'all_datasource_access')
merge_perm(sm, 'all_database_access', 'all_database_access')
perms = db.session.query(ab_models.PermissionView).all()
perms = [p for p in perms if p.permission and p.view_menu]
logging.info("Syncing admin perms")
for p in perms:
# admin has all_database_access and all_datasource_access
if is_user_defined_permission(p):
sm.del_permission_role(admin, p)
else:
sm.add_permission_role(admin, p)
logging.info("Syncing alpha perms")
for p in perms:
# alpha has all_database_access and all_datasource_access
if is_user_defined_permission(p):
sm.del_permission_role(alpha, p)
elif ((p.view_menu.name not in ADMIN_ONLY_VIEW_MENUES
and p.permission.name not in ADMIN_ONLY_PERMISSIONS)
or (p.permission.name, p.view_menu.name) in READ_ONLY_PRODUCT):
sm.add_permission_role(alpha, p)
else:
sm.del_permission_role(alpha, p)
logging.info("Syncing gamma perms and public if specified")
PUBLIC_ROLE_LIKE_GAMMA = conf.get('PUBLIC_ROLE_LIKE_GAMMA', False)
for p in perms:
if ((p.view_menu.name not in ADMIN_ONLY_VIEW_MENUES
and p.view_menu.name not in GAMMA_READ_ONLY_MODELVIEWS
and p.permission.name not in ADMIN_ONLY_PERMISSIONS
and p.permission.name not in ALPHA_ONLY_PERMISSIONS) or
(p.permission.name, p.view_menu.name) in GAMMA_READ_ONLY_PRODUCT):
sm.add_permission_role(gamma, p)
if PUBLIC_ROLE_LIKE_GAMMA:
sm.add_permission_role(public, p)
else:
sm.del_permission_role(gamma, p)
sm.del_permission_role(public, p)
logging.info("Syncing sql_lab perms")
for p in perms:
if (p.view_menu.name in {'SQL Lab'} or p.permission.name
in {'can_sql_json', 'can_csv', 'can_search_queries'}):
sm.add_permission_role(sql_lab, p)
else:
sm.del_permission_role(sql_lab, p)
logging.info("Syncing granter perms")
for p in perms:
if (p.permission.name
in {'can_override_role_permissions', 'can_aprove'}):
sm.add_permission_role(granter, p)
else:
sm.del_permission_role(granter, p)
logging.info("Syncing dashboard_access perms")
for p in perms:
if (p.view_menu.name in {'Dashboards'} or p.permission.name in {
'can_explore', 'can_explore_json', 'can_slice',
'can_created_dashboards', 'can_fave_dashboards',
'all_datasource_access', 'all_database_access', 'can_profile'
} or (p.permission.name in {'can_list', 'can_show', 'can_download'}
and p.view_menu.name in {'DashboardModelView'}) or
(p.permission.name in {'can_list', 'can_show', 'can_download'}
and p.view_menu.name in {'DashboardModelViewAsync'})
or (p.permission.name in {
'can_show', 'can_edit', 'can_download', 'can_userinfo',
'resetmypassword', 'userinfoedit'
} and p.view_menu.name in {'UserDBModelView'})):
sm.add_permission_role(dashboard_access, p)
else:
sm.del_permission_role(dashboard_access, p)
logging.info("Syncing dashboard_edit perms")
for p in perms:
if (p.view_menu.name in {'Dashboards'} or p.permission.name in {
'can_explore', 'can_explore_json', 'can_slice',
'can_created_dashboards', 'can_fave_dashboards',
'all_datasource_access', 'all_database_access', 'can_profile'
} or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete', 'muldelete',
'can_edit', 'can_download', 'mulexport'
} and p.view_menu.name in {'DashboardModelView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'muldelete', 'can_edit', 'can_download', 'mulexport'
} and p.view_menu.name in {'DashboardModelViewAsync'})
or (p.permission.name in {
'can_show', 'can_edit', 'can_download', 'can_userinfo',
'resetmypassword', 'userinfoedit'
} and p.view_menu.name in {'UserDBModelView'})):
sm.add_permission_role(dashboard_edit, p)
else:
sm.del_permission_role(dashboard_edit, p)
logging.info("Syncing slice_access perms")
for p in perms:
if (p.view_menu.name in {'Slices'} or p.permission.name in {
'can_explore', 'can_explore_json', 'can_slice',
'can_created_slices', 'can_fave_slices',
'all_datasource_access', 'all_database_access', 'can_profile'
} or (p.permission.name in {'can_list', 'can_show', 'can_download'}
and p.view_menu.name in {'SliceModelView'}) or
(p.permission.name in {'can_list', 'can_show', 'can_download'}
and p.view_menu.name in {'SliceAsync'}) or (p.permission.name in {
'can_show', 'can_edit', 'can_userinfo', 'resetmypassword',
'userinfoedit'
} and p.view_menu.name in {'UserDBModelView'})):
sm.add_permission_role(slice_access, p)
else:
sm.del_permission_role(slice_access, p)
logging.info("Syncing slice_edit perms")
for p in perms:
if (p.view_menu.name in {'Slices'} or p.permission.name in {
'can_explore', 'can_explore_json', 'can_slice',
'can_created_slices', 'can_fave_slices', 'can_add_slices',
'all_datasource_access', 'all_database_access', 'can_profile'
} or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete', 'muldelete',
'can_edit', 'can_download'
} and p.view_menu.name in {'SliceModelView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'muldelete', 'can_edit', 'can_download'
} and p.view_menu.name in {'SliceAsync'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'muldelete', 'can_edit', 'can_download'
} and p.view_menu.name in {'SliceAddView'})
or (p.permission.name in {
'can_show', 'can_edit', 'can_userinfo', 'resetmypassword',
'userinfoedit'
} and p.view_menu.name in {'UserDBModelView'})):
sm.add_permission_role(slice_edit, p)
else:
sm.del_permission_role(slice_edit, p)
logging.info("Syncing datasource_access perms")
for p in perms:
if (p.view_menu.name in {
'Sources', 'Databases', 'Tables', 'Druid Clusters',
'Druid Datasources'
} or p.permission.name in {
'can_explore', 'can_explore_json', 'all_datasource_access',
'all_database_access', 'can_profile'
} or (p.permission.name in {'can_list', 'can_show'}
and p.view_menu.name in {'DatabaseView'})
or (p.permission.name in {'can_list', 'can_show'}
and p.view_menu.name in {'DatabaseAsync'})
or (p.permission.name in {'can_list', 'can_show'}
and p.view_menu.name in {'TableModelView'})
or (p.permission.name in {'can_list', 'can_show'}
and p.view_menu.name in {'DatabaseTableAsync'})
or (p.permission.name in {'can_list', 'can_show'}
and p.view_menu.name in {'DruidDatasourceModelView'})
or (p.permission.name in {'can_list', 'can_show'}
and p.view_menu.name in {'DruidClusterModelView'})
or (p.permission.name in {
'can_show', 'can_edit', 'can_userinfo', 'resetmypassword',
'userinfoedit'
} and p.view_menu.name in {'UserDBModelView'})):
sm.add_permission_role(datasource_access, p)
else:
sm.del_permission_role(datasource_access, p)
logging.info("Syncing datasource_edit perms")
for p in perms:
if (p.view_menu.name in {
'Sources', 'Databases', 'Tables', 'Druid Clusters',
'Druid Datasources', 'Refresh Druid Metadata',
'TableColumnInlineView', 'SqlMetricInlineView'
} or p.permission.name in {
'can_explore', 'can_explore_json', 'can_testconn',
'can_checkbox', 'can_refresh_datasources',
'all_datasource_access', 'all_database_access', 'can_profile'
} or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete', 'muldelete',
'can_edit', 'can_download'
} and p.view_menu.name in {'DatabaseView'}) or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete', 'muldelete',
'can_edit', 'can_download'
} and p.view_menu.name in {'DatabaseAsync'}) or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete', 'muldelete',
'can_edit', 'can_download'
} and p.view_menu.name in {'TableModelView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'muldelete', 'can_edit', 'can_download'
} and p.view_menu.name in {'DatabaseTablesAsync'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'muldelete', 'can_edit', 'can_download'
} and p.view_menu.name in {'DruidDatasourceModelView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'muldelete', 'can_edit', 'can_download'
} and p.view_menu.name in {'DruidClusterModelView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'can_edit', 'can_download'
} and p.view_menu.name in {'TableColumnInlineView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'can_edit', 'can_download'
} and p.view_menu.name in {'SqlMetricInlineView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'can_edit', 'can_download'
} and p.view_menu.name in {'DruidColumnInlineView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'can_edit', 'can_download'
} and p.view_menu.name in {'DruidMetricInlineView'})
or (p.permission.name in {
'can_show', 'can_edit', 'can_userinfo', 'resetmypassword',
'userinfoedit'
} and p.view_menu.name in {'UserDBModelView'})):
sm.add_permission_role(datasource_edit, p)
else:
sm.del_permission_role(datasource_edit, p)
logging.info("Syncing manage_edit perms")
for p in perms:
if (p.view_menu.name
in {'Manage', 'Import Dashboards', 'Queries', 'CSS Templates'}
or p.permission.name in {'can_profile'}
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'can_edit', 'can_download'
} and p.view_menu.name in {'QueryView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'can_edit', 'can_download'
} and p.view_menu.name in {'CssTemplateModelView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'can_edit', 'can_download'
} and p.view_menu.name in {'CssTemplateAsyncModelView'})
or (p.permission.name in {'can_add'}
and p.view_menu.name in {'DashboardModelView'})
or (p.permission.name in {'can_add'}
and p.view_menu.name in {'SliceAddView'})
or (p.permission.name in {
'can_show', 'can_edit', 'can_userinfo', 'resetmypassword',
'userinfoedit'
} and p.view_menu.name in {'UserDBModelView'})):
sm.add_permission_role(manage_edit, p)
else:
sm.del_permission_role(manage_edit, p)
logging.info("Syncing user_role_edit perms")
for p in perms:
if (p.view_menu.name in {
'Security', 'List Users', 'List Roles', "User's Statistics",
'Base Permissions', 'Views/Menus', 'Permission on Views/Menus',
'Access requests', 'Action Log'
} or p.permission.name in {'can_recent_activity', 'can_profile'}
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'muldelete', 'can_edit', 'can_download', 'can_userinfo',
'resetmypassword', 'resetpasswords', 'userinfoedit'
} and p.view_menu.name in {'UserDBModelView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'muldelete', 'can_edit', 'can_download', 'Copy Role',
'can_update_role', 'can_override_role_permissions'
} and p.view_menu.name in {'RoleModelView'})
or (p.permission.name in {'can_chart'}
and p.view_menu.name in {'UserStatsChartView'})
or (p.permission.name in {'can_list'}
and p.view_menu.name in {'PermissionModelView'})
or (p.permission.name in {'can_list'}
and p.view_menu.name in {'ViewMenuModelView'})
or (p.permission.name in {'can_list'}
and p.view_menu.name in {'PermissionViewModelView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'can_edit', 'can_download', 'muldelete'
} and p.view_menu.name in {'AccessRequestsModelView'})
or (p.permission.name in {
'can_list', 'can_show', 'can_add', 'can_delete',
'can_edit', 'can_download'
} and p.view_menu.name in {'LogModelView'}) or
(p.permission.name in {'can_this_form_post', 'can_this_form_get'}
and p.view_menu.name in {'ResetMyPasswordView'}) or
(p.permission.name in {'can_this_form_post', 'can_this_form_get'}
and p.view_menu.name in {'ResetPasswordView'}) or
(p.permission.name in {'can_this_form_post', 'can_this_form_get'}
and p.view_menu.name in {'UserInfoEditView'})):
sm.add_permission_role(user_role_edit, p)
else:
sm.del_permission_role(user_role_edit, p)
logging.info("Making sure all data source perms have been created")
session = db.session()
datasources = [o for o in session.query(models.SqlaTable).all()]
datasources += [o for o in session.query(models.DruidDatasource).all()]
for datasource in datasources:
perm = datasource.get_perm()
merge_perm(sm, 'datasource_access', perm)
if datasource.schema:
merge_perm(sm, 'schema_access', datasource.schema_perm)
if perm != datasource.perm:
datasource.perm = perm
logging.info("Making sure all database perms have been created")
databases = [o for o in session.query(models.Database).all()]
for database in databases:
perm = database.get_perm()
if perm != database.perm:
database.perm = perm
merge_perm(sm, 'database_access', perm)
session.commit()
logging.info("Making sure all dashboard perms have been created")
dashboards = [o for o in session.query(models.Dashboard).all()]
for dashboard in dashboards:
perm = dashboard.get_dashboard_title()
sm.add_permission_view_menu('dashboard_access', perm)
session.commit()
logging.info("Making sure all metrics perms exist")
models.init_metrics_perm()
class CsvResponse(Response):
"""
Override Response to take into account csv encoding from config.py
"""
charset = conf.get('CSV_EXPORT').get('encoding', 'utf-8')
def get_main_database():
from superset import conf
return get_or_create_db("main", conf.get("SQLALCHEMY_DATABASE_URI"))
class XlsxResponse(Response):
"""
Override Response to take into account xlsx encoding from config.py
"""
charset = conf.get('XLSX_EXPORT').get('encoding', 'utf-8')
def get_table_access_link(self, tables):
from superset import conf
return conf.get('PERMISSION_INSTRUCTIONS_LINK')
def get_user_roles():
if g.user.is_anonymous():
public_role = conf.get('AUTH_ROLE_PUBLIC')
return [security_manager.find_role(public_role)] if public_role else []
return g.user.roles
def get_user_roles():
if g.user.is_anonymous():
public_role = conf.get('AUTH_ROLE_PUBLIC')
return [appbuilder.sm.find_role(public_role)] if public_role else []
return g.user.roles
def sync_role_definitions():
"""Inits the Superset application with security roles and such"""
logging.info("Syncing role definition")
# Creating default roles
alpha = sm.add_role("Alpha")
admin = sm.add_role("Admin")
gamma = sm.add_role("Gamma")
public = sm.add_role("Public")
sql_lab = sm.add_role("sql_lab")
granter = sm.add_role("granter")
get_or_create_main_db()
# Global perms
merge_perm(sm, 'all_datasource_access', 'all_datasource_access')
merge_perm(sm, 'all_database_access', 'all_database_access')
perms = db.session.query(ab_models.PermissionView).all()
perms = [p for p in perms if p.permission and p.view_menu]
logging.info("Syncing admin perms")
for p in perms:
# admin has all_database_access and all_datasource_access
if is_user_defined_permission(p):
sm.del_permission_role(admin, p)
else:
sm.add_permission_role(admin, p)
logging.info("Syncing alpha perms")
for p in perms:
# alpha has all_database_access and all_datasource_access
if is_user_defined_permission(p):
sm.del_permission_role(alpha, p)
elif ((p.view_menu.name not in ADMIN_ONLY_VIEW_MENUES
and p.permission.name not in ADMIN_ONLY_PERMISSIONS)
or (p.permission.name, p.view_menu.name) in READ_ONLY_PRODUCT):
sm.add_permission_role(alpha, p)
else:
sm.del_permission_role(alpha, p)
logging.info("Syncing gamma perms and public if specified")
PUBLIC_ROLE_LIKE_GAMMA = conf.get('PUBLIC_ROLE_LIKE_GAMMA', False)
for p in perms:
if ((p.view_menu.name not in ADMIN_ONLY_VIEW_MENUES
and p.view_menu.name not in GAMMA_READ_ONLY_MODELVIEWS
and p.permission.name not in ADMIN_ONLY_PERMISSIONS
and p.permission.name not in ALPHA_ONLY_PERMISSIONS) or
(p.permission.name, p.view_menu.name) in GAMMA_READ_ONLY_PRODUCT):
sm.add_permission_role(gamma, p)
if PUBLIC_ROLE_LIKE_GAMMA:
sm.add_permission_role(public, p)
else:
sm.del_permission_role(gamma, p)
sm.del_permission_role(public, p)
logging.info("Syncing sql_lab perms")
for p in perms:
if (p.view_menu.name in {'SQL Lab'} or p.permission.name
in {'can_sql_json', 'can_csv', 'can_search_queries'}):
sm.add_permission_role(sql_lab, p)
else:
sm.del_permission_role(sql_lab, p)
logging.info("Syncing granter perms")
for p in perms:
if (p.permission.name
in {'can_override_role_permissions', 'can_aprove'}):
sm.add_permission_role(granter, p)
else:
sm.del_permission_role(granter, p)
logging.info("Making sure all data source perms have been created")
session = db.session()
datasources = [o for o in session.query(models.SqlaTable).all()]
datasources += [o for o in session.query(models.DruidDatasource).all()]
for datasource in datasources:
perm = datasource.get_perm()
merge_perm(sm, 'datasource_access', perm)
if datasource.schema:
merge_perm(sm, 'schema_access', datasource.schema_perm)
if perm != datasource.perm:
datasource.perm = perm
logging.info("Making sure all database perms have been created")
databases = [o for o in session.query(models.Database).all()]
for database in databases:
perm = database.get_perm()
if perm != database.perm:
database.perm = perm
merge_perm(sm, 'database_access', perm)
session.commit()
logging.info("Making sure all metrics perms exist")
models.init_metrics_perm()
"""
Superset allows to redirect user to a default dashboard
but the only way to set this dashboard is by code
"""
from flask_appbuilder.security.sqla.models import User
from superset import conf, db
from superset.models.core import Dashboard
from superset.models.user_attributes import UserAttribute
def set_welcome_dashboard(id, username):
# Get default user
user = db.session.query(User).filter_by(username=username).first()
# Make sure welcome dashboard exists
dashboard = db.session.query(Dashboard).filter_by(id=id).first()
# Set dashboard as default
extra_attributes = UserAttribute(
user_id=user.id,
welcome_dashboard_id=dashboard.id,
)
db.session.add(extra_attributes)
db.session.commit()
set_welcome_dashboard(conf.get('DEFAULT_DASHBOARD_ID'),
conf.get('DEFAULT_USERNAME'))
def get_user_roles():
if g.user.is_anonymous():
public_role = conf.get('AUTH_ROLE_PUBLIC')
return [appbuilder.sm.find_role(public_role)] if public_role else []
return g.user.roles
def sync_role_definitions():
"""Inits the Superset application with security roles and such"""
logging.info("Syncing role definition")
# Creating default roles
alpha = sm.add_role("Alpha")
admin = sm.add_role("Admin")
gamma = sm.add_role("Gamma")
public = sm.add_role("Public")
sql_lab = sm.add_role("sql_lab")
granter = sm.add_role("granter")
get_or_create_main_db()
# Global perms
sm.add_permission_view_menu(
'all_datasource_access', 'all_datasource_access')
sm.add_permission_view_menu('all_database_access', 'all_database_access')
perms = db.session.query(ab_models.PermissionView).all()
perms = [p for p in perms if p.permission and p.view_menu]
logging.info("Syncing admin perms")
for p in perms:
# admin has all_database_access and all_datasource_access
if is_user_defined_permission(p):
sm.del_permission_role(admin, p)
else:
sm.add_permission_role(admin, p)
logging.info("Syncing alpha perms")
for p in perms:
# alpha has all_database_access and all_datasource_access
if is_user_defined_permission(p):
sm.del_permission_role(alpha, p)
elif (
(
p.view_menu.name not in ADMIN_ONLY_VIEW_MENUES and
p.permission.name not in ADMIN_ONLY_PERMISSIONS
) or
(p.permission.name, p.view_menu.name) in READ_ONLY_PRODUCT
):
sm.add_permission_role(alpha, p)
else:
sm.del_permission_role(alpha, p)
logging.info("Syncing gamma perms and public if specified")
PUBLIC_ROLE_LIKE_GAMMA = conf.get('PUBLIC_ROLE_LIKE_GAMMA', False)
for p in perms:
if (
(
p.view_menu.name not in ADMIN_ONLY_VIEW_MENUES and
p.permission.name not in ADMIN_ONLY_PERMISSIONS and
p.permission.name not in ALPHA_ONLY_PERMISSIONS
) or
(p.permission.name, p.view_menu.name) in READ_ONLY_PRODUCT
):
sm.add_permission_role(gamma, p)
if PUBLIC_ROLE_LIKE_GAMMA:
sm.add_permission_role(public, p)
else:
sm.del_permission_role(gamma, p)
sm.del_permission_role(public, p)
logging.info("Syncing sql_lab perms")
for p in perms:
if (
p.view_menu.name in {'SQL Lab'} or
p.permission.name in {
'can_sql_json', 'can_csv', 'can_search_queries'}
):
sm.add_permission_role(sql_lab, p)
else:
sm.del_permission_role(sql_lab, p)
logging.info("Syncing granter perms")
for p in perms:
if (
p.permission.name in {
'can_override_role_permissions', 'can_aprove'}
):
sm.add_permission_role(granter, p)
else:
sm.del_permission_role(granter, p)
logging.info("Making sure all data source perms have been created")
session = db.session()
datasources = [
o for o in session.query(models.SqlaTable).all()]
datasources += [
o for o in session.query(models.DruidDatasource).all()]
for datasource in datasources:
perm = datasource.get_perm()
sm.add_permission_view_menu('datasource_access', perm)
if perm != datasource.perm:
datasource.perm = perm
logging.info("Making sure all database perms have been created")
databases = [o for o in session.query(models.Database).all()]
for database in databases:
perm = database.get_perm()
if perm != database.perm:
database.perm = perm
sm.add_permission_view_menu('database_access', perm)
session.commit()
logging.info("Making sure all metrics perms exist")
models.init_metrics_perm()
def get_datasource_access_link(self, datasource):
from superset import conf
return conf.get('PERMISSION_INSTRUCTIONS_LINK')
def get_table_access_link(self, tables):
from superset import conf
return conf.get('PERMISSION_INSTRUCTIONS_LINK')
import sqlalchemy as sqla
from sqlalchemy import select
from sqlalchemy.engine import create_engine
from sqlalchemy.engine.url import make_url
from sqlalchemy.sql import text
import sqlparse
import unicodecsv
from werkzeug.utils import secure_filename
from superset import app, cache_util, conf, db, utils
from superset.exceptions import SupersetTemplateException
from superset.utils import QueryStatus
config = app.config
tracking_url_trans = conf.get('TRACKING_URL_TRANSFORMER')
hive_poll_interval = conf.get('HIVE_POLL_INTERVAL')
Grain = namedtuple('Grain', 'name label function')
class LimitMethod(object):
"""Enum the ways that limits can be applied"""
FETCH_MANY = 'fetch_many'
WRAP_SQL = 'wrap_sql'
class BaseEngineSpec(object):
"""Abstract class for database engine specific configurations"""
def get_datasource_access_link(self, datasource):
from superset import conf
return conf.get('PERMISSION_INSTRUCTIONS_LINK')
)
from pydruid.utils.having import Aggregation
from flask import Markup, escape
from flask_appbuilder.models.decorators import renders
from flask_appbuilder import Model
from flask_babel import lazy_gettext as _
from superset import conf, db, import_util, utils, sm, get_session
from superset.utils import (flasher, MetricPermException, DimSelector,
DTTM_ALIAS)
from superset.connectors.base import BaseDatasource, BaseColumn, BaseMetric
from superset.models.helpers import AuditMixinNullable, QueryResult, set_perm
DRUID_TZ = conf.get("DRUID_TZ")
class JavascriptPostAggregator(Postaggregator):
def __init__(self, name, field_names, function):
self.post_aggregator = {
'type': 'javascript',
'fieldNames': field_names,
'name': name,
'function': function,
}
self.name = name
class DruidCluster(Model, AuditMixinNullable):
"""ORM object referencing the Druid clusters"""
def get_user_roles():
if g.user.is_anonymous:
public_role = conf.get('AUTH_ROLE_PUBLIC')
return [security_manager.find_role(public_role)] if public_role else []
return g.user.roles
from pydruid.utils.having import Aggregation
from flask import Markup, escape
from flask_appbuilder.models.decorators import renders
from flask_appbuilder import Model
from flask_babel import lazy_gettext as _
from superset import conf, db, import_util, utils, sm, get_session
from superset.utils import (
flasher, MetricPermException, DimSelector, DTTM_ALIAS
)
from superset.connectors.base.models import BaseDatasource, BaseColumn, BaseMetric
from superset.models.helpers import AuditMixinNullable, QueryResult, set_perm
DRUID_TZ = conf.get("DRUID_TZ")
# Function wrapper because bound methods cannot
# be passed to processes
def _fetch_metadata_for(datasource):
return datasource.latest_metadata()
class JavascriptPostAggregator(Postaggregator):
def __init__(self, name, field_names, function):
self.post_aggregator = {
'type': 'javascript',
'fieldNames': field_names,
'name': name,
'function': function,
def get_main_database() -> "Database":
from superset import conf
db_uri = conf.get("SQLALCHEMY_DATABASE_URI")
return get_or_create_db("main", db_uri)
def is_user_admin() -> bool:
user_roles = [
role.name.lower() for role in security_manager.get_user_roles()
]
admin_role = conf.get("AUTH_ROLE_ADMIN").lower()
return admin_role in user_roles
def get_example_database():
from superset import conf
db_uri = conf.get("SQLALCHEMY_EXAMPLES_URI") or conf.get(
"SQLALCHEMY_DATABASE_URI")
return get_or_create_db("examples", db_uri)
def run_query( # noqa / druid
self,
groupby, metrics,
granularity,
from_dttm, to_dttm,
filter=None, # noqa
is_timeseries=True,
timeseries_limit=None,
timeseries_limit_metric=None,
row_limit=None,
inner_from_dttm=None, inner_to_dttm=None,
orderby=None,
extras=None, # noqa
columns=None, phase=2, client=None,
order_desc=True,
prequeries=None,
is_prequery=False,
):
"""Runs a query against Druid and returns a dataframe.
"""
# TODO refactor into using a TBD Query object
client = client or self.cluster.get_pydruid_client()
row_limit = row_limit or conf.get('ROW_LIMIT')
if not is_timeseries:
granularity = 'all'
if (
granularity == 'all' or
timeseries_limit is None or
timeseries_limit == 0):
phase = 1
inner_from_dttm = inner_from_dttm or from_dttm
inner_to_dttm = inner_to_dttm or to_dttm
timezone = from_dttm.tzname() if from_dttm else None
query_str = ''
metrics_dict = {m.metric_name: m for m in self.metrics}
columns_dict = {c.column_name: c for c in self.columns}
all_metrics, post_aggs = DruidDatasource.metrics_and_post_aggs(
metrics,
metrics_dict)
aggregations = self.get_aggregations(all_metrics)
self.check_restricted_metrics(aggregations)
# the dimensions list with dimensionSpecs expanded
dimensions = self.get_dimensions(groupby, columns_dict)
extras = extras or {}
qry = dict(
datasource=self.datasource_name,
dimensions=dimensions,
aggregations=aggregations,
granularity=DruidDatasource.granularity(
granularity,
timezone=timezone,
origin=extras.get('druid_time_origin'),
),
post_aggregations=post_aggs,
intervals=self.intervals_from_dttms(from_dttm, to_dttm),
)
filters = DruidDatasource.get_filters(filter, self.num_cols)
if filters:
qry['filter'] = filters
having_filters = self.get_having_filters(extras.get('having_druid'))
if having_filters:
qry['having'] = having_filters
order_direction = 'descending' if order_desc else 'ascending'
if columns:
del qry['post_aggregations']
del qry['aggregations']
qry['dimensions'] = columns
qry['metrics'] = []
qry['granularity'] = 'all'
qry['limit'] = row_limit
client.scan(**qry)
elif len(groupby) == 0 and not having_filters:
logging.info('Running timeseries query for no groupby values')
del qry['dimensions']
client.timeseries(**qry)
elif (
not having_filters and
len(groupby) == 1 and
order_desc
):
dim = list(qry.get('dimensions'))[0]
logging.info('Running two-phase topn query for dimension [{}]'.format(dim))
pre_qry = deepcopy(qry)
if timeseries_limit_metric:
order_by = timeseries_limit_metric
pre_qry['aggregations'] = self.get_aggregations([timeseries_limit_metric])
else:
order_by = list(qry['aggregations'].keys())[0]
# Limit on the number of timeseries, doing a two-phases query
pre_qry['granularity'] = 'all'
pre_qry['threshold'] = min(row_limit,
timeseries_limit or row_limit)
pre_qry['metric'] = order_by
pre_qry['dimension'] = self._dimensions_to_values(qry.get('dimensions'))[0]
del pre_qry['dimensions']
client.topn(**pre_qry)
logging.info('Phase 1 Complete')
query_str += '// Two phase query\n// Phase 1\n'
query_str += json.dumps(
client.query_builder.last_query.query_dict, indent=2)
query_str += '\n'
if phase == 1:
return query_str
query_str += (
"// Phase 2 (built based on phase one's results)\n")
df = client.export_pandas()
qry['filter'] = self._add_filter_from_pre_query_data(
df,
[pre_qry['dimension']],
filters)
qry['threshold'] = timeseries_limit or 1000
if row_limit and granularity == 'all':
qry['threshold'] = row_limit
qry['dimension'] = dim
del qry['dimensions']
qry['metric'] = list(qry['aggregations'].keys())[0]
client.topn(**qry)
logging.info('Phase 2 Complete')
elif len(groupby) > 0 or having_filters:
# If grouping on multiple fields or using a having filter
# we have to force a groupby query
logging.info('Running groupby query for dimensions [{}]'.format(dimensions))
if timeseries_limit and is_timeseries:
logging.info('Running two-phase query for timeseries')
pre_qry = deepcopy(qry)
pre_qry_dims = self._dimensions_to_values(qry['dimensions'])
pre_qry['dimensions'] = list(set(pre_qry_dims))
order_by = metrics[0] if metrics else pre_qry_dims[0]
if timeseries_limit_metric:
order_by = timeseries_limit_metric
# Limit on the number of timeseries, doing a two-phases query
pre_qry['granularity'] = 'all'
pre_qry['limit_spec'] = {
'type': 'default',
'limit': min(timeseries_limit, row_limit),
'intervals': self.intervals_from_dttms(
inner_from_dttm, inner_to_dttm),
'columns': [{
'dimension': order_by,
'direction': order_direction,
}],
}
client.groupby(**pre_qry)
logging.info('Phase 1 Complete')
query_str += '// Two phase query\n// Phase 1\n'
query_str += json.dumps(
client.query_builder.last_query.query_dict, indent=2)
query_str += '\n'
if phase == 1:
return query_str
query_str += (
"// Phase 2 (built based on phase one's results)\n")
df = client.export_pandas()
qry['filter'] = self._add_filter_from_pre_query_data(
df,
pre_qry['dimensions'],
filters,
)
qry['limit_spec'] = None
if row_limit:
dimension_values = self._dimensions_to_values(dimensions)
qry['limit_spec'] = {
'type': 'default',
'limit': row_limit,
'columns': [{
'dimension': (
metrics[0] if metrics else dimension_values[0]),
'direction': order_direction,
}],
}
client.groupby(**qry)
logging.info('Query Complete')
query_str += json.dumps(
client.query_builder.last_query.query_dict, indent=2)
return query_str
from superset.db_engine_specs.base import BaseEngineSpec
from superset.db_engine_specs.presto import PrestoEngineSpec
from superset.exceptions import SupersetException
from superset.models.sql_lab import Query
from superset.sql_parse import Table
from superset.utils import core as utils
if TYPE_CHECKING:
# prevent circular imports
from superset.models.core import Database # pylint: disable=unused-import
QueryStatus = utils.QueryStatus
config = app.config
logger = logging.getLogger(__name__)
tracking_url_trans = conf.get("TRACKING_URL_TRANSFORMER")
hive_poll_interval = conf.get("HIVE_POLL_INTERVAL")
def upload_to_s3(filename: str, upload_prefix: str, table: Table) -> str:
# Optional dependency
import boto3 # pylint: disable=import-error
bucket_path = config["CSV_TO_HIVE_UPLOAD_S3_BUCKET"]
if not bucket_path:
logger.info("No upload bucket specified")
raise Exception(
"No upload bucket specified. You can specify one in the config file."
)
class CsvResponse(Response):
"""
Override Response to take into account csv encoding from config.py
"""
charset = conf.get("CSV_EXPORT").get("encoding", "utf-8")
def get_user_roles() -> List[Role]:
if g.user.is_anonymous:
public_role = conf.get("AUTH_ROLE_PUBLIC")
return [security_manager.find_role(public_role)] if public_role else []
return g.user.roles
import sqlalchemy as sa
from sqlalchemy import (
Boolean, Column, DateTime, ForeignKey, Integer, or_, String, Text, UniqueConstraint,
)
from sqlalchemy.orm import backref, relationship
from superset import conf, db, import_util, sm, utils
from superset.connectors.base.models import BaseColumn, BaseDatasource, BaseMetric
from superset.models.helpers import (
AuditMixinNullable, ImportMixin, QueryResult, set_perm,
)
from superset.utils import (
DimSelector, DTTM_ALIAS, flasher, MetricPermException,
)
DRUID_TZ = conf.get('DRUID_TZ')
# Function wrapper because bound methods cannot
# be passed to processes
def _fetch_metadata_for(datasource):
return datasource.latest_metadata()
class JavascriptPostAggregator(Postaggregator):
def __init__(self, name, field_names, function):
self.post_aggregator = {
'type': 'javascript',
'fieldNames': field_names,
'name': name,
'function': function,
def run_query( # noqa / druid
self,
groupby, metrics,
granularity,
from_dttm, to_dttm,
filter=None, # noqa
is_timeseries=True,
timeseries_limit=None,
timeseries_limit_metric=None,
row_limit=None,
inner_from_dttm=None, inner_to_dttm=None,
orderby=None,
extras=None, # noqa
columns=None, phase=2, client=None, form_data=None,
order_desc=True,
prequeries=None,
is_prequery=False,
):
"""Runs a query against Druid and returns a dataframe.
"""
# TODO refactor into using a TBD Query object
client = client or self.cluster.get_pydruid_client()
row_limit = row_limit or conf.get('ROW_LIMIT')
if not is_timeseries:
granularity = 'all'
if (
granularity == 'all' or
timeseries_limit is None or
timeseries_limit == 0):
phase = 1
inner_from_dttm = inner_from_dttm or from_dttm
inner_to_dttm = inner_to_dttm or to_dttm
timezone = from_dttm.tzname() if from_dttm else None
query_str = ''
metrics_dict = {m.metric_name: m for m in self.metrics}
columns_dict = {c.column_name: c for c in self.columns}
all_metrics, post_aggs = DruidDatasource.metrics_and_post_aggs(
metrics,
metrics_dict)
aggregations = self.get_aggregations(all_metrics)
self.check_restricted_metrics(aggregations)
# the dimensions list with dimensionSpecs expanded
dimensions = self.get_dimensions(groupby, columns_dict)
extras = extras or {}
qry = dict(
datasource=self.datasource_name,
dimensions=dimensions,
aggregations=aggregations,
granularity=DruidDatasource.granularity(
granularity,
timezone=timezone,
origin=extras.get('druid_time_origin'),
),
post_aggregations=post_aggs,
intervals=self.intervals_from_dttms(from_dttm, to_dttm),
)
filters = DruidDatasource.get_filters(filter, self.num_cols)
if filters:
qry['filter'] = filters
having_filters = self.get_having_filters(extras.get('having_druid'))
if having_filters:
qry['having'] = having_filters
order_direction = 'descending' if order_desc else 'ascending'
if columns:
del qry['post_aggregations']
del qry['aggregations']
qry['dimensions'] = columns
qry['metrics'] = []
qry['granularity'] = 'all'
qry['limit'] = row_limit
client.scan(**qry)
elif len(groupby) == 0 and not having_filters:
logging.info('Running timeseries query for no groupby values')
del qry['dimensions']
client.timeseries(**qry)
elif (
not having_filters and
len(groupby) == 1 and
order_desc
):
dim = list(qry.get('dimensions'))[0]
logging.info('Running two-phase topn query for dimension [{}]'.format(dim))
if timeseries_limit_metric:
order_by = timeseries_limit_metric
else:
order_by = list(qry['aggregations'].keys())[0]
# Limit on the number of timeseries, doing a two-phases query
pre_qry = deepcopy(qry)
pre_qry['granularity'] = 'all'
pre_qry['threshold'] = min(row_limit,
timeseries_limit or row_limit)
pre_qry['metric'] = order_by
if isinstance(dim, dict):
if 'dimension' in dim:
pre_qry['dimension'] = dim['dimension']
else:
pre_qry['dimension'] = dim
del pre_qry['dimensions']
client.topn(**pre_qry)
logging.info('Phase 1 Complete')
query_str += '// Two phase query\n// Phase 1\n'
query_str += json.dumps(
client.query_builder.last_query.query_dict, indent=2)
query_str += '\n'
if phase == 1:
return query_str
query_str += (
"// Phase 2 (built based on phase one's results)\n")
df = client.export_pandas()
qry['filter'] = self._add_filter_from_pre_query_data(
df,
[pre_qry['dimension']],
filters)
qry['threshold'] = timeseries_limit or 1000
if row_limit and granularity == 'all':
qry['threshold'] = row_limit
qry['dimension'] = dim
del qry['dimensions']
qry['metric'] = list(qry['aggregations'].keys())[0]
client.topn(**qry)
logging.info('Phase 2 Complete')
elif len(groupby) > 0 or having_filters:
# If grouping on multiple fields or using a having filter
# we have to force a groupby query
logging.info('Running groupby query for dimensions [{}]'.format(dimensions))
if timeseries_limit and is_timeseries:
logging.info('Running two-phase query for timeseries')
order_by = metrics[0] if metrics else self.metrics[0]
if timeseries_limit_metric:
order_by = timeseries_limit_metric
# Limit on the number of timeseries, doing a two-phases query
pre_qry = deepcopy(qry)
pre_qry['granularity'] = 'all'
pre_qry['limit_spec'] = {
'type': 'default',
'limit': min(timeseries_limit, row_limit),
'intervals': self.intervals_from_dttms(
inner_from_dttm, inner_to_dttm),
'columns': [{
'dimension': order_by,
'direction': order_direction,
}],
}
pre_qry_dims = []
# Replace dimensions specs with their `dimension`
# values, and ignore those without
for dim in qry['dimensions']:
if isinstance(dim, dict):
if 'dimension' in dim:
pre_qry_dims.append(dim['dimension'])
else:
pre_qry_dims.append(dim)
pre_qry['dimensions'] = list(set(pre_qry_dims))
client.groupby(**pre_qry)
logging.info('Phase 1 Complete')
query_str += '// Two phase query\n// Phase 1\n'
query_str += json.dumps(
client.query_builder.last_query.query_dict, indent=2)
query_str += '\n'
if phase == 1:
return query_str
query_str += (
"// Phase 2 (built based on phase one's results)\n")
df = client.export_pandas()
qry['filter'] = self._add_filter_from_pre_query_data(
df,
pre_qry['dimensions'],
filters,
)
qry['limit_spec'] = None
if row_limit:
qry['limit_spec'] = {
'type': 'default',
'limit': row_limit,
'columns': [{
'dimension': (
metrics[0] if metrics else self.metrics[0]),
'direction': order_direction,
}],
}
client.groupby(**qry)
logging.info('Query Complete')
query_str += json.dumps(
client.query_builder.last_query.query_dict, indent=2)
return query_str
import sqlalchemy as sa
from sqlalchemy import (
Boolean, Column, DateTime, ForeignKey, Integer, or_, String, Text, UniqueConstraint,
)
from sqlalchemy.orm import backref, relationship
from superset import conf, db, import_util, sm, utils
from superset.connectors.base.models import BaseColumn, BaseDatasource, BaseMetric
from superset.models.helpers import (
AuditMixinNullable, ImportMixin, QueryResult, set_perm,
)
from superset.utils import (
DimSelector, DTTM_ALIAS, flasher, MetricPermException,
)
DRUID_TZ = conf.get('DRUID_TZ')
# Function wrapper because bound methods cannot
# be passed to processes
def _fetch_metadata_for(datasource):
return datasource.latest_metadata()
class JavascriptPostAggregator(Postaggregator):
def __init__(self, name, field_names, function):
self.post_aggregator = {
'type': 'javascript',
'fieldNames': field_names,
'name': name,
'function': function,
from flask import g
from flask_babel import lazy_gettext as _
import pandas
from sqlalchemy import select
from sqlalchemy.engine import create_engine
from sqlalchemy.engine.url import make_url
from sqlalchemy.sql import text
import sqlparse
from werkzeug.utils import secure_filename
from superset import app, cache_util, conf, db, utils
from superset.utils import QueryStatus, SupersetTemplateException
config = app.config
tracking_url_trans = conf.get('TRACKING_URL_TRANSFORMER')
Grain = namedtuple('Grain', 'name label function')
class LimitMethod(object):
"""Enum the ways that limits can be applied"""
FETCH_MANY = 'fetch_many'
WRAP_SQL = 'wrap_sql'
class BaseEngineSpec(object):
"""Abstract class for database engine specific configurations"""
engine = 'base' # str as defined in sqlalchemy.engine.engine
def get_since_until(
time_range: Optional[str] = None,
since: Optional[str] = None,
until: Optional[str] = None,
time_shift: Optional[str] = None,
relative_start: Optional[str] = None,
relative_end: Optional[str] = None,
) -> Tuple[datetime, datetime]:
"""Return `since` and `until` date time tuple from string representations of
time_range, since, until and time_shift.
This functiom supports both reading the keys separately (from `since` and
`until`), as well as the new `time_range` key. Valid formats are:
- ISO 8601
- X days/years/hours/day/year/weeks
- X days/years/hours/day/year/weeks ago
- X days/years/hours/day/year/weeks from now
- freeform
Additionally, for `time_range` (these specify both `since` and `until`):
- Last day
- Last week
- Last month
- Last quarter
- Last year
- No filter
- Last X seconds/minutes/hours/days/weeks/months/years
- Next X seconds/minutes/hours/days/weeks/months/years
"""
from superset import conf
separator = " : "
relative_start = parse_human_datetime(
relative_start if relative_start else "today")
relative_end = parse_human_datetime(
relative_end if relative_end else "today")
utc_dt = datetime.utcnow().replace(tzinfo=timezone.utc)
TIME_ZONE = conf.get("TIME_ZONE")
common_time_frames = {
"Today":
(relative_start, utc_dt.astimezone(timezone(
timedelta(hours=TIME_ZONE))).replace(tzinfo=None)),
"Last day": (
relative_start - relativedelta(days=1), # type: ignore
relative_end,
),
"Last week": (
relative_start - relativedelta(weeks=1), # type: ignore
relative_end,
),
"Last month": (
relative_start - relativedelta(months=1), # type: ignore
relative_end,
),
"Last quarter": (
relative_start - relativedelta(months=3), # type: ignore
relative_end,
),
"Last year": (
relative_start - relativedelta(years=1), # type: ignore
relative_end,
),
}
if time_range:
if separator in time_range:
since, until = time_range.split(separator, 1)
if since and since not in common_time_frames:
since = add_ago_to_since(since)
since = parse_human_datetime(since)
until = parse_human_datetime(until)
elif time_range in common_time_frames:
since, until = common_time_frames[time_range]
elif time_range == "No filter":
since = until = None
else:
rel, num, grain = time_range.split()
if rel == "Last":
since = relative_start - relativedelta( # type: ignore
**{grain: int(num)})
until = relative_end
else: # rel == 'Next'
since = relative_start
until = relative_end + relativedelta( # type: ignore
**{grain: int(num)})
else:
since = since or ""
if since:
since = add_ago_to_since(since)
since = parse_human_datetime(since)
until = parse_human_datetime(until) if until else relative_end
if time_shift:
time_delta = parse_past_timedelta(time_shift)
since = since if since is None else (since -
time_delta) # type: ignore
until = until if until is None else (until -
time_delta) # type: ignore
if since and until and since > until:
raise ValueError(_("From date cannot be larger than to date"))
return since, until # type: ignore
