コード例 #1
0
ファイル: base.py プロジェクト: bkyryliuk/caravel
 def common_bootsrap_payload(self):
     """Common data always sent to the client"""
     messages = get_flashed_messages(with_categories=True)
     locale = str(get_locale())
     return {
         'flash_messages': messages,
         'conf': {k: conf.get(k) for k in FRONTEND_CONF_KEYS},
         'locale': locale,
         'language_pack': get_language_pack(locale),
         'feature_flags': conf.get('FEATURE_FLAGS'),
     }
コード例 #2
0
ファイル: security.py プロジェクト: kiskong/superset
def sync_role_definitions():
    """Inits the Superset application with security roles and such"""
    logging.info("Syncing role definition")

    get_or_create_main_db()
    create_custom_permissions()

    pvms = db.session.query(ab_models.PermissionView).all()
    pvms = [p for p in pvms if p.permission and p.view_menu]

    # cleanup
    pvms_to_delete = [p for p in pvms if not (p.permission and p.view_menu)]

    for pvm_to_delete in pvms_to_delete:
        sm.get_session.delete(pvm_to_delete)

    # Creating default roles
    set_role('Admin', pvms, is_admin_pvm)
    set_role('Alpha', pvms, is_alpha_pvm)
    set_role('Gamma', pvms, is_gamma_pvm)
    set_role('granter', pvms, is_granter_pvm)
    set_role('sql_lab', pvms, is_sql_lab_pvm)

    if conf.get('PUBLIC_ROLE_LIKE_GAMMA', False):
        set_role('Public', pvms, is_gamma_pvm)

    view_menu_set = db.session.query(models.SqlaTable).all()
    create_missing_datasource_perms(view_menu_set)
    create_missing_database_perms(view_menu_set)
    create_missing_metrics_perm(view_menu_set)

    # commit role and view menu updates
    sm.get_session.commit()
コード例 #3
0
ファイル: security.py プロジェクト: kiskong/superset
def get_or_create_main_db():
    logging.info("Creating database reference")
    dbobj = (
        db.session.query(models.Database)
        .filter_by(database_name='main')
        .first()
    )
    if not dbobj:
        dbobj = models.Database(database_name="main")
    logging.info(conf.get("SQLALCHEMY_DATABASE_URI"))
    dbobj.set_sqlalchemy_uri(conf.get("SQLALCHEMY_DATABASE_URI"))
    dbobj.expose_in_sqllab = True
    dbobj.allow_run_sync = True
    db.session.add(dbobj)
    db.session.commit()
    return dbobj
コード例 #4
0
def get_error_msg():
    if conf.get("SHOW_STACKTRACE"):
        error_msg = traceback.format_exc()
    else:
        error_msg = "FATAL ERROR \n"
        error_msg += (
            "Stacktrace is hidden. Change the SHOW_STACKTRACE "
            "configuration setting to enable it")
    return error_msg
コード例 #5
0
ファイル: models.py プロジェクト: avsolatorio/caravel
 def refresh_datasources(self, datasource_name=None, merge_flag=False):
     """Refresh metadata of all datasources in the cluster
     If ``datasource_name`` is specified, only that datasource is updated
     """
     self.druid_version = self.get_druid_version()
     for datasource in self.get_datasources():
         if datasource not in conf.get('DRUID_DATA_SOURCE_BLACKLIST', []):
             if not datasource_name or datasource_name == datasource:
                 DruidDatasource.sync_to_db(datasource, self, merge_flag)
コード例 #6
0
ファイル: base.py プロジェクト: neuroradiology/caravel
def get_error_msg():
    if conf.get('SHOW_STACKTRACE'):
        error_msg = traceback.format_exc()
    else:
        error_msg = 'FATAL ERROR \n'
        error_msg += (
            'Stacktrace is hidden. Change the SHOW_STACKTRACE '
            'configuration setting to enable it')
    return error_msg
コード例 #7
0
ファイル: models.py プロジェクト: avsolatorio/caravel
 def latest_metadata(self):
     """Returns segment metadata from the latest segment"""
     client = self.cluster.get_pydruid_client()
     results = client.time_boundary(datasource=self.datasource_name)
     if not results:
         return
     max_time = results[0]['result']['maxTime']
     max_time = dparse(max_time)
     # Query segmentMetadata for 7 days back. However, due to a bug,
     # we need to set this interval to more than 1 day ago to exclude
     # realtime segments, which triggered a bug (fixed in druid 0.8.2).
     # https://groups.google.com/forum/#!topic/druid-user/gVCqqspHqOQ
     lbound = (max_time - timedelta(days=7)).isoformat()
     rbound = max_time.isoformat()
     if not self.version_higher(self.cluster.druid_version, '0.8.2'):
         rbound = (max_time - timedelta(1)).isoformat()
     segment_metadata = None
     try:
         segment_metadata = client.segment_metadata(
             datasource=self.datasource_name,
             intervals=lbound + '/' + rbound,
             merge=self.merge_flag,
             analysisTypes=conf.get('DRUID_ANALYSIS_TYPES'))
     except Exception as e:
         logging.warning("Failed first attempt to get latest segment")
         logging.exception(e)
     if not segment_metadata:
         # if no segments in the past 7 days, look at all segments
         lbound = datetime(1901, 1, 1).isoformat()[:10]
         rbound = datetime(2050, 1, 1).isoformat()[:10]
         if not self.version_higher(self.cluster.druid_version, '0.8.2'):
             rbound = datetime.now().isoformat()[:10]
         try:
             segment_metadata = client.segment_metadata(
                 datasource=self.datasource_name,
                 intervals=lbound + '/' + rbound,
                 merge=self.merge_flag,
                 analysisTypes=conf.get('DRUID_ANALYSIS_TYPES'))
         except Exception as e:
             logging.warning("Failed 2nd attempt to get latest segment")
             logging.exception(e)
     if segment_metadata:
         return segment_metadata[-1]['columns']
コード例 #8
0
ファイル: core.py プロジェクト: bkyryliuk/caravel
def get_or_create_main_db():
    from superset import conf, db
    from superset.models import core as models

    logging.info('Creating database reference')
    dbobj = get_main_database(db.session)
    if not dbobj:
        dbobj = models.Database(database_name='main')
    dbobj.set_sqlalchemy_uri(conf.get('SQLALCHEMY_DATABASE_URI'))
    dbobj.expose_in_sqllab = True
    dbobj.allow_csv_upload = True
    db.session.add(dbobj)
    db.session.commit()
    return dbobj
コード例 #9
0
ファイル: models.py プロジェクト: ythzx/incubator-superset
 def refresh_datasources(
         self,
         datasource_name=None,
         merge_flag=True,
         refreshAll=True):
     """Refresh metadata of all datasources in the cluster
     If ``datasource_name`` is specified, only that datasource is updated
     """
     self.druid_version = self.get_druid_version()
     ds_list = self.get_datasources()
     blacklist = conf.get('DRUID_DATA_SOURCE_BLACKLIST', [])
     ds_refresh = []
     if not datasource_name:
         ds_refresh = list(filter(lambda ds: ds not in blacklist, ds_list))
     elif datasource_name not in blacklist and datasource_name in ds_list:
         ds_refresh.append(datasource_name)
     else:
         return
     self.refresh_async(ds_refresh, merge_flag, refreshAll)
コード例 #10
0
def sync_role_definitions():
    """Inits the Superset application with security roles and such"""
    logging.info("Syncing role definition")

    get_or_create_main_db()
    create_custom_permissions()

    # Creating default roles
    set_role('Admin', is_admin_pvm)
    set_role('Alpha', is_alpha_pvm)
    set_role('Gamma', is_gamma_pvm)
    set_role('granter', is_granter_pvm)
    set_role('sql_lab', is_sql_lab_pvm)

    if conf.get('PUBLIC_ROLE_LIKE_GAMMA', False):
        set_role('Public', is_gamma_pvm)

    create_missing_perms()

    # commit role and view menu updates
    sm.get_session.commit()
コード例 #11
0
ファイル: security.py プロジェクト: tan31989/caravel
    def sync_role_definitions(self):
        """Inits the Superset application with security roles and such"""
        from superset import conf
        logging.info('Syncing role definition')

        self.create_custom_permissions()

        # Creating default roles
        self.set_role('Admin', self.is_admin_pvm)
        self.set_role('Alpha', self.is_alpha_pvm)
        self.set_role('Gamma', self.is_gamma_pvm)
        self.set_role('granter', self.is_granter_pvm)
        self.set_role('sql_lab', self.is_sql_lab_pvm)

        if conf.get('PUBLIC_ROLE_LIKE_GAMMA', False):
            self.set_role('Public', self.is_gamma_pvm)

        self.create_missing_perms()

        # commit role and view menu updates
        self.get_session.commit()
        self.clean_perms()
コード例 #12
0
ファイル: db_engine_specs.py プロジェクト: fplxp3s/superset
from flask import g
from flask_babel import lazy_gettext as _
import pandas
from sqlalchemy import select
from sqlalchemy.engine import create_engine
from sqlalchemy.engine.url import make_url
from sqlalchemy.sql import text
import sqlparse
from werkzeug.utils import secure_filename

from superset import app, cache_util, conf, db, utils
from superset.utils import QueryStatus, SupersetTemplateException

config = app.config

tracking_url_trans = conf.get('TRACKING_URL_TRANSFORMER')

Grain = namedtuple('Grain', 'name label function')


class LimitMethod(object):
    """Enum the ways that limits can be applied"""
    FETCH_MANY = 'fetch_many'
    WRAP_SQL = 'wrap_sql'


class BaseEngineSpec(object):
    """Abstract class for database engine specific configurations"""

    engine = 'base'  # str as defined in sqlalchemy.engine.engine
    cursor_execute_kwargs = {}
コード例 #13
0
ファイル: security.py プロジェクト: gaffeylaw/superset
def sync_role_definitions():
    """Inits the Superset application with security roles and such"""
    logging.info("Syncing role definition")

    # Creating default roles
    alpha = sm.add_role("Alpha")
    admin = sm.add_role("Admin")
    gamma = sm.add_role("Gamma")
    public = sm.add_role("Public")
    sql_lab = sm.add_role("sql_lab")
    granter = sm.add_role("granter")
    dashboard_access = sm.add_role("dashboard_access")
    dashboard_edit = sm.add_role("dashboard_edit")
    slice_access = sm.add_role("slice_access")
    slice_edit = sm.add_role("slice_edit")
    datasource_access = sm.add_role("datasource_access")
    datasource_edit = sm.add_role("datasource_edit")
    manage_edit = sm.add_role("manage_edit")
    user_role_edit = sm.add_role("user_role_edit")

    get_or_create_main_db()

    # Global perms
    merge_perm(sm, 'all_datasource_access', 'all_datasource_access')
    merge_perm(sm, 'all_database_access', 'all_database_access')

    perms = db.session.query(ab_models.PermissionView).all()
    perms = [p for p in perms if p.permission and p.view_menu]

    logging.info("Syncing admin perms")
    for p in perms:
        # admin has all_database_access and all_datasource_access
        if is_user_defined_permission(p):
            sm.del_permission_role(admin, p)
        else:
            sm.add_permission_role(admin, p)

    logging.info("Syncing alpha perms")
    for p in perms:
        # alpha has all_database_access and all_datasource_access
        if is_user_defined_permission(p):
            sm.del_permission_role(alpha, p)
        elif ((p.view_menu.name not in ADMIN_ONLY_VIEW_MENUES
               and p.permission.name not in ADMIN_ONLY_PERMISSIONS)
              or (p.permission.name, p.view_menu.name) in READ_ONLY_PRODUCT):
            sm.add_permission_role(alpha, p)
        else:
            sm.del_permission_role(alpha, p)

    logging.info("Syncing gamma perms and public if specified")
    PUBLIC_ROLE_LIKE_GAMMA = conf.get('PUBLIC_ROLE_LIKE_GAMMA', False)
    for p in perms:
        if ((p.view_menu.name not in ADMIN_ONLY_VIEW_MENUES
             and p.view_menu.name not in GAMMA_READ_ONLY_MODELVIEWS
             and p.permission.name not in ADMIN_ONLY_PERMISSIONS
             and p.permission.name not in ALPHA_ONLY_PERMISSIONS) or
            (p.permission.name, p.view_menu.name) in GAMMA_READ_ONLY_PRODUCT):
            sm.add_permission_role(gamma, p)
            if PUBLIC_ROLE_LIKE_GAMMA:
                sm.add_permission_role(public, p)
        else:
            sm.del_permission_role(gamma, p)
            sm.del_permission_role(public, p)

    logging.info("Syncing sql_lab perms")
    for p in perms:
        if (p.view_menu.name in {'SQL Lab'} or p.permission.name
                in {'can_sql_json', 'can_csv', 'can_search_queries'}):
            sm.add_permission_role(sql_lab, p)
        else:
            sm.del_permission_role(sql_lab, p)

    logging.info("Syncing granter perms")
    for p in perms:
        if (p.permission.name
                in {'can_override_role_permissions', 'can_aprove'}):
            sm.add_permission_role(granter, p)
        else:
            sm.del_permission_role(granter, p)

    logging.info("Syncing dashboard_access perms")
    for p in perms:
        if (p.view_menu.name in {'Dashboards'} or p.permission.name in {
                'can_explore', 'can_explore_json', 'can_slice',
                'can_created_dashboards', 'can_fave_dashboards',
                'all_datasource_access', 'all_database_access', 'can_profile'
        } or (p.permission.name in {'can_list', 'can_show', 'can_download'}
              and p.view_menu.name in {'DashboardModelView'}) or
            (p.permission.name in {'can_list', 'can_show', 'can_download'}
             and p.view_menu.name in {'DashboardModelViewAsync'})
                or (p.permission.name in {
                    'can_show', 'can_edit', 'can_download', 'can_userinfo',
                    'resetmypassword', 'userinfoedit'
                } and p.view_menu.name in {'UserDBModelView'})):
            sm.add_permission_role(dashboard_access, p)
        else:
            sm.del_permission_role(dashboard_access, p)

    logging.info("Syncing dashboard_edit perms")
    for p in perms:
        if (p.view_menu.name in {'Dashboards'} or p.permission.name in {
                'can_explore', 'can_explore_json', 'can_slice',
                'can_created_dashboards', 'can_fave_dashboards',
                'all_datasource_access', 'all_database_access', 'can_profile'
        } or (p.permission.name in {
                'can_list', 'can_show', 'can_add', 'can_delete', 'muldelete',
                'can_edit', 'can_download', 'mulexport'
        } and p.view_menu.name in {'DashboardModelView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'muldelete', 'can_edit', 'can_download', 'mulexport'
                } and p.view_menu.name in {'DashboardModelViewAsync'})
                or (p.permission.name in {
                    'can_show', 'can_edit', 'can_download', 'can_userinfo',
                    'resetmypassword', 'userinfoedit'
                } and p.view_menu.name in {'UserDBModelView'})):
            sm.add_permission_role(dashboard_edit, p)
        else:
            sm.del_permission_role(dashboard_edit, p)

    logging.info("Syncing slice_access perms")
    for p in perms:
        if (p.view_menu.name in {'Slices'} or p.permission.name in {
                'can_explore', 'can_explore_json', 'can_slice',
                'can_created_slices', 'can_fave_slices',
                'all_datasource_access', 'all_database_access', 'can_profile'
        } or (p.permission.name in {'can_list', 'can_show', 'can_download'}
              and p.view_menu.name in {'SliceModelView'}) or
            (p.permission.name in {'can_list', 'can_show', 'can_download'}
             and p.view_menu.name in {'SliceAsync'}) or (p.permission.name in {
                 'can_show', 'can_edit', 'can_userinfo', 'resetmypassword',
                 'userinfoedit'
             } and p.view_menu.name in {'UserDBModelView'})):
            sm.add_permission_role(slice_access, p)
        else:
            sm.del_permission_role(slice_access, p)

    logging.info("Syncing slice_edit perms")
    for p in perms:
        if (p.view_menu.name in {'Slices'} or p.permission.name in {
                'can_explore', 'can_explore_json', 'can_slice',
                'can_created_slices', 'can_fave_slices', 'can_add_slices',
                'all_datasource_access', 'all_database_access', 'can_profile'
        } or (p.permission.name in {
                'can_list', 'can_show', 'can_add', 'can_delete', 'muldelete',
                'can_edit', 'can_download'
        } and p.view_menu.name in {'SliceModelView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'muldelete', 'can_edit', 'can_download'
                } and p.view_menu.name in {'SliceAsync'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'muldelete', 'can_edit', 'can_download'
                } and p.view_menu.name in {'SliceAddView'})
                or (p.permission.name in {
                    'can_show', 'can_edit', 'can_userinfo', 'resetmypassword',
                    'userinfoedit'
                } and p.view_menu.name in {'UserDBModelView'})):
            sm.add_permission_role(slice_edit, p)
        else:
            sm.del_permission_role(slice_edit, p)

    logging.info("Syncing datasource_access perms")
    for p in perms:
        if (p.view_menu.name in {
                'Sources', 'Databases', 'Tables', 'Druid Clusters',
                'Druid Datasources'
        } or p.permission.name in {
                'can_explore', 'can_explore_json', 'all_datasource_access',
                'all_database_access', 'can_profile'
        } or (p.permission.name in {'can_list', 'can_show'}
              and p.view_menu.name in {'DatabaseView'})
                or (p.permission.name in {'can_list', 'can_show'}
                    and p.view_menu.name in {'DatabaseAsync'})
                or (p.permission.name in {'can_list', 'can_show'}
                    and p.view_menu.name in {'TableModelView'})
                or (p.permission.name in {'can_list', 'can_show'}
                    and p.view_menu.name in {'DatabaseTableAsync'})
                or (p.permission.name in {'can_list', 'can_show'}
                    and p.view_menu.name in {'DruidDatasourceModelView'})
                or (p.permission.name in {'can_list', 'can_show'}
                    and p.view_menu.name in {'DruidClusterModelView'})
                or (p.permission.name in {
                    'can_show', 'can_edit', 'can_userinfo', 'resetmypassword',
                    'userinfoedit'
                } and p.view_menu.name in {'UserDBModelView'})):
            sm.add_permission_role(datasource_access, p)
        else:
            sm.del_permission_role(datasource_access, p)

    logging.info("Syncing datasource_edit perms")
    for p in perms:
        if (p.view_menu.name in {
                'Sources', 'Databases', 'Tables', 'Druid Clusters',
                'Druid Datasources', 'Refresh Druid Metadata',
                'TableColumnInlineView', 'SqlMetricInlineView'
        } or p.permission.name in {
                'can_explore', 'can_explore_json', 'can_testconn',
                'can_checkbox', 'can_refresh_datasources',
                'all_datasource_access', 'all_database_access', 'can_profile'
        } or (p.permission.name in {
                'can_list', 'can_show', 'can_add', 'can_delete', 'muldelete',
                'can_edit', 'can_download'
        } and p.view_menu.name in {'DatabaseView'}) or (p.permission.name in {
                'can_list', 'can_show', 'can_add', 'can_delete', 'muldelete',
                'can_edit', 'can_download'
        } and p.view_menu.name in {'DatabaseAsync'}) or (p.permission.name in {
                'can_list', 'can_show', 'can_add', 'can_delete', 'muldelete',
                'can_edit', 'can_download'
        } and p.view_menu.name in {'TableModelView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'muldelete', 'can_edit', 'can_download'
                } and p.view_menu.name in {'DatabaseTablesAsync'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'muldelete', 'can_edit', 'can_download'
                } and p.view_menu.name in {'DruidDatasourceModelView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'muldelete', 'can_edit', 'can_download'
                } and p.view_menu.name in {'DruidClusterModelView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'can_edit', 'can_download'
                } and p.view_menu.name in {'TableColumnInlineView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'can_edit', 'can_download'
                } and p.view_menu.name in {'SqlMetricInlineView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'can_edit', 'can_download'
                } and p.view_menu.name in {'DruidColumnInlineView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'can_edit', 'can_download'
                } and p.view_menu.name in {'DruidMetricInlineView'})
                or (p.permission.name in {
                    'can_show', 'can_edit', 'can_userinfo', 'resetmypassword',
                    'userinfoedit'
                } and p.view_menu.name in {'UserDBModelView'})):
            sm.add_permission_role(datasource_edit, p)
        else:
            sm.del_permission_role(datasource_edit, p)

    logging.info("Syncing manage_edit perms")
    for p in perms:
        if (p.view_menu.name
                in {'Manage', 'Import Dashboards', 'Queries', 'CSS Templates'}
                or p.permission.name in {'can_profile'}
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'can_edit', 'can_download'
                } and p.view_menu.name in {'QueryView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'can_edit', 'can_download'
                } and p.view_menu.name in {'CssTemplateModelView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'can_edit', 'can_download'
                } and p.view_menu.name in {'CssTemplateAsyncModelView'})
                or (p.permission.name in {'can_add'}
                    and p.view_menu.name in {'DashboardModelView'})
                or (p.permission.name in {'can_add'}
                    and p.view_menu.name in {'SliceAddView'})
                or (p.permission.name in {
                    'can_show', 'can_edit', 'can_userinfo', 'resetmypassword',
                    'userinfoedit'
                } and p.view_menu.name in {'UserDBModelView'})):
            sm.add_permission_role(manage_edit, p)
        else:
            sm.del_permission_role(manage_edit, p)

    logging.info("Syncing user_role_edit perms")
    for p in perms:
        if (p.view_menu.name in {
                'Security', 'List Users', 'List Roles', "User's Statistics",
                'Base Permissions', 'Views/Menus', 'Permission on Views/Menus',
                'Access requests', 'Action Log'
        } or p.permission.name in {'can_recent_activity', 'can_profile'}
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'muldelete', 'can_edit', 'can_download', 'can_userinfo',
                    'resetmypassword', 'resetpasswords', 'userinfoedit'
                } and p.view_menu.name in {'UserDBModelView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'muldelete', 'can_edit', 'can_download', 'Copy Role',
                    'can_update_role', 'can_override_role_permissions'
                } and p.view_menu.name in {'RoleModelView'})
                or (p.permission.name in {'can_chart'}
                    and p.view_menu.name in {'UserStatsChartView'})
                or (p.permission.name in {'can_list'}
                    and p.view_menu.name in {'PermissionModelView'})
                or (p.permission.name in {'can_list'}
                    and p.view_menu.name in {'ViewMenuModelView'})
                or (p.permission.name in {'can_list'}
                    and p.view_menu.name in {'PermissionViewModelView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'can_edit', 'can_download', 'muldelete'
                } and p.view_menu.name in {'AccessRequestsModelView'})
                or (p.permission.name in {
                    'can_list', 'can_show', 'can_add', 'can_delete',
                    'can_edit', 'can_download'
                } and p.view_menu.name in {'LogModelView'}) or
            (p.permission.name in {'can_this_form_post', 'can_this_form_get'}
             and p.view_menu.name in {'ResetMyPasswordView'}) or
            (p.permission.name in {'can_this_form_post', 'can_this_form_get'}
             and p.view_menu.name in {'ResetPasswordView'}) or
            (p.permission.name in {'can_this_form_post', 'can_this_form_get'}
             and p.view_menu.name in {'UserInfoEditView'})):
            sm.add_permission_role(user_role_edit, p)
        else:
            sm.del_permission_role(user_role_edit, p)

    logging.info("Making sure all data source perms have been created")
    session = db.session()
    datasources = [o for o in session.query(models.SqlaTable).all()]
    datasources += [o for o in session.query(models.DruidDatasource).all()]
    for datasource in datasources:
        perm = datasource.get_perm()
        merge_perm(sm, 'datasource_access', perm)
        if datasource.schema:
            merge_perm(sm, 'schema_access', datasource.schema_perm)
        if perm != datasource.perm:
            datasource.perm = perm

    logging.info("Making sure all database perms have been created")
    databases = [o for o in session.query(models.Database).all()]
    for database in databases:
        perm = database.get_perm()
        if perm != database.perm:
            database.perm = perm
        merge_perm(sm, 'database_access', perm)
    session.commit()

    logging.info("Making sure all dashboard perms have been created")
    dashboards = [o for o in session.query(models.Dashboard).all()]
    for dashboard in dashboards:
        perm = dashboard.get_dashboard_title()
        sm.add_permission_view_menu('dashboard_access', perm)
    session.commit()

    logging.info("Making sure all metrics perms exist")
    models.init_metrics_perm()
コード例 #14
0
ファイル: base.py プロジェクト: ducminh0705/Superset
class CsvResponse(Response):
    """
    Override Response to take into account csv encoding from config.py
    """
    charset = conf.get('CSV_EXPORT').get('encoding', 'utf-8')
コード例 #15
0
def get_main_database():
    from superset import conf

    return get_or_create_db("main", conf.get("SQLALCHEMY_DATABASE_URI"))
コード例 #16
0
ファイル: base.py プロジェクト: bijanmoudi/incubator-superset
class XlsxResponse(Response):
    """
       Override Response to take into account xlsx encoding from config.py
    """
    charset = conf.get('XLSX_EXPORT').get('encoding', 'utf-8')
コード例 #17
0
 def get_table_access_link(self, tables):
     from superset import conf
     return conf.get('PERMISSION_INSTRUCTIONS_LINK')
コード例 #18
0
def get_user_roles():
    if g.user.is_anonymous():
        public_role = conf.get('AUTH_ROLE_PUBLIC')
        return [security_manager.find_role(public_role)] if public_role else []
    return g.user.roles
コード例 #19
0
def get_user_roles():
    if g.user.is_anonymous():
        public_role = conf.get('AUTH_ROLE_PUBLIC')
        return [appbuilder.sm.find_role(public_role)] if public_role else []
    return g.user.roles
コード例 #20
0
def sync_role_definitions():
    """Inits the Superset application with security roles and such"""
    logging.info("Syncing role definition")

    # Creating default roles
    alpha = sm.add_role("Alpha")
    admin = sm.add_role("Admin")
    gamma = sm.add_role("Gamma")
    public = sm.add_role("Public")
    sql_lab = sm.add_role("sql_lab")
    granter = sm.add_role("granter")

    get_or_create_main_db()

    # Global perms
    merge_perm(sm, 'all_datasource_access', 'all_datasource_access')
    merge_perm(sm, 'all_database_access', 'all_database_access')

    perms = db.session.query(ab_models.PermissionView).all()
    perms = [p for p in perms if p.permission and p.view_menu]

    logging.info("Syncing admin perms")
    for p in perms:
        # admin has all_database_access and all_datasource_access
        if is_user_defined_permission(p):
            sm.del_permission_role(admin, p)
        else:
            sm.add_permission_role(admin, p)

    logging.info("Syncing alpha perms")
    for p in perms:
        # alpha has all_database_access and all_datasource_access
        if is_user_defined_permission(p):
            sm.del_permission_role(alpha, p)
        elif ((p.view_menu.name not in ADMIN_ONLY_VIEW_MENUES
               and p.permission.name not in ADMIN_ONLY_PERMISSIONS)
              or (p.permission.name, p.view_menu.name) in READ_ONLY_PRODUCT):
            sm.add_permission_role(alpha, p)
        else:
            sm.del_permission_role(alpha, p)

    logging.info("Syncing gamma perms and public if specified")
    PUBLIC_ROLE_LIKE_GAMMA = conf.get('PUBLIC_ROLE_LIKE_GAMMA', False)
    for p in perms:
        if ((p.view_menu.name not in ADMIN_ONLY_VIEW_MENUES
             and p.view_menu.name not in GAMMA_READ_ONLY_MODELVIEWS
             and p.permission.name not in ADMIN_ONLY_PERMISSIONS
             and p.permission.name not in ALPHA_ONLY_PERMISSIONS) or
            (p.permission.name, p.view_menu.name) in GAMMA_READ_ONLY_PRODUCT):
            sm.add_permission_role(gamma, p)
            if PUBLIC_ROLE_LIKE_GAMMA:
                sm.add_permission_role(public, p)
        else:
            sm.del_permission_role(gamma, p)
            sm.del_permission_role(public, p)

    logging.info("Syncing sql_lab perms")
    for p in perms:
        if (p.view_menu.name in {'SQL Lab'} or p.permission.name
                in {'can_sql_json', 'can_csv', 'can_search_queries'}):
            sm.add_permission_role(sql_lab, p)
        else:
            sm.del_permission_role(sql_lab, p)

    logging.info("Syncing granter perms")
    for p in perms:
        if (p.permission.name
                in {'can_override_role_permissions', 'can_aprove'}):
            sm.add_permission_role(granter, p)
        else:
            sm.del_permission_role(granter, p)

    logging.info("Making sure all data source perms have been created")
    session = db.session()
    datasources = [o for o in session.query(models.SqlaTable).all()]
    datasources += [o for o in session.query(models.DruidDatasource).all()]
    for datasource in datasources:
        perm = datasource.get_perm()
        merge_perm(sm, 'datasource_access', perm)
        if datasource.schema:
            merge_perm(sm, 'schema_access', datasource.schema_perm)
        if perm != datasource.perm:
            datasource.perm = perm

    logging.info("Making sure all database perms have been created")
    databases = [o for o in session.query(models.Database).all()]
    for database in databases:
        perm = database.get_perm()
        if perm != database.perm:
            database.perm = perm
        merge_perm(sm, 'database_access', perm)
    session.commit()

    logging.info("Making sure all metrics perms exist")
    models.init_metrics_perm()
コード例 #21
0
"""
Superset allows to redirect user to a default dashboard
but the only way to set this dashboard is by code
"""

from flask_appbuilder.security.sqla.models import User

from superset import conf, db
from superset.models.core import Dashboard
from superset.models.user_attributes import UserAttribute


def set_welcome_dashboard(id, username):
    # Get default user
    user = db.session.query(User).filter_by(username=username).first()

    # Make sure welcome dashboard exists
    dashboard = db.session.query(Dashboard).filter_by(id=id).first()

    # Set dashboard as default
    extra_attributes = UserAttribute(
        user_id=user.id,
        welcome_dashboard_id=dashboard.id,
    )
    db.session.add(extra_attributes)
    db.session.commit()


set_welcome_dashboard(conf.get('DEFAULT_DASHBOARD_ID'),
                      conf.get('DEFAULT_USERNAME'))
コード例 #22
0
ファイル: base.py プロジェクト: herotan24/incubator-superset
def get_user_roles():
    if g.user.is_anonymous():
        public_role = conf.get('AUTH_ROLE_PUBLIC')
        return [appbuilder.sm.find_role(public_role)] if public_role else []
    return g.user.roles
コード例 #23
0
ファイル: security.py プロジェクト: the-dcruz/caravel
def sync_role_definitions():
    """Inits the Superset application with security roles and such"""
    logging.info("Syncing role definition")

    # Creating default roles
    alpha = sm.add_role("Alpha")
    admin = sm.add_role("Admin")
    gamma = sm.add_role("Gamma")
    public = sm.add_role("Public")
    sql_lab = sm.add_role("sql_lab")
    granter = sm.add_role("granter")

    get_or_create_main_db()

    # Global perms
    sm.add_permission_view_menu(
        'all_datasource_access', 'all_datasource_access')
    sm.add_permission_view_menu('all_database_access', 'all_database_access')

    perms = db.session.query(ab_models.PermissionView).all()
    perms = [p for p in perms if p.permission and p.view_menu]

    logging.info("Syncing admin perms")
    for p in perms:
        # admin has all_database_access and all_datasource_access
        if is_user_defined_permission(p):
            sm.del_permission_role(admin, p)
        else:
            sm.add_permission_role(admin, p)

    logging.info("Syncing alpha perms")
    for p in perms:
        # alpha has all_database_access and all_datasource_access
        if is_user_defined_permission(p):
            sm.del_permission_role(alpha, p)
        elif (
                (
                    p.view_menu.name not in ADMIN_ONLY_VIEW_MENUES and
                    p.permission.name not in ADMIN_ONLY_PERMISSIONS
                ) or
                (p.permission.name, p.view_menu.name) in READ_ONLY_PRODUCT
        ):
            sm.add_permission_role(alpha, p)
        else:
            sm.del_permission_role(alpha, p)

    logging.info("Syncing gamma perms and public if specified")
    PUBLIC_ROLE_LIKE_GAMMA = conf.get('PUBLIC_ROLE_LIKE_GAMMA', False)
    for p in perms:
        if (
                (
                    p.view_menu.name not in ADMIN_ONLY_VIEW_MENUES and
                    p.permission.name not in ADMIN_ONLY_PERMISSIONS and
                    p.permission.name not in ALPHA_ONLY_PERMISSIONS
                ) or
                (p.permission.name, p.view_menu.name) in READ_ONLY_PRODUCT
        ):
            sm.add_permission_role(gamma, p)
            if PUBLIC_ROLE_LIKE_GAMMA:
                sm.add_permission_role(public, p)
        else:
            sm.del_permission_role(gamma, p)
            sm.del_permission_role(public, p)

    logging.info("Syncing sql_lab perms")
    for p in perms:
        if (
                p.view_menu.name in {'SQL Lab'} or
                p.permission.name in {
                    'can_sql_json', 'can_csv', 'can_search_queries'}
        ):
            sm.add_permission_role(sql_lab, p)
        else:
            sm.del_permission_role(sql_lab, p)

    logging.info("Syncing granter perms")
    for p in perms:
        if (
                p.permission.name in {
                    'can_override_role_permissions', 'can_aprove'}
        ):
            sm.add_permission_role(granter, p)
        else:
            sm.del_permission_role(granter, p)

    logging.info("Making sure all data source perms have been created")
    session = db.session()
    datasources = [
        o for o in session.query(models.SqlaTable).all()]
    datasources += [
        o for o in session.query(models.DruidDatasource).all()]
    for datasource in datasources:
        perm = datasource.get_perm()
        sm.add_permission_view_menu('datasource_access', perm)
        if perm != datasource.perm:
            datasource.perm = perm

    logging.info("Making sure all database perms have been created")
    databases = [o for o in session.query(models.Database).all()]
    for database in databases:
        perm = database.get_perm()
        if perm != database.perm:
            database.perm = perm
        sm.add_permission_view_menu('database_access', perm)
    session.commit()

    logging.info("Making sure all metrics perms exist")
    models.init_metrics_perm()
コード例 #24
0
ファイル: security.py プロジェクト: tan31989/caravel
 def get_datasource_access_link(self, datasource):
     from superset import conf
     return conf.get('PERMISSION_INSTRUCTIONS_LINK')
コード例 #25
0
ファイル: security.py プロジェクト: tan31989/caravel
 def get_table_access_link(self, tables):
     from superset import conf
     return conf.get('PERMISSION_INSTRUCTIONS_LINK')
コード例 #26
0
import sqlalchemy as sqla
from sqlalchemy import select
from sqlalchemy.engine import create_engine
from sqlalchemy.engine.url import make_url
from sqlalchemy.sql import text
import sqlparse
import unicodecsv
from werkzeug.utils import secure_filename

from superset import app, cache_util, conf, db, utils
from superset.exceptions import SupersetTemplateException
from superset.utils import QueryStatus

config = app.config

tracking_url_trans = conf.get('TRACKING_URL_TRANSFORMER')
hive_poll_interval = conf.get('HIVE_POLL_INTERVAL')

Grain = namedtuple('Grain', 'name label function')


class LimitMethod(object):
    """Enum the ways that limits can be applied"""
    FETCH_MANY = 'fetch_many'
    WRAP_SQL = 'wrap_sql'


class BaseEngineSpec(object):

    """Abstract class for database engine specific configurations"""
コード例 #27
0
 def get_datasource_access_link(self, datasource):
     from superset import conf
     return conf.get('PERMISSION_INSTRUCTIONS_LINK')
コード例 #28
0
)
from pydruid.utils.having import Aggregation

from flask import Markup, escape
from flask_appbuilder.models.decorators import renders
from flask_appbuilder import Model

from flask_babel import lazy_gettext as _

from superset import conf, db, import_util, utils, sm, get_session
from superset.utils import (flasher, MetricPermException, DimSelector,
                            DTTM_ALIAS)
from superset.connectors.base import BaseDatasource, BaseColumn, BaseMetric
from superset.models.helpers import AuditMixinNullable, QueryResult, set_perm

DRUID_TZ = conf.get("DRUID_TZ")


class JavascriptPostAggregator(Postaggregator):
    def __init__(self, name, field_names, function):
        self.post_aggregator = {
            'type': 'javascript',
            'fieldNames': field_names,
            'name': name,
            'function': function,
        }
        self.name = name


class DruidCluster(Model, AuditMixinNullable):
    """ORM object referencing the Druid clusters"""
コード例 #29
0
ファイル: base.py プロジェクト: neuroradiology/caravel
def get_user_roles():
    if g.user.is_anonymous:
        public_role = conf.get('AUTH_ROLE_PUBLIC')
        return [security_manager.find_role(public_role)] if public_role else []
    return g.user.roles
コード例 #30
0
ファイル: models.py プロジェクト: ythzx/incubator-superset
from pydruid.utils.having import Aggregation

from flask import Markup, escape
from flask_appbuilder.models.decorators import renders
from flask_appbuilder import Model

from flask_babel import lazy_gettext as _

from superset import conf, db, import_util, utils, sm, get_session
from superset.utils import (
    flasher, MetricPermException, DimSelector, DTTM_ALIAS
)
from superset.connectors.base.models import BaseDatasource, BaseColumn, BaseMetric
from superset.models.helpers import AuditMixinNullable, QueryResult, set_perm

DRUID_TZ = conf.get("DRUID_TZ")


# Function wrapper because bound methods cannot
# be passed to processes
def _fetch_metadata_for(datasource):
    return datasource.latest_metadata()


class JavascriptPostAggregator(Postaggregator):
    def __init__(self, name, field_names, function):
        self.post_aggregator = {
            'type': 'javascript',
            'fieldNames': field_names,
            'name': name,
            'function': function,
コード例 #31
0
def get_main_database() -> "Database":
    from superset import conf

    db_uri = conf.get("SQLALCHEMY_DATABASE_URI")
    return get_or_create_db("main", db_uri)
コード例 #32
0
def is_user_admin() -> bool:
    user_roles = [
        role.name.lower() for role in security_manager.get_user_roles()
    ]
    admin_role = conf.get("AUTH_ROLE_ADMIN").lower()
    return admin_role in user_roles
コード例 #33
0
def get_example_database():
    from superset import conf

    db_uri = conf.get("SQLALCHEMY_EXAMPLES_URI") or conf.get(
        "SQLALCHEMY_DATABASE_URI")
    return get_or_create_db("examples", db_uri)
コード例 #34
0
ファイル: models.py プロジェクト: johnsonc/caravel
    def run_query(  # noqa / druid
            self,
            groupby, metrics,
            granularity,
            from_dttm, to_dttm,
            filter=None,  # noqa
            is_timeseries=True,
            timeseries_limit=None,
            timeseries_limit_metric=None,
            row_limit=None,
            inner_from_dttm=None, inner_to_dttm=None,
            orderby=None,
            extras=None,  # noqa
            columns=None, phase=2, client=None,
            order_desc=True,
            prequeries=None,
            is_prequery=False,
        ):
        """Runs a query against Druid and returns a dataframe.
        """
        # TODO refactor into using a TBD Query object
        client = client or self.cluster.get_pydruid_client()
        row_limit = row_limit or conf.get('ROW_LIMIT')

        if not is_timeseries:
            granularity = 'all'

        if (
                granularity == 'all' or
                timeseries_limit is None or
                timeseries_limit == 0):
            phase = 1
        inner_from_dttm = inner_from_dttm or from_dttm
        inner_to_dttm = inner_to_dttm or to_dttm

        timezone = from_dttm.tzname() if from_dttm else None

        query_str = ''
        metrics_dict = {m.metric_name: m for m in self.metrics}
        columns_dict = {c.column_name: c for c in self.columns}

        all_metrics, post_aggs = DruidDatasource.metrics_and_post_aggs(
            metrics,
            metrics_dict)

        aggregations = self.get_aggregations(all_metrics)
        self.check_restricted_metrics(aggregations)

        # the dimensions list with dimensionSpecs expanded
        dimensions = self.get_dimensions(groupby, columns_dict)
        extras = extras or {}
        qry = dict(
            datasource=self.datasource_name,
            dimensions=dimensions,
            aggregations=aggregations,
            granularity=DruidDatasource.granularity(
                granularity,
                timezone=timezone,
                origin=extras.get('druid_time_origin'),
            ),
            post_aggregations=post_aggs,
            intervals=self.intervals_from_dttms(from_dttm, to_dttm),
        )

        filters = DruidDatasource.get_filters(filter, self.num_cols)
        if filters:
            qry['filter'] = filters

        having_filters = self.get_having_filters(extras.get('having_druid'))
        if having_filters:
            qry['having'] = having_filters

        order_direction = 'descending' if order_desc else 'ascending'

        if columns:
            del qry['post_aggregations']
            del qry['aggregations']
            qry['dimensions'] = columns
            qry['metrics'] = []
            qry['granularity'] = 'all'
            qry['limit'] = row_limit
            client.scan(**qry)
        elif len(groupby) == 0 and not having_filters:
            logging.info('Running timeseries query for no groupby values')
            del qry['dimensions']
            client.timeseries(**qry)
        elif (
                not having_filters and
                len(groupby) == 1 and
                order_desc
        ):
            dim = list(qry.get('dimensions'))[0]
            logging.info('Running two-phase topn query for dimension [{}]'.format(dim))
            pre_qry = deepcopy(qry)
            if timeseries_limit_metric:
                order_by = timeseries_limit_metric
                pre_qry['aggregations'] = self.get_aggregations([timeseries_limit_metric])
            else:
                order_by = list(qry['aggregations'].keys())[0]
            # Limit on the number of timeseries, doing a two-phases query
            pre_qry['granularity'] = 'all'
            pre_qry['threshold'] = min(row_limit,
                                       timeseries_limit or row_limit)
            pre_qry['metric'] = order_by
            pre_qry['dimension'] = self._dimensions_to_values(qry.get('dimensions'))[0]
            del pre_qry['dimensions']

            client.topn(**pre_qry)
            logging.info('Phase 1 Complete')
            query_str += '// Two phase query\n// Phase 1\n'
            query_str += json.dumps(
                client.query_builder.last_query.query_dict, indent=2)
            query_str += '\n'
            if phase == 1:
                return query_str
            query_str += (
                "// Phase 2 (built based on phase one's results)\n")
            df = client.export_pandas()
            qry['filter'] = self._add_filter_from_pre_query_data(
                df,
                [pre_qry['dimension']],
                filters)
            qry['threshold'] = timeseries_limit or 1000
            if row_limit and granularity == 'all':
                qry['threshold'] = row_limit
            qry['dimension'] = dim
            del qry['dimensions']
            qry['metric'] = list(qry['aggregations'].keys())[0]
            client.topn(**qry)
            logging.info('Phase 2 Complete')
        elif len(groupby) > 0 or having_filters:
            # If grouping on multiple fields or using a having filter
            # we have to force a groupby query
            logging.info('Running groupby query for dimensions [{}]'.format(dimensions))
            if timeseries_limit and is_timeseries:
                logging.info('Running two-phase query for timeseries')

                pre_qry = deepcopy(qry)
                pre_qry_dims = self._dimensions_to_values(qry['dimensions'])
                pre_qry['dimensions'] = list(set(pre_qry_dims))

                order_by = metrics[0] if metrics else pre_qry_dims[0]

                if timeseries_limit_metric:
                    order_by = timeseries_limit_metric

                # Limit on the number of timeseries, doing a two-phases query
                pre_qry['granularity'] = 'all'
                pre_qry['limit_spec'] = {
                    'type': 'default',
                    'limit': min(timeseries_limit, row_limit),
                    'intervals': self.intervals_from_dttms(
                        inner_from_dttm, inner_to_dttm),
                    'columns': [{
                        'dimension': order_by,
                        'direction': order_direction,
                    }],
                }
                client.groupby(**pre_qry)
                logging.info('Phase 1 Complete')
                query_str += '// Two phase query\n// Phase 1\n'
                query_str += json.dumps(
                    client.query_builder.last_query.query_dict, indent=2)
                query_str += '\n'
                if phase == 1:
                    return query_str
                query_str += (
                    "// Phase 2 (built based on phase one's results)\n")
                df = client.export_pandas()
                qry['filter'] = self._add_filter_from_pre_query_data(
                    df,
                    pre_qry['dimensions'],
                    filters,
                )
                qry['limit_spec'] = None
            if row_limit:
                dimension_values = self._dimensions_to_values(dimensions)
                qry['limit_spec'] = {
                    'type': 'default',
                    'limit': row_limit,
                    'columns': [{
                        'dimension': (
                            metrics[0] if metrics else dimension_values[0]),
                        'direction': order_direction,
                    }],
                }
            client.groupby(**qry)
            logging.info('Query Complete')
        query_str += json.dumps(
            client.query_builder.last_query.query_dict, indent=2)
        return query_str
コード例 #35
0
from superset.db_engine_specs.base import BaseEngineSpec
from superset.db_engine_specs.presto import PrestoEngineSpec
from superset.exceptions import SupersetException
from superset.models.sql_lab import Query
from superset.sql_parse import Table
from superset.utils import core as utils

if TYPE_CHECKING:
    # prevent circular imports
    from superset.models.core import Database  # pylint: disable=unused-import

QueryStatus = utils.QueryStatus
config = app.config
logger = logging.getLogger(__name__)

tracking_url_trans = conf.get("TRACKING_URL_TRANSFORMER")
hive_poll_interval = conf.get("HIVE_POLL_INTERVAL")


def upload_to_s3(filename: str, upload_prefix: str, table: Table) -> str:
    # Optional dependency
    import boto3  # pylint: disable=import-error

    bucket_path = config["CSV_TO_HIVE_UPLOAD_S3_BUCKET"]

    if not bucket_path:
        logger.info("No upload bucket specified")
        raise Exception(
            "No upload bucket specified. You can specify one in the config file."
        )
コード例 #36
0
class CsvResponse(Response):
    """
    Override Response to take into account csv encoding from config.py
    """

    charset = conf.get("CSV_EXPORT").get("encoding", "utf-8")
コード例 #37
0
ファイル: base.py プロジェクト: takadaZ/superset
def get_user_roles() -> List[Role]:
    if g.user.is_anonymous:
        public_role = conf.get("AUTH_ROLE_PUBLIC")
        return [security_manager.find_role(public_role)] if public_role else []
    return g.user.roles
コード例 #38
0
ファイル: models.py プロジェクト: chkui/superset
import sqlalchemy as sa
from sqlalchemy import (
    Boolean, Column, DateTime, ForeignKey, Integer, or_, String, Text, UniqueConstraint,
)
from sqlalchemy.orm import backref, relationship

from superset import conf, db, import_util, sm, utils
from superset.connectors.base.models import BaseColumn, BaseDatasource, BaseMetric
from superset.models.helpers import (
  AuditMixinNullable, ImportMixin, QueryResult, set_perm,
)
from superset.utils import (
    DimSelector, DTTM_ALIAS, flasher, MetricPermException,
)

DRUID_TZ = conf.get('DRUID_TZ')


# Function wrapper because bound methods cannot
# be passed to processes
def _fetch_metadata_for(datasource):
    return datasource.latest_metadata()


class JavascriptPostAggregator(Postaggregator):
    def __init__(self, name, field_names, function):
        self.post_aggregator = {
            'type': 'javascript',
            'fieldNames': field_names,
            'name': name,
            'function': function,
コード例 #39
0
ファイル: models.py プロジェクト: chkui/superset
    def run_query(  # noqa / druid
            self,
            groupby, metrics,
            granularity,
            from_dttm, to_dttm,
            filter=None,  # noqa
            is_timeseries=True,
            timeseries_limit=None,
            timeseries_limit_metric=None,
            row_limit=None,
            inner_from_dttm=None, inner_to_dttm=None,
            orderby=None,
            extras=None,  # noqa
            columns=None, phase=2, client=None, form_data=None,
            order_desc=True,
            prequeries=None,
            is_prequery=False,
        ):
        """Runs a query against Druid and returns a dataframe.
        """
        # TODO refactor into using a TBD Query object
        client = client or self.cluster.get_pydruid_client()
        row_limit = row_limit or conf.get('ROW_LIMIT')

        if not is_timeseries:
            granularity = 'all'

        if (
                granularity == 'all' or
                timeseries_limit is None or
                timeseries_limit == 0):
            phase = 1
        inner_from_dttm = inner_from_dttm or from_dttm
        inner_to_dttm = inner_to_dttm or to_dttm

        timezone = from_dttm.tzname() if from_dttm else None

        query_str = ''
        metrics_dict = {m.metric_name: m for m in self.metrics}
        columns_dict = {c.column_name: c for c in self.columns}

        all_metrics, post_aggs = DruidDatasource.metrics_and_post_aggs(
            metrics,
            metrics_dict)

        aggregations = self.get_aggregations(all_metrics)
        self.check_restricted_metrics(aggregations)

        # the dimensions list with dimensionSpecs expanded
        dimensions = self.get_dimensions(groupby, columns_dict)
        extras = extras or {}
        qry = dict(
            datasource=self.datasource_name,
            dimensions=dimensions,
            aggregations=aggregations,
            granularity=DruidDatasource.granularity(
                granularity,
                timezone=timezone,
                origin=extras.get('druid_time_origin'),
            ),
            post_aggregations=post_aggs,
            intervals=self.intervals_from_dttms(from_dttm, to_dttm),
        )

        filters = DruidDatasource.get_filters(filter, self.num_cols)
        if filters:
            qry['filter'] = filters

        having_filters = self.get_having_filters(extras.get('having_druid'))
        if having_filters:
            qry['having'] = having_filters

        order_direction = 'descending' if order_desc else 'ascending'

        if columns:
            del qry['post_aggregations']
            del qry['aggregations']
            qry['dimensions'] = columns
            qry['metrics'] = []
            qry['granularity'] = 'all'
            qry['limit'] = row_limit
            client.scan(**qry)
        elif len(groupby) == 0 and not having_filters:
            logging.info('Running timeseries query for no groupby values')
            del qry['dimensions']
            client.timeseries(**qry)
        elif (
                not having_filters and
                len(groupby) == 1 and
                order_desc
        ):
            dim = list(qry.get('dimensions'))[0]
            logging.info('Running two-phase topn query for dimension [{}]'.format(dim))
            if timeseries_limit_metric:
                order_by = timeseries_limit_metric
            else:
                order_by = list(qry['aggregations'].keys())[0]
            # Limit on the number of timeseries, doing a two-phases query
            pre_qry = deepcopy(qry)
            pre_qry['granularity'] = 'all'
            pre_qry['threshold'] = min(row_limit,
                                       timeseries_limit or row_limit)
            pre_qry['metric'] = order_by
            if isinstance(dim, dict):
                if 'dimension' in dim:
                    pre_qry['dimension'] = dim['dimension']
            else:
                pre_qry['dimension'] = dim
            del pre_qry['dimensions']
            client.topn(**pre_qry)
            logging.info('Phase 1 Complete')
            query_str += '// Two phase query\n// Phase 1\n'
            query_str += json.dumps(
                client.query_builder.last_query.query_dict, indent=2)
            query_str += '\n'
            if phase == 1:
                return query_str
            query_str += (
                "// Phase 2 (built based on phase one's results)\n")
            df = client.export_pandas()
            qry['filter'] = self._add_filter_from_pre_query_data(
                df,
                [pre_qry['dimension']],
                filters)
            qry['threshold'] = timeseries_limit or 1000
            if row_limit and granularity == 'all':
                qry['threshold'] = row_limit
            qry['dimension'] = dim
            del qry['dimensions']
            qry['metric'] = list(qry['aggregations'].keys())[0]
            client.topn(**qry)
            logging.info('Phase 2 Complete')
        elif len(groupby) > 0 or having_filters:
            # If grouping on multiple fields or using a having filter
            # we have to force a groupby query
            logging.info('Running groupby query for dimensions [{}]'.format(dimensions))
            if timeseries_limit and is_timeseries:
                logging.info('Running two-phase query for timeseries')
                order_by = metrics[0] if metrics else self.metrics[0]
                if timeseries_limit_metric:
                    order_by = timeseries_limit_metric
                # Limit on the number of timeseries, doing a two-phases query
                pre_qry = deepcopy(qry)
                pre_qry['granularity'] = 'all'
                pre_qry['limit_spec'] = {
                    'type': 'default',
                    'limit': min(timeseries_limit, row_limit),
                    'intervals': self.intervals_from_dttms(
                        inner_from_dttm, inner_to_dttm),
                    'columns': [{
                        'dimension': order_by,
                        'direction': order_direction,
                    }],
                }
                pre_qry_dims = []
                # Replace dimensions specs with their `dimension`
                # values, and ignore those without
                for dim in qry['dimensions']:
                    if isinstance(dim, dict):
                        if 'dimension' in dim:
                            pre_qry_dims.append(dim['dimension'])
                    else:
                        pre_qry_dims.append(dim)
                pre_qry['dimensions'] = list(set(pre_qry_dims))
                client.groupby(**pre_qry)
                logging.info('Phase 1 Complete')
                query_str += '// Two phase query\n// Phase 1\n'
                query_str += json.dumps(
                    client.query_builder.last_query.query_dict, indent=2)
                query_str += '\n'
                if phase == 1:
                    return query_str
                query_str += (
                    "// Phase 2 (built based on phase one's results)\n")
                df = client.export_pandas()
                qry['filter'] = self._add_filter_from_pre_query_data(
                    df,
                    pre_qry['dimensions'],
                    filters,
                )
                qry['limit_spec'] = None
            if row_limit:
                qry['limit_spec'] = {
                    'type': 'default',
                    'limit': row_limit,
                    'columns': [{
                        'dimension': (
                            metrics[0] if metrics else self.metrics[0]),
                        'direction': order_direction,
                    }],
                }
            client.groupby(**qry)
            logging.info('Query Complete')
        query_str += json.dumps(
            client.query_builder.last_query.query_dict, indent=2)
        return query_str
コード例 #40
0
ファイル: models.py プロジェクト: johnsonc/caravel
import sqlalchemy as sa
from sqlalchemy import (
    Boolean, Column, DateTime, ForeignKey, Integer, or_, String, Text, UniqueConstraint,
)
from sqlalchemy.orm import backref, relationship

from superset import conf, db, import_util, sm, utils
from superset.connectors.base.models import BaseColumn, BaseDatasource, BaseMetric
from superset.models.helpers import (
  AuditMixinNullable, ImportMixin, QueryResult, set_perm,
)
from superset.utils import (
    DimSelector, DTTM_ALIAS, flasher, MetricPermException,
)

DRUID_TZ = conf.get('DRUID_TZ')


# Function wrapper because bound methods cannot
# be passed to processes
def _fetch_metadata_for(datasource):
    return datasource.latest_metadata()


class JavascriptPostAggregator(Postaggregator):
    def __init__(self, name, field_names, function):
        self.post_aggregator = {
            'type': 'javascript',
            'fieldNames': field_names,
            'name': name,
            'function': function,
コード例 #41
0
from flask import g
from flask_babel import lazy_gettext as _
import pandas
from sqlalchemy import select
from sqlalchemy.engine import create_engine
from sqlalchemy.engine.url import make_url
from sqlalchemy.sql import text
import sqlparse
from werkzeug.utils import secure_filename

from superset import app, cache_util, conf, db, utils
from superset.utils import QueryStatus, SupersetTemplateException

config = app.config

tracking_url_trans = conf.get('TRACKING_URL_TRANSFORMER')

Grain = namedtuple('Grain', 'name label function')


class LimitMethod(object):
    """Enum the ways that limits can be applied"""
    FETCH_MANY = 'fetch_many'
    WRAP_SQL = 'wrap_sql'


class BaseEngineSpec(object):

    """Abstract class for database engine specific configurations"""

    engine = 'base'  # str as defined in sqlalchemy.engine.engine
コード例 #42
0
def get_since_until(
    time_range: Optional[str] = None,
    since: Optional[str] = None,
    until: Optional[str] = None,
    time_shift: Optional[str] = None,
    relative_start: Optional[str] = None,
    relative_end: Optional[str] = None,
) -> Tuple[datetime, datetime]:
    """Return `since` and `until` date time tuple from string representations of
    time_range, since, until and time_shift.

    This functiom supports both reading the keys separately (from `since` and
    `until`), as well as the new `time_range` key. Valid formats are:

        - ISO 8601
        - X days/years/hours/day/year/weeks
        - X days/years/hours/day/year/weeks ago
        - X days/years/hours/day/year/weeks from now
        - freeform

    Additionally, for `time_range` (these specify both `since` and `until`):

        - Last day
        - Last week
        - Last month
        - Last quarter
        - Last year
        - No filter
        - Last X seconds/minutes/hours/days/weeks/months/years
        - Next X seconds/minutes/hours/days/weeks/months/years

    """
    from superset import conf

    separator = " : "
    relative_start = parse_human_datetime(
        relative_start if relative_start else "today")
    relative_end = parse_human_datetime(
        relative_end if relative_end else "today")
    utc_dt = datetime.utcnow().replace(tzinfo=timezone.utc)
    TIME_ZONE = conf.get("TIME_ZONE")
    common_time_frames = {
        "Today":
        (relative_start, utc_dt.astimezone(timezone(
            timedelta(hours=TIME_ZONE))).replace(tzinfo=None)),
        "Last day": (
            relative_start - relativedelta(days=1),  # type: ignore
            relative_end,
        ),
        "Last week": (
            relative_start - relativedelta(weeks=1),  # type: ignore
            relative_end,
        ),
        "Last month": (
            relative_start - relativedelta(months=1),  # type: ignore
            relative_end,
        ),
        "Last quarter": (
            relative_start - relativedelta(months=3),  # type: ignore
            relative_end,
        ),
        "Last year": (
            relative_start - relativedelta(years=1),  # type: ignore
            relative_end,
        ),
    }

    if time_range:
        if separator in time_range:
            since, until = time_range.split(separator, 1)
            if since and since not in common_time_frames:
                since = add_ago_to_since(since)
            since = parse_human_datetime(since)
            until = parse_human_datetime(until)
        elif time_range in common_time_frames:
            since, until = common_time_frames[time_range]
        elif time_range == "No filter":
            since = until = None
        else:
            rel, num, grain = time_range.split()
            if rel == "Last":
                since = relative_start - relativedelta(  # type: ignore
                    **{grain: int(num)})
                until = relative_end
            else:  # rel == 'Next'
                since = relative_start
                until = relative_end + relativedelta(  # type: ignore
                    **{grain: int(num)})
    else:
        since = since or ""
        if since:
            since = add_ago_to_since(since)
        since = parse_human_datetime(since)
        until = parse_human_datetime(until) if until else relative_end

    if time_shift:
        time_delta = parse_past_timedelta(time_shift)
        since = since if since is None else (since -
                                             time_delta)  # type: ignore
        until = until if until is None else (until -
                                             time_delta)  # type: ignore

    if since and until and since > until:
        raise ValueError(_("From date cannot be larger than to date"))

    return since, until  # type: ignore