示例#1
0
def import_provided_stores():
    provided_result_stores = {}
    for (
            store_name,
        (store_path, reader_name, uploader_name),
    ) in PROVIDED_RESULT_STORES_PATHS.items():
        store_module = import_module_with_default(store_path, default=None)
        if store_module is not None:
            reader = getattr(store_module, reader_name)
            uploader = getattr(store_module, uploader_name)

            if reader and uploader:
                provided_result_stores[store_name] = ResultStore(
                    reader, uploader)
    return provided_result_stores
示例#2
0
    ("lib.result_store.stores.file_store", "FileReader", "FileUploader"),
}
ResultStore = namedtuple("ResultStore", ["reader", "uploader"])


def import_provided_stores():
    provided_result_stores = {}
    for (
            store_name,
        (store_path, reader_name, uploader_name),
    ) in PROVIDED_RESULT_STORES_PATHS.items():
        store_module = import_module_with_default(store_path, default=None)
        if store_module is not None:
            reader = getattr(store_module, reader_name)
            uploader = getattr(store_module, uploader_name)

            if reader and uploader:
                provided_result_stores[store_name] = ResultStore(
                    reader, uploader)
    return provided_result_stores


PROVIDED_RESULT_STORES = import_provided_stores()
ALL_PLUGIN_RESULT_STORES = import_module_with_default(
    "result_store_plugin", "ALL_PLUGIN_RESULT_STORES", default={})

ALL_RESULT_STORES = {
    **PROVIDED_RESULT_STORES,
    **ALL_PLUGIN_RESULT_STORES,
}
示例#3
0
from typing import Dict

from app.db import with_session
from logic.admin import get_query_engine_by_id
from lib.utils.import_helper import import_module_with_default

from .base_exporter import BaseTableUploadExporter

ALL_PLUGIN_EXPORTERS: Dict[
    str, BaseTableUploadExporter] = import_module_with_default(
        "table_uploader_plugin",
        "ALL_PLUGIN_TABLE_UPLOAD_EXPORTERS",
        default={})

ALL_TABLE_UPLOAD_EXPORTER_BY_NAME: Dict[str, BaseTableUploadExporter] = {}

SqlalchemyExporter = import_module_with_default(
    "lib.table_upload.exporter.sqlalchemy_exporter",
    "SqlalchemyExporter",
    default=None)
if SqlalchemyExporter:
    ALL_TABLE_UPLOAD_EXPORTER_BY_NAME[
        "SqlalchemyExporter"] = SqlalchemyExporter()

ALL_TABLE_UPLOAD_EXPORTER_BY_NAME |= ALL_PLUGIN_EXPORTERS


@with_session
def get_table_upload_exporter(engine_id,
                              session=None) -> BaseTableUploadExporter:
    query_engine = get_query_engine_by_id(engine_id, session=session)
示例#4
0
from lib.utils.import_helper import import_module_with_default
from .notifier.email_notifier import EmailNotifier

ALL_PLUGIN_NOTIFIERS = import_module_with_default(
    "notifier_plugin",
    "ALL_PLUGIN_NOTIFIERS",
    default=[
        EmailNotifier(),
    ],
)

ALL_NOTIFIERS = ALL_PLUGIN_NOTIFIERS

DEFAULT_NOTIFIER = ALL_NOTIFIERS[0].notifier_name if ALL_NOTIFIERS else None


def get_notifier_class(name: str):
    for notifier in ALL_PLUGIN_NOTIFIERS:
        if notifier.notifier_name == name:
            return notifier
    raise ValueError(f"Unknown notifier name {name}")
示例#5
0
from lib.utils.import_helper import import_module_with_default

ALL_PLUGIN_JOBS = import_module_with_default("job_plugin",
                                             "ALL_PLUGIN_JOBS",
                                             default={})

ALL_JOBS = {**{}, **ALL_PLUGIN_JOBS}
示例#6
0
def load_auth():
    global auth
    auth = import_module_with_default(QuerybookSettings.AUTH_BACKEND)
    get_login_config()
示例#7
0
from lib.utils.import_helper import import_module_with_default
from .base_checker import BaseEngineStatusChecker
from .connection_checker import ConnectionChecker
from .select_one_checker import SelectOneChecker
from .null_checker import NullChecker

ALL_PLUGIN_ENGINE_STATUS_CHECKERS = import_module_with_default(
    "engine_status_checker_plugin", "ALL_PLUGIN_ENGINE_STATUS_CHECKERS", default=[]
)

ALL_ENGINE_STATUS_CHECKERS = [
    ConnectionChecker,
    SelectOneChecker,
    NullChecker,
] + ALL_PLUGIN_ENGINE_STATUS_CHECKERS


def get_engine_checker_class(name: str) -> BaseEngineStatusChecker:
    for checker in ALL_ENGINE_STATUS_CHECKERS:
        if checker.NAME() == name:
            return checker
    raise ValueError(f"Unknown checker name {name}")
示例#8
0
from lib.logger import get_logger

from .executors.sqlalchemy import GenericSqlAlchemyQueryExecutor

LOG = get_logger(__file__)

PROVIDED_EXECUTORS = import_modules([
    ("lib.query_executor.executors.hive", "HiveQueryExecutor"),
    ("lib.query_executor.executors.presto", "PrestoQueryExecutor"),
    ("lib.query_executor.executors.bigquery", "BigQueryQueryExecutor"),
    ("lib.query_executor.executors.snowflake", "SnowflakeQueryExecutor"),
    ("lib.query_executor.executors.trino", "TrinoQueryExecutor"),
    ("lib.query_executor.executors.redshift", "RedshiftQueryExecutor"),
])
ALL_PLUGIN_EXECUTORS = import_module_with_default("executor_plugin",
                                                  "ALL_PLUGIN_EXECUTORS",
                                                  default=[])
ALL_EXECUTORS = ([GenericSqlAlchemyQueryExecutor] + PROVIDED_EXECUTORS +
                 ALL_PLUGIN_EXECUTORS)


def get_executor_class(language: str, name: str):
    for executor in ALL_EXECUTORS:
        if executor.match(language, name):
            return executor
    raise ValueError(f"Unknown executor {name} with language {language}")


def get_flattened_executor_template() -> Dict:
    """A query executor may correspond to multiple languages, in this case we duplicate
       the number of available executor template per language per executor.
示例#9
0
from lib.utils.import_helper import import_module_with_default, import_modules

PROVIDED_METASTORE_LOADERS = import_modules(
    [
        ("lib.metastore.loaders.hive_metastore_loader", "HMSMetastoreLoader"),
        ("lib.metastore.loaders.mysql_metastore_loader", "MysqlMetastoreLoader"),
        (
            "lib.metastore.loaders.thrifthive_metastore_loader",
            "HMSThriftMetastoreLoader",
        ),
        (
            "lib.metastore.loaders.sqlalchemy_metastore_loader",
            "SqlAlchemyMetastoreLoader",
        ),
        ("lib.metastore.loaders.glue_data_catalog_loader", "GlueDataCatalogLoader"),
    ]
)

ALL_PLUGIN_METASTORE_LOADERS = import_module_with_default(
    "metastore_plugin", "ALL_PLUGIN_METASTORE_LOADERS", default=[]
)


ALL_METASTORE_LOADERS = PROVIDED_METASTORE_LOADERS + ALL_PLUGIN_METASTORE_LOADERS
示例#10
0
from env import QuerybookSettings

from lib.utils.import_helper import import_module_with_default

lineage = import_module_with_default(QuerybookSettings.DATA_LINEAGE_BACKEND)
示例#11
0
from lib.utils.import_helper import import_module_with_default

ALL_PLUGIN_EXPORTERS = import_module_with_default("exporter_plugin",
                                                  "ALL_PLUGIN_EXPORTERS",
                                                  default=[])

# No default exporter is provided
ALL_EXPORTERS = ALL_PLUGIN_EXPORTERS


def get_exporter(name: str):
    for exporter in ALL_EXPORTERS:
        if exporter.exporter_name == name:
            return exporter
    raise ValueError(f"Unknown exporter name {name}")