def import_provided_stores(): provided_result_stores = {} for ( store_name, (store_path, reader_name, uploader_name), ) in PROVIDED_RESULT_STORES_PATHS.items(): store_module = import_module_with_default(store_path, default=None) if store_module is not None: reader = getattr(store_module, reader_name) uploader = getattr(store_module, uploader_name) if reader and uploader: provided_result_stores[store_name] = ResultStore( reader, uploader) return provided_result_stores
("lib.result_store.stores.file_store", "FileReader", "FileUploader"), } ResultStore = namedtuple("ResultStore", ["reader", "uploader"]) def import_provided_stores(): provided_result_stores = {} for ( store_name, (store_path, reader_name, uploader_name), ) in PROVIDED_RESULT_STORES_PATHS.items(): store_module = import_module_with_default(store_path, default=None) if store_module is not None: reader = getattr(store_module, reader_name) uploader = getattr(store_module, uploader_name) if reader and uploader: provided_result_stores[store_name] = ResultStore( reader, uploader) return provided_result_stores PROVIDED_RESULT_STORES = import_provided_stores() ALL_PLUGIN_RESULT_STORES = import_module_with_default( "result_store_plugin", "ALL_PLUGIN_RESULT_STORES", default={}) ALL_RESULT_STORES = { **PROVIDED_RESULT_STORES, **ALL_PLUGIN_RESULT_STORES, }
from typing import Dict from app.db import with_session from logic.admin import get_query_engine_by_id from lib.utils.import_helper import import_module_with_default from .base_exporter import BaseTableUploadExporter ALL_PLUGIN_EXPORTERS: Dict[ str, BaseTableUploadExporter] = import_module_with_default( "table_uploader_plugin", "ALL_PLUGIN_TABLE_UPLOAD_EXPORTERS", default={}) ALL_TABLE_UPLOAD_EXPORTER_BY_NAME: Dict[str, BaseTableUploadExporter] = {} SqlalchemyExporter = import_module_with_default( "lib.table_upload.exporter.sqlalchemy_exporter", "SqlalchemyExporter", default=None) if SqlalchemyExporter: ALL_TABLE_UPLOAD_EXPORTER_BY_NAME[ "SqlalchemyExporter"] = SqlalchemyExporter() ALL_TABLE_UPLOAD_EXPORTER_BY_NAME |= ALL_PLUGIN_EXPORTERS @with_session def get_table_upload_exporter(engine_id, session=None) -> BaseTableUploadExporter: query_engine = get_query_engine_by_id(engine_id, session=session)
from lib.utils.import_helper import import_module_with_default from .notifier.email_notifier import EmailNotifier ALL_PLUGIN_NOTIFIERS = import_module_with_default( "notifier_plugin", "ALL_PLUGIN_NOTIFIERS", default=[ EmailNotifier(), ], ) ALL_NOTIFIERS = ALL_PLUGIN_NOTIFIERS DEFAULT_NOTIFIER = ALL_NOTIFIERS[0].notifier_name if ALL_NOTIFIERS else None def get_notifier_class(name: str): for notifier in ALL_PLUGIN_NOTIFIERS: if notifier.notifier_name == name: return notifier raise ValueError(f"Unknown notifier name {name}")
from lib.utils.import_helper import import_module_with_default ALL_PLUGIN_JOBS = import_module_with_default("job_plugin", "ALL_PLUGIN_JOBS", default={}) ALL_JOBS = {**{}, **ALL_PLUGIN_JOBS}
def load_auth(): global auth auth = import_module_with_default(QuerybookSettings.AUTH_BACKEND) get_login_config()
from lib.utils.import_helper import import_module_with_default from .base_checker import BaseEngineStatusChecker from .connection_checker import ConnectionChecker from .select_one_checker import SelectOneChecker from .null_checker import NullChecker ALL_PLUGIN_ENGINE_STATUS_CHECKERS = import_module_with_default( "engine_status_checker_plugin", "ALL_PLUGIN_ENGINE_STATUS_CHECKERS", default=[] ) ALL_ENGINE_STATUS_CHECKERS = [ ConnectionChecker, SelectOneChecker, NullChecker, ] + ALL_PLUGIN_ENGINE_STATUS_CHECKERS def get_engine_checker_class(name: str) -> BaseEngineStatusChecker: for checker in ALL_ENGINE_STATUS_CHECKERS: if checker.NAME() == name: return checker raise ValueError(f"Unknown checker name {name}")
from lib.logger import get_logger from .executors.sqlalchemy import GenericSqlAlchemyQueryExecutor LOG = get_logger(__file__) PROVIDED_EXECUTORS = import_modules([ ("lib.query_executor.executors.hive", "HiveQueryExecutor"), ("lib.query_executor.executors.presto", "PrestoQueryExecutor"), ("lib.query_executor.executors.bigquery", "BigQueryQueryExecutor"), ("lib.query_executor.executors.snowflake", "SnowflakeQueryExecutor"), ("lib.query_executor.executors.trino", "TrinoQueryExecutor"), ("lib.query_executor.executors.redshift", "RedshiftQueryExecutor"), ]) ALL_PLUGIN_EXECUTORS = import_module_with_default("executor_plugin", "ALL_PLUGIN_EXECUTORS", default=[]) ALL_EXECUTORS = ([GenericSqlAlchemyQueryExecutor] + PROVIDED_EXECUTORS + ALL_PLUGIN_EXECUTORS) def get_executor_class(language: str, name: str): for executor in ALL_EXECUTORS: if executor.match(language, name): return executor raise ValueError(f"Unknown executor {name} with language {language}") def get_flattened_executor_template() -> Dict: """A query executor may correspond to multiple languages, in this case we duplicate the number of available executor template per language per executor.
from lib.utils.import_helper import import_module_with_default, import_modules PROVIDED_METASTORE_LOADERS = import_modules( [ ("lib.metastore.loaders.hive_metastore_loader", "HMSMetastoreLoader"), ("lib.metastore.loaders.mysql_metastore_loader", "MysqlMetastoreLoader"), ( "lib.metastore.loaders.thrifthive_metastore_loader", "HMSThriftMetastoreLoader", ), ( "lib.metastore.loaders.sqlalchemy_metastore_loader", "SqlAlchemyMetastoreLoader", ), ("lib.metastore.loaders.glue_data_catalog_loader", "GlueDataCatalogLoader"), ] ) ALL_PLUGIN_METASTORE_LOADERS = import_module_with_default( "metastore_plugin", "ALL_PLUGIN_METASTORE_LOADERS", default=[] ) ALL_METASTORE_LOADERS = PROVIDED_METASTORE_LOADERS + ALL_PLUGIN_METASTORE_LOADERS
from env import QuerybookSettings from lib.utils.import_helper import import_module_with_default lineage = import_module_with_default(QuerybookSettings.DATA_LINEAGE_BACKEND)
from lib.utils.import_helper import import_module_with_default ALL_PLUGIN_EXPORTERS = import_module_with_default("exporter_plugin", "ALL_PLUGIN_EXPORTERS", default=[]) # No default exporter is provided ALL_EXPORTERS = ALL_PLUGIN_EXPORTERS def get_exporter(name: str): for exporter in ALL_EXPORTERS: if exporter.exporter_name == name: return exporter raise ValueError(f"Unknown exporter name {name}")