from lib.utils.plugin import import_plugin
ALL_PLUGIN_JOBS = import_plugin("job_plugin", "ALL_PLUGIN_JOBS", {})
ALL_JOBS = {**{}, **ALL_PLUGIN_JOBS}
from env import QuerybookSettings from lib.utils.plugin import import_plugin lineage = import_plugin(QuerybookSettings.DATA_LINEAGE_BACKEND)
from collections import namedtuple
from lib.utils.plugin import import_plugin
from .stores.db_store import DBReader, DBUploader
from .stores.s3_store import S3Reader, S3Uploader
from .stores.google_store import GoogleReader, GoogleUploader
from .stores.file_store import FileReader, FileUploader
ALL_PLUGIN_RESULT_STORES = import_plugin("result_store_plugin",
"ALL_PLUGIN_RESULT_STORES", {})
ResultStore = namedtuple("ResultStore", ["reader", "uploader"])
ALL_RESULT_STORES = {
"db": ResultStore(DBReader, DBUploader),
"s3": ResultStore(S3Reader, S3Uploader),
"gcs": ResultStore(GoogleReader, GoogleUploader), # Google Cloud Storage
"file": ResultStore(FileReader, FileUploader),
**ALL_PLUGIN_RESULT_STORES,
}
def load_auth():
global auth
auth = import_plugin(QuerybookSettings.AUTH_BACKEND)
get_login_config()
from lib.utils.plugin import import_plugin
from .base_executor import parse_exception
from .executors.hive import HiveQueryExecutor
from .executors.presto import PrestoQueryExecutor
from .executors.sqlalchemy import (
MysqlQueryExecutor,
DruidQueryExecutor,
SqliteQueryExecutor,
SnowflakeQueryExecutor,
)
from .executors.bigquery import BigQueryQueryExecutor
ALL_PLUGIN_EXECUTORS = import_plugin("executor_plugin", "ALL_PLUGIN_EXECUTORS",
[])
ALL_EXECUTORS = [
HiveQueryExecutor,
PrestoQueryExecutor,
MysqlQueryExecutor,
DruidQueryExecutor,
SqliteQueryExecutor,
BigQueryQueryExecutor,
SnowflakeQueryExecutor,
] + ALL_PLUGIN_EXECUTORS
def get_executor_class(language: str, name: str):
for executor in ALL_EXECUTORS:
if (executor.EXECUTOR_LANGUAGE() == language
and executor.EXECUTOR_NAME() == name):
from lib.utils.plugin import import_plugin
from .hive_metastore_loader import HMSMetastoreLoader
from .mysql_metastore_loader import MysqlMetastoreLoader
from .thrifthive_metastore_loader import HMSThriftMetastoreLoader
from .sqlalchemy_metastore_loader import SqlAlchemyMetastoreLoader
from .glue_data_catalog_loader import GlueDataCatalogLoader
ALL_PLUGIN_METASTORE_LOADERS = import_plugin(
"metastore_plugin", "ALL_PLUGIN_METASTORE_LOADERS", []
)
ALL_METASTORE_LOADERS = [
HMSMetastoreLoader,
MysqlMetastoreLoader,
HMSThriftMetastoreLoader,
SqlAlchemyMetastoreLoader,
GlueDataCatalogLoader,
] + ALL_PLUGIN_METASTORE_LOADERS
from lib.utils.plugin import import_plugin
ALL_PLUGIN_EXPORTERS = import_plugin("exporter_plugin", "ALL_PLUGIN_EXPORTERS", [])
# No default exporter is provided
ALL_EXPORTERS = ALL_PLUGIN_EXPORTERS
def get_exporter(name: str):
for exporter in ALL_EXPORTERS:
if exporter.exporter_name == name:
return exporter
raise ValueError(f"Unknown exporter name {name}")
from lib.utils.plugin import import_plugin
from .notifier.email_notifier import EmailNotifier
ALL_PLUGIN_NOTIFIERS = import_plugin(
"notifier_plugin",
"ALL_PLUGIN_NOTIFIERS",
[
EmailNotifier(),
],
)
ALL_NOTIFIERS = ALL_PLUGIN_NOTIFIERS
DEFAULT_NOTIFIER = ALL_NOTIFIERS[0].notifier_name if ALL_NOTIFIERS else None
def get_notifier_class(name: str):
for notifier in ALL_PLUGIN_NOTIFIERS:
if notifier.notifier_name == name:
return notifier
raise ValueError(f"Unknown notifier name {name}")
from lib.utils.plugin import import_plugin
from .connection_checker import ConnectionChecker
from .select_one_checker import SelectOneChecker
from .null_checker import NullChecker
ALL_PLUGIN_ENGINE_STATUS_CHECKERS = import_plugin(
"engine_status_checker_plugin", "ALL_PLUGIN_ENGINE_STATUS_CHECKERS", [])
ALL_ENGINE_STATUS_CHECKERS = [
ConnectionChecker,
SelectOneChecker,
NullChecker,
] + ALL_PLUGIN_ENGINE_STATUS_CHECKERS
def get_engine_checker_class(name: str):
for checker in ALL_ENGINE_STATUS_CHECKERS:
if checker.NAME() == name:
return checker
raise ValueError(f"Unknown checker name {name}")