def get_dags_from_databand(custom_operator_class: Optional[type] = None): if environ_enabled(ENV_DBND_DISABLE_SCHEDULED_DAGS_LOAD): return None from dbnd._core.errors.base import DatabandApiError, DatabandConnectionException try: # let be sure that we are loaded config.load_system_configs() if not config.get("core", "databand_url"): return {} default_retries = config.getint("scheduler", "default_retries") dags = DbndSchedulerDBDagsProvider( default_retries=default_retries, custom_operator_class=custom_operator_class).get_dags() if not in_quiet_mode(): logger.info("providing %s dags from scheduled jobs" % len(dags)) return {dag.dag_id: dag for dag in dags} except (DatabandConnectionException, DatabandApiError) as e: logger.error(str(e)) raise e except Exception as e: logging.exception("Failed to get dags form databand server") raise e
def project_init(ctx, no_init_db, overwrite, dbnd_home, dbnd_system): """Initialize the project structure and local db""" from dbnd._core.errors import DatabandSystemError from dbnd import databand_lib_path os.environ["SKIP_DAGS_PARSING"] = "True" # Exclude airflow dag examples conf_folder = databand_lib_path("conf/project_init") project_name = os.path.basename(dbnd_home) output_dir = os.path.dirname(dbnd_home) if os.path.exists(os.path.join(dbnd_home, "project.cfg")): if not overwrite: raise DatabandSystemError( "You are trying to re-initialize your project. You already have dbnd configuration at %s. " "You can force project-init by providing --overwrite flag. " "If you need to create/update database use `dbnd db init` instead" % dbnd_system) logger.warning( "You are re-initializing your project, all files at %s are going to be over written!" % dbnd_home) copy_tree(conf_folder, dbnd_home) click.echo("Databand project has been initialized at %s" % dbnd_home) config.load_system_configs(force=True) if no_init_db: click.echo("Don't forget to run `dbnd db init` ") return if is_web_enabled(): from dbnd_web.cmd_db import init as db_init ctx.invoke(db_init)
def patched_create_app(*args, **kwargs): from dbnd._core.configuration.dbnd_config import config logger.info("Setting SQL from databand configuration.") config.load_system_configs() from dbnd_airflow.bootstrap import dbnd_airflow_bootstrap dbnd_airflow_bootstrap() res = create_app_func(*args, **kwargs) try: use_databand_airflow_dagbag() except Exception: logger.info("Failed to apply dbnd versioned dagbag") return res
def get_dags(): if environ_enabled(ENV_DBND_DISABLE_SCHEDULED_DAGS_LOAD): return None from dbnd._core.errors.base import DatabandConnectionException, DatabandApiError try: # let be sure that we are loaded config.load_system_configs() dags = DbndSchedulerDBDagsProvider().get_dags() if not in_quiet_mode(): logger.info("providing %s dags from scheduled jobs" % len(dags)) return dags except (DatabandConnectionException, DatabandApiError) as e: logger.error(str(e)) except Exception as e: raise e
def get_dags_from_file(): if environ_enabled(ENV_DBND_DISABLE_SCHEDULED_DAGS_LOAD): return None try: # let be sure that we are loaded config.load_system_configs() config_file = config.get("scheduler", "config_file") if not config_file: logger.info("No dags file has been defined at scheduler.config_file") return {} default_retries = config.getint("scheduler", "default_retries") active_by_default = config.getboolean("scheduler", "active_by_default") dags = DbndAirflowDagsProviderFromFile( config_file=config_file, active_by_default=active_by_default, default_retries=default_retries, ).get_dags() return {dag.dag_id: dag for dag in dags} except Exception as e: logging.exception("Failed to get dags from the file") raise e
def __init__(self, config_file=None): config.load_system_configs() self.config_file = (config_file if config_file else config.get( "scheduler", "config_file")) self.active_by_default = config.get("scheduler", "active_by_default")
def __init__(self, config_file, active_by_default=False, default_retries=1): config.load_system_configs() self.config_file = config_file self.active_by_default = active_by_default self.default_retries = default_retries
import json from collections import namedtuple from typing import Any, Dict from dbnd._core.configuration.dbnd_config import config from dbnd.api.api_utils import ApiClient from dbnd.api.shared_schemas.scheduled_job_schema import ScheduledJobSchemaV2 config.load_system_configs() api_client = ApiClient( config.get("core", "databand_url"), auth=True, user=config.get("scheduler", "dbnd_user"), password=config.get("scheduler", "dbnd_password"), ) schema = ScheduledJobSchemaV2(strict=False) ScheduledJobNamedTuple = namedtuple("ScheduledJobNamedTuple", schema.fields.keys()) ScheduledJobNamedTuple.__new__.__defaults__ = (None, ) * len( ScheduledJobNamedTuple._fields) def post_scheduled_job(scheduled_job_dict): data, _ = schema.dump({"DbndScheduledJob": scheduled_job_dict}) response = api_client.api_request("/api/v1/scheduled_jobs", data, method="POST", no_prefix=True)