def get_dags_from_databand(custom_operator_class: Optional[type] = None):
if environ_enabled(ENV_DBND_DISABLE_SCHEDULED_DAGS_LOAD):
return None
from dbnd._core.errors.base import DatabandApiError, DatabandConnectionException
try:
# let be sure that we are loaded
config.load_system_configs()
if not config.get("core", "databand_url"):
return {}
default_retries = config.getint("scheduler", "default_retries")
dags = DbndSchedulerDBDagsProvider(
default_retries=default_retries,
custom_operator_class=custom_operator_class).get_dags()
if not in_quiet_mode():
logger.info("providing %s dags from scheduled jobs" % len(dags))
return {dag.dag_id: dag for dag in dags}
except (DatabandConnectionException, DatabandApiError) as e:
logger.error(str(e))
raise e
except Exception as e:
logging.exception("Failed to get dags form databand server")
raise e
def __init__(self):
self.scheduled_jobs = []
if (config.getboolean("scheduler", "always_file_sync") or
("scheduler" in sys.argv)) and not config.getboolean(
"scheduler", "never_file_sync"):
self.file_config_loader = SchedulerFileConfigLoader()
logger.debug("scheduler file syncing active")
else:
self.file_config_loader = None
logger.debug("scheduler file syncing disabled")
self.default_retries = config.getint("scheduler", "default_retries")
def get_dags_from_file():
if environ_enabled(ENV_DBND_DISABLE_SCHEDULED_DAGS_LOAD):
return None
try:
# let be sure that we are loaded
config.load_system_configs()
config_file = config.get("scheduler", "config_file")
if not config_file:
logger.info("No dags file has been defined at scheduler.config_file")
return {}
default_retries = config.getint("scheduler", "default_retries")
active_by_default = config.getboolean("scheduler", "active_by_default")
dags = DbndAirflowDagsProviderFromFile(
config_file=config_file,
active_by_default=active_by_default,
default_retries=default_retries,
).get_dags()
return {dag.dag_id: dag for dag in dags}
except Exception as e:
logging.exception("Failed to get dags from the file")
raise e