def get_dags_from_databand(custom_operator_class: Optional[type] = None):
if environ_enabled(ENV_DBND_DISABLE_SCHEDULED_DAGS_LOAD):
return None
from dbnd._core.errors.base import DatabandApiError, DatabandConnectionException
try:
# let be sure that we are loaded
config.load_system_configs()
if not config.get("core", "databand_url"):
return {}
default_retries = config.getint("scheduler", "default_retries")
dags = DbndSchedulerDBDagsProvider(
default_retries=default_retries,
custom_operator_class=custom_operator_class).get_dags()
if not in_quiet_mode():
logger.info("providing %s dags from scheduled jobs" % len(dags))
return {dag.dag_id: dag for dag in dags}
except (DatabandConnectionException, DatabandApiError) as e:
logger.error(str(e))
raise e
except Exception as e:
logging.exception("Failed to get dags form databand server")
raise e
def project_init(ctx, no_init_db, overwrite, dbnd_home, dbnd_system):
"""Initialize the project structure and local db"""
from dbnd._core.errors import DatabandSystemError
from dbnd import databand_lib_path
os.environ["SKIP_DAGS_PARSING"] = "True" # Exclude airflow dag examples
conf_folder = databand_lib_path("conf/project_init")
project_name = os.path.basename(dbnd_home)
output_dir = os.path.dirname(dbnd_home)
if os.path.exists(os.path.join(dbnd_home, "project.cfg")):
if not overwrite:
raise DatabandSystemError(
"You are trying to re-initialize your project. You already have dbnd configuration at %s. "
"You can force project-init by providing --overwrite flag. "
"If you need to create/update database use `dbnd db init` instead"
% dbnd_system)
logger.warning(
"You are re-initializing your project, all files at %s are going to be over written!"
% dbnd_home)
copy_tree(conf_folder, dbnd_home)
click.echo("Databand project has been initialized at %s" % dbnd_home)
config.load_system_configs(force=True)
if no_init_db:
click.echo("Don't forget to run `dbnd db init` ")
return
if is_web_enabled():
from dbnd_web.cmd_db import init as db_init
ctx.invoke(db_init)
def patched_create_app(*args, **kwargs):
from dbnd._core.configuration.dbnd_config import config
logger.info("Setting SQL from databand configuration.")
config.load_system_configs()
from dbnd_airflow.bootstrap import dbnd_airflow_bootstrap
dbnd_airflow_bootstrap()
res = create_app_func(*args, **kwargs)
try:
use_databand_airflow_dagbag()
except Exception:
logger.info("Failed to apply dbnd versioned dagbag")
return res
def get_dags():
if environ_enabled(ENV_DBND_DISABLE_SCHEDULED_DAGS_LOAD):
return None
from dbnd._core.errors.base import DatabandConnectionException, DatabandApiError
try:
# let be sure that we are loaded
config.load_system_configs()
dags = DbndSchedulerDBDagsProvider().get_dags()
if not in_quiet_mode():
logger.info("providing %s dags from scheduled jobs" % len(dags))
return dags
except (DatabandConnectionException, DatabandApiError) as e:
logger.error(str(e))
except Exception as e:
raise e
def get_dags_from_file():
if environ_enabled(ENV_DBND_DISABLE_SCHEDULED_DAGS_LOAD):
return None
try:
# let be sure that we are loaded
config.load_system_configs()
config_file = config.get("scheduler", "config_file")
if not config_file:
logger.info("No dags file has been defined at scheduler.config_file")
return {}
default_retries = config.getint("scheduler", "default_retries")
active_by_default = config.getboolean("scheduler", "active_by_default")
dags = DbndAirflowDagsProviderFromFile(
config_file=config_file,
active_by_default=active_by_default,
default_retries=default_retries,
).get_dags()
return {dag.dag_id: dag for dag in dags}
except Exception as e:
logging.exception("Failed to get dags from the file")
raise e
def __init__(self, config_file=None):
config.load_system_configs()
self.config_file = (config_file if config_file else config.get(
"scheduler", "config_file"))
self.active_by_default = config.get("scheduler", "active_by_default")
def __init__(self, config_file, active_by_default=False, default_retries=1):
config.load_system_configs()
self.config_file = config_file
self.active_by_default = active_by_default
self.default_retries = default_retries
import json
from collections import namedtuple
from typing import Any, Dict
from dbnd._core.configuration.dbnd_config import config
from dbnd.api.api_utils import ApiClient
from dbnd.api.shared_schemas.scheduled_job_schema import ScheduledJobSchemaV2
config.load_system_configs()
api_client = ApiClient(
config.get("core", "databand_url"),
auth=True,
user=config.get("scheduler", "dbnd_user"),
password=config.get("scheduler", "dbnd_password"),
)
schema = ScheduledJobSchemaV2(strict=False)
ScheduledJobNamedTuple = namedtuple("ScheduledJobNamedTuple",
schema.fields.keys())
ScheduledJobNamedTuple.__new__.__defaults__ = (None, ) * len(
ScheduledJobNamedTuple._fields)
def post_scheduled_job(scheduled_job_dict):
data, _ = schema.dump({"DbndScheduledJob": scheduled_job_dict})
response = api_client.api_request("/api/v1/scheduled_jobs",
data,
method="POST",
no_prefix=True)
