def _new_config_layer(self,
config_values,
source=None,
override=False,
merge_settings=None):
# let validate that we are initialized
# user can call this function out of no-where, so we will create a layer, and will override it
# the moment we create more layers on config.system_load
dbnd_system_bootstrap()
if not config_values:
return self.config_layer
if not isinstance(config_values, _ConfigStore):
if not source:
source = "{sig}".format(sig=id(config_values))
config_values = parse_and_build_config_store(
config_values=config_values, source=source,
override=override) # type: _ConfigStore
source = source or config_values.source
if not source:
source = "{sig}".format(sig=id(config_values))
return self.config_layer.create_layer(name=source,
config_values=config_values,
merge_settings=merge_settings)
def main(args=None):
# from dbnd._core.log.config import configure_basic_logging
# configure_basic_logging(None)
dbnd_system_bootstrap()
# LET'S PATCH AIRFLOW FIRST
from dbnd_airflow.bootstrap import dbnd_airflow_bootstrap
dbnd_airflow_bootstrap()
from airflow.bin.cli import CLIFactory
from airflow.configuration import conf
from dbnd_airflow.plugins.setup_plugins import (
setup_scheduled_dags,
setup_versioned_dags,
)
# ORIGINAL CODE from airflow/bin/airflow
if conf.get("core", "security") == "kerberos":
os.environ["KRB5CCNAME"] = conf.get("kerberos", "ccache")
os.environ["KRB5_KTNAME"] = conf.get("kerberos", "keytab")
import argcomplete
from dbnd_airflow.scheduler.zombies import find_and_kill_dagrun_zombies
CLIFactory.subparsers_dict[find_and_kill_dagrun_zombies.__name__] = {
"func": find_and_kill_dagrun_zombies,
"help": "Clean up BackfillJob zombie tasks",
"args": tuple(),
}
parser = CLIFactory.get_parser()
argcomplete.autocomplete(parser)
args = parser.parse_args(args=args)
func_name = args.func.__name__
# DBND PATCH:
if dbnd_config.getboolean("airflow",
"auto_add_scheduled_dags") and func_name in [
"scheduler",
"webserver",
]:
setup_scheduled_dags()
if dbnd_config.getboolean(
"airflow",
"auto_add_versioned_dags") and func_name in ["webserver"]:
setup_versioned_dags()
args.func(args)
if func_name in ["resetdb", "initdb"]:
pool_name = dbnd_config.get("airflow", "dbnd_pool")
if pool_name == "dbnd_pool":
create_airflow_pool(pool_name)
def main(args=None):
# from dbnd._core.log.config import configure_basic_logging
# configure_basic_logging(None)
from dbnd import dbnd_config
from dbnd._core.configuration.environ_config import set_quiet_mode
from dbnd._core.context.bootstrap import dbnd_system_bootstrap
set_quiet_mode()
dbnd_system_bootstrap()
# LET'S PATCH AIRFLOW FIRST
from dbnd_airflow.bootstrap import airflow_bootstrap
airflow_bootstrap()
from airflow.bin.cli import CLIFactory
from airflow.configuration import conf
from dbnd_airflow.plugins.setup_plugins import (
setup_scheduled_dags,
setup_versioned_dags,
)
# ORIGINAL CODE from airflow/bin/airflow
if conf.get("core", "security") == "kerberos":
os.environ["KRB5CCNAME"] = conf.get("kerberos", "ccache")
os.environ["KRB5_KTNAME"] = conf.get("kerberos", "keytab")
parser = CLIFactory.get_parser()
argcomplete.autocomplete(parser)
args = parser.parse_args(args=args)
func_name = args.func.__name__
# DBND PATCH:
if dbnd_config.getboolean("airflow",
"auto_add_scheduled_dags") and func_name in [
"scheduler",
"webserver",
]:
setup_scheduled_dags()
if dbnd_config.getboolean(
"airflow",
"auto_add_versioned_dags") and func_name in ["webserver"]:
setup_versioned_dags()
args.func(args)