def _on_enter(self):
pm.hook.dbnd_on_pre_init_context(ctx=self)
run_user_func(config.get("core", "user_pre_init"))
# if we are deserialized - we don't need to run this code again.
if not self.initialized_context:
# noinspection PyTypeChecker
if self._module:
load_python_module(self._module, "--module")
module_from_config = config.get("databand", "module")
if self._autoload_modules and module_from_config:
load_python_module(module_from_config,
"config file (see [databand].module)")
# will be called from singleton context manager
# we want to be able to catch all "new" inline airflow operators
self.system_settings = DatabandSystemConfig()
if self.system_settings.conf:
self.config.set_values(self.system_settings.conf,
source="dbnd.conf")
if self.system_settings.conf_file:
conf_file = read_from_config_files(
self.system_settings.conf_file)
self.config.set_values(conf_file,
source="dbnd__databand__conf")
from dbnd._core.settings import DatabandSettings
self.settings = DatabandSettings(databand_context=self)
self.env = self.settings.get_env_config(self.system_settings.env)
self.config.set_values(
config_values={"task": {
"task_env": self.system_settings.env
}},
source="context",
)
pm.hook.dbnd_on_new_context(ctx=self)
# RUN USER SETUP FUNCTIONS
_run_user_func(
self.settings.core.__class__.user_driver_init,
self.settings.core.user_driver_init,
)
self.task_run_env = RunInfoConfig().build_task_run_info()
self.initialized_context = True
else:
# we get here if we are running at sub process that recreates the Context
pm.hook.dbnd_on_existing_context(ctx=self)
# we do it every time we go into databand_config
self.configure_targets()
self.settings.log.configure_dbnd_logging()
_run_user_func(self.settings.core.__class__.user_init,
self.settings.core.user_init)
pm.hook.dbnd_post_enter_context(ctx=self)
def dbnd_bootstrap():
global _dbnd_bootstrap
global _dbnd_bootstrap_started
if _dbnd_bootstrap_started:
return
_dbnd_bootstrap_started = True
dbnd_system_bootstrap()
from targets.marshalling import register_basic_data_marshallers
register_basic_data_marshallers()
_surpress_loggers()
_suppress_warnings()
enable_osx_forked_request_calls()
if is_airflow_enabled():
from dbnd_airflow.bootstrap import airflow_bootstrap
airflow_bootstrap()
register_dbnd_plugins()
from dbnd._core.configuration import environ_config
from dbnd._core.utils.basics.load_python_module import run_user_func
from dbnd._core.plugin.dbnd_plugins import pm
from dbnd._core.configuration.dbnd_config import config
user_plugins = config.get("core", "plugins", None)
if user_plugins:
register_dbnd_user_plugins(user_plugins.split(","))
if is_unit_test_mode():
pm.hook.dbnd_setup_unittest()
pm.hook.dbnd_setup_plugin()
if is_sigquit_handler_on():
from dbnd._core.utils.basics.signal_utils import (
register_sigquit_stack_dump_handler,
)
register_sigquit_stack_dump_handler()
# now we can run user code ( at driver/task)
user_preinit = environ_config.get_user_preinit()
if user_preinit:
run_user_func(user_preinit)
# if for any reason there will be code that calls dbnd_bootstrap, this will prevent endless recursion
_dbnd_bootstrap = True
def _interpolate_from_str(self, value):
# when we parse string, if it's has @ - we always will want to "de-reference" it
# and read from disk
# it's very important for primitive types, as we don't know
# if we need to parse("str") or read_from_disk("str")
# @ in the beginning of the string supported by @@
if isinstance(value, six.string_types) and value.startswith("@"):
value = value[1:] # we shift first @
if value.startswith("@"):
# this is the way to provide string value with @
# we return value minus "@"
return value
# so this is @/somepath case and not @@
if value.lower() == "none":
return None
from targets import target
if value.startswith("python://"):
value = value[9:]
value = run_user_func(value)
else:
if value.startswith("target:"):
value = value[7:]
value = target(value)
else:
# the path is a target, something like @/path/to/file/with/data
# this might be dangerous, we need to have more "explicit" way
# to understand that the path is a path
value = self.load_from_target(target(value))
return value
def _on_enter(self):
pm.hook.dbnd_on_pre_init_context(ctx=self)
run_user_func(config.get("core", "user_pre_init"))
# if we are deserialized - we don't need to run this code again.
if not self._is_initialized:
# will be called from singleton context manager
self.system_settings = DatabandSystemConfig()
if self.system_settings.conf:
self.config.set_values(self.system_settings.conf,
source="[databand]conf")
if self.system_settings.conf_file:
conf_file = read_from_config_files(
self.system_settings.conf_file)
self.config.set_values(conf_file, source="[databand]conf")
from dbnd._core.settings import DatabandSettings
self.settings = DatabandSettings(databand_context=self)
self.env = self.settings.get_env_config(self.system_settings.env)
self.config.set_values(
config_values={"task": {
"task_env": self.system_settings.env
}},
source="context",
)
pm.hook.dbnd_on_new_context(ctx=self)
# RUN USER SETUP FUNCTIONS
_run_user_func(
self.settings.core.__class__.user_driver_init,
self.settings.core.user_driver_init,
)
self.task_run_env = RunInfoConfig().build_task_run_info()
self._is_initialized = True
else:
# we get here if we are running at sub process that recreates the Context
pm.hook.dbnd_on_existing_context(ctx=self)
# we do it every time we go into databand_config
self.configure_targets()
self.settings.log.configure_dbnd_logging()
_run_user_func(self.settings.core.__class__.user_init,
self.settings.core.user_init)
pm.hook.dbnd_post_enter_context(ctx=self)
def _interpolate_from_str(self, value):
# when we parse string, if it's has @ - we always will want to "de-reference" it
# and read from disk
# it's very important for primitive types, as we don't know
# if we need to parse("str") or read_from_disk("str")
# @ in the beginning of the string supported by @@
if isinstance(value, six.string_types) and value.startswith("@"):
value = value[1:]
if value.startswith("@"):
# this is the way to provide string value with @
pass
else:
# so this is @/somepath case and not @@
# TODO: we need to move it into target wrapper
if value.startswith("python://"):
value = value[9:]
value = run_user_func(value)
else:
from targets import target
value = self.load_from_target(target(value))
return value
def _run_user_func(param, value):
if not value:
return
return run_user_func(value)
