def _on_enter(self):
pm.hook.dbnd_on_pre_init_context(ctx=self)
run_user_func(config.get("core", "user_pre_init"))
# if we are deserialized - we don't need to run this code again.
if not self.initialized_context:
# noinspection PyTypeChecker
if self._module:
load_python_module(self._module, "--module")
module_from_config = config.get("databand", "module")
if self._autoload_modules and module_from_config:
load_python_module(module_from_config,
"config file (see [databand].module)")
# will be called from singleton context manager
# we want to be able to catch all "new" inline airflow operators
self.system_settings = DatabandSystemConfig()
if self.system_settings.conf:
self.config.set_values(self.system_settings.conf,
source="dbnd.conf")
if self.system_settings.conf_file:
conf_file = read_from_config_files(
self.system_settings.conf_file)
self.config.set_values(conf_file,
source="dbnd__databand__conf")
from dbnd._core.settings import DatabandSettings
self.settings = DatabandSettings(databand_context=self)
self.env = self.settings.get_env_config(self.system_settings.env)
self.config.set_values(
config_values={"task": {
"task_env": self.system_settings.env
}},
source="context",
)
pm.hook.dbnd_on_new_context(ctx=self)
# RUN USER SETUP FUNCTIONS
_run_user_func(
self.settings.core.__class__.user_driver_init,
self.settings.core.user_driver_init,
)
self.task_run_env = RunInfoConfig().build_task_run_info()
self.initialized_context = True
else:
# we get here if we are running at sub process that recreates the Context
pm.hook.dbnd_on_existing_context(ctx=self)
# we do it every time we go into databand_config
self.configure_targets()
self.settings.log.configure_dbnd_logging()
_run_user_func(self.settings.core.__class__.user_init,
self.settings.core.user_init)
pm.hook.dbnd_post_enter_context(ctx=self)
def load_user_modules(dbnd_config, modules=None):
# type: (DbndConfig, List[str]) -> None
# loading user modules
module_from_config = dbnd_config.get("databand", "module")
if module_from_config:
load_python_module(module_from_config,
"config file (see [databand].module)")
if modules:
for m in modules:
load_python_module(m, "--module")
def _get_task_cls(self, task_name):
from dbnd._core.utils.basics.load_python_module import load_python_module
task_cls = self._get_registered_task_cls(task_name)
if task_cls:
return task_cls
# we are going to check if we have override/definition in config
config_task_type = config.get(task_name, "_type", None)
if config_task_type:
_validate_no_recursion_in_config(task_name, config_task_type,
"_type")
try:
return self._get_task_cls(config_task_type)
except Exception:
logger.error(
"Failed to load type required by [%s] using _type=%s",
task_name,
config_task_type,
)
raise
config_task_type = config.get(task_name, "_from", None)
if config_task_type:
_validate_no_recursion_in_config(task_name, config_task_type,
"_from")
return self._get_task_cls(config_task_type)
if "." in task_name:
parts = task_name.split(".")
possible_root_task = parts.pop()
possible_module = ".".join(parts)
# Try to load module and check again for existance
load_python_module(possible_module, "task name '%s'" % task_name)
task_cls = self._get_registered_task_cls(task_name)
if task_cls:
return task_cls
# Check if task exists but user forgot to decorate method with @task
task_module = sys.modules.get(possible_module)
if task_module and hasattr(task_module, possible_root_task):
user_func = getattr(task_module, possible_root_task)
if callable(user_func):
# Non-decorated function was found - decorate and return it
from dbnd._core.decorator import dbnd_func_proxy
decorated_task = dbnd_func_proxy.task(user_func)
setattr(task_module, possible_root_task, decorated_task)
logger.warning(
"Found non-decorated task: %s. "
"Please decorate this task with the proper symbol @pipeline \ @task.\n"
"Auto-decorating and treating it as @task ...",
task_name,
)
return decorated_task.task
if is_airflow_enabled():
from dbnd_airflow.dbnd_task_executor.airflow_operator_as_dbnd import (
AirflowDagAsDbndTask, )
dag = self._get_aiflow_dag(task_name)
if dag:
return AirflowDagAsDbndTask
return None
