def _on_enter(self): pm.hook.dbnd_on_pre_init_context(ctx=self) run_user_func(config.get("core", "user_pre_init")) # if we are deserialized - we don't need to run this code again. if not self.initialized_context: # noinspection PyTypeChecker if self._module: load_python_module(self._module, "--module") module_from_config = config.get("databand", "module") if self._autoload_modules and module_from_config: load_python_module(module_from_config, "config file (see [databand].module)") # will be called from singleton context manager # we want to be able to catch all "new" inline airflow operators self.system_settings = DatabandSystemConfig() if self.system_settings.conf: self.config.set_values(self.system_settings.conf, source="dbnd.conf") if self.system_settings.conf_file: conf_file = read_from_config_files( self.system_settings.conf_file) self.config.set_values(conf_file, source="dbnd__databand__conf") from dbnd._core.settings import DatabandSettings self.settings = DatabandSettings(databand_context=self) self.env = self.settings.get_env_config(self.system_settings.env) self.config.set_values( config_values={"task": { "task_env": self.system_settings.env }}, source="context", ) pm.hook.dbnd_on_new_context(ctx=self) # RUN USER SETUP FUNCTIONS _run_user_func( self.settings.core.__class__.user_driver_init, self.settings.core.user_driver_init, ) self.task_run_env = RunInfoConfig().build_task_run_info() self.initialized_context = True else: # we get here if we are running at sub process that recreates the Context pm.hook.dbnd_on_existing_context(ctx=self) # we do it every time we go into databand_config self.configure_targets() self.settings.log.configure_dbnd_logging() _run_user_func(self.settings.core.__class__.user_init, self.settings.core.user_init) pm.hook.dbnd_post_enter_context(ctx=self)
def dbnd_bootstrap(): global _dbnd_bootstrap global _dbnd_bootstrap_started if _dbnd_bootstrap_started: return _dbnd_bootstrap_started = True dbnd_system_bootstrap() from targets.marshalling import register_basic_data_marshallers register_basic_data_marshallers() _surpress_loggers() _suppress_warnings() enable_osx_forked_request_calls() if is_airflow_enabled(): from dbnd_airflow.bootstrap import airflow_bootstrap airflow_bootstrap() register_dbnd_plugins() from dbnd._core.configuration import environ_config from dbnd._core.utils.basics.load_python_module import run_user_func from dbnd._core.plugin.dbnd_plugins import pm from dbnd._core.configuration.dbnd_config import config user_plugins = config.get("core", "plugins", None) if user_plugins: register_dbnd_user_plugins(user_plugins.split(",")) if is_unit_test_mode(): pm.hook.dbnd_setup_unittest() pm.hook.dbnd_setup_plugin() if is_sigquit_handler_on(): from dbnd._core.utils.basics.signal_utils import ( register_sigquit_stack_dump_handler, ) register_sigquit_stack_dump_handler() # now we can run user code ( at driver/task) user_preinit = environ_config.get_user_preinit() if user_preinit: run_user_func(user_preinit) # if for any reason there will be code that calls dbnd_bootstrap, this will prevent endless recursion _dbnd_bootstrap = True
def _interpolate_from_str(self, value): # when we parse string, if it's has @ - we always will want to "de-reference" it # and read from disk # it's very important for primitive types, as we don't know # if we need to parse("str") or read_from_disk("str") # @ in the beginning of the string supported by @@ if isinstance(value, six.string_types) and value.startswith("@"): value = value[1:] # we shift first @ if value.startswith("@"): # this is the way to provide string value with @ # we return value minus "@" return value # so this is @/somepath case and not @@ if value.lower() == "none": return None from targets import target if value.startswith("python://"): value = value[9:] value = run_user_func(value) else: if value.startswith("target:"): value = value[7:] value = target(value) else: # the path is a target, something like @/path/to/file/with/data # this might be dangerous, we need to have more "explicit" way # to understand that the path is a path value = self.load_from_target(target(value)) return value
def _on_enter(self): pm.hook.dbnd_on_pre_init_context(ctx=self) run_user_func(config.get("core", "user_pre_init")) # if we are deserialized - we don't need to run this code again. if not self._is_initialized: # will be called from singleton context manager self.system_settings = DatabandSystemConfig() if self.system_settings.conf: self.config.set_values(self.system_settings.conf, source="[databand]conf") if self.system_settings.conf_file: conf_file = read_from_config_files( self.system_settings.conf_file) self.config.set_values(conf_file, source="[databand]conf") from dbnd._core.settings import DatabandSettings self.settings = DatabandSettings(databand_context=self) self.env = self.settings.get_env_config(self.system_settings.env) self.config.set_values( config_values={"task": { "task_env": self.system_settings.env }}, source="context", ) pm.hook.dbnd_on_new_context(ctx=self) # RUN USER SETUP FUNCTIONS _run_user_func( self.settings.core.__class__.user_driver_init, self.settings.core.user_driver_init, ) self.task_run_env = RunInfoConfig().build_task_run_info() self._is_initialized = True else: # we get here if we are running at sub process that recreates the Context pm.hook.dbnd_on_existing_context(ctx=self) # we do it every time we go into databand_config self.configure_targets() self.settings.log.configure_dbnd_logging() _run_user_func(self.settings.core.__class__.user_init, self.settings.core.user_init) pm.hook.dbnd_post_enter_context(ctx=self)
def _interpolate_from_str(self, value): # when we parse string, if it's has @ - we always will want to "de-reference" it # and read from disk # it's very important for primitive types, as we don't know # if we need to parse("str") or read_from_disk("str") # @ in the beginning of the string supported by @@ if isinstance(value, six.string_types) and value.startswith("@"): value = value[1:] if value.startswith("@"): # this is the way to provide string value with @ pass else: # so this is @/somepath case and not @@ # TODO: we need to move it into target wrapper if value.startswith("python://"): value = value[9:] value = run_user_func(value) else: from targets import target value = self.load_from_target(target(value)) return value
def _run_user_func(param, value): if not value: return return run_user_func(value)