def task_context(self, phase):
# we don't want logs/user wrappers at this stage
with nested(
task_context(self.task, phase),
TaskContextFilter.task_context(self.task.task_id),
):
yield
def create_dbnd_task(self):
# create task meta
self._log_build_step(
"Resolving task params with %s" % self.task_config_sections
)
# let apply all "override" values in Task(override={})
if self.task_config_override:
self.config.set_values(
source=self._source_name("override"),
config_values=self.task_config_override,
override=True,
)
if self.verbose_build:
# and issubclass(
# self.task_definition.task_class, _TaskParamContainer
# ):
self._log_config()
self.ctor_kwargs = self._build_task_ctor_kwargs(
self.task_args__ctor, self.task_kwargs
)
try:
task_meta = self.build_task_meta()
except Exception:
if not self.verbose_build:
self._log_config(force_log=True)
raise
self._log_build_step("Task Meta with obj_id = %s" % str(task_meta.obj_key))
# If a Task has already been instantiated with the same parameters,
# the previous instance is returned to reduce number of object instances.
tic = self.dbnd_context.task_instance_cache
task = tic.get_task_obj_by_id(task_meta.obj_key.id)
if not task or hasattr(task, "_dbnd_no_cache"):
task = self.new_task_factory(task_meta)
tic.task_obj_instances[task.task_meta.obj_key.id] = task
# now the task is created - all nested constructors will see it as parent
with task_context(task, TaskContextPhase.BUILD):
task._initialize()
task._validate()
task.task_meta.config_layer = self.config.config_layer
tic.register_task_object(task)
if self.parent_task and hasattr(task, "task_id"):
if isinstance(task, _TaskParamContainer):
self.parent_task.task_meta.add_child(task.task_id)
return task
def _create_task(cls, args, kwargs):
task_definition = cls.task_definition
# we need to have context initialized before we start to run all logic in config() scope
# update config with current class defaults
# we apply them to config only if there are no values (this is defaults)
with config(
config_values=task_definition.task_defaults_config_store,
source=task_definition.task_passport.format_source_name(
"defaults"),
merge_settings=ConfigMergeSettings.on_non_exists_only,
) as task_config:
tracking_mode = TaskEssence.TRACKING.is_included(cls)
# create task meta first
task_meta_factory = (TrackedTaskMetaFactory
if tracking_mode else TaskMetaFactory)
factory = task_meta_factory(config=task_config,
task_cls=cls,
task_args=args,
task_kwargs=kwargs)
task_meta = factory.create_dbnd_task_meta()
# If a Task has already been instantiated with the same parameters,
# the previous instance is returned to reduce number of object instances.
tic = get_databand_context().task_instance_cache
task = tic.get_task_obj_by_id(task_meta.obj_key.id)
if not task or tracking_mode or hasattr(task, "_dbnd_no_cache"):
task = cls._build_task_obj(task_meta)
tic.register_task_obj_instance(task)
# now the task is created - all nested constructors will see it as parent
with task_context(task, TaskContextPhase.BUILD):
task._initialize()
task._validate()
task.task_meta.config_layer = config.config_layer
tic.register_task_instance(task)
parent_task = try_get_current_task()
if (parent_task and hasattr(task, "task_id")
and (task.task_essence != TaskEssence.CONFIG)):
parent_task.descendants.add_child(task.task_id)
return task
def create_dbnd_task(config, new_task_factory, task_cls, task_args, task_kwargs):
# type:(DbndConfig, Any, Type[_BaseTask], Any, Any, bool)->None
tracking_mode = task_cls.is_tracking_mode
task_meta_factory = TrackedTaskMetaFactory if tracking_mode else TaskMetaFactory
factory = task_meta_factory(
config=config, task_cls=task_cls, task_args=task_args, task_kwargs=task_kwargs,
)
task_meta = factory.create_dbnd_task_meta()
# If a Task has already been instantiated with the same parameters,
# the previous instance is returned to reduce number of object instances.
tic = get_databand_context().task_instance_cache
task = tic.get_task_obj_by_id(task_meta.obj_key.id)
if not task or tracking_mode or hasattr(task, "_dbnd_no_cache"):
task = new_task_factory(task_meta)
tic.register_task_obj_instance(task)
# now the task is created - all nested constructors will see it as parent
with task_context(task, TaskContextPhase.BUILD):
task._initialize()
task._validate()
task.task_meta.config_layer = config.config_layer
tic.register_task_instance(task)
parent_task = try_get_current_task()
if (
parent_task
and hasattr(task, "task_id")
and isinstance(task, _TaskParamContainer)
):
parent_task.task_meta.add_child(task.task_id)
return task
def build_task_object(self, task_metaclass):
databand_context = get_databand_context()
# convert args to kwargs, validate values
self.task_kwargs = self._build_and_validate_task_ctor_kwargs(
self.task_args__ctor, self.task_kwargs)
self._log_build_step("Resolving task params with %s" %
self.config_sections)
try:
task_param_values = self._build_task_param_values()
task_params = Parameters(source=self._ctor_as_str,
param_values=task_param_values)
except Exception:
self._log_config(force_log=True)
raise
task_enabled = True
if self.parent_task and not self.parent_task.ctrl.should_run():
task_enabled = False
# load from task_band if exists
task_band_param = task_params.get_param_value(TASK_BAND_PARAMETER_NAME)
if task_band_param and task_band_param.value:
task_band = task_band_param.value
# we are going to load all task parameters from task_band
task_params = self.load_task_params_from_task_band(
task_band, task_params)
params = task_params.get_params_signatures(
ParameterFilters.SIGNIFICANT_INPUTS)
# we add override to Object Cache signature
override_signature = self._get_override_params_signature()
# task schema id is unique per Class definition.
# so if we have new implementation - we will not a problem with rerunning it
full_task_name = "%s@%s(object=%s)" % (
self.task_name,
self.task_definition.full_task_family,
str(id(self.task_definition)),
)
# now we don't know the real signature - so we calculate signature based on all known params
cache_object_signature = build_signature(
name=full_task_name,
params=params,
extra={"task_override": override_signature},
)
self._log_build_step("Task task_signature %s" %
str(cache_object_signature.signature))
# If a Task has already been instantiated with the same parameters,
# the previous instance is returned to reduce number of object instances.
tic = databand_context.task_instance_cache
cached_task_object = tic.get_cached_task_obj(cache_object_signature)
if cached_task_object and not hasattr(cached_task_object,
"_dbnd_no_cache"):
return cached_task_object
# we want to have task id immediately, so we can initialize outputs/use by user
# we should switch to SIGNIFICANT_INPUT here
task_signature_obj = build_signature(
name=self.task_name,
params=params,
extra=self.task_definition.task_signature_extra,
)
task_children_scope_params = self._calculate_task_children_scope_params(
task_params=task_params)
task = task_metaclass._build_task_obj(
task_definition=self.task_definition,
task_name=self.task_name,
task_params=task_params,
task_signature_obj=task_signature_obj,
task_config_override=self.task_config_override,
task_config_layer=self.config.config_layer,
task_enabled=task_enabled,
task_sections=self.config_sections,
task_children_scope_params=task_children_scope_params,
)
tic.register_task_obj_cache_instance(
task, task_obj_cache_signature=cache_object_signature)
task.task_call_source = [
databand_context.user_code_detector.find_user_side_frame(2)
]
if task.task_call_source and self.parent_task:
task.task_call_source.extend(self.parent_task.task_call_source)
# now the task is created - all nested constructors will see it as parent
with task_context(task, TaskContextPhase.BUILD):
task._initialize()
task._validate()
# it might be that config has been changed even more
task.task_config_layer = self.config.config_layer
# only now we know "task_id" so we can register in "publicaly facing cache
tic.register_task_instance(task)
return task