def _build_filesystem_target(luigi_target, task_id, is_output):
from dbnd._core.task_ctrl.task_relations import traverse_and_set_target
from targets.base_target import TargetSource
dbnd_target = target(luigi_target.path)
fixed_target = traverse_and_set_target(dbnd_target,
TargetSource(task_id=task_id))
target_param = (
get_dbnd_param_by_luigi_name("TargetParameter").target_config(
fixed_target.config).default(fixed_target.path)())
if is_output:
target_param = target_param.output()
return target_param
def initialize_outputs(self):
"""
The default output that this Task produces. Use outputs! Override only if you are writing "base" class
"""
task = self.task
outputs = {"user": {}, "system": {}}
for p, value in self.params.get_params_with_value(
ParameterFilters.OUTPUTS):
if is_not_defined(value):
value = p.build_output(task=task)
setattr(self.task, p.name, value)
if isinstance(p, FuncResultParameter):
continue
value = traverse_and_set_target(value, p._target_source(self.task))
outputs[_section(p)][p.name] = value
custom_outputs = self.task._output()
if custom_outputs:
if outputs["user"]:
warnings.warn(
"Task %s has custom outputs in _output() function, all other outputs will be removed: %s"
% (task, outputs["user"]),
stacklevel=2,
)
outputs["user"] = custom_outputs
# take ownership of all outputs and clean it, just in case
# usually all outputs are assigned to task
# just in case we have some "outputs" with Tasks
outputs = to_targets(outputs)
self.task_outputs = traverse_and_set_target(
outputs, target_source=TargetSource(task_id=self.task_id))
def _target_source(self, task):
return TargetSource(
task_id=task.task_id, parameter_name=self.name, name=self.name
)
def build_tracking_output(self, p):
return InMemoryTarget(
path="memory://{value_type}:{task}.{p_name}".format(
value_type=p.value_type, task=self.task_id, p_name=p.name),
source=TargetSource(task_id=self.task_id, parameter_name=p.name),
)