def __new__(mcs, classname, bases, classdict):
"""
Custom class creation for namespacing.
Also register all subclasses.
When the set or inherited namespace evaluates to ``None``, set the task namespace to
whatever the currently declared namespace is.
"""
cls = super(TaskMetaclass,
mcs).__new__(mcs, classname, bases,
classdict) # type: typing.Type[_BaseTask]
# we are starting from "not clean" classdict ->
# A. it's deserialization
# B. it was calculated before
if classdict.get("task_definition") is not None:
return cls
cls.task_definition = TaskDefinition.from_task_cls(task_class=cls,
classdict=classdict)
# now we will assign all calculated parameters
# so instead of ParameterFactory, we will have ParameterDefinition
for k, v in six.iteritems(cls.task_definition.task_param_defs):
setattr(cls, k, v)
# every time we see new implementation, we want it to have an priority over old implementation
# we need to switch to dict() and store history else where
r = get_task_registry()
r.register_task(cls)
return cls
def test_task_definition_as_func(self):
@task
def a():
pass
td = TaskDefinition(a.task, {}, "td")
assert td
def __new__(mcs, classname, bases, classdict):
"""
Custom class creation for namespacing.
Also register all subclasses.
When the set or inherited namespace evaluates to ``None``, set the task namespace to
whatever the currently declared namespace is.
"""
cls = super(TaskMetaclass, mcs).__new__(
mcs, classname, bases, classdict
) # type: typing.Type[_BaseTask]
# we are starting from "not clean" classdict -> it's deserialization
if classdict.get("task_definition") is not None:
return cls
td = cls.task_definition = TaskDefinition(cls, classdict)
# now we will assign all params
set_params = td.class_params if cls.is_tracking_mode else td.all_task_params
for k, v in six.iteritems(set_params):
setattr(cls, k, v)
# every time we see new implementation, we want it to have an priority over old implementation
# we need to switch to dict() and store history else where
r = get_task_registry()
r.register_task(cls)
return cls
def test_task_definition_as_func(self):
@task
def a():
pass
td = TaskDefinition.from_task_cls(a.task, {})
assert td
def _build_inline_root_task(root_task_name):
# create "root task" with default name as current process executable file name
task_definition = TaskDefinition(
task_passport=TaskPassport.from_module(
TrackingTask.__module__), # we need to fix that
source_code=TaskSourceCode.from_callstack(),
)
root_task = TrackingTask(
task_name=root_task_name,
task_definition=task_definition,
task_params=Parameters(source="inline_root_task", param_values=[]),
)
root_task.ctrl.task_repr.task_command_line = list2cmdline(sys.argv)
root_task.ctrl.task_repr.task_functional_call = "bash_cmd(args=%s)" % repr(
sys.argv)
return root_task
def for_user_params(
cls,
task_definition_uid, # type: UUID
task_name, # type: str
user_params, # type: Dict[str,Any]
task_passport, # type: TaskPassport
source_code=None, # type: TaskSourceCode
result=True, # type: Optional[bool]
):
# type: (...) -> TrackingTask
"""
Creating a customize task from the required params.
Here we build the params from runtime values and use them to build the task definition.
"""
param_values = []
for key, value in six.iteritems(user_params):
p = build_user_parameter_value(
name=key,
value=value,
source=task_passport.full_task_family_short)
param_values.append(p)
if result:
result_param_value = build_result_param(task_passport)
param_values.append(result_param_value)
task_params = Parameters(source=task_passport.full_task_family_short,
param_values=param_values)
task_definition = TaskDefinition(
task_passport=task_passport,
source_code=source_code,
external_parameters=task_params,
task_definition_uid=task_definition_uid,
)
return cls(
task_name=task_name,
task_definition=task_definition,
task_params=task_params,
)
def _build_luigi_task_definition(luigi_task_cls):
# type: (Type[luigi.Task]) -> Type[_LuigiTask]
"""
build a classdict as needed in the creation of TaskMetaclass
"""
task_family = luigi_task_cls.get_task_family()
task_passport = TaskPassport.build_task_passport(
task_family=task_family,
task_namespace=luigi_task_cls.get_task_namespace(),
cls_name=luigi_task_cls.__name__,
module_name=luigi_task_cls.__module__,
)
task_source = TaskSourceCode.from_callable(luigi_task_cls)
params_defs = extract_luigi_params(luigi_task_cls)
td = TaskDefinition(task_passport=task_passport,
source_code=task_source,
classdict=params_defs)
# we can't support outputs/inputs - they are dynamic per task
# params_defs.update(_extract_targets_dedup(luigi_task_cls, attributes))
return td
def _build_tracking_task_definition(self):
return TaskDefinition.from_task_decorator(
task_decorator=self.task_decorator)
def test_task_definition_as_class(self):
class TdTask(Task):
pass
td = TaskDefinition(TdTask, {}, "td")
assert td
def test_task_definition_as_class(self):
class TdTask(Task):
pass
td = TaskDefinition.from_task_cls(TdTask, {})
assert td