def __new__(mcs, classname, bases, classdict): """ Custom class creation for namespacing. Also register all subclasses. When the set or inherited namespace evaluates to ``None``, set the task namespace to whatever the currently declared namespace is. """ cls = super(TaskMetaclass, mcs).__new__(mcs, classname, bases, classdict) # type: typing.Type[_BaseTask] # we are starting from "not clean" classdict -> # A. it's deserialization # B. it was calculated before if classdict.get("task_definition") is not None: return cls cls.task_definition = TaskDefinition.from_task_cls(task_class=cls, classdict=classdict) # now we will assign all calculated parameters # so instead of ParameterFactory, we will have ParameterDefinition for k, v in six.iteritems(cls.task_definition.task_param_defs): setattr(cls, k, v) # every time we see new implementation, we want it to have an priority over old implementation # we need to switch to dict() and store history else where r = get_task_registry() r.register_task(cls) return cls
def test_task_definition_as_func(self): @task def a(): pass td = TaskDefinition(a.task, {}, "td") assert td
def __new__(mcs, classname, bases, classdict): """ Custom class creation for namespacing. Also register all subclasses. When the set or inherited namespace evaluates to ``None``, set the task namespace to whatever the currently declared namespace is. """ cls = super(TaskMetaclass, mcs).__new__( mcs, classname, bases, classdict ) # type: typing.Type[_BaseTask] # we are starting from "not clean" classdict -> it's deserialization if classdict.get("task_definition") is not None: return cls td = cls.task_definition = TaskDefinition(cls, classdict) # now we will assign all params set_params = td.class_params if cls.is_tracking_mode else td.all_task_params for k, v in six.iteritems(set_params): setattr(cls, k, v) # every time we see new implementation, we want it to have an priority over old implementation # we need to switch to dict() and store history else where r = get_task_registry() r.register_task(cls) return cls
def test_task_definition_as_func(self): @task def a(): pass td = TaskDefinition.from_task_cls(a.task, {}) assert td
def _build_inline_root_task(root_task_name): # create "root task" with default name as current process executable file name task_definition = TaskDefinition( task_passport=TaskPassport.from_module( TrackingTask.__module__), # we need to fix that source_code=TaskSourceCode.from_callstack(), ) root_task = TrackingTask( task_name=root_task_name, task_definition=task_definition, task_params=Parameters(source="inline_root_task", param_values=[]), ) root_task.ctrl.task_repr.task_command_line = list2cmdline(sys.argv) root_task.ctrl.task_repr.task_functional_call = "bash_cmd(args=%s)" % repr( sys.argv) return root_task
def for_user_params( cls, task_definition_uid, # type: UUID task_name, # type: str user_params, # type: Dict[str,Any] task_passport, # type: TaskPassport source_code=None, # type: TaskSourceCode result=True, # type: Optional[bool] ): # type: (...) -> TrackingTask """ Creating a customize task from the required params. Here we build the params from runtime values and use them to build the task definition. """ param_values = [] for key, value in six.iteritems(user_params): p = build_user_parameter_value( name=key, value=value, source=task_passport.full_task_family_short) param_values.append(p) if result: result_param_value = build_result_param(task_passport) param_values.append(result_param_value) task_params = Parameters(source=task_passport.full_task_family_short, param_values=param_values) task_definition = TaskDefinition( task_passport=task_passport, source_code=source_code, external_parameters=task_params, task_definition_uid=task_definition_uid, ) return cls( task_name=task_name, task_definition=task_definition, task_params=task_params, )
def _build_luigi_task_definition(luigi_task_cls): # type: (Type[luigi.Task]) -> Type[_LuigiTask] """ build a classdict as needed in the creation of TaskMetaclass """ task_family = luigi_task_cls.get_task_family() task_passport = TaskPassport.build_task_passport( task_family=task_family, task_namespace=luigi_task_cls.get_task_namespace(), cls_name=luigi_task_cls.__name__, module_name=luigi_task_cls.__module__, ) task_source = TaskSourceCode.from_callable(luigi_task_cls) params_defs = extract_luigi_params(luigi_task_cls) td = TaskDefinition(task_passport=task_passport, source_code=task_source, classdict=params_defs) # we can't support outputs/inputs - they are dynamic per task # params_defs.update(_extract_targets_dedup(luigi_task_cls, attributes)) return td
def _build_tracking_task_definition(self): return TaskDefinition.from_task_decorator( task_decorator=self.task_decorator)
def test_task_definition_as_class(self): class TdTask(Task): pass td = TaskDefinition(TdTask, {}, "td") assert td
def test_task_definition_as_class(self): class TdTask(Task): pass td = TaskDefinition.from_task_cls(TdTask, {}) assert td