class _BaseTask(object):
# override to change get_task_family() -> changes task_family
_conf__task_family = None
_conf__track_source_code = True # module/class/function user source code tracking
task_namespace = NOTHING
_conf__task_type_name = TaskType.python
_conf__task_ui_color = None
#####
# override output path format
_conf__base_output_path_fmt = None
# stores call spec for the @task definition
_conf__decorator_spec = None # type: Optional[_TaskDecoratorSpec]
_conf__tracked = True # track task changes with TrackingStore
_conf__no_child_params = False # disable child scope params
_conf_auto_read_params = True # enables autoread of params.
_conf_confirm_on_kill_msg = None # get user confirmation on task kill if not empty
_conf__require_run_dump_file = False
# this is the state of autoread
_task_auto_read_original = None
_task_auto_read = None
####
# execution
# will be used by Registry
task_definition = None # type: TaskDefinition
# user can override this with his configuration
defaults = None # type: Dict[ParameterDefinition, any()]
validate_no_extra_params = parameter.enum(ParamValidation).system(
description="validate that all configured keys for a task have a matching parameter definition"
)
@classmethod
def get_task_family(cls):
return cls.task_definition.task_family
@classmethod
def get_full_task_family(cls):
return cls.task_definition.full_task_family
def __init__(self, **kwargs):
super(_BaseTask, self).__init__()
# most of the time we will use it as TaskMetaCtrl - we want this to be type hint!
self.task_meta = kwargs["task_meta"] # type: TaskMeta
self._params = TaskParameters(self)
for p_value in self.task_meta.task_params:
setattr(self, p_value.name, p_value.value)
self._task_auto_read_current = None
self._task_auto_read_origin = None
@property
def task_id(self):
return self.task_meta.task_id
@property
def task_signature(self):
return self.task_meta.task_signature
@property
def task_name(self):
return self.task_meta.task_name
def __eq__(self, other):
return (
self.__class__ == other.__class__
and self._params.get_param_values() == other._params.get_param_values()
)
def __hash__(self):
return hash(self.task_id)
def __repr__(self):
"""
Build a task representation like `MyTask(param1=1.5, param2='5')`
"""
return "%s" % self.task_meta.task_id
def __str__(self):
"""
Build a task representation like `MyTask(param1=1.5, param2='5')`
"""
return self.task_meta.task_id
@property
def friendly_task_name(self):
if self.task_name != self.task_meta.task_family:
return "%s[%s]" % (self.task_name, self.task_meta.task_family)
return self.task_name
def __getattribute__(self, name):
def _get(n):
return super(_BaseTask, self).__getattribute__(n)
value = _get(name)
try:
_task_auto_read = _get("_task_auto_read")
except Exception:
return value
# already cached
if _task_auto_read is None or name in _task_auto_read:
return value
parameter = _get("_params").get_param(name)
# we are not parameter
# or there is nothing to "deferefence"
# TODO: rebase : value is None
if not parameter:
return value
runtime_value = parameter.calc_runtime_value(value, task=self)
if parameter.is_output():
# if it's outpus, we should not "cache" it
# otherwise we will try to save it on autosave ( as it was changed)
return runtime_value
# for the cache, so next time we don't need to calculate it
setattr(self, name, runtime_value)
_task_auto_read.add(name)
return runtime_value
def _auto_load_save_params(
self, auto_read=False, save_on_change=False, normalize_on_change=False
):
c = TaskAutoParamsReadWrite(
task=self,
auto_read=auto_read,
save_on_change=save_on_change,
normalize_on_change=normalize_on_change,
)
return c.auto_load_save_params()
def clone(self, cls=None, output_params_to_clone=None, **kwargs):
"""
Creates a new instance from an existing instance where some of the args have changed.
There's at least two scenarios where this is useful (see test/clone_test.py):
* remove a lot of boiler plate when you have recursive dependencies and lots of args
* there's task inheritance and some logic is on the base class
:param cls:
:param kwargs:
:return:
"""
if cls is None:
cls = self.__class__
output_params_to_clone = output_params_to_clone or []
new_k = {}
for param_name, param_class in six.iteritems(cls.task_definition.task_params):
if param_class.is_output() and param_name not in output_params_to_clone:
continue
if param_name in kwargs:
new_k[param_name] = kwargs[param_name]
elif hasattr(self, param_name):
new_k[param_name] = getattr(self, param_name)
new_k["task_name"] = self.task_name
return cls(**new_k)
@property
def settings(self):
# type: () -> DatabandSettings
return self.task_meta.dbnd_context.settings
def __iter__(self):
raise friendly_error.task_build.iteration_over_task(self)
def _initialize(self):
pass
def _task_banner(self, banner, verbosity):
"""
customize task banner
"""
return
def _validate(self):
"""
will be called after after object is created
:return:
"""
return
def simple_params_dict(self):
return {k: p.value for k, p in self._params._param_meta_map.items()}
class _TaskWithParams(_BaseTask):
_conf__scoped_params = True # enable/disable ParameterScope.CHILDREN scope params
_conf_auto_read_params = True # enables auto-read value of data params.
def __init__(
self,
task_name,
task_definition,
task_signature_obj,
task_params, # type: Parameters
task_config_layer,
task_config_override,
task_enabled=True,
task_sections=None,
task_call_source=None,
task_children_scope_params=None, # type: Dict[str, ParameterValue]
):
super(_TaskWithParams, self).__init__(
task_name=task_name,
task_definition=task_definition,
task_signature_obj=task_signature_obj,
task_params=task_params,
)
self.task_enabled = task_enabled # relevant only for orchestration
# configuration data
self.task_sections = task_sections
self.task_config_override = task_config_override
self.task_config_layer = task_config_layer
self.task_children_scope_params = task_children_scope_params
self.task_call_source = task_call_source
for param_value in self.task_params.get_param_values():
param_value.task = self
object.__setattr__(self, param_value.name, param_value.value)
task_definition = None # type: TaskDefinition
# user can override this with his configuration
defaults = None # type: Dict[ParameterDefinition, Any]
validate_no_extra_params = parameter.enum(ParamValidation).system(
default=ParamValidation.disabled,
description=
"validate that all configured keys for a task have a matching parameter definition",
)
def __setattr__(self, name, value):
p = (self.task_params.get_param_value(name) if hasattr(
self, "task_params") else None)
if p:
p._update_param_value_from_task_set(value)
object.__setattr__(self, name, value)
def _update_property_on_parameter_value_set(self, name, value):
"""called from param.update_value() , we don't want to call back p._update_param_value_from_task_set()"""
object.__setattr__(self, name, value)
def _initialize(self):
pass
def _validate(self):
"""
will be called after after object is created
:return:
"""
return
def clone(self, cls=None, output_params_to_clone=None, **kwargs):
"""
Creates a new instance from an existing instance where some of the args have changed.
There's at least two scenarios where this is useful (see test/clone_test.py):
* remove a lot of boiler plate when you have recursive dependencies and lots of args
* there's task inheritance and some logic is on the base class
:param cls:
:param kwargs:
:return:
"""
if cls is None:
cls = self.__class__
output_params_to_clone = output_params_to_clone or []
new_k = {}
# copy only fields that exists in target class
for param_name, param_class in six.iteritems(
cls.task_definition.task_param_defs):
if param_class.is_output(
) and param_name not in output_params_to_clone:
continue
if hasattr(self, param_name):
new_k[param_name] = getattr(self, param_name)
new_k["task_name"] = self.task_name
new_k.update(kwargs)
return cls(**new_k)
@classmethod
def get_task_family(cls):
return cls.task_definition.task_family
@classmethod
def get_full_task_family(cls):
return cls.task_definition.full_task_family