def load_task_params_from_task_band(self, task_band, task_params):
task_band_value = target(task_band).as_object.read_json()
new_params = {}
found = []
source = "task_band.json"
for name, p_value in iteritems(task_params):
if name not in task_band_value or name == RESULT_PARAM:
new_params[name] = p_value
continue
value = p_value.parameter.calc_init_value(task_band_value[name])
found.append(name)
new_parameter_value = ParameterValue(
parameter=p_value.parameter,
source=source,
source_value=value,
value=value,
)
new_params[new_parameter_value.name] = new_parameter_value
logger.info("Loading task '{task_family}' from {task_band}:\n"
"\tfields taken:\t{found}".format(
task_family=self.task_family,
task_band=task_band,
found=",".join(found)))
return new_params
def load_task_params_from_task_band(self, task_band, task_params):
task_band_value = target(task_band).as_object.read_json()
new_params = []
found = []
source = "task_band.json"
for p_value in task_params:
if p_value.name not in task_band_value or p_value.name == "result":
new_params.append(p_value)
continue
value = p_value.parameter.calc_init_value(
task_band_value[p_value.name])
found.append(p_value.name)
new_parameter_value = ParameterValue(
parameter=p_value.parameter,
source=source,
source_value=value,
value=value,
)
new_params.append(new_parameter_value)
logger.info("Loading task '{task_family}' from {task_band}:\n"
"\tfields taken:\t{found}".format(
task_family=self.task_family,
task_band=task_band,
found=",".join(found)))
return new_params
def build_user_parameter_value(name, value, source):
"""
Build parameter value for user defined name and value
"""
value_type = get_value_type_of_obj(
value, default_value_type=DefaultObjectValueType)
param_f = get_parameter_for_value_type(value_type)
param = build_parameter(param_f)
param.name = name
if value is NOTHING:
parameter, warnings = param, []
actual_value = param.default
else:
parameter, warnings = infer_parameter_value_type(param, value)
actual_value = value
return ParameterValue(
parameter=parameter,
source=source,
source_value=value,
value=actual_value,
parsed=False,
warnings=warnings,
)
def build_parameter_value(self, param_def):
# This is the place we calculate param_def value
# based on Class(defaults, constructor, overrides, root)
# and Config(env, cmd line, config) state
# used for target_format update
# change param_def definition based on config state
param_def = self._update_param_def_target_config(param_def=param_def)
param_name = param_def.name
p_config_value = self._get_param_config_value(param_def)
if p_config_value and p_config_value.override:
cf_value = p_config_value
elif param_name in self.ctor_kwargs:
cf_value = ConfigValue(self.ctor_kwargs.get(param_name),
source=self._source_name("ctor"))
elif p_config_value:
cf_value = p_config_value
elif param_def.is_output():
# outputs can be none, we "generate" their values later
cf_value = None
else:
err_msg = "No value defined for '{name}' at {context_str}".format(
name=param_name, context_str=self._exc_desc)
raise MissingParameterError(
err_msg,
help_msg=param_def._get_help_message(
sections=self.task_config_sections),
)
if cf_value:
param_def, p_value = build_parameter_value(param_def, cf_value)
p_value = ParameterValue(
parameter=param_def,
source=cf_value.source,
source_value=cf_value.value,
value=p_value,
parsed=cf_value.require_parse,
)
else:
p_value = ParameterValue(
parameter=param_def,
source=None,
source_value=NOTHING,
value=NOTHING,
parsed=False,
)
return p_value
def _build_user_parameter_value(self, param_def, value):
if value is NOTHING:
parameter, warnings = param_def, []
actual_value = param_def.default
else:
parameter, warnings = infer_parameter_value_type(param_def, value)
actual_value = value
return ParameterValue(
parameter=parameter,
source=self.task_definition.full_task_family_short,
source_value=value,
value=actual_value,
parsed=False,
warnings=warnings,
)
def build_result_param(task_passport, param_def=None, name=RESULT_PARAM):
# type: (TaskPassport, Optional[ParameterDefinition], str) -> ParameterValue
"""
Build results parameter for the task definition, if parameter definition is not specify it will build a naive one.
"""
if not param_def:
from targets.values import ObjectValueType
# naive creation of result param definition - default named "result" and single value
param_def = parameter.modify(
name=name,
value_type=ObjectValueType).output.build_parameter("inline")
return ParameterValue(
parameter=param_def,
source=task_passport.full_task_family_short,
source_value=None,
value=NOTHING,
parsed=False,
)
def build_parameter_value(parameter, cf_value):
# type: (ParameterDefinition, ConfigValue) -> ParameterValue
if not cf_value:
return ParameterValue(
parameter=parameter,
source=None,
source_value=NOTHING,
value=NOTHING,
parsed=False,
)
warnings = []
value = cf_value.value
try:
if value is not None and not parameter.is_output():
updated_value_type = _update_parameter_from_runtime_value_type(
parameter, value)
message = ("{parameter}: type of the value at runtime '{runtime}"
" doesn't match user defined type '{compile}'".format(
parameter=parameter,
runtime=updated_value_type,
compile=parameter.value_type,
))
if updated_value_type:
if isinstance(parameter.value_type, DefaultObjectValueType):
# we are going to update
parameter = attr.evolve(
parameter,
value_type=updated_value_type,
load_on_build=updated_value_type.load_on_build,
)
message = "%s: updating parameter with the runtime info" % (
message)
# warn anyway
warnings.append(message)
except Exception as ex:
# we don't want to fail user code on failed value discovery
# we only print message from "friendly exception" and show real stack
logger.exception("Failed to discover runtime for %s", parameter)
try:
p_val = parameter.calc_init_value(value)
except Exception as ex:
raise parameter.parameter_exception("calculate value from '%s'" %
safe_string(value, 100),
ex=ex)
# we need to break strong reference between tasks
# otherwise we will have pointer from task to another task
# if p_val is task, that's ok, but let minimize the risk by patching cf_value
if isinstance(value, _TaskParamContainer):
cf_value.value = str(cf_value)
try:
if p_val is not None and not isinstance(p_val, Target):
parameter.validate(p_val)
except Exception as ex:
raise parameter.parameter_exception("validate value='%s'" %
safe_string(p_val),
ex=ex)
p_value = ParameterValue(
parameter=parameter,
source=cf_value.source,
source_value=cf_value.value,
value=p_val,
parsed=cf_value.require_parse,
warnings=warnings + cf_value.warnings,
)
return p_value
def _build_parameter_value(self, param_def):
# type: (ParameterDefinition) -> ParameterValue
"""
-= MAIN FUNCTION for Parameter Calculation =-
This is the place we calculate param_def value
based on Class(defaults, constructor, overrides, root)
and Config(env, cmd line, config) state
Build parameter value from config_value and param_definition.
Considerate - priority, constructor values, param type and so on.
"""
config_values_stack = self._get_config_value_stack_for_param(param_def)
param_name = param_def.name
# if we will use config values we will need to reduce them to a single value
# if there is only one - reduce will get it
# other wise we need combine the values using `fold_parameter_value`
# may raise if can't fold two values together!!
has_overrides_in_config = any(
cf for cf in config_values_stack
if cf and cf.priority >= ConfigValuePriority.OVERRIDE)
# first check for override (HIGHEST PRIORITY)
if has_overrides_in_config:
config_values_as_param_values = [
build_parameter_value(param_def, cf)
for cf in config_values_stack
]
return functools.reduce(fold_parameter_value,
config_values_as_param_values)
# second using kwargs we received from the user
# do we need to do it for tracking?
elif param_name in self.task_kwargs:
return build_parameter_value(
param_def,
ConfigValue(self.task_kwargs.get(param_name),
source=self._source_name("ctor")),
)
if config_values_stack:
config_values_as_param_values = [
build_parameter_value(param_def, cf)
for cf in config_values_stack
]
return functools.reduce(fold_parameter_value,
config_values_as_param_values)
if (self.parent_task
and param_name in self.parent_task.task_children_scope_params):
# we have parent task with param = parameter(scope=ParameterScope.children)
# the priority is lower than config (as this one is more like "default" than "explicit config")
# see _calculate_task_children_scope_params implementation and ParameterScope.children
parameter_value = self.parent_task.task_children_scope_params[
param_name]
return build_parameter_value(
param_def,
ConfigValue(value=parameter_value.value,
source=parameter_value.source),
)
if param_def.from_task_env_config:
# param = parameter(from_task_env_config=True)
# we check task.task_env.param for the value
if not self.task_env_config:
raise friendly_error.task_parameters.task_env_param_with_no_env(
context=self._ctor_as_str, key=param_name)
param_env_config_value = self.task_env_config.task_params.get_param_value(
param_name)
if param_env_config_value is None:
raise friendly_error.task_parameters.task_env_param_not_exists_in_env(
context=self._ctor_as_str,
key=param_name,
env_config=self.task_env_config,
)
return build_parameter_value(
param_def,
ConfigValue(
value=param_env_config_value.value,
source=param_env_config_value.source,
),
)
if param_name in self.task_definition.param_defaults:
# we can't "add" defaults to the current config and rely on configuration system
# we don't know what section name to use, and it my clash with current config
return build_parameter_value(
param_def,
ConfigValue(
self.task_definition.param_defaults.get(param_name),
source=self._source_name("default"),
),
)
if not param_def.is_output():
# outputs can be none, we "generate" their values later
err_msg = "No value defined for '{name}' at {context_str}".format(
name=param_name, context_str=self._ctor_as_str)
raise MissingParameterError(
err_msg,
help_msg=param_def._get_help_message(
sections=self.config_sections),
)
# returning empty Output value
return ParameterValue(
parameter=param_def,
source="",
source_value=NOTHING,
value=NOTHING,
parsed=False,
)
import pytest
from dbnd import parameter
from dbnd._core.parameter.parameter_value import ParameterValue, fold_parameter_value
list_of_ints = parameter[List[int]].build_parameter("context")
str_to_init_map = parameter[Dict[str, int]].build_parameter("context")
just_int = parameter[int].build_parameter("context")
@pytest.mark.parametrize(
"left, right, expected",
[
pytest.param(
ParameterValue(parameter=list_of_ints,
value=[1],
source="test",
source_value=[1]),
None,
ParameterValue(parameter=list_of_ints,
value=[1],
source="test",
source_value=[1]),
id="Fold value with None",
),
pytest.param(
ParameterValue(parameter=list_of_ints,
value=[1],
source="test",
source_value=[1]),
ParameterValue(parameter=list_of_ints,
value=[2],