def build_parameter_value(self, param_def):
# This is the place we calculate param_def value
# based on Class(defaults, constructor, overrides, root)
# and Config(env, cmd line, config) state
# used for target_format update
# change param_def definition based on config state
param_def = self._update_param_def_target_config(param_def=param_def)
param_name = param_def.name
p_config_value = self._get_param_config_value(param_def)
if p_config_value and p_config_value.override:
cf_value = p_config_value
elif param_name in self.ctor_kwargs:
cf_value = ConfigValue(self.ctor_kwargs.get(param_name),
source=self._source_name("ctor"))
elif p_config_value:
cf_value = p_config_value
elif param_def.is_output():
# outputs can be none, we "generate" their values later
cf_value = None
else:
err_msg = "No value defined for '{name}' at {context_str}".format(
name=param_name, context_str=self._exc_desc)
raise MissingParameterError(
err_msg,
help_msg=param_def._get_help_message(
sections=self.task_config_sections),
)
return build_parameter_value(param_def, cf_value)
def read_environ_config():
"""
Read configuration from process environment
Every env var in following format will be added to config
$DBND__SECTION__KEY=value (please notice double underscore "__")
:return:
"""
dbnd_environ = _ConfigStore()
for key, value in six.iteritems(os.environ):
if not key.startswith("DBND__"):
continue
# must have format DBND__{SECTION}__{KEY} (note double underscore)
dbnd_key_var = _DBND_ENVIRON_RE.match(key)
if dbnd_key_var:
section, key = dbnd_key_var.group(1), dbnd_key_var.group(2)
dbnd_environ.set_config_value(
section,
key,
ConfigValue(
value, source="environ[{}]".format(key), require_parse=True
),
)
else:
# check that it's known name, or print error
pass
return dbnd_environ
def update_section(self, section, param_values, source):
# we take values using names only
defaults_store = _ConfigStore()
for key, value in param_values:
previous_value = self.config.get(section, key)
if previous_value != value:
cf = ConfigValue(value=value, source=source)
defaults_store.set_config_value(section, key, cf)
# we apply set on change only in the for loop, so we can optimize and not run all these code
if defaults_store:
self.config.set_values(config_values=defaults_store, source=source)
def read_from_config_stream(config_fp, source="<stream>"):
"""
Read config from config file (.ini, .cfg)
"""
parser = ConfigParser()
parser._read(config_fp, source)
source = "config[{file_name}]".format(
file_name=os.path.basename(str(source)))
new_config = _ConfigStore()
for section in parser.sections():
for option in parser.options(section):
value = parser.get(section, option)
new_config.set_config_value(
section, option, ConfigValue(value, source,
require_parse=True))
return new_config
def get_environ_config_from_dict(env_dict, source_prefix):
dbnd_environ = _ConfigStore()
for key, value in six.iteritems(env_dict):
if not key.startswith("DBND__"):
continue
# must have format DBND__{SECTION}__{KEY} (note double underscore)
dbnd_key_var = _DBND_ENVIRON_RE.match(key)
if dbnd_key_var:
section, key = dbnd_key_var.group(1), dbnd_key_var.group(2)
dbnd_environ.set_config_value(
section,
key,
ConfigValue(
value,
source="{}[{}]".format(source_prefix, key),
require_parse=True,
),
)
else:
# check that it's known name, or print error
pass
return dbnd_environ
def override(value):
return ConfigValue(value=value, source=None, override=True)
def parse_and_build_config_store(
source,
config_values,
override=False,
auto_section_parse=False,
set_if_not_exists_only=False,
):
# type:(str, Mapping[str, Mapping[str, Any]], bool, bool , bool)->_ConfigStore
"""
Read user defined values. Following format are supported:
1. SomeTask.some_param [ParameterDefinition] : value
2. { "section" : { "key" : "value" }}
3 ? "SomeTask.some_param" [str] : value
"""
if isinstance(config_values, _ConfigStore):
return config_values
new_config = _ConfigStore()
new_config.source = source
for section, section_values in six.iteritems(config_values):
if isinstance(section, six.string_types):
if auto_section_parse:
m = _SECTION_NAME_RE.match(section)
if m: # section contains key!
section, key = m.group(1), m.group(2)
section_values = {key: section_values}
if not isinstance(section_values, Mapping):
raise DatabandConfigError(
"can't convert '%s' to configuration " % config_values)
elif isinstance(section, ParameterDefinition):
# this is parameter -> Spark.jars = ["jars"]
section_values = {section.name: section_values}
section = section.task_config_section
else:
raise Exception("section='%s' not supported" % section)
new_section = new_config[section]
for key, value in six.iteritems(section_values):
if key in new_section:
raise Exception(
"multiple definition of {section}.{key} at {config}".
format(section=section, key=key, config=config_values))
if isinstance(key, ParameterDefinition):
key = key.name
if not isinstance(value, ConfigValue):
value = ConfigValue(
value=value,
source=source,
require_parse=False,
override=override,
set_if_not_exists_only=set_if_not_exists_only,
)
else:
# we can have override values without source
if value.source is None:
value = attr.evolve(value, source=source)
new_config.set_config_value(section, key, value)
return new_config
def _build_parameter_value(self, param_def):
# type: (ParameterDefinition) -> ParameterValue
"""
-= MAIN FUNCTION for Parameter Calculation =-
This is the place we calculate param_def value
based on Class(defaults, constructor, overrides, root)
and Config(env, cmd line, config) state
Build parameter value from config_value and param_definition.
Considerate - priority, constructor values, param type and so on.
"""
config_values_stack = self._get_config_value_stack_for_param(param_def)
param_name = param_def.name
# if we will use config values we will need to reduce them to a single value
# if there is only one - reduce will get it
# other wise we need combine the values using `fold_parameter_value`
# may raise if can't fold two values together!!
has_overrides_in_config = any(
cf for cf in config_values_stack
if cf and cf.priority >= ConfigValuePriority.OVERRIDE)
# first check for override (HIGHEST PRIORITY)
if has_overrides_in_config:
config_values_as_param_values = [
build_parameter_value(param_def, cf)
for cf in config_values_stack
]
return functools.reduce(fold_parameter_value,
config_values_as_param_values)
# second using kwargs we received from the user
# do we need to do it for tracking?
elif param_name in self.task_kwargs:
return build_parameter_value(
param_def,
ConfigValue(self.task_kwargs.get(param_name),
source=self._source_name("ctor")),
)
if config_values_stack:
config_values_as_param_values = [
build_parameter_value(param_def, cf)
for cf in config_values_stack
]
return functools.reduce(fold_parameter_value,
config_values_as_param_values)
if (self.parent_task
and param_name in self.parent_task.task_children_scope_params):
# we have parent task with param = parameter(scope=ParameterScope.children)
# the priority is lower than config (as this one is more like "default" than "explicit config")
# see _calculate_task_children_scope_params implementation and ParameterScope.children
parameter_value = self.parent_task.task_children_scope_params[
param_name]
return build_parameter_value(
param_def,
ConfigValue(value=parameter_value.value,
source=parameter_value.source),
)
if param_def.from_task_env_config:
# param = parameter(from_task_env_config=True)
# we check task.task_env.param for the value
if not self.task_env_config:
raise friendly_error.task_parameters.task_env_param_with_no_env(
context=self._ctor_as_str, key=param_name)
param_env_config_value = self.task_env_config.task_params.get_param_value(
param_name)
if param_env_config_value is None:
raise friendly_error.task_parameters.task_env_param_not_exists_in_env(
context=self._ctor_as_str,
key=param_name,
env_config=self.task_env_config,
)
return build_parameter_value(
param_def,
ConfigValue(
value=param_env_config_value.value,
source=param_env_config_value.source,
),
)
if param_name in self.task_definition.param_defaults:
# we can't "add" defaults to the current config and rely on configuration system
# we don't know what section name to use, and it my clash with current config
return build_parameter_value(
param_def,
ConfigValue(
self.task_definition.param_defaults.get(param_name),
source=self._source_name("default"),
),
)
if not param_def.is_output():
# outputs can be none, we "generate" their values later
err_msg = "No value defined for '{name}' at {context_str}".format(
name=param_name, context_str=self._ctor_as_str)
raise MissingParameterError(
err_msg,
help_msg=param_def._get_help_message(
sections=self.config_sections),
)
# returning empty Output value
return ParameterValue(
parameter=param_def,
source="",
source_value=NOTHING,
value=NOTHING,
parsed=False,
)