def _build_func_spec_params(self, decorator_kwargs_params):
params = {}
# let go over all kwargs of the functions
for k in self.callable_spec.args:
if (
k in self.exclude or k in decorator_kwargs_params
): # excluded or processed already
continue
context = self._get_param_context(k)
default = self.callable_spec.defaults.get(k, NOTHING)
if isinstance(default, ParameterFactory):
# it's inplace defition
# user_param= parameter[str]
params[k] = build_parameter(default, context=context)
elif k in self.base_params:
# we have param definition from "base class"
# so we just need to provide a default value for the parameter
# otherwise - do nothing, as we are good
if is_defined(default):
params[k] = default
else:
try:
# regular value
param_value_type = guess_func_arg_value_type(
self.callable_spec, k, default
)
if param_value_type is None:
# fallback to "object"
param_value_type = DefaultObjectValueType()
param = get_parameter_for_value_type(param_value_type)
param = param.default(default)
params[k] = build_parameter(param, context=context)
except Exception:
logger.exception("Failed to analyze function arg %s", context)
raise
if self.callable_spec.varargs:
# create a param with the name of `*args` argument of the function
params[self.callable_spec.varargs] = build_parameter(
parameter[list], context=self._get_param_context("*args")
)
if self.callable_spec.varkw:
# create a param with the name of `**kwargs` argument of the function
params[self.callable_spec.varkw] = build_parameter(
parameter[typing.Dict[str, typing.Any]],
context=self._get_param_context("**kwargs"),
)
return params
def build_user_parameter_value(name, value, source):
"""
Build parameter value for user defined name and value
"""
value_type = get_value_type_of_obj(
value, default_value_type=DefaultObjectValueType)
param_f = get_parameter_for_value_type(value_type)
param = build_parameter(param_f)
param.name = name
if value is NOTHING:
parameter, warnings = param, []
actual_value = param.default
else:
parameter, warnings = infer_parameter_value_type(param, value)
actual_value = value
return ParameterValue(
parameter=parameter,
source=source,
source_value=value,
value=actual_value,
parsed=False,
warnings=warnings,
)
def _build_multiple_outputs_result(self, result_deco_spec):
# type: (List[ParameterFactory]) -> (List[ParameterDefinition], ParameterDefinition)
context = "{}.{}".format(self.decorator_spec.name, RESULT_PARAM)
res = []
for i, lv in enumerate(result_deco_spec):
lv = build_parameter(lv, context="%s.%s" % (context, i))
if not lv.name:
raise DatabandBuildError("{}[{}]: {} should have name".format(
context, i, type(lv)))
if not lv.is_output():
raise DatabandBuildError(
"{}[{}]: {} should marked as output".format(
context, i, type(lv)))
res.append(lv)
param = FuncResultParameter(
schema=res,
value_type=DefaultObjectValueType(),
name=RESULT_PARAM,
significant=False,
kind=_ParameterKind.task_output,
)
return param
def _build_decorator_kwargs_params(self):
params = {}
for k, param in six.iteritems(self.task_decorator.decorator_kwargs):
if k in self.exclude: # we'll take care of result param later
continue
if param is None:
self.exclude.add(k)
continue
context = self._get_param_context(k)
if k not in self.callable_spec.args and k not in self.base_params:
# we have parameter which is not part of real function signature
# @task(some_unknown_parameter=parameter)
logger.info(
"{} is not part of parameters, creating hidden parameter".format(
context
)
)
if k in self.callable_spec.defaults:
if isinstance(self.callable_spec.defaults[k], ParameterFactory):
raise DatabandBuildError(
"{}: {} has conlficted definition in function and in decorator itself".format(
context, k
)
)
if is_defined(param.parameter.default):
logger.warning(
"Default value conflict between function and @task decorator"
)
param = param.default(self.callable_spec.defaults.get(k))
if k not in self.base_params or isinstance(param, ParameterFactory):
# we are going to build a new parameter
param = build_parameter(param, context=context)
params[k] = param
return params
def _spec_params(self):
"""
We process only regular params here, not the "result" params
:return:
"""
params = {}
exclude = {RESULT_PARAM, "self"}
for k, param in six.iteritems(self.decorator_kwargs):
if k in exclude: # we'll take care of result param later
continue
if param is None:
exclude.add(k)
continue
context = "%s.%s" % (self.decorator_spec.name, k)
if k not in self.decorator_spec.args and k not in self.base_params:
# we have parameter which is not part of real function signature
# @task(some_unknown_parameter=parameter)
logger.info(
"{} is not part of parameters, creating hidden parameter".
format(context))
if k in self.decorator_spec.defaults:
if isinstance(self.decorator_spec.defaults[k],
ParameterFactory):
raise DatabandBuildError(
"{}: {} has conlficted definition in function and in decorator itself"
.format(context, k))
if is_defined(param.parameter.default):
logger.warning(
"Default value conflict between function and @task decorator"
)
param = param.default(self.decorator_spec.defaults.get(k))
if k in self.base_params and not isinstance(
param, ParameterFactory):
# just override value
params[k] = param
else:
# we are going to build a new parameter
params[k] = build_parameter(param, context=context)
# let go over all kwargs of the functions
for k in self.decorator_spec.args:
if k in exclude or k in params: # excluded or processed already
continue
context = "%s.%s" % (self.decorator_spec.name, k)
default = self.decorator_spec.defaults.get(k, NOTHING)
if isinstance(default, ParameterFactory):
# it's inplace defition
# user_param= parameter[str]
params[k] = build_parameter(default, context=context)
elif k in self.base_params:
# we have param definition from "base class"
# so we just need to provide a default value for the parameter
# otherwise - do nothing, as we are good
if is_defined(default):
params[k] = default
else:
try:
# regular value
param_value_type = guess_func_arg_value_type(
self.decorator_spec, k, default)
if param_value_type is None:
# fallback to "object"
param_value_type = DefaultObjectValueType()
param = get_parameter_for_value_type(param_value_type)
param = param.default(default)
params[k] = build_parameter(param, context=context)
except Exception:
logger.exception("Failed to analyze function arg %s",
context)
raise
return params
def _get_result_parameter(self):
context = "{}.{}".format(self.decorator_spec.name, RESULT_PARAM)
return_spec = guess_func_return_type(self.decorator_spec)
deco_spec = None
# first of all , let parse the definition we have
if RESULT_PARAM in self.decorator_kwargs:
# @task(result=...)
deco_spec = self.decorator_kwargs[RESULT_PARAM]
if isinstance(deco_spec, dict):
raise friendly_error.task_parameters.dict_in_result_definition(
deco_spec)
# @task(result=None)
if deco_spec is None:
# user explicitly don't want to have result value
return {}
if isinstance(deco_spec, six.string_types):
# we have result = "output1,output2"
# support both space and comma
deco_spec = deco_spec.replace(",", " ").split()
if len(deco_spec) == 1:
deco_spec = deco_spec[0]
elif isinstance(deco_spec, tuple):
deco_spec = list(deco_spec)
# user didn't specify - so we don't have any "hints"
if is_not_defined(return_spec):
return_spec = None
elif return_spec is not None:
# we will use type hints from "-> ..." spec only if it's has exact match to our params
return_spec = self._validate_return_spec(
deco_spec, return_spec)
else:
# we don't have @task(result=)
if return_spec is None:
# .. -> None
# user explicitly don't want to have result value
return {}
# let return default parameter ( pickle in @task)
if is_not_defined(return_spec):
return build_parameter(self.decorator_spec.default_result,
context)
# so we have something in return speck, let use it
if isinstance(return_spec, list):
# we can get names from ->
deco_spec = [r[0] for r in return_spec]
else:
# or we just use default name
deco_spec = RESULT_PARAM
# so now we have 2 cases
# 1. we have list of results -->
if isinstance(deco_spec, list):
result = []
for i, deco_p in enumerate(deco_spec):
value_type_hint = None
if return_spec:
_, value_type_hint = return_spec[i]
deco_p = self._get_result_parameter_part(
p=deco_p,
name_hint="result_%s" % i,
value_type_hint=value_type_hint)
result.append(deco_p)
param = self._build_multiple_outputs_result(result)
# 2. we have only one result-->
else:
param = self._get_result_parameter_part(
p=deco_spec,
name_hint=RESULT_PARAM,
value_type_hint=return_spec)
return build_parameter(param, context)