def _build_multiple_outputs_result(self, result_deco_spec):
# type: (List[ParameterFactory]) -> (List[ParameterDefinition], ParameterDefinition)
context = "{}.{}".format(self.decorator_spec.name, RESULT_PARAM)
res = []
for i, lv in enumerate(result_deco_spec):
lv = build_parameter(lv, context="%s.%s" % (context, i))
if not lv.name:
raise DatabandBuildError("{}[{}]: {} should have name".format(
context, i, type(lv)))
if not lv.is_output():
raise DatabandBuildError(
"{}[{}]: {} should marked as output".format(
context, i, type(lv)))
res.append(lv)
param = FuncResultParameter(
schema=res,
value_type=DefaultObjectValueType(),
name=RESULT_PARAM,
significant=False,
kind=_ParameterKind.task_output,
)
return param
def result_and_params_have_same_keys(context, conflict_keys):
return DatabandBuildError(
"{context} have same keys in result schema and function args: {conflict_keys}".format(
context=context, conflict_keys=conflict_keys
),
help_msg="Please, check %s and rename conflicted keys" % context,
)
def parameter_exception(self, reason, ex):
err_msg = "Failed to {reason} for parameter '{name}' at {task_family}()".format(
reason=reason, name=self.name, task_family=self.task_family)
log_exception(err_msg, ex, logger)
raise DatabandBuildError(err_msg,
nested_exceptions=[ex],
help_msg=self._get_help_message())
def run(self):
# we don't actually have run function except for inline
#
# we don't want to read params automatically
# most likely our run function just creates a command line to run on remote compute
raise DatabandBuildError(
"You should not call or override run functions")
def value_is_not_parameter_cls(value, context=None):
return DatabandBuildError(
"Parameter {context} is '{value}', while it should be defined via 'parameter'"
.format(value=value, context=context),
help_msg=
"Please, check your implementation (see relevant lines in exception message)",
)
def sub_type_with_non_structural_value(context, value_type, sub_type):
return DatabandBuildError(
"Sub type {sub_type} is not supported by main value type :{value_type}."
.format(sub_type=sub_type, value_type=value_type),
help_msg=
" Use main value type like List/Set/Dict: {context} = parameter.sub_type(int)[List]",
)
def failed_to_build_parameter(context, parameter_state, ex):
return DatabandBuildError(
"Failed to build parameter '{context}' defined as '{parameter}': {ex}".
format(context=context, parameter=parameter_state, ex=ex),
help_msg="Validate parameter implementation at {}".format(context),
nested_exceptions=[ex],
)
def failed_to_convert_value_to_target(value):
return DatabandBuildError(
"Can't convert '%s' of type %s to target." %
(safe_value(value), type(value)),
help_msg="You can convert only values with type of "
"Target, Tasks and strings to the Path/PathStr/Target",
)
def unknown_args_in_task_call(parent_task, cls, func_params):
return DatabandBuildError(
"You are trying to create %s from %s with *args, please use named arguments only: args=%s"
% (_run_name(cls), _band_call_str(parent_task), func_params[0]),
show_exc_info=True,
help_msg="Check your %s logic" % _band_call_str(parent_task),
)
def failed_to_call(ex, task_cls):
return DatabandBuildError(
"Failed to invoke '%s': %s" % (_band_call_str(task_cls), ex),
show_exc_info=False,
nested_exceptions=[ex],
help_msg="Check your %s logic" % _band_call_str(task_cls),
)
def failed_to_assign_param_value_at_band(ex, param, value, task):
return DatabandBuildError(
"Failed to assign '{value}' to parameter {param.name} at '{band}': {ex}"
.format(band=_band_call_str(task), value=value, ex=ex, param=param),
show_exc_info=False,
nested_exceptions=[ex],
help_msg="Check your %s logic" % _band_call_str(task),
)
def failed_to_import_pyspark(task, ex):
return DatabandBuildError(
"Tried to create spark session for a task {task} but failed to import pyspark"
.format(task=task.get_task_family),
show_exc_info=False,
nested_exceptions=[ex],
help_msg="Check your environment for pyspark installation.",
)
def pipeline_task_has_unassigned_outputs(task, param):
return DatabandBuildError(
"You have unassigned output '{param.name}' in task '{task}'".format(
param=param, task=task),
help_msg=
"Check your {band} logic, Add self.{param.name} = SOME_TASK_OUTPUT".
format(band=_band_call_str(task), param=param),
)
def dict_in_result_definition(result_deco):
return DatabandBuildError(
"Result definition should be tuple/list, we don't support dict in definition, got: {}"
.format(result_deco),
help_msg="You can use dict for output of the task "
"{'features': some_output, 'scores': some_outputs} "
"but not in result=<DEFINITION>. For example: result=('features', 'scores')",
)
def unknown_value_type_in_parameter(type_):
from targets.values import get_types_registry
return DatabandBuildError(
"The parameter type '{}' is unknown, ".format(type_),
help_msg=
"Use one of: {}, or register the new one via 'register_custom_parameter'"
.format(get_types_registry().list_known_types()),
)
def failed_to_convert_to_target_type(param, x, ex):
return DatabandBuildError(
"Failed to convert '{value}' to {param_type}".format(
value=safe_value(x), param=param,
param_type=param.value_type.type),
show_exc_info=False,
nested_exceptions=[ex],
help_msg="Check your %s logic" % param.name,
)
def task_env_param_not_exists_in_env(context, key, env_config):
return DatabandBuildError(
"{context} can't calculate {key}, as it doesn't exists in {env_config}"
.format(context=context, key=key, env_config=env_config),
help_msg=
"Please, check {context} definition, only parameters defined at {env_config} "
"can have 'from_task_env_config=True'".format(context=context,
env_config=env_config),
)
def failed_to_build_output_target(p_name, task, ex):
help_msg = ("Make sure that output parameter '{p_name}' is "
"configured without keyword [parameter]!").format(
p_name=p_name)
return DatabandBuildError(
("Failed to build output target for parameter '{p_name}' in task '{task}'! "
"Exception: {ex}").format(p_name=p_name, task=task, ex=ex),
help_msg=help_msg,
)
def failed_to_calculate_task_parameter_value(ex, task_family, p_name,
exc_desc):
return DatabandBuildError(
"Failed to process parameter %s for %s: %s" % (p_name, exc_desc, ex),
nested_exceptions=[ex],
help_msg=
"Please validate your config/commandline/.band that creates runs %s" %
exc_desc,
)
def no_value_type_defined_in_parameter(context):
from targets.values import get_types_registry
return DatabandBuildError(
"The parameter {context} doesn't have type! Please use parameter[YOUR_TYPE] or parameter[object]"
.format(context=context),
help_msg=
"Use one of: {}, or register the new type via 'register_custom_parameter'"
.format(get_types_registry().list_known_types()),
)
def failed_to_create_task(exc_desc, nested_exceptions):
msg = "Failed to create task %s" % exc_desc
if nested_exceptions:
msg += ": "
msg += ",".join([str(e) for e in nested_exceptions[:3]])
if len(nested_exceptions) > 3:
msg += " (showing 3 errors out of %s)" % len(nested_exceptions)
return DatabandBuildError(msg,
show_exc_info=False,
nested_exceptions=nested_exceptions)
def no_value_type_from_default(default_value, context):
from targets.values import get_types_registry
return DatabandBuildError(
"The parameter '{parameter}' has default value '{default_value}' "
"that cannot be resolved into known type!"
"Please use '{parameter} = parameter[YOUR_TYPE]' or '{parameter} = parameter[object]'"
.format(default_value=default_value, parameter=context),
help_msg=
"Use one of: {}, or register the new type via 'register_custom_parameter'"
.format(get_types_registry().list_known_types()),
)
def incomplete_output_found_for_task(task_name, complete_outputs,
incomplete_outputs):
return DatabandBuildError(
"Task {} has incomplete outputs! "
"This means the task might fail every time. "
"Complete outputs: {} "
"Incomplete outputs: {} "
"Hint: clean the environment or overwrite the output. "
"To ignore this error, turn off 'validate_task_outputs_on_build' in the '[run]' configuration section"
.format(
task_name,
", ".join(complete_outputs),
", ".join(incomplete_outputs),
))
def validate_git_policy(self):
if not self.enforce_clean:
return
if is_git_dirty():
if self.allow_dirty:
logger.warning("Runing with not commited files")
return
raise DatabandBuildError(
help_msg="Git workspace must be clean."
"\nYou see this message because enforce_clean in git section is enabled."
"\nTo temporarily disable this message use --git-allow-dirty."
)
def cyclic_graph_detected(task, cyclic_nodes):
# print the line of tasks
circle_trail = _find_cirlce(cyclic_nodes)
from dbnd._core.task_ctrl.task_dag_describe import tasks_trail
logger.warning("Cyclic graph detected: %s", tasks_trail(circle_trail))
return DatabandBuildError(
"A cyclic dependency occurred %s: {short_trail}.. ({num_of_tasks} tasks)".format(
task_call="in '{task_call}'" % _band_call_str(task) if task else "",
short_trail=tasks_trail(circle_trail[:3]),
num_of_tasks=len(circle_trail),
),
show_exc_info=False,
help_msg="Check your %s logic, see the full circle path in log. "
"For better visability, use run.recheck_circle_dependencies=True"
% _band_call_str(task),
)
def _build_decorator_kwargs_params(self):
params = {}
for k, param in six.iteritems(self.task_decorator.decorator_kwargs):
if k in self.exclude: # we'll take care of result param later
continue
if param is None:
self.exclude.add(k)
continue
context = self._get_param_context(k)
if k not in self.callable_spec.args and k not in self.base_params:
# we have parameter which is not part of real function signature
# @task(some_unknown_parameter=parameter)
logger.info(
"{} is not part of parameters, creating hidden parameter".format(
context
)
)
if k in self.callable_spec.defaults:
if isinstance(self.callable_spec.defaults[k], ParameterFactory):
raise DatabandBuildError(
"{}: {} has conlficted definition in function and in decorator itself".format(
context, k
)
)
if is_defined(param.parameter.default):
logger.warning(
"Default value conflict between function and @task decorator"
)
param = param.default(self.callable_spec.defaults.get(k))
if k not in self.base_params or isinstance(param, ParameterFactory):
# we are going to build a new parameter
param = build_parameter(param, context=context)
params[k] = param
return params
def no_databand_context():
return DatabandBuildError(
"You are trying to create task without having active Databand context! "
"You should have databand context while building/running tasks. "
"You can create one inplace by adding `init_dbnd()` call")
def wrong_result_definition(result_deco):
return DatabandBuildError(
"{} should be Parameter of task_output kind , "
"or one of list/tuple of parameters or str".format(result_deco))
def failed_to_access_dbnd_home(dbnd_home, ex):
return DatabandBuildError((
"Failed to access DBND_HOME '{dbnd_home}'. "
"Check that folder exists and a process has sufficient permissions to write there. "
"Exception: {ex}").format(dbnd_home=dbnd_home, ex=ex))
def task_env_param_with_no_env(context, key):
return DatabandBuildError(
"{context} can't calculate {key} and env is not defined".format(
context=context, key=key),
help_msg="Please, check that %s has task_env parameter" % context,
)
