def _build_multiple_outputs_result(self, result_deco_spec): # type: (List[ParameterFactory]) -> (List[ParameterDefinition], ParameterDefinition) context = "{}.{}".format(self.decorator_spec.name, RESULT_PARAM) res = [] for i, lv in enumerate(result_deco_spec): lv = build_parameter(lv, context="%s.%s" % (context, i)) if not lv.name: raise DatabandBuildError("{}[{}]: {} should have name".format( context, i, type(lv))) if not lv.is_output(): raise DatabandBuildError( "{}[{}]: {} should marked as output".format( context, i, type(lv))) res.append(lv) param = FuncResultParameter( schema=res, value_type=DefaultObjectValueType(), name=RESULT_PARAM, significant=False, kind=_ParameterKind.task_output, ) return param
def result_and_params_have_same_keys(context, conflict_keys): return DatabandBuildError( "{context} have same keys in result schema and function args: {conflict_keys}".format( context=context, conflict_keys=conflict_keys ), help_msg="Please, check %s and rename conflicted keys" % context, )
def parameter_exception(self, reason, ex): err_msg = "Failed to {reason} for parameter '{name}' at {task_family}()".format( reason=reason, name=self.name, task_family=self.task_family) log_exception(err_msg, ex, logger) raise DatabandBuildError(err_msg, nested_exceptions=[ex], help_msg=self._get_help_message())
def run(self): # we don't actually have run function except for inline # # we don't want to read params automatically # most likely our run function just creates a command line to run on remote compute raise DatabandBuildError( "You should not call or override run functions")
def value_is_not_parameter_cls(value, context=None): return DatabandBuildError( "Parameter {context} is '{value}', while it should be defined via 'parameter'" .format(value=value, context=context), help_msg= "Please, check your implementation (see relevant lines in exception message)", )
def sub_type_with_non_structural_value(context, value_type, sub_type): return DatabandBuildError( "Sub type {sub_type} is not supported by main value type :{value_type}." .format(sub_type=sub_type, value_type=value_type), help_msg= " Use main value type like List/Set/Dict: {context} = parameter.sub_type(int)[List]", )
def failed_to_build_parameter(context, parameter_state, ex): return DatabandBuildError( "Failed to build parameter '{context}' defined as '{parameter}': {ex}". format(context=context, parameter=parameter_state, ex=ex), help_msg="Validate parameter implementation at {}".format(context), nested_exceptions=[ex], )
def failed_to_convert_value_to_target(value): return DatabandBuildError( "Can't convert '%s' of type %s to target." % (safe_value(value), type(value)), help_msg="You can convert only values with type of " "Target, Tasks and strings to the Path/PathStr/Target", )
def unknown_args_in_task_call(parent_task, cls, func_params): return DatabandBuildError( "You are trying to create %s from %s with *args, please use named arguments only: args=%s" % (_run_name(cls), _band_call_str(parent_task), func_params[0]), show_exc_info=True, help_msg="Check your %s logic" % _band_call_str(parent_task), )
def failed_to_call(ex, task_cls): return DatabandBuildError( "Failed to invoke '%s': %s" % (_band_call_str(task_cls), ex), show_exc_info=False, nested_exceptions=[ex], help_msg="Check your %s logic" % _band_call_str(task_cls), )
def failed_to_assign_param_value_at_band(ex, param, value, task): return DatabandBuildError( "Failed to assign '{value}' to parameter {param.name} at '{band}': {ex}" .format(band=_band_call_str(task), value=value, ex=ex, param=param), show_exc_info=False, nested_exceptions=[ex], help_msg="Check your %s logic" % _band_call_str(task), )
def failed_to_import_pyspark(task, ex): return DatabandBuildError( "Tried to create spark session for a task {task} but failed to import pyspark" .format(task=task.get_task_family), show_exc_info=False, nested_exceptions=[ex], help_msg="Check your environment for pyspark installation.", )
def pipeline_task_has_unassigned_outputs(task, param): return DatabandBuildError( "You have unassigned output '{param.name}' in task '{task}'".format( param=param, task=task), help_msg= "Check your {band} logic, Add self.{param.name} = SOME_TASK_OUTPUT". format(band=_band_call_str(task), param=param), )
def dict_in_result_definition(result_deco): return DatabandBuildError( "Result definition should be tuple/list, we don't support dict in definition, got: {}" .format(result_deco), help_msg="You can use dict for output of the task " "{'features': some_output, 'scores': some_outputs} " "but not in result=<DEFINITION>. For example: result=('features', 'scores')", )
def unknown_value_type_in_parameter(type_): from targets.values import get_types_registry return DatabandBuildError( "The parameter type '{}' is unknown, ".format(type_), help_msg= "Use one of: {}, or register the new one via 'register_custom_parameter'" .format(get_types_registry().list_known_types()), )
def failed_to_convert_to_target_type(param, x, ex): return DatabandBuildError( "Failed to convert '{value}' to {param_type}".format( value=safe_value(x), param=param, param_type=param.value_type.type), show_exc_info=False, nested_exceptions=[ex], help_msg="Check your %s logic" % param.name, )
def task_env_param_not_exists_in_env(context, key, env_config): return DatabandBuildError( "{context} can't calculate {key}, as it doesn't exists in {env_config}" .format(context=context, key=key, env_config=env_config), help_msg= "Please, check {context} definition, only parameters defined at {env_config} " "can have 'from_task_env_config=True'".format(context=context, env_config=env_config), )
def failed_to_build_output_target(p_name, task, ex): help_msg = ("Make sure that output parameter '{p_name}' is " "configured without keyword [parameter]!").format( p_name=p_name) return DatabandBuildError( ("Failed to build output target for parameter '{p_name}' in task '{task}'! " "Exception: {ex}").format(p_name=p_name, task=task, ex=ex), help_msg=help_msg, )
def failed_to_calculate_task_parameter_value(ex, task_family, p_name, exc_desc): return DatabandBuildError( "Failed to process parameter %s for %s: %s" % (p_name, exc_desc, ex), nested_exceptions=[ex], help_msg= "Please validate your config/commandline/.band that creates runs %s" % exc_desc, )
def no_value_type_defined_in_parameter(context): from targets.values import get_types_registry return DatabandBuildError( "The parameter {context} doesn't have type! Please use parameter[YOUR_TYPE] or parameter[object]" .format(context=context), help_msg= "Use one of: {}, or register the new type via 'register_custom_parameter'" .format(get_types_registry().list_known_types()), )
def failed_to_create_task(exc_desc, nested_exceptions): msg = "Failed to create task %s" % exc_desc if nested_exceptions: msg += ": " msg += ",".join([str(e) for e in nested_exceptions[:3]]) if len(nested_exceptions) > 3: msg += " (showing 3 errors out of %s)" % len(nested_exceptions) return DatabandBuildError(msg, show_exc_info=False, nested_exceptions=nested_exceptions)
def no_value_type_from_default(default_value, context): from targets.values import get_types_registry return DatabandBuildError( "The parameter '{parameter}' has default value '{default_value}' " "that cannot be resolved into known type!" "Please use '{parameter} = parameter[YOUR_TYPE]' or '{parameter} = parameter[object]'" .format(default_value=default_value, parameter=context), help_msg= "Use one of: {}, or register the new type via 'register_custom_parameter'" .format(get_types_registry().list_known_types()), )
def incomplete_output_found_for_task(task_name, complete_outputs, incomplete_outputs): return DatabandBuildError( "Task {} has incomplete outputs! " "This means the task might fail every time. " "Complete outputs: {} " "Incomplete outputs: {} " "Hint: clean the environment or overwrite the output. " "To ignore this error, turn off 'validate_task_outputs_on_build' in the '[run]' configuration section" .format( task_name, ", ".join(complete_outputs), ", ".join(incomplete_outputs), ))
def validate_git_policy(self): if not self.enforce_clean: return if is_git_dirty(): if self.allow_dirty: logger.warning("Runing with not commited files") return raise DatabandBuildError( help_msg="Git workspace must be clean." "\nYou see this message because enforce_clean in git section is enabled." "\nTo temporarily disable this message use --git-allow-dirty." )
def cyclic_graph_detected(task, cyclic_nodes): # print the line of tasks circle_trail = _find_cirlce(cyclic_nodes) from dbnd._core.task_ctrl.task_dag_describe import tasks_trail logger.warning("Cyclic graph detected: %s", tasks_trail(circle_trail)) return DatabandBuildError( "A cyclic dependency occurred %s: {short_trail}.. ({num_of_tasks} tasks)".format( task_call="in '{task_call}'" % _band_call_str(task) if task else "", short_trail=tasks_trail(circle_trail[:3]), num_of_tasks=len(circle_trail), ), show_exc_info=False, help_msg="Check your %s logic, see the full circle path in log. " "For better visability, use run.recheck_circle_dependencies=True" % _band_call_str(task), )
def _build_decorator_kwargs_params(self): params = {} for k, param in six.iteritems(self.task_decorator.decorator_kwargs): if k in self.exclude: # we'll take care of result param later continue if param is None: self.exclude.add(k) continue context = self._get_param_context(k) if k not in self.callable_spec.args and k not in self.base_params: # we have parameter which is not part of real function signature # @task(some_unknown_parameter=parameter) logger.info( "{} is not part of parameters, creating hidden parameter".format( context ) ) if k in self.callable_spec.defaults: if isinstance(self.callable_spec.defaults[k], ParameterFactory): raise DatabandBuildError( "{}: {} has conlficted definition in function and in decorator itself".format( context, k ) ) if is_defined(param.parameter.default): logger.warning( "Default value conflict between function and @task decorator" ) param = param.default(self.callable_spec.defaults.get(k)) if k not in self.base_params or isinstance(param, ParameterFactory): # we are going to build a new parameter param = build_parameter(param, context=context) params[k] = param return params
def no_databand_context(): return DatabandBuildError( "You are trying to create task without having active Databand context! " "You should have databand context while building/running tasks. " "You can create one inplace by adding `init_dbnd()` call")
def wrong_result_definition(result_deco): return DatabandBuildError( "{} should be Parameter of task_output kind , " "or one of list/tuple of parameters or str".format(result_deco))
def failed_to_access_dbnd_home(dbnd_home, ex): return DatabandBuildError(( "Failed to access DBND_HOME '{dbnd_home}'. " "Check that folder exists and a process has sufficient permissions to write there. " "Exception: {ex}").format(dbnd_home=dbnd_home, ex=ex))
def task_env_param_with_no_env(context, key): return DatabandBuildError( "{context} can't calculate {key} and env is not defined".format( context=context, key=key), help_msg="Please, check that %s has task_env parameter" % context, )