def _prepare(self, task_id, re_execute: bool = False) -> Executable: sess = self.sess() task = self._get_task(sess, task_id) fn: Executable = _REG.get(task.func_name) # function existence check if fn is None: raise PPLException(ErrorCode.FUNCTION_NOT_EXISTS, task.func_name) # param existence check if task.parameter is None: raise PPLException(ErrorCode.TASK_HAS_NO_PARAM, task.id) # Executability check if not re_execute: if not task.is_executable(): raise PPLException(ErrorCode.TASK_NOT_EXECUTABLE, task.id) # Parent completion check parent = self._parent(sess, task) if parent is not None and not parent.is_completed(): raise PPLException(ErrorCode.TASK_PARENT_UNFINISHED, task.id) # init task status to RUNNING task.status = TaskStatus.RUNNING.value # load parameter into function if task.parameter.data is not None and task.parameter.data.data is not None: fn.load_param(task.parameter.data.data) else: fn.load_param('{}') sess.commit() return fn
def _add_task(self, sess: Session, name: str, func_name: str, ppl_id: int, parent_id: int): # param check. When pipeline is non-empty, parent id must present when # adding tasks ppl: Pipeline = self._get_pipeline(sess, ppl_id=ppl_id) if len(ppl.tasks) > 0 and parent_id is None: raise ValueError( "Parent task id must be specified when appending to a non-empty pipeline {}" .format(ppl_id)) # Check parent existence parent = self.task_manager._get_task(sess, parent_id) if len(ppl.tasks) > 0 and parent is None: raise PPLException(ErrorCode.TASK_PARENT_NOT_EXISTS, parent_id) # Check function existence func = _REG.get(func_name) if func is None: raise PPLException(ErrorCode.FUNCTION_NOT_EXISTS, func_name) t = Task.new(name, func_name, ppl) sess.add(t) if parent is not None: parent.next = t return t
def __init__(self) -> None: try: param_type = type(self).Param result_type = type(self).Result except AttributeError: raise PPLException(ErrorCode.META_EXEC_LACK_PARAM_RES) if (not issubclass(param_type, BaseModel)) or (not issubclass( result_type, BaseModel)): raise PPLException(ErrorCode.PARAM_RES_INVALID_TYPE) super().__init__(param_type, result_type)
def register_task(name: str, func: Executable): if _REG.get(name) is not None: logger.warn("Repeated registration: %s", name) if not isinstance(func, Executable): raise PPLException(ErrorCode.FUNCTION_TYPE_INVALID, type(func)) _REG[name] = func
def _task_head(self, sess, ppl_id) -> Task: ppl = self._get_pipeline(sess, ppl_id) tasks = ppl.tasks res = [] for t in tasks: if self.task_manager._parent(sess, t) is None: res.append(t) if len(res) > 1: raise PPLException(ErrorCode.PIPELINE_MULTIPLE_HEAD, ppl_id) if len(res) == 0: raise PPLException(ErrorCode.PIPELINE_CIRCULAR_DEP) return res[0]
def _rollback_task(self, task: Task): fn: Executable = _REG.get(task.func_name) if fn is None: raise PPLException(ErrorCode.FUNCTION_NOT_EXISTS, task.func_name) fn.load_param(task.parameter.data.data) fn.load_result(task.result.data.data) fn.rollback()
def abort_task(self, task_id: int): sess: Session = self.trasient_sess() task = self._get_task(sess, task_id) if task.is_abortable(): task.status = TaskStatus.ABORTED.value else: raise PPLException(ErrorCode.TASK_NOT_ABORTABLE, task_id) sess.commit()
def _add_parameter(self, sess: Session, task: Task, param: BaseModel): if task is None: raise PPLException(ErrorCode.TASK_NOT_EXISTS, None) data = Data() data.data = param.json() sess.add(data) if task.parameter is not None: db_param = task.parameter else: db_param = Parameter() db_param.data = data db_param.task = task sess.add(db_param)
def _dump_result(self, sess: Session, task_id: int, result: BaseModel): task = self._get_task(sess, task_id) if task is None: raise PPLException(ErrorCode.TASK_NOT_EXISTS, task_id) data = Data() data.data = result.json() sess.add(data) if task.result is not None: db_result = task.result else: db_result = Result() db_result.task = task db_result.data = data sess.add(db_result)
def test_disp(self): try: raise PPLException(ErrorCode.TASK_PARENT_NOT_EXISTS, "task1") except PPLException as e: print(e)