def _prepare(self, task_id, re_execute: bool = False) -> Executable:
sess = self.sess()
task = self._get_task(sess, task_id)
fn: Executable = _REG.get(task.func_name)
# function existence check
if fn is None:
raise PPLException(ErrorCode.FUNCTION_NOT_EXISTS, task.func_name)
# param existence check
if task.parameter is None:
raise PPLException(ErrorCode.TASK_HAS_NO_PARAM, task.id)
# Executability check
if not re_execute:
if not task.is_executable():
raise PPLException(ErrorCode.TASK_NOT_EXECUTABLE, task.id)
# Parent completion check
parent = self._parent(sess, task)
if parent is not None and not parent.is_completed():
raise PPLException(ErrorCode.TASK_PARENT_UNFINISHED, task.id)
# init task status to RUNNING
task.status = TaskStatus.RUNNING.value
# load parameter into function
if task.parameter.data is not None and task.parameter.data.data is not None:
fn.load_param(task.parameter.data.data)
else:
fn.load_param('{}')
sess.commit()
return fn
def _add_task(self, sess: Session, name: str, func_name: str, ppl_id: int,
parent_id: int):
# param check. When pipeline is non-empty, parent id must present when
# adding tasks
ppl: Pipeline = self._get_pipeline(sess, ppl_id=ppl_id)
if len(ppl.tasks) > 0 and parent_id is None:
raise ValueError(
"Parent task id must be specified when appending to a non-empty pipeline {}"
.format(ppl_id))
# Check parent existence
parent = self.task_manager._get_task(sess, parent_id)
if len(ppl.tasks) > 0 and parent is None:
raise PPLException(ErrorCode.TASK_PARENT_NOT_EXISTS, parent_id)
# Check function existence
func = _REG.get(func_name)
if func is None:
raise PPLException(ErrorCode.FUNCTION_NOT_EXISTS, func_name)
t = Task.new(name, func_name, ppl)
sess.add(t)
if parent is not None:
parent.next = t
return t
def __init__(self) -> None:
try:
param_type = type(self).Param
result_type = type(self).Result
except AttributeError:
raise PPLException(ErrorCode.META_EXEC_LACK_PARAM_RES)
if (not issubclass(param_type, BaseModel)) or (not issubclass(
result_type, BaseModel)):
raise PPLException(ErrorCode.PARAM_RES_INVALID_TYPE)
super().__init__(param_type, result_type)
def register_task(name: str, func: Executable):
if _REG.get(name) is not None:
logger.warn("Repeated registration: %s", name)
if not isinstance(func, Executable):
raise PPLException(ErrorCode.FUNCTION_TYPE_INVALID, type(func))
_REG[name] = func
def _task_head(self, sess, ppl_id) -> Task:
ppl = self._get_pipeline(sess, ppl_id)
tasks = ppl.tasks
res = []
for t in tasks:
if self.task_manager._parent(sess, t) is None:
res.append(t)
if len(res) > 1:
raise PPLException(ErrorCode.PIPELINE_MULTIPLE_HEAD, ppl_id)
if len(res) == 0:
raise PPLException(ErrorCode.PIPELINE_CIRCULAR_DEP)
return res[0]
def _rollback_task(self, task: Task):
fn: Executable = _REG.get(task.func_name)
if fn is None:
raise PPLException(ErrorCode.FUNCTION_NOT_EXISTS, task.func_name)
fn.load_param(task.parameter.data.data)
fn.load_result(task.result.data.data)
fn.rollback()
def abort_task(self, task_id: int):
sess: Session = self.trasient_sess()
task = self._get_task(sess, task_id)
if task.is_abortable():
task.status = TaskStatus.ABORTED.value
else:
raise PPLException(ErrorCode.TASK_NOT_ABORTABLE, task_id)
sess.commit()
def _add_parameter(self, sess: Session, task: Task, param: BaseModel):
if task is None:
raise PPLException(ErrorCode.TASK_NOT_EXISTS, None)
data = Data()
data.data = param.json()
sess.add(data)
if task.parameter is not None:
db_param = task.parameter
else:
db_param = Parameter()
db_param.data = data
db_param.task = task
sess.add(db_param)
def _dump_result(self, sess: Session, task_id: int, result: BaseModel):
task = self._get_task(sess, task_id)
if task is None:
raise PPLException(ErrorCode.TASK_NOT_EXISTS, task_id)
data = Data()
data.data = result.json()
sess.add(data)
if task.result is not None:
db_result = task.result
else:
db_result = Result()
db_result.task = task
db_result.data = data
sess.add(db_result)
def test_disp(self):
try:
raise PPLException(ErrorCode.TASK_PARENT_NOT_EXISTS, "task1")
except PPLException as e:
print(e)