def get_task_status(request, task_id, bk_biz_id):
try:
task = TaskFlowInstance.objects.get(pk=task_id, business__cc_id=bk_biz_id)
task_status = task.get_status()
result = {
'result': True,
'data': task_status
}
return JsonResponse(result)
# 请求子流程的状态，直接通过pipeline api查询
except (ValueError, TaskFlowInstance.DoesNotExist):
logger.info('task[id=%s] does not exist' % task_id)
except Exception as e:
message = 'task[id=%s] get status error: %s' % (task_id, e)
logger.error(message)
result = {'result': False, 'message': message}
return JsonResponse(result)
try:
task_status = pipeline_api.get_status_tree(task_id, max_depth=99)
TaskFlowInstance.format_pipeline_status(task_status)
except Exception as e:
message = 'task[id=%s] get status error: %s' % (task_id, e)
logger.error(message)
result = {'result': False, 'message': message}
return JsonResponse(result)
result = {
'result': True,
'data': task_status
}
return JsonResponse(result)
def status(request, biz_cc_id):
instance_id = request.GET.get('instance_id')
try:
task = TaskFlowInstance.objects.get(pk=instance_id,
business__cc_id=biz_cc_id)
task_status = task.get_status()
ctx = {'result': True, 'data': task_status}
return JsonResponse(ctx)
# 请求子流程的状态，直接通过pipeline api查询
except (ValueError, TaskFlowInstance.DoesNotExist):
logger.info('taskflow[id=%s] does not exist' % instance_id)
except Exception as e:
message = 'taskflow[id=%s] get status error: %s' % (instance_id, e)
logger.error(message)
ctx = {'result': False, 'message': message}
return JsonResponse(ctx)
try:
task_status = pipeline_api.get_status_tree(instance_id, max_depth=99)
TaskFlowInstance.format_pipeline_status(task_status)
ctx = {'result': True, 'data': task_status}
# subprocess pipeline has not executed
except exceptions.InvalidOperationException:
ctx = {'result': True, 'data': {'state': states.CREATED}}
except Exception as e:
message = 'taskflow[id=%s] get status error: %s' % (instance_id, e)
logger.error(message)
ctx = {'result': False, 'message': message}
return JsonResponse(ctx)
def get_task_status(request, task_id, project_id):
project = request.project
try:
task = TaskFlowInstance.objects.get(pk=task_id, project_id=project.id, is_deleted=False)
task_status = task.get_status()
result = {
'result': True,
'data': task_status
}
return JsonResponse(result)
# 请求子流程的状态，直接通过pipeline api查询
except (ValueError, TaskFlowInstance.DoesNotExist):
logger.info('task[id=%s] does not exist' % task_id)
except Exception as e:
message = 'task[id={task_id}] get status error: {error}'.format(task_id=task_id, error=e)
logger.error(message)
result = {'result': False, 'message': message}
return JsonResponse(result)
try:
task_status = pipeline_api.get_status_tree(task_id, max_depth=99)
TaskFlowInstance.format_pipeline_status(task_status)
except Exception as e:
message = 'task[id={task_id}] get status error: {error}'.format(task_id=task_id, error=e)
logger.error(message)
result = {'result': False, 'message': message}
return JsonResponse(result)
result = {
'result': True,
'data': task_status
}
return JsonResponse(result)
def status(request, project_id):
instance_id = request.GET.get('instance_id')
subprocess_id = request.GET.get('subprocess_id')
if not subprocess_id:
try:
task = TaskFlowInstance.objects.get(pk=instance_id,
project_id=project_id)
task_status = task.get_status()
ctx = {'result': True, 'data': task_status}
return JsonResponse(ctx)
except Exception as e:
message = 'taskflow[id=%s] get status error: %s' % (instance_id, e)
logger.error(message)
ctx = {'result': False, 'message': message}
return JsonResponse(ctx)
# 请求子流程的状态，直接通过pipeline api查询
try:
task_status = pipeline_api.get_status_tree(subprocess_id, max_depth=99)
TaskFlowInstance.format_pipeline_status(task_status)
ctx = {'result': True, 'data': task_status}
# subprocess pipeline has not executed
except exceptions.InvalidOperationException:
ctx = {'result': True, 'data': {'state': states.CREATED}}
except Exception as e:
message = 'taskflow[id=%s] get status error: %s' % (instance_id, e)
logger.error(message)
ctx = {'result': False, 'message': message}
return JsonResponse(ctx)
def pipeline_post_save_handler(sender, instance, created, **kwargs):
# 任务必须是执行完成，由 celery 触发
if not created and instance.is_finished:
# 获得任务实例的执行树
status_tree = get_status_tree(instance.instance_id, 99)
# 删除原有原子数据
ComponentExecuteData.objects.filter(instance_id=instance.id).delete()
# 获得任务实例的执行数据
data = instance.execution_data
component_list = []
with transaction.atomic():
try:
# act_id 节点 act 原子数据
for act_id, act in data[PE.activities].items():
is_retry = False
if act['type'] == PE.ServiceActivity:
# 原子重试
if status_tree["children"][act_id]["retry"] > 0:
# 需要通过执行历史获得
history_list = get_activity_histories(act_id)
for history in history_list:
start_time = history["started_time"]
archived_time = history["archived_time"]
elapsed_time = history["elapsed_time"]
is_retry = True
else:
# 原子没有重试
# 执行树的相关内容
start_time = status_tree["started_time"]
archived_time = status_tree["archived_time"]
elapsed_time = status_tree["elapsed_time"]
status = True if status_tree[
"state"] == "FINISHED" else False
# 创建对象
component = ComponentExecuteData(
component_code=act['component']['code'],
instance_id=instance.instance_id,
node_id=act_id,
started_time=start_time,
archived_time=archived_time,
elapsed_time=elapsed_time,
status=status,
is_skip=status_tree["skip"],
is_retry=is_retry)
component_list.append(component)
else:
# 传递流程数据
children_tree_dict = status_tree["children"][act_id][
"children"]
component_list = recursive_subprocess_tree(
children_tree_dict, act_id, instance.instance_id,
component_list, act[PE.pipeline][PE.activities],
None)
ComponentExecuteData.objects.bulk_create(component_list)
except Exception as e:
logger.exception(
u"instance_post_save_handler raise error: %s" % e)
def get_state(node_id):
tree = api.get_status_tree(node_id, max_depth=100)
res = _map(tree)
# collect all atom
descendants = {}
_collect_descendants(tree, descendants)
res['children'] = descendants
# return
return res
def get_status(self):
if not self.pipeline_instance.is_started:
return {
"start_time": None,
"state": "CREATED",
"retry": 0,
"skip": 0,
"finish_time": None,
"children": {}
}
status_tree = pipeline_api.get_status_tree(
self.pipeline_instance.instance_id, max_depth=99)
TaskFlowInstance.format_pipeline_status(status_tree)
return status_tree
def get_node_detail_v1(
self,
username: str,
subprocess_stack: List[str],
component_code: Optional[str] = None,
loop: Optional[int] = None,
**kwargs,
) -> dict:
act_start = True
detail = {}
# 首先获取最新一次执行详情
try:
detail = pipeline_api.get_status_tree(self.node_id)
except pipeline_exceptions.InvalidOperationException:
act_start = False
if not act_start:
pipeline_instance = kwargs["pipeline_instance"]
node = self._get_node_info(
node_id=self.node_id, pipeline=pipeline_instance.execution_data, subprocess_stack=subprocess_stack
)
detail.update(
{
"name": node["name"],
"error_ignorable": node.get("error_ignorable", False),
"state": pipeline_states.READY,
}
)
else:
format_pipeline_status(detail)
# 默认只请求最后一次循环结果
if loop is None or int(loop) >= detail["loop"]:
loop = detail["loop"]
detail["history_id"] = -1
detail["histories"] = pipeline_api.get_activity_histories(node_id=self.node_id, loop=loop)
# 如果用户传了 loop 参数，并且 loop 小于当前节点已循环次数 detail['loop']，则从历史数据获取结果
else:
histories = pipeline_api.get_activity_histories(node_id=self.node_id, loop=loop)
self._assemble_histroy_detail(detail=detail, histories=histories)
detail["history_id"] = histories[-1]["history_id"]
for hist in detail["histories"]:
# 重试记录必然是因为失败才重试
hist.setdefault("state", pipeline_states.FAILED)
format_pipeline_status(hist)
if "error_ignorable" in detail:
detail["error_ignored"] = detail["error_ignorable"]
return {"result": True, "data": detail, "message": "", "code": err_code.SUCCESS.code}
def get_node_detail(self, node_id, component_code=None, subprocess_stack=None):
try:
detail = pipeline_api.get_status_tree(node_id)
except exceptions.InvalidOperationException as e:
return {'result': False, 'message': e.message}
TaskFlowInstance.format_pipeline_status(detail)
data = self.get_node_data(node_id, component_code, subprocess_stack)
if not data['result']:
return data
detail['histories'] = pipeline_api.get_activity_histories(node_id)
for his in detail['histories']:
his.setdefault('state', 'FAILED')
TaskFlowInstance.format_pipeline_status(his)
detail.update(data['data'])
return {'result': True, 'data': detail}
def get_node_detail(self,
node_id,
component_code=None,
subprocess_stack=None):
if not self.has_node(node_id):
message = 'node[node_id={node_id}] not found in task[task_id={task_id}]'.format(
node_id=node_id, task_id=self.id)
return {'result': False, 'message': message, 'data': {}}
ret_data = self.get_node_data(node_id, component_code,
subprocess_stack)
try:
detail = pipeline_api.get_status_tree(node_id)
except exceptions.InvalidOperationException as e:
return {'result': False, 'message': e.message, 'data': {}}
TaskFlowInstance.format_pipeline_status(detail)
detail['histories'] = pipeline_api.get_activity_histories(node_id)
for his in detail['histories']:
his.setdefault('state', 'FAILED')
TaskFlowInstance.format_pipeline_status(his)
detail.update(ret_data['data'])
return {'result': True, 'data': detail, 'message': ''}
def pipeline_post_save_handler(sender, instance, created, **kwargs):
# 任务必须是执行完成，由 celery 触发
if not created and instance.is_finished:
# 获得任务实例的执行树
status_tree = get_status_tree(instance.instance_id, 99)
# 删除原有标准插件数据
ComponentExecuteData.objects.filter(
instance_id=instance.instance_id).delete()
# 获得任务实例的执行数据
data = instance.execution_data
component_list = []
with transaction.atomic():
try:
# act_id 节点 act 标准插件数据
for act_id, act in data[PE.activities].items():
is_retry = False
if act['type'] == PE.ServiceActivity:
# 标准插件重试
status_act = status_tree["children"].get(act_id)
if status_act is None:
continue
if status_act["retry"] > 0:
# 需要通过执行历史获得
history_list = get_activity_histories(act_id)
for history in history_list:
start_time = history["started_time"]
archived_time = history["archived_time"]
elapsed_time = history["elapsed_time"]
is_retry = True
else:
# 标准插件没有重试
# 执行树的相关内容
start_time = status_tree["started_time"]
archived_time = status_tree["archived_time"]
elapsed_time = status_tree["elapsed_time"]
status = True if status_tree[
"state"] == "FINISHED" else False
# 创建对象
component = ComponentExecuteData(
component_code=act['component']['code'],
instance_id=instance.instance_id,
node_id=act_id,
started_time=start_time,
archived_time=archived_time,
elapsed_time=elapsed_time,
status=status,
is_skip=status_tree["skip"],
is_retry=is_retry)
component_list.append(component)
else:
# 传递流程数据
children_tree_dict = status_tree["children"][act_id][
"children"]
component_list = recursive_subprocess_tree(
children_tree_dict, act_id, instance.instance_id,
component_list, act[PE.pipeline][PE.activities],
None)
ComponentExecuteData.objects.bulk_create(component_list)
except Exception as e:
logger.exception(
u"instance_post_save_handler raise error: %s" % e)
# 统计流程标准插件个数，子流程个数，网关个数
instance_id = instance.instance_id
# 获取pipeline_tree
pipeline_tree = instance.execution_data
# 初始化插入值
atom_total = 0
subprocess_total = 0
# 获得pipeline_tree
tree_activities = pipeline_tree["activities"]
# 获取网关数量
gateways_total = len(pipeline_tree["gateways"])
# 遍历activities节点
for activity in tree_activities:
activity_type = tree_activities[activity]["type"]
if activity_type == "ServiceActivity":
atom_total += 1
elif activity_type == "SubProcess":
subprocess_total += 1
try:
InstanceInPipeline.objects.create(
instance_id=instance_id,
atom_total=atom_total,
subprocess_total=subprocess_total,
gateways_total=gateways_total)
except Exception as e:
logger.exception(
u"instance_post_save_handler raise error: %s" % e)
def test_status_tree(self):
s1 = Status.objects.create(id=uniqid(),
name='s1',
state=states.FINISHED,
started_time=timezone.now(),
archived_time=timezone.now() +
timedelta(seconds=3))
s2 = Status.objects.create(id=uniqid(),
name='s2',
state=states.FINISHED,
started_time=timezone.now(),
archived_time=timezone.now() +
timedelta(seconds=3))
s3 = Status.objects.create(id=uniqid(),
name='s3',
state=states.FINISHED,
started_time=timezone.now(),
archived_time=timezone.now() +
timedelta(seconds=3))
s4 = Status.objects.create(id=uniqid(),
name='s4',
state=states.FINISHED,
started_time=timezone.now(),
archived_time=timezone.now() +
timedelta(seconds=3))
s5 = Status.objects.create(id=uniqid(),
name='s5',
state=states.FINISHED,
started_time=timezone.now(),
archived_time=timezone.now() +
timedelta(seconds=3))
s6 = Status.objects.create(id=uniqid(),
name='s6',
state=states.FINISHED,
started_time=timezone.now(),
archived_time=timezone.now() +
timedelta(seconds=3))
NodeRelationship.objects.build_relationship(s1.id, s1.id)
NodeRelationship.objects.build_relationship(s2.id, s2.id)
NodeRelationship.objects.build_relationship(s3.id, s3.id)
NodeRelationship.objects.build_relationship(s4.id, s4.id)
NodeRelationship.objects.build_relationship(s5.id, s5.id)
NodeRelationship.objects.build_relationship(s6.id, s6.id)
NodeRelationship.objects.build_relationship(s1.id, s2.id)
NodeRelationship.objects.build_relationship(s1.id, s3.id)
NodeRelationship.objects.build_relationship(s2.id, s4.id)
NodeRelationship.objects.build_relationship(s4.id, s5.id)
NodeRelationship.objects.build_relationship(s4.id, s6.id)
# refresh from db, sync datetime
s1.refresh_from_db()
s2.refresh_from_db()
s3.refresh_from_db()
s4.refresh_from_db()
s5.refresh_from_db()
s6.refresh_from_db()
def get_status_dict_with_children(s, children):
return {
'archived_time':
s.archived_time,
'created_time':
s.created_time,
'elapsed_time':
calculate_elapsed_time(s.started_time, s.archived_time),
'error_ignorable':
s.error_ignorable,
'id':
s.id,
'loop':
s.loop,
'name':
s.name,
'retry':
s.retry,
'skip':
s.skip,
'started_time':
s.started_time,
'state':
s.state,
'version':
s.version,
'children':
children
}
tree_depth_1 = get_status_dict_with_children(
s1,
children={
s2.id: get_status_dict_with_children(s2, children={}),
s3.id: get_status_dict_with_children(s3, children={})
})
tree = api.get_status_tree(s1.id, 1)
self.assertDictEqual(tree, tree_depth_1)
tree_depth_2 = get_status_dict_with_children(
s1,
children={
s2.id:
get_status_dict_with_children(
s2,
children={s4.id: get_status_dict_with_children(s4, {})}),
s3.id:
get_status_dict_with_children(s3, {})
})
tree = api.get_status_tree(s1.id, 2)
self.assertDictEqual(tree, tree_depth_2)
tree_depth_3 = get_status_dict_with_children(
s1,
children={
s2.id:
get_status_dict_with_children(
s2,
children={
s4.id:
get_status_dict_with_children(
s4,
children={
s5.id: get_status_dict_with_children(s5, {}),
s6.id: get_status_dict_with_children(s6, {})
})
}),
s3.id:
get_status_dict_with_children(s3, children={})
}) # noqa
tree = api.get_status_tree(s1.id, 3)
self.assertDictEqual(tree, tree_depth_3)
tree = api.get_status_tree(s1.id, 4)
self.assertDictEqual(tree, tree_depth_3)
def get_task_status_v1(self, subprocess_id: Optional[str],
with_ex_data: bool) -> dict:
if self.pipeline_instance.is_expired:
return {
"result": True,
"data": {
"state": "EXPIRED"
},
"message": "",
"code": err_code.SUCCESS.code
}
if not self.pipeline_instance.is_started:
return {
"result": True,
"data": self.CREATED_STATUS,
"message": "",
"code": err_code.SUCCESS.code,
}
if not subprocess_id:
try:
task_status = pipeline_api.get_status_tree(
self.pipeline_instance.instance_id, max_depth=99)
format_pipeline_status(task_status)
return {
"result": True,
"data": task_status,
"message": "",
"code": err_code.SUCCESS.code,
}
except pipeline_exceptions.InvalidOperationException as e:
logger.error(f"node relationship does not exist: {e}")
task_status = self.CREATED_STATUS
except Exception:
logger.exception("task.get_status fail")
return {
"result": False,
"message": "task.get_status fail",
"data": {},
"code": err_code.UNKNOWN_ERROR.code,
}
else:
try:
task_status = pipeline_api.get_status_tree(subprocess_id,
max_depth=99)
format_pipeline_status(task_status)
except pipeline_exceptions.InvalidOperationException:
# do not raise error when subprocess not exist or has not been executed
task_status = self.CREATED_STATUS
except Exception:
logger.exception(
"pipeline_api.get_status_tree(subprocess_id:{}) fail".
format(subprocess_id))
return {
"result": False,
"message":
"pipeline_api.get_status_tree(subprocess_id:{}) fail",
"data": {},
"code": err_code.UNKNOWN_ERROR.code,
}
# 返回失败节点和对应调试信息
if with_ex_data and task_status["state"] == bamboo_engine_states.FAILED:
failed_nodes = self._collect_fail_nodes(task_status)
task_status["ex_data"] = {}
failed_nodes_outputs = pipeline_api.get_batch_outputs(failed_nodes)
for node_id in failed_nodes:
task_status["ex_data"][node_id] = failed_nodes_outputs[
node_id]["ex_data"]
return {
"result": True,
"data": task_status,
"code": err_code.SUCCESS.code,
"message": ""
}
def get_node_data_v1(
self,
username: str,
subprocess_stack: List[str],
component_code: Optional[str] = None,
loop: Optional[int] = None,
**kwargs,
) -> dict:
node_started = True
inputs = {}
outputs = {}
try:
detail = pipeline_api.get_status_tree(self.node_id)
except pipeline_exceptions.InvalidOperationException:
node_started = False
else:
# 最新 loop 执行记录，直接通过接口获取
if loop is None or int(loop) >= detail["loop"]:
try:
inputs = pipeline_api.get_inputs(self.node_id)
except pipeline_engine_models.Data.DoesNotExist:
logger.exception("shield DoesNotExist in pipeline engine layer")
inputs = {}
try:
outputs = pipeline_api.get_outputs(self.node_id)
except pipeline_engine_models.Data.DoesNotExist:
logger.exception("shield DoesNotExist in pipeline engine layer")
outputs = {}
# 历史 loop 记录，需要从 histories 获取，并取最新一次操作数据（如手动重试时重新填参）
else:
his_data = pipeline_api.get_activity_histories(node_id=self.node_id, loop=loop)
inputs = his_data[-1]["inputs"]
outputs = {"outputs": his_data[-1]["outputs"], "ex_data": his_data[-1]["ex_data"]}
pipeline_instance = kwargs["pipeline_instance"]
if not node_started:
node_info = self._get_node_info(
node_id=self.node_id, pipeline=pipeline_instance.execution_data, subprocess_stack=subprocess_stack
)
if node_info["type"] != "ServiceActivity":
return {
"result": True,
"data": {"inputs": {}, "outputs": [], "ex_data": ""},
"message": "",
"code": err_code.SUCCESS.code,
}
success, err, inputs, outputs = self._prerender_node_data(
pipeline_instance=pipeline_instance, subprocess_stack=subprocess_stack, username=username
)
if not success:
return {
"result": False,
"data": {},
"message": err,
"code": err_code.UNKNOWN_ERROR.code,
}
# 根据传入的 component_code 对输出进行格式化
success, err, outputs_table = self._format_outputs(
outputs=outputs,
component_code=component_code,
pipeline_instance=pipeline_instance,
subprocess_stack=subprocess_stack,
)
if not success:
return {
"result": False,
"data": {},
"message": err,
"code": err_code.UNKNOWN_ERROR.code,
}
data = {"inputs": inputs, "outputs": outputs_table, "ex_data": outputs.pop("ex_data", "")}
return {"result": True, "data": data, "message": "", "code": err_code.SUCCESS.code}
