def get_node_log_v2(self, history_id: int) -> dict:
runtime = BambooDjangoRuntime()
return {
"result": True,
"data": handle_plain_log(runtime.get_plain_log_for_node(node_id=self.node_id, history_id=history_id)),
"message": "",
}
def callback_v2(self, operator: str, **kwargs) -> dict:
# 兼容 pipeline 引擎时期 callback 可以不传 version 的请求
runtime = BambooDjangoRuntime()
version = kwargs.get("version")
if not version:
version = runtime.get_state(self.node_id).version
return bamboo_engine_api.callback(runtime=runtime, node_id=self.node_id, version=version, data=kwargs["data"])
def skip_cpg_v2(self, operator: str, **kwargs) -> dict:
return bamboo_engine_api.skip_conditional_parallel_gateway(
runtime=BambooDjangoRuntime(),
node_id=self.node_id,
flow_ids=kwargs["flow_ids"],
converge_gateway_id=kwargs["converge_gateway_id"],
)
def render_current_constants_v2(self):
runtime = BambooDjangoRuntime()
context_values = runtime.get_context(
self.pipeline_instance.instance_id)
try:
root_pipeline_inputs = {
key: di.value
for key, di in runtime.get_data_inputs(
self.pipeline_instance.instance_id).items()
}
except bamboo_engine_exceptions.NotFoundError:
return {
"result": False,
"data": None,
"code": err_code.CONTENT_NOT_EXIST.code,
"message": "data not found, task is not running",
}
context = Context(runtime, context_values, root_pipeline_inputs)
try:
hydrated_context = context.hydrate()
except Exception as e:
logger.exception(
"[render_current_constants_v2] error occurred at context hydrate"
)
return {
"result": False,
"data": None,
"code": err_code.UNKNOWN_ERROR.code,
"message": "context hydrate error: %s" % str(e),
}
data = []
for key, value in hydrated_context.items():
if isinstance(value, SystemObject):
data.append({"key": key, "value": value.__dict__})
else:
data.append({"key": key, "value": value})
return {
"result": True,
"data": data,
"code": err_code.SUCCESS.code,
"message": ""
}
def task_pause(self):
"""
手动暂停正在运行的pipeline任务
"""
pause_info = api.pause_pipeline(runtime=BambooDjangoRuntime(),
pipeline_id=self.pipeline_id)
if not pause_info.result:
logger.error("暂停任务失败,错误信息:{}，pipeline_id :{}".format(
pause_info.exc, self.pipeline_id))
return pause_info.result
def task_resume(self):
"""
手动启动已暂停的pipeline任务
"""
resume_info = api.resume_pipeline(runtime=BambooDjangoRuntime(),
pipeline_id=self.pipeline_id)
if not resume_info.result:
logger.error("重新启动任务失败,错误信息:{}，pipeline_id :{}".format(
resume_info.exc, self.pipeline_id))
return resume_info.result
def get_task_states(self):
"""
根据pipeline_id 获取流程树状态
"""
if self.pipeline_id:
data = api.get_pipeline_states(runtime=BambooDjangoRuntime(),
root_id=self.pipeline_id).data
return data
else:
return None
def get_node_output(self):
"""
根据node_id 获取活动节点的输出日志
"""
if self.node_id:
data = api.get_execution_data_outputs(
runtime=BambooDjangoRuntime(),
node_id=self.node_id).data.get('result_message', 'running')
return data
else:
logger.error("no node_id")
return None
def get_outputs_v2(self):
runtime = BambooDjangoRuntime()
outputs_result = bamboo_engine_api.get_execution_data_outputs(runtime=runtime, node_id=self.node_id)
if not outputs_result.result:
logger.exception("bamboo_engine_api.get_execution_data_outputs fail")
return {
"result": False,
"data": {},
"message": "{}: {}".format(outputs_result.message, outputs_result.exc),
"code": err_code.UNKNOWN_ERROR.code,
}
return {"result": True, "data": outputs_result.data, "message": "", "code": err_code.SUCCESS.code}
def build_bamboo(self):
"""
build_bamboo方法：建立流程任务
"""
self.pipe.extend(self.end_act)
bamboo_task = build_tree(self.start_act, data=self.global_data)
update_record_detail(self.task_id, {"pipeline_id": bamboo_task['id']})
update_record_detail(self.task_id, {"pipeline_tree": json.dumps(self.act_tree)})
if not api.run_pipeline(runtime=BambooDjangoRuntime(), pipeline=bamboo_task).result:
logger.error("部署bamboo流程任务创建失败，任务结束")
return False
return True
def _ensure_node_can_retry(node_id, engine_ver):
count = 0
while count < 3:
if engine_ver == EngineConfig.ENGINE_VER_V1:
if PipelineProcess.objects.filter(current_node_id=node_id,
is_sleep=True).exists():
return True
elif engine_ver == EngineConfig.ENGINE_VER_V2:
if BambooDjangoRuntime(
).get_sleep_process_info_with_current_node_id(node_id):
return True
else:
raise ValueError("invalid engine_ver: %s" % engine_ver)
time.sleep(0.1)
count += 1
return False
def start_v2(self, executor: str) -> dict:
# CAS
update_success = PipelineInstance.objects.filter(
instance_id=self.pipeline_instance.instance_id,
is_started=False).update(start_time=timezone.now(),
is_started=True,
executor=executor)
self.pipeline_instance.calculate_tree_info()
PipelineInstance.objects.filter(
instance_id=self.pipeline_instance.instance_id).update(
tree_info_id=self.pipeline_instance.tree_info.id)
if not update_success:
return {
"result": False,
"message": "task already started",
"code": err_code.INVALID_OPERATION.code
}
try:
# convert web pipeline to pipeline
pipeline = format_web_data_to_pipeline(
self.pipeline_instance.execution_data)
root_pipeline_data = get_pipeline_context(self.pipeline_instance,
obj_type="instance",
data_type="data",
username=executor)
system_obj = SystemObject(root_pipeline_data)
root_pipeline_context = {"${_system}": system_obj}
root_pipeline_context.update(
get_project_constants_context(self.project_id))
# run pipeline
result = bamboo_engine_api.run_pipeline(
runtime=BambooDjangoRuntime(),
pipeline=pipeline,
root_pipeline_data=root_pipeline_data,
root_pipeline_context=root_pipeline_context,
subprocess_context=root_pipeline_context,
queue=self.queue,
cycle_tolerate=True,
)
except Exception as e:
logger.exception("run pipeline failed")
PipelineInstance.objects.filter(
instance_id=self.pipeline_instance.instance_id,
is_started=True).update(
start_time=None,
is_started=False,
executor="",
)
return {
"result": False,
"message": "run pipeline failed: {}".format(e),
"code": err_code.UNKNOWN_ERROR.code,
}
if not result.result:
PipelineInstance.objects.filter(
instance_id=self.pipeline_instance.instance_id,
is_started=True).update(
start_time=None,
is_started=False,
executor="",
)
logger.error("run_pipeline fail: {}, exception: {}".format(
result.message, result.exc_trace))
else:
taskflow_started.send(sender=self.__class__,
task_id=self.taskflow_id)
dict_result = {
"result":
result.result,
"message":
result.message,
"code":
err_code.SUCCESS.code
if result.result else err_code.UNKNOWN_ERROR.code,
}
return dict_result
def resume_v2(self, operator: str, **kwargs) -> dict:
return bamboo_engine_api.resume_node_appoint(runtime=BambooDjangoRuntime(), node_id=self.node_id)
def resume_subproc_v2(self, operator: str, **kwargs) -> dict:
return bamboo_engine_api.resume_pipeline(runtime=BambooDjangoRuntime(), pipeline_id=self.node_id)
def forced_fail_v2(self, operator: str, **kwargs) -> dict:
return bamboo_engine_api.forced_fail_activity(
runtime=BambooDjangoRuntime(), node_id=self.node_id, ex_data="forced fail by {}".format(operator)
)
def retry_subprocess_v2(self, operator: str, **kwargs) -> dict:
return bamboo_engine_api.retry_subprocess(runtime=BambooDjangoRuntime(), node_id=self.node_id)
def get(self, request, project_id, task_id, node_id, version):
runtime = BambooDjangoRuntime()
logs = handle_plain_log(runtime.get_plain_log_for_node(node_id=node_id, version=version))
return Response({"result": True, "message": "success", "data": logs})
def skip_exg_v2(self, operator: str, **kwargs) -> dict:
return bamboo_engine_api.skip_exclusive_gateway(
runtime=BambooDjangoRuntime(), node_id=self.node_id, flow_id=kwargs["flow_id"]
)
def get_node_data_v2(
self,
username: str,
subprocess_stack: List[str],
component_code: Optional[str] = None,
loop: Optional[int] = None,
**kwargs,
) -> dict:
runtime = BambooDjangoRuntime()
result = bamboo_engine_api.get_children_states(runtime=runtime, node_id=self.node_id)
if not result.result:
logger.exception("bamboo_engine_api.get_children_states fail")
return {
"result": False,
"data": {},
"message": "{}: {}".format(result.message, result.exc),
"code": err_code.UNKNOWN_ERROR.code,
}
state = result.data
# 已执行的节点直接获取执行数据
inputs = {}
outputs = {}
pipeline_instance = kwargs["pipeline_instance"]
if state:
# 获取最新的执行数据
if loop is None or int(loop) >= state[self.node_id]["loop"]:
result = bamboo_engine_api.get_execution_data(runtime=runtime, node_id=self.node_id)
if not result.result:
logger.exception("bamboo_engine_api.get_execution_data fail")
# 对上层屏蔽执行数据不存在的场景
if isinstance(result.exc, bamboo_exceptions.NotFoundError):
return {
"result": True,
"data": {"inputs": {}, "outputs": [], "ex_data": ""},
"message": "",
"code": err_code.SUCCESS.code,
}
else:
return {
"result": False,
"data": {},
"message": "{}: {}".format(result.message, result.exc),
"code": err_code.UNKNOWN_ERROR.code,
}
data = result.data
inputs = data["inputs"]
outputs = data["outputs"]
outputs = {"outputs": outputs, "ex_data": outputs.get("ex_data")}
# 读取历史记录
else:
result = bamboo_engine_api.get_node_histories(runtime=runtime, node_id=self.node_id, loop=loop)
if not result.result:
logger.exception("bamboo_engine_api.get_node_histories fail")
return {
"result": False,
"data": {},
"message": "{}: {}".format(result.message, result.exc),
"code": err_code.UNKNOWN_ERROR.code,
}
hist = result.data
if hist:
inputs = hist[-1]["inputs"]
outputs = hist[-1]["outputs"]
outputs = {"outputs": outputs, "ex_data": outputs.get("ex_data")}
# 未执行节点需要实时渲染
else:
node_info = self._get_node_info(
node_id=self.node_id, pipeline=pipeline_instance.execution_data, subprocess_stack=subprocess_stack
)
if node_info["type"] != "ServiceActivity":
return {
"result": True,
"data": {"inputs": {}, "outputs": [], "ex_data": ""},
"message": "",
"code": err_code.SUCCESS.code,
}
try:
root_pipeline_data = get_pipeline_context(
pipeline_instance, obj_type="instance", data_type="data", username=username
)
system_obj = SystemObject(root_pipeline_data)
root_pipeline_context = {"${_system}": {"type": "plain", "value": system_obj}}
root_pipeline_context.update(
{
key: {"type": "plain", "value": value}
for key, value in get_project_constants_context(kwargs["project_id"]).items()
}
)
formatted_pipeline = format_web_data_to_pipeline(pipeline_instance.execution_data)
preview_result = bamboo_engine_api.preview_node_inputs(
runtime=runtime,
pipeline=formatted_pipeline,
node_id=self.node_id,
subprocess_stack=subprocess_stack,
root_pipeline_data=root_pipeline_data,
parent_params=root_pipeline_context,
)
if not preview_result.result:
return {
"result": False,
"data": {},
"message": preview_result.message,
"code": err_code.UNKNOWN_ERROR.code,
}
inputs = preview_result.data
except Exception as err:
return {
"result": False,
"data": {},
"message": err,
"code": err_code.UNKNOWN_ERROR.code,
}
# 根据传入的 component_code 对输出进行格式化
success, err, outputs_table = self._format_outputs(
outputs=outputs,
component_code=component_code,
pipeline_instance=pipeline_instance,
subprocess_stack=subprocess_stack,
)
if not success:
return {
"result": False,
"data": {},
"message": err,
"code": err_code.UNKNOWN_ERROR.code,
}
data = {"inputs": inputs, "outputs": outputs_table, "ex_data": outputs.pop("ex_data", "")}
return {
"result": True,
"data": data,
"message": "",
"code": err_code.SUCCESS.code,
}
def revoke_v2(self, operator: str) -> dict:
return bamboo_engine_api.revoke_pipeline(
runtime=BambooDjangoRuntime(),
pipeline_id=self.pipeline_instance.instance_id)
def get_node_detail_v2(
self,
username: str,
subprocess_stack: List[str],
component_code: Optional[str] = None,
loop: Optional[int] = None,
**kwargs,
) -> dict:
runtime = BambooDjangoRuntime()
result = bamboo_engine_api.get_children_states(runtime=runtime, node_id=self.node_id)
if not result.result:
logger.exception("bamboo_engine_api.get_children_states fail")
return {
"result": False,
"data": {},
"message": "{}: {}".format(result.message, result.exc),
"code": err_code.UNKNOWN_ERROR.code,
}
detail = result.data
# 节点已经执行
if detail:
detail = detail[self.node_id]
# 默认只请求最后一次循环结果
format_pipeline_status(detail)
if loop is None or int(loop) >= detail["loop"]:
loop = detail["loop"]
hist_result = bamboo_engine_api.get_node_histories(runtime=runtime, node_id=self.node_id, loop=loop)
if not hist_result:
logger.exception("bamboo_engine_api.get_node_histories fail")
return {
"result": False,
"data": {},
"message": "{}: {}".format(hist_result.message, hist_result.exc),
"code": err_code.UNKNOWN_ERROR.code,
}
for hist in hist_result.data:
hist["ex_data"] = hist.get("outputs", {}).get("ex_data", "")
detail["histories"] = hist_result.data
detail["history_id"] = -1
# 如果用户传了 loop 参数，并且 loop 小于当前节点已循环次数，则从历史数据获取结果
else:
hist_result = bamboo_engine_api.get_node_histories(runtime=runtime, node_id=self.node_id, loop=loop)
if not hist_result:
logger.exception("bamboo_engine_api.get_node_histories fail")
return {
"result": False,
"data": {},
"message": "{}: {}".format(hist_result.message, hist_result.exc),
"code": err_code.UNKNOWN_ERROR.code,
}
self._assemble_histroy_detail(detail=detail, histories=hist_result.data)
detail["history_id"] = hist_result.data[-1]["id"]
detail["version"] = hist_result.data[-1]["version"]
for hist in detail["histories"]:
# 重试记录必然是因为失败才重试
hist.setdefault("state", bamboo_engine_states.FAILED)
hist["history_id"] = hist["id"]
format_pipeline_status(hist)
# 节点未执行
else:
pipeline_instance = kwargs["pipeline_instance"]
node = self._get_node_info(
node_id=self.node_id, pipeline=pipeline_instance.execution_data, subprocess_stack=subprocess_stack
)
detail.update(
{
"name": node["name"],
"error_ignorable": node.get("error_ignorable", False),
"state": bamboo_engine_states.READY,
}
)
return {"result": True, "data": detail, "message": "", "code": err_code.SUCCESS.code}
def get_task_status_v2(self, subprocess_id: Optional[str],
with_ex_data: bool) -> dict:
if self.pipeline_instance.is_expired:
return {
"result": True,
"data": {
"state": "EXPIRED"
},
"message": "",
"code": err_code.SUCCESS.code
}
if not self.pipeline_instance.is_started:
return {
"result": True,
"data": self.CREATED_STATUS,
"message": "",
"code": err_code.SUCCESS.code,
}
runtime = BambooDjangoRuntime()
status_result = bamboo_engine_api.get_pipeline_states(
runtime=runtime,
root_id=self.pipeline_instance.instance_id,
flat_children=False)
if not status_result:
logger.exception("bamboo_engine_api.get_pipeline_states fail")
return {
"result":
False,
"data": {},
"message":
"{}: {}".format(status_result.message, status_result.exc),
"code":
err_code.UNKNOWN_ERROR.code,
}
task_status = status_result.data
if not task_status:
return {
"result": True,
"data": self.CREATED_STATUS,
"message": "",
"code": err_code.SUCCESS.code,
}
task_status = task_status[self.pipeline_instance.instance_id]
def get_subprocess_status(task_status: dict,
subprocess_id: str) -> dict:
for child in task_status["children"].values():
if child["id"] == subprocess_id:
return child
if child["children"]:
status = get_subprocess_status(child, subprocess_id)
if status is not None:
return status
if subprocess_id:
task_status = get_subprocess_status(task_status, subprocess_id)
# subprocess not been executed
task_status = task_status or self.CREATED_STATUS
format_bamboo_engine_status(task_status)
# 返回失败节点和对应调试信息
if with_ex_data and task_status["state"] == bamboo_engine_states.FAILED:
failed_nodes = self._collect_fail_nodes(task_status)
task_status["ex_data"] = {}
for node_id in failed_nodes:
data_result = bamboo_engine_api.get_execution_data_outputs(
runtime=runtime, node_id=node_id)
if not data_result:
task_status["ex_data"][
node_id] = "get ex_data fail: {}".format(
data_result.exc)
else:
task_status["ex_data"][node_id] = data_result.data.get(
"ex_data")
return {
"result": True,
"data": task_status,
"code": err_code.SUCCESS.code,
"message": ""
}
def retry_v2(self, operator: str, **kwargs) -> dict:
# 数据为空的情况传入 None, v2 engine api 不认为 {} 是空数据
return bamboo_engine_api.retry_node(
runtime=BambooDjangoRuntime(), node_id=self.node_id, data=kwargs["inputs"] or None
)
def recursive_collect_components_execution(activities,
status_tree,
task_instance,
engine_ver=1,
stack=None):
"""
@summary 递归流程树，获取所有执行结束的插件TaskflowExecutedNodeStatistics对象列表（成功/失败）
@param activities: 当前流程树的任务节点信息
@param status_tree: 当前流程树的任务节点状态
@param task_instance: 根流程实例TaskFlowInstance
@param stack: 子流程堆栈
@param engine_ver: 流程引擎版本
"""
instance = task_instance.pipeline_instance
task_instance_id = task_instance.id
task_template = TaskTemplate.objects.get(
pipeline_template=instance.template)
if stack is None:
stack = []
is_sub = False
else:
is_sub = True
component_list = []
for act_id, act in activities.items():
if act_id in status_tree:
exec_act = status_tree[act_id]
# 标准插件节点
if act[PE.type] == PE.ServiceActivity:
# 结束、失败、撤销
if exec_act["state"] in states.ARCHIVED_STATES:
component_code = act["component"]["code"]
component_version = act["component"].get(
"version", LEGACY_PLUGINS_VERSION)
is_remote = False
if component_code == "remote_plugin":
component_code = act["component"]["data"][
"plugin_code"]["value"]
component_version = act["component"]["data"][
"plugin_version"]["value"]
is_remote = True
component_kwargs = {
"component_code":
component_code,
"instance_id":
instance.id,
"task_instance_id":
task_instance_id,
"is_sub":
is_sub,
"node_id":
act_id,
"subprocess_stack":
json.dumps(stack),
"started_time":
format_date_time(exec_act["start_time"]),
"archived_time":
format_date_time(exec_act["finish_time"]),
"elapsed_time":
exec_act.get(
"elapsed_time",
calculate_elapsed_time(
format_date_time(exec_act["start_time"]),
format_date_time(exec_act["finish_time"])),
),
"is_skip":
exec_act["skip"],
"is_retry":
False,
"status":
exec_act["state"] == "FINISHED",
"version":
component_version,
"template_id":
instance.template.id,
"task_template_id":
task_template.id,
"project_id":
task_template.project.id,
"instance_create_time":
instance.create_time,
"instance_start_time":
instance.start_time,
"instance_finish_time":
instance.finish_time,
"is_remote":
is_remote,
}
component_list.append(
TaskflowExecutedNodeStatistics(**component_kwargs))
if exec_act["retry"] > 0:
# 有重试记录，需要从执行历史中获取数据
if engine_ver == 1:
history_list = pipeline_api.get_activity_histories(
act_id)
else:
history_list_result = bamboo_engine_api.get_node_short_histories(
runtime=BambooDjangoRuntime(), node_id=act_id)
history_list = history_list_result.data if history_list_result.result else []
for history in history_list:
component_kwargs.update({
"started_time":
history["started_time"],
"archived_time":
history["archived_time"],
"elapsed_time":
history.get(
"elapsed_time",
calculate_elapsed_time(
history["started_time"],
history["archived_time"]),
),
"is_retry":
True,
"is_skip":
False,
"status":
False,
})
component_list.append(
TaskflowExecutedNodeStatistics(
**component_kwargs))
# 子流程的执行堆栈（子流程的执行过程）
elif act[PE.type] == PE.SubProcess:
sub_activities = act[PE.pipeline][PE.activities]
# 防止stack共用
copied_stack = deepcopy(stack)
copied_stack.insert(0, act_id)
component_list += recursive_collect_components_execution(
activities=sub_activities,
status_tree=exec_act["children"],
task_instance=task_instance,
stack=copied_stack,
engine_ver=engine_ver,
)
return component_list
def skip_v2(self, operator: str, **kwargs) -> dict:
return bamboo_engine_api.skip_node(runtime=BambooDjangoRuntime(), node_id=self.node_id)