def validate(self, request, *args, **kwargs):
valid, err = super().validate(request, *args, **kwargs)
if not valid:
return valid, err
try:
params = json.loads(request.body)
except Exception:
return False, "invalid json format"
try:
pipeline_tree = params["pipeline_tree"]
standardize_pipeline_node_name(pipeline_tree)
pipeline_tree.setdefault("gateways", {})
pipeline_tree.setdefault("constants", {})
pipeline_tree.setdefault("outputs", [])
draw_pipeline(pipeline_tree)
validate_web_pipeline_tree(pipeline_tree)
except Exception as e:
message = "[API] fast_create_task get invalid pipeline_tree: %s" % str(
e)
logger.warning(message)
return False, message
# 校验流程树中子流程是否在当前项目下
has_common_subprocess = params.get("has_common_subprocess", False)
templates_in_task = set()
pipeline_tree = params["pipeline_tree"]
for activity in pipeline_tree["activities"].values():
if "template_id" in activity:
templates_in_task.add(activity["template_id"])
if not has_common_subprocess:
project_templates = set(
TaskTemplate.objects.filter(
project_id=request.project.id).values_list("id",
flat=True))
if not templates_in_task.issubset(project_templates):
invalid_template_ids = [
str(template)
for template in list(templates_in_task - project_templates)
]
message = (
"[API] fast_create_task get invalid template_id: {}, template_id "
"should belong to current project.".format(
",".join(invalid_template_ids)))
logger.warning(message)
return False, message
setattr(request, "params_json", params)
return True, ""
def handle_task_name_attr(data):
data["name"] = standardize_name(data["name"], TASK_NAME_MAX_LENGTH)
standardize_pipeline_node_name(data["pipeline_tree"])
def create_pipeline(
self,
name: str,
creator: str,
pipeline_tree: dict,
description: str = "",
) -> dict:
"""
创建 pipeline 层模板
:param name: 模板名
:type name: str
:param creator: 创建者
:type creator: str
:param pipeline_tree: 模板数据
:type pipeline_tree: dict
:param description: 模板描述, defaults to ""
:type description: str, optional
:return: [description]
:rtype: dict
"""
name = standardize_name(name, TEMPLATE_NODE_NAME_MAX_LENGTH)
standardize_pipeline_node_name(pipeline_tree)
try:
validate_web_pipeline_tree(pipeline_tree)
except PipelineException as e:
return {
"result":
False,
"data":
None,
"message":
"[TemplateManager]validate_web_pipeline_tree failed: {}".
format(str(e)),
"verbose_message":
"[TemplateManager]validate_web_pipeline_tree failed: {}".
format(traceback.format_exc()),
}
create_template_kwargs = {
"name": name,
"creator": creator,
"pipeline_tree": pipeline_tree,
"description": description,
}
try:
pipeline_template = self.template_model_cls.objects.create_pipeline_template(
**create_template_kwargs)
except Exception as e:
return {
"result":
False,
"data":
None,
"message":
"[TemplateManager]create_pipeline_template({kwargs}) failed: {e}"
.format(kwargs=create_template_kwargs, e=str(e)),
"verbose_message":
"[TemplateManager]create_pipeline_template({kwargs}) failed: {trace}"
.format(kwargs=create_template_kwargs,
trace=traceback.format_exc()),
}
return {
"result": True,
"data": pipeline_template,
"message": "success",
"verbose_message": "success"
}
def create_task(request, template_id, project_id):
params = json.loads(request.body)
project = request.project
template_source = params.get("template_source", PROJECT)
logger.info(
"[API] create_task info, template_id: {template_id}, project_id: {project_id}, params: {params}"
.format(template_id=template_id, project_id=project.id, params=params))
# 兼容老版本的接口调用
if template_source in NON_COMMON_TEMPLATE_TYPES:
template_source = PROJECT
try:
tmpl = TaskTemplate.objects.select_related(
"pipeline_template").get(id=template_id,
project_id=project.id,
is_deleted=False)
except TaskTemplate.DoesNotExist:
result = {
"result":
False,
"message":
"template[id={template_id}] of project[project_id={project_id},biz_id={biz_id}] "
"does not exist".format(template_id=template_id,
project_id=project.id,
biz_id=project.bk_biz_id),
"code":
err_code.CONTENT_NOT_EXIST.code,
}
return result
else:
try:
tmpl = CommonTemplate.objects.select_related(
"pipeline_template").get(id=template_id, is_deleted=False)
except CommonTemplate.DoesNotExist:
result = {
"result":
False,
"message":
"common template[id={template_id}] does not exist".format(
template_id=template_id),
"code":
err_code.CONTENT_NOT_EXIST.code,
}
return result
app_code = getattr(request.jwt.app, settings.APIGW_APP_CODE_KEY)
if not app_code:
message = "app_code cannot be empty, make sure api gateway has sent correct params"
return {
"result": False,
"message": message,
"code": err_code.CONTENT_NOT_EXIST.code
}
try:
params.setdefault("flow_type", "common")
params.setdefault("constants", {})
params.setdefault("exclude_task_nodes_id", [])
params.setdefault("simplify_vars", [])
params.setdefault("execute_task_nodes_id", [])
jsonschema.validate(params, APIGW_CREATE_TASK_PARAMS)
except jsonschema.ValidationError as e:
logger.exception("[API] create_task raise prams error: %s" % e)
message = "task params is invalid: %s" % e
return {
"result": False,
"message": message,
"code": err_code.REQUEST_PARAM_INVALID.code
}
create_with_tree = "pipeline_tree" in params
pipeline_instance_kwargs = {
"name": params["name"],
"creator": request.user.username,
"description": params.get("description", ""),
}
if create_with_tree:
try:
pipeline_tree = params["pipeline_tree"]
for key, value in params["constants"].items():
if key in pipeline_tree["constants"]:
if pipeline_tree["constants"][key].get("is_meta", False):
meta = copy.deepcopy(pipeline_tree["constants"][key])
pipeline_tree["constants"][key]["meta"] = meta
pipeline_tree["constants"][key]["value"] = value
standardize_pipeline_node_name(pipeline_tree)
validate_web_pipeline_tree(pipeline_tree)
except Exception as e:
message = "[API] create_task get invalid pipeline_tree: %s" % str(
e)
logger.exception(message)
return {
"result": False,
"message": message,
"code": err_code.UNKNOWN_ERROR.code
}
pipeline_instance_kwargs["pipeline_tree"] = pipeline_tree
try:
data = TaskFlowInstance.objects.create_pipeline_instance(
template=tmpl, **pipeline_instance_kwargs)
except PipelineException as e:
message = "[API] create_task create pipeline error: %s" % str(e)
logger.exception(message)
return {
"result": False,
"message": message,
"code": err_code.UNKNOWN_ERROR.code
}
else:
# 如果请求参数中含有非空的execute_task_nodes_id(要执行的节点)，就将其转换为exclude_task_nodes_id(要排除的节点)
if not params["execute_task_nodes_id"]:
exclude_task_nodes_id = params["exclude_task_nodes_id"]
else:
exclude_task_nodes_id = get_exclude_nodes_by_execute_nodes(
params["execute_task_nodes_id"], tmpl)
try:
data = TaskFlowInstance.objects.create_pipeline_instance_exclude_task_nodes(
tmpl,
pipeline_instance_kwargs,
params["constants"],
exclude_task_nodes_id,
params["simplify_vars"],
)
except Exception as e:
message = f"[API] create_task create pipeline without tree error: {e}"
logger.exception(message)
return {
"result": False,
"message": message,
"code": err_code.UNKNOWN_ERROR.code
}
task = TaskFlowInstance.objects.create(
project=project,
pipeline_instance=data,
category=tmpl.category,
template_id=template_id,
template_source=template_source,
create_method="api",
create_info=app_code,
flow_type=params.get("flow_type", "common"),
current_flow="execute_task"
if params.get("flow_type", "common") == "common" else "func_claim",
engine_ver=EngineConfig.objects.get_engine_ver(
project_id=project.id,
template_id=template_id,
template_source=template_source),
)
# crete auto retry strategy
arn_creator = AutoRetryNodeStrategyCreator(
taskflow_id=task.id,
root_pipeline_id=task.pipeline_instance.instance_id)
arn_creator.batch_create_strategy(task.pipeline_instance.execution_data)
# create timeout config
TimeoutNodeConfig.objects.batch_create_node_timeout_config(
taskflow_id=task.id,
root_pipeline_id=task.pipeline_instance.instance_id,
pipeline_tree=task.pipeline_instance.execution_data,
)
return {
"result": True,
"data": {
"task_id": task.id,
"task_url": task.url,
"pipeline_tree": task.pipeline_tree
},
"code": err_code.SUCCESS.code,
}
def update_pipeline(
self,
pipeline_template: PipelineTemplate,
editor: str,
name: str = "",
pipeline_tree: str = None,
description: str = "",
) -> dict:
"""
更新 pipeline 层模板
:param pipeline_template: pipeline 模板对象
:type pipeline_template: PipelineTemplate
:param editor: 编辑者
:type editor: str
:param name: 模板名, defaults to ""
:type name: str, optional
:param pipeline_tree: 模板结构, defaults to None
:type pipeline_tree: str, optional
:param description: 模板描述, defaults to ""
:type description: str, optional
:return: [description]
:rtype: dict
"""
update_kwargs = {"editor": editor}
if name:
update_kwargs["name"] = standardize_name(
name, TEMPLATE_NODE_NAME_MAX_LENGTH)
if description:
update_kwargs["description"] = description
if pipeline_tree:
standardize_pipeline_node_name(pipeline_tree)
try:
validate_web_pipeline_tree(pipeline_tree)
except PipelineException as e:
return {
"result":
False,
"data":
None,
"message":
"[TemplateManager]validate_web_pipeline_tree failed: {}".
format(str(e)),
"verbose_message":
"[TemplateManager]validate_web_pipeline_tree failed: {}".
format(traceback.format_exc()),
}
replace_template_id(self.template_model_cls, pipeline_tree)
pipeline_web_tree = PipelineWebTreeCleaner(pipeline_tree)
pipeline_web_tree.clean()
update_kwargs["structure_data"] = pipeline_tree
try:
pipeline_template.update_template(**update_kwargs)
except Exception as e:
return {
"result":
False,
"data":
None,
"message":
"[TemplateManager]update_template({update_kwargs}) failed: {e}"
.format(update_kwargs=update_kwargs, e=str(e)),
"verbose_message":
"[TemplateManager]update_template({update_kwargs}) failed: {trace}"
.format(update_kwargs=update_kwargs,
trace=traceback.format_exc()),
}
# create node in template
NodeInTemplate.objects.update_nodes_in_template(
pipeline_template, pipeline_web_tree.origin_data)
else:
for k, v in update_kwargs.items():
setattr(pipeline_template, k, v)
pipeline_template.save()
return {
"result": True,
"data": pipeline_template,
"message": "success",
"verbose_message": "success"
}