def get_pipeline_tree_by_version(self, version=None):
tree = self.pipeline_template.data_for_version(version)
replace_template_id(self.__class__, tree, reverse=True)
# add nodes attr
pipeline_web_clean = PipelineWebTreeCleaner(tree)
nodes = NodeInTemplate.objects.filter(template_id=self.pipeline_template.template_id, version=self.version)
nodes_attr = NodeAttr.get_nodes_attr(nodes, "template")
pipeline_web_clean.to_web(nodes_attr)
return tree
def find_deprecated_plugins_in_unfold_tree(tree, template_model, phases=None):
"""查找子流程未展开的树中已经下线的插件
:param tree: 子流程未展开的树
:type tree: dict
:param template_model: 子流程引用的模板模型
:type template_model: Model
:return: {
"found": True or False,
"plugins": {
"activities": [
{
"id": "act_id",
"name": "act_name",
"component": "component_code",
"version": "component_version",
"subprocess": "subprocess_name"
},
...
],
"variables": [
{
"key": "var_key",
"name": "var_name",
"custom_type": "var_code",
"version": "var version",
"subprocess": "subprocess_name"
},
...
]
}
}
:rtype: dict
"""
check_tree = copy.deepcopy(tree)
# replace template id to pipeline id
replace_template_id(template_model, check_tree)
# unfold subprocess reference
PipelineTemplateWebWrapper.unfold_subprocess(check_tree, template_model)
phases = phases or [DeprecatedPlugin.PLUGIN_PHASE_DEPRECATED]
return find_deprecated_plugins_in_spread_tree(tree=check_tree,
phases=phases)
def _unfold_subprocess(pipeline_data, template_model):
"""内部递归调用函数
:param pipeline_data: pipeline tree
:type pipeline_data: dict
:param template_model: 用于获取子流程 tree 的 Model
:type template_model: TaskTemplate or CommonTemplate
"""
activities = pipeline_data[PWE.activities]
for act_id, act in list(activities.items()):
if act[PWE.type] == PWE.SubProcess:
always_use_latest = act.get("always_use_latest", False)
if always_use_latest:
version = None
else:
version = act.get("version")
subproc_data = template_model.objects.get(
pipeline_template__template_id=act["template_id"]
).get_pipeline_tree_by_version(version)
if "constants" in pipeline_data:
subproc_inputs = act.pop("constants")
# replace show constants with inputs
subproc_constants = {}
for key, info in subproc_inputs.items():
if "form" in info:
info.pop("form")
subproc_constants[key] = info
subproc_data["constants"].update(subproc_constants)
replace_template_id(template_model, subproc_data)
# 需要将父流程中修改的 constants 传到子流程的 act constants 中
# 根据执行方案创建子流程实例
scheme_id_list = act.get("scheme_id_list", [])
exclude_task_nodes_id = PipelineTemplateWebPreviewer.get_template_exclude_task_nodes_with_schemes(
subproc_data, scheme_id_list)
PipelineTemplateWebPreviewer.preview_pipeline_tree_exclude_task_nodes(
subproc_data, exclude_task_nodes_id, False)
_unfold_subprocess(subproc_data, template_model)
subproc_data["id"] = act_id
act["pipeline"] = subproc_data
def create_pipeline_template(self, **kwargs):
pipeline_tree = kwargs["pipeline_tree"]
replace_template_id(self.model, pipeline_tree)
pipeline_template_data = {
"name": kwargs["name"],
"creator": kwargs["creator"],
"description": kwargs["description"],
}
pipeline_web_tree = PipelineWebTreeCleaner(pipeline_tree)
pipeline_web_tree.clean()
pipeline_template = PipelineTemplate.objects.create_model(pipeline_tree, **pipeline_template_data)
# create node in template
NodeInTemplate.objects.create_nodes_in_template(pipeline_template, pipeline_web_tree.origin_data)
return pipeline_template
def get_clone_pipeline_tree(self):
clone_tree = self.pipeline_template.clone_data()
replace_template_id(self.__class__, clone_tree, reverse=True)
return clone_tree
def obj_create(self, bundle, **kwargs):
try:
template_id = bundle.data.pop("template_id")
template_source = bundle.data.get("template_source", PROJECT)
name = bundle.data.pop("name")
cron = bundle.data.pop("cron")
pipeline_tree = json.loads(bundle.data.pop("pipeline_tree"))
except (KeyError, ValueError) as e:
message = "create periodic_task params error: %s" % e.message
logger.error(message)
raise BadRequest(message)
if not isinstance(cron, dict):
raise BadRequest("cron must be a object json string")
try:
project = ProjectResource().get_via_uri(bundle.data.get("project"),
request=bundle.request)
except NotFound:
raise BadRequest("project [uri=%s] does not exist" %
bundle.data.get("project"))
# check if the periodic task of the project reach the limit
periodic_task_limit = env.PERIODIC_TASK_PROJECT_MAX_NUMBER
project_config = ProjectConfig.objects.filter(
project_id=project.id).only("max_periodic_task_num").first()
if project_config and project_config.max_periodic_task_num > 0:
periodic_task_limit = project_config.max_periodic_task_num
if PeriodicTask.objects.filter(
project__id=project.id).count() >= periodic_task_limit:
raise BadRequest("Periodic task number reaches limit: {}".format(
periodic_task_limit))
if template_source == PROJECT:
try:
template = TaskTemplate.objects.get(id=template_id,
project=project,
is_deleted=False)
except TaskTemplate.DoesNotExist:
raise BadRequest(
"template[id={template_id}] of project[{project_id}] does not exist"
.format(template_id=template_id, project_id=project.id))
allow_or_raise_immediate_response(
iam=iam,
system=IAMMeta.SYSTEM_ID,
subject=Subject("user", bundle.request.user.username),
action=Action(IAMMeta.FLOW_CREATE_PERIODIC_TASK_ACTION),
resources=res_factory.resources_for_flow_obj(template),
)
try:
replace_template_id(TaskTemplate, pipeline_tree)
except TaskTemplate.DoesNotExist:
raise BadRequest(
"invalid subprocess, check subprocess node please")
elif template_source == COMMON:
try:
template = CommonTemplate.objects.get(id=template_id,
is_deleted=False)
except CommonTemplate.DoesNotExist:
raise BadRequest("common template[id=%s] does not exist" %
template_id)
allow_or_raise_immediate_response(
iam=iam,
system=IAMMeta.SYSTEM_ID,
subject=Subject("user", bundle.request.user.username),
action=Action(IAMMeta.COMMON_FLOW_CREATE_PERIODIC_TASK_ACTION),
resources=[
res_factory.resources_for_project_obj(project)[0],
res_factory.resources_for_common_flow_obj(template)[0],
],
)
try:
replace_template_id(CommonTemplate, pipeline_tree)
except TaskTemplate.DoesNotExist:
raise BadRequest(
"invalid subprocess, check subprocess node please")
else:
raise BadRequest("invalid template_source[%s]" % template_source)
# XSS handle
name = standardize_name(name, PERIOD_TASK_NAME_MAX_LENGTH)
creator = bundle.request.user.username
# validate pipeline tree
try:
validate_web_pipeline_tree(pipeline_tree)
except PipelineException as e:
raise BadRequest(str(e))
kwargs["template_id"] = template_id
kwargs["template_source"] = template_source
try:
kwargs["task"] = PeriodicTask.objects.create_pipeline_task(
project=project,
template=template,
name=name,
cron=cron,
pipeline_tree=pipeline_tree,
creator=creator,
template_source=template_source,
)
except Exception as e:
logger.warning(traceback.format_exc())
raise BadRequest(str(e))
response = super(PeriodicTaskResource,
self).obj_create(bundle, **kwargs)
response.obj.set_enabled(True)
return response
def create_periodic_task(request, template_id, project_id):
project = request.project
# check if the periodic task of the project reach the limit
periodic_task_limit = env.PERIODIC_TASK_PROJECT_MAX_NUMBER
project_config = ProjectConfig.objects.filter(
project_id=project.id).only("max_periodic_task_num").first()
if project_config and project_config.max_periodic_task_num > 0:
periodic_task_limit = project_config.max_periodic_task_num
if PeriodicTask.objects.filter(
project__id=project.id).count() >= periodic_task_limit:
message = "Periodic task number reaches limit: {}".format(
periodic_task_limit)
return {
"result": False,
"message": message,
"code": err_code.INVALID_OPERATION.code
}
params = json.loads(request.body)
template_source = params.get("template_source", PROJECT)
logger.info(
"[API] apigw create_periodic_task info, "
"template_id: {template_id}, project_id: {project_id}, params: {params}"
.format(template_id=template_id, project_id=project.id, params=params))
if template_source in NON_COMMON_TEMPLATE_TYPES:
template_source = PROJECT
try:
template = TaskTemplate.objects.get(pk=template_id,
project_id=project.id,
is_deleted=False)
except TaskTemplate.DoesNotExist:
result = {
"result":
False,
"message":
"template[id={template_id}] of project[project_id={project_id} , biz_id{biz_id}] "
"does not exist".format(
template_id=template_id,
project_id=project.id,
biz_id=project.bk_biz_id,
),
"code":
err_code.CONTENT_NOT_EXIST.code,
}
return result
else:
try:
template = CommonTemplate.objects.get(id=template_id,
is_deleted=False)
except CommonTemplate.DoesNotExist:
result = {
"result":
False,
"message":
"common template[id={template_id}] does not exist".format(
template_id=template_id),
"code":
err_code.CONTENT_NOT_EXIST.code,
}
return result
params.setdefault("constants", {})
params.setdefault("exclude_task_nodes_id", [])
try:
jsonschema.validate(params, APIGW_CREATE_PERIODIC_TASK_PARAMS)
except jsonschema.ValidationError as e:
logger.warning("[API] create_periodic_task raise prams error: %s" % e)
message = "task params is invalid: %s" % e
return {
"result": False,
"message": message,
"code": err_code.REQUEST_PARAM_INVALID.code
}
exclude_task_nodes_id = params["exclude_task_nodes_id"]
pipeline_tree = template.pipeline_tree
try:
PipelineTemplateWebPreviewer.preview_pipeline_tree_exclude_task_nodes(
pipeline_tree, exclude_task_nodes_id)
except Exception as e:
logger.exception(
"[API] create_periodic_task preview tree error: {}".format(e))
return {
"result": False,
"message": str(e),
"code": err_code.UNKNOWN_ERROR.code
}
for key, val in list(params["constants"].items()):
if key in pipeline_tree["constants"]:
pipeline_tree["constants"][key]["value"] = val
name = params["name"]
cron = params["cron"]
try:
replace_template_id(TaskTemplate, pipeline_tree)
except Exception as e:
logger.exception(
"[API] create_periodic_task replace id error: {}".format(e))
return {
"result": False,
"message": str(e),
"code": err_code.UNKNOWN_ERROR.code
}
try:
task = PeriodicTask.objects.create(
project=project,
template=template,
template_source=template_source,
name=name,
cron=cron,
pipeline_tree=pipeline_tree,
creator=request.user.username,
)
except Exception as e:
logger.exception(
"[API] create_periodic_task create error: {}".format(e))
return {
"result": False,
"message": str(e),
"code": err_code.UNKNOWN_ERROR.code
}
data = info_data_from_period_task(task)
return {"result": True, "data": data, "code": err_code.SUCCESS.code}
def update_pipeline(
self,
pipeline_template: PipelineTemplate,
editor: str,
name: str = "",
pipeline_tree: str = None,
description: str = "",
) -> dict:
"""
更新 pipeline 层模板
:param pipeline_template: pipeline 模板对象
:type pipeline_template: PipelineTemplate
:param editor: 编辑者
:type editor: str
:param name: 模板名, defaults to ""
:type name: str, optional
:param pipeline_tree: 模板结构, defaults to None
:type pipeline_tree: str, optional
:param description: 模板描述, defaults to ""
:type description: str, optional
:return: [description]
:rtype: dict
"""
update_kwargs = {"editor": editor}
if name:
update_kwargs["name"] = standardize_name(
name, TEMPLATE_NODE_NAME_MAX_LENGTH)
if description:
update_kwargs["description"] = description
if pipeline_tree:
standardize_pipeline_node_name(pipeline_tree)
try:
validate_web_pipeline_tree(pipeline_tree)
except PipelineException as e:
return {
"result":
False,
"data":
None,
"message":
"[TemplateManager]validate_web_pipeline_tree failed: {}".
format(str(e)),
"verbose_message":
"[TemplateManager]validate_web_pipeline_tree failed: {}".
format(traceback.format_exc()),
}
replace_template_id(self.template_model_cls, pipeline_tree)
pipeline_web_tree = PipelineWebTreeCleaner(pipeline_tree)
pipeline_web_tree.clean()
update_kwargs["structure_data"] = pipeline_tree
try:
pipeline_template.update_template(**update_kwargs)
except Exception as e:
return {
"result":
False,
"data":
None,
"message":
"[TemplateManager]update_template({update_kwargs}) failed: {e}"
.format(update_kwargs=update_kwargs, e=str(e)),
"verbose_message":
"[TemplateManager]update_template({update_kwargs}) failed: {trace}"
.format(update_kwargs=update_kwargs,
trace=traceback.format_exc()),
}
# create node in template
NodeInTemplate.objects.update_nodes_in_template(
pipeline_template, pipeline_web_tree.origin_data)
else:
for k, v in update_kwargs.items():
setattr(pipeline_template, k, v)
pipeline_template.save()
return {
"result": True,
"data": pipeline_template,
"message": "success",
"verbose_message": "success"
}