def format_template_data(template, project=None):
pipeline_tree = template.pipeline_tree
pipeline_tree.pop('line')
pipeline_tree.pop('location')
varschema.add_schema_for_input_vars(pipeline_tree)
data = {
'id': template.id,
'name': template.pipeline_template.name,
'creator': template.pipeline_template.creator,
'create_time': format_datetime(template.pipeline_template.create_time),
'editor': template.pipeline_template.editor,
'edit_time': format_datetime(template.pipeline_template.edit_time),
'category': template.category,
'pipeline_tree': pipeline_tree
}
if project:
data.update({
'project_id': project.id,
'project_name': project.name,
'bk_biz_id': project.bk_biz_id,
'bk_biz_name': project.name if project.from_cmdb else None
})
return data
def format_template_list_data(templates, project=None):
data = []
for tmpl in templates:
item = {
'id': tmpl.id,
'name': tmpl.pipeline_template.name,
'creator': tmpl.pipeline_template.creator,
'create_time': format_datetime(tmpl.pipeline_template.create_time),
'editor': tmpl.pipeline_template.editor,
'edit_time': format_datetime(tmpl.pipeline_template.edit_time),
'category': tmpl.category,
}
if project:
item.update({
'project_id': project.id,
'project_name': project.name,
'bk_biz_id': project.bk_biz_id,
'bk_biz_name': project.name if project.from_cmdb else None
})
data.append(item)
return data
def get_task_detail(self):
data = {
'id': self.id,
'business_id': int(self.business.cc_id),
'business_name': self.business.cc_name,
'name': self.name,
'create_time': format_datetime(self.create_time),
'creator': self.creator,
'create_method': self.create_method,
'template_id': int(self.template_id),
'start_time': format_datetime(self.start_time),
'finish_time': format_datetime(self.finish_time),
'executor': self.executor,
'elapsed_time': self.elapsed_time
}
exec_data = self.pipeline_instance.execution_data
# inputs data
constants = exec_data['constants']
data['constants'] = constants
# outputs data, if task has not executed, outputs is empty list
instance_id = self.pipeline_instance.instance_id
try:
outputs = pipeline_api.get_outputs(instance_id)
except Data.DoesNotExist:
outputs = {}
outputs_table = [{
'key': key,
'value': val
} for key, val in outputs.get('outputs', {}).items()]
for out in outputs_table:
out['name'] = constants[out['key']]['name']
data.update({
'outputs': outputs_table,
'ex_data': outputs.get('ex_data', '')
})
return data
def info_data_from_period_task(task, detail=True):
info = {
'id': task.id,
'name': task.name,
'template_id': task.template_id,
'creator': task.creator,
'cron': task.cron,
'enabled': task.enabled,
'last_run_at': format_datetime(task.last_run_at),
'total_run_count': task.total_run_count,
}
if detail:
info['form'] = task.form
info['pipeline_tree'] = task.pipeline_tree
return info
def extend_classified_count(self,
group_by,
filters=None,
page=None,
limit=None):
"""
@summary: 兼容按照任务状态分类的扩展
@param group_by:
@param filters:
@param page:
@param limit:
@return:
"""
# 获得所有类型的dict列表
category_dict = dict(TASK_CATEGORY)
if filters is None:
filters = {}
prefix_filters = {}
for cond, value in filters.items():
# component_code不加入查询条件中
if value in ['None', ''
] or cond in ['component_code', 'order_by', 'type']:
continue
if TEMPLATE_REGEX.match(cond):
filter_cond = 'pipeline_template__%s' % cond
# 时间需要大于小于
if cond == 'create_time':
filter_cond = '%s__gte' % filter_cond
prefix_filters.update(
{filter_cond: timestamp_to_datetime(value)})
continue
elif cond == 'finish_time':
filter_cond = 'pipeline_template__create_time__lt'
prefix_filters.update({
filter_cond:
timestamp_to_datetime(value) +
datetime.timedelta(days=1)
})
continue
else:
filter_cond = cond
prefix_filters.update({filter_cond: value})
# 获得标准插件dict列表
component_dict = ComponentModel.objects.get_component_dict()
try:
tasktmpl = self.filter(**prefix_filters)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (
filters, e)
return False, message
if group_by == AE.state:
total = tasktmpl.count()
groups = [{
'code':
'CREATED',
'name':
_(u"未执行"),
'value':
tasktmpl.filter(pipeline_template__is_started=False).count()
}, {
'code':
'EXECUTING',
'name':
_(u"执行中"),
'value':
tasktmpl.filter(pipeline_template__is_started=True,
pipeline_template__is_finished=False).count()
}, {
'code':
'FINISHED',
'name':
_(u"已完成"),
'value':
tasktmpl.filter(pipeline_template__is_finished=True).count()
}]
elif group_by == AE.business__cc_id:
total = tasktmpl.count()
template_list = tasktmpl.values(
AE.business__cc_id, AE.business__cc_name).annotate(
value=Count(group_by)).order_by("value")
groups = []
for data in template_list:
groups.append({
'code': data.get(AE.business__cc_id),
'name': data.get(AE.business__cc_name),
'value': data.get('value', 0)
})
elif group_by == AE.atom_cite:
template_list = tasktmpl.values_list(
"pipeline_template__template_id")
component_list = ComponentModel.objects.filter(
status=True).values("code")
# 用 template_id 列表获取所有符合条件的总数
other_component_list = ComponentInTemplate.objects.filter(
template_id__in=template_list).values(
"component_code").annotate(
value=Count("component_code")).order_by()
components_dict = {}
total = 0
for component in other_component_list:
value = component["value"]
components_dict[component["component_code"]] = value
# 总数不能通过查询获得，需要通过循环计数
total += value
groups = []
# 循环聚合信息
for data in component_list:
code = data.get("code")
groups.append({
'code': code,
'name': component_dict.get(code, None),
'value': components_dict.get(code, 0)
})
elif group_by == AE.atom_template:
# 按起始时间、业务（可选）、类型（可选）、标准插件查询被引用的流程模板列表(dataTable)
# 获取标准插件code
component_code = filters.get("component_code")
# 获取到组件code对应的template_id_list
if component_code:
template_id_list = ComponentInTemplate.objects.filter(
component_code=component_code).distinct().values_list(
"template_id")
else:
template_id_list = ComponentInTemplate.objects.all(
).values_list("template_id")
template_list = tasktmpl.filter(
pipeline_template__template_id__in=template_id_list)
total = template_list.count()
template_list = template_list.values(
"id", "business__cc_id", "business__cc_name",
"pipeline_template__name", "category",
"pipeline_template__edit_time",
"pipeline_template__editor")[(page - 1) * limit:page * limit]
groups = []
# 循环聚合信息
for data in template_list:
groups.append({
'templateId':
data.get("id"),
'businessId':
data.get("business__cc_id"),
'businessName':
data.get("business__cc_name"),
'templateName':
data.get("pipeline_template__name"),
'category':
category_dict[data.get("category")], # 需要将code转为名称
"editTime":
format_datetime(data.get("pipeline_template__edit_time")),
"editor":
data.get("pipeline_template__editor")
})
elif group_by == AE.atom_execute:
# 需要获得符合的查询的对应 template_id 列表
# 获取标准插件code
component_code = filters.get("component_code")
# 获取到组件code对应的template_id
template_id_list = ComponentInTemplate.objects.filter(
component_code=component_code).values_list("template_id")
total = template_id_list.count()
template_list = tasktmpl.filter(
pipeline_template__template_id__in=template_id_list).values(
"id", "business__cc_name", "business__cc_id",
"pipeline_template__name", "category",
"pipeline_template__edit_time",
"pipeline_template__editor")[(page - 1) * limit:page *
limit]
groups = []
# 循环聚合信息
for data in template_list:
groups.append({
'templateId':
data.get("id"),
'businessId':
data.get("business__cc_id"),
'businessName':
data.get("business__cc_name"),
'templateName':
data.get("pipeline_template__name"),
'category':
category_dict[data.get("category")],
"editTime":
data.get("pipeline_template__edit_time").strftime(
"%Y-%m-%d %H:%M:%S"),
"editor":
data.get("pipeline_template__editor")
})
elif group_by == AE.template_cite:
# 按起始时间、业务（可选）、类型（可选）查询各流程模板被引用为子流程个数、创建轻应用个数、创建任务实例个数
template_list = tasktmpl
id_list = template_list.values(
"pipeline_template__id", "id")[(page - 1) * limit:page * limit]
template_id_map = {
template['pipeline_template__id']: template['id']
for template in id_list
}
t_id_list = [
x[0] for x in template_list.values_list(
"pipeline_template__template_id")[(page - 1) * limit:page *
limit]
]
appmaker_list = template_list.values("id", "appmaker").annotate(
appmaker_total=Count("appmaker")).order_by("-id")
taskflow_list = PipelineInstance.objects.filter(
template_id__in=template_id_map.keys()).values(
"template_id").annotate(
instance_total=Count("template_id")).order_by()
relationship_list = TemplateRelationship.objects.filter(
descendant_template_id__in=t_id_list
).values("descendant_template_id").annotate(
relationship_total=Count("descendant_template_id")).order_by()
# 排序
order_by = filters.get("order_by", "-templateId")
# 使用驼峰转下划线进行转换order_by
camel_order_by = camel_case_to_underscore_naming(order_by)
if order_by == "appmakerTotal":
appmaker_list = appmaker_list.order_by(camel_order_by)
elif order_by == "instanceTotal":
taskflow_list = taskflow_list.order_by(camel_order_by)
elif order_by == "relationshipTotal":
relationship_list = relationship_list.order_by(camel_order_by)
appmaker_list = appmaker_list[(page - 1) * limit:page * limit]
relationship_list = relationship_list
# 获得对应的dict数据
appmaker_dict = {}
for appmaker in appmaker_list:
appmaker_dict[appmaker["id"]] = appmaker["appmaker_total"]
taskflow_dict = {}
for taskflow in taskflow_list:
taskflow_dict[template_id_map[
taskflow["template_id"]]] = taskflow["instance_total"]
relationship_dict = {}
for relationship in relationship_list:
relationship_dict[
relationship["descendant_template_id"]] = relationship[
"relationship_total"]
groups = []
for template in template_list[(page - 1) * limit:page * limit]:
template_id = template.id
groups.append({
'templateId':
template.id,
'templateName':
template.name,
'businessId':
template.business.cc_id,
'appmakerTotal':
appmaker_dict.get(template_id, 0),
'relationshipTotal':
relationship_dict.get(
template.pipeline_template.template_id, 0),
'instanceTotal':
taskflow_dict.get(template_id, 0)
})
total = template_list.count()
if order_by[0] == "-":
# 需要去除负号
order_by = order_by[1:]
groups = sorted(groups, key=lambda group: -group.get(order_by))
else:
groups = sorted(groups, key=lambda group: group.get(order_by))
elif group_by == AE.template_node:
# 按起始时间、业务（可选）、类型（可选）查询各流程模板标准插件节点个数、子流程节点个数、网关节点数
total = tasktmpl.count()
groups = []
# 排序
template_id_list = tasktmpl.values(
"pipeline_template__template_id")
template_pipeline_data = TemplateInPipeline.objects.filter(
template_id__in=template_id_list)
order_by = filters.get("order_by", "-templateId")
# 使用驼峰转下划线进行转换order_by
camel_order_by = camel_case_to_underscore_naming(order_by)
# 排列获取分页后的数据
pipeline_data = template_pipeline_data.order_by(
camel_order_by)[(page - 1) * limit:page * limit]
template_id_list = [
template.template_id for template in pipeline_data
]
tasktmpl = tasktmpl.filter(
pipeline_template__template_id__in=template_id_list)
pipeline_dict = {}
for pipeline in pipeline_data:
pipeline_dict[pipeline.template_id] = {
"atom_total": pipeline.atom_total,
"subprocess_total": pipeline.subprocess_total,
"gateways_total": pipeline.gateways_total
}
# 需要循环执行计算相关节点
for template in tasktmpl:
pipeline_template = template.pipeline_template
template_id = template.id
pipeline_template_id = pipeline_template.template_id
# 插入信息
groups.append({
'templateId':
template_id,
'businessId':
template.business.cc_id,
'businessName':
template.business.cc_name,
'templateName':
pipeline_template.name,
'category':
category_dict[template.category],
"createTime":
format_datetime(pipeline_template.create_time),
"creator":
pipeline_template.creator,
"atomTotal":
pipeline_dict[pipeline_template_id]["atom_total"],
"subprocessTotal":
pipeline_dict[pipeline_template_id]["subprocess_total"],
"gatewaysTotal":
pipeline_dict[pipeline_template_id]["gateways_total"]
})
if order_by[0] == "-":
# 需要去除负号
order_by = order_by[1:]
groups = sorted(groups, key=lambda group: -group.get(order_by))
else:
groups = sorted(groups, key=lambda group: group.get(order_by))
elif group_by in [AE.category, AE.create_method, AE.flow_type]:
try:
total, groups = self.classified_count(prefix_filters, group_by)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (
filters, e)
return False, message
else:
total, groups = 0, []
data = {'total': total, 'groups': groups}
return True, data
def extend_classified_count(self, group_by, filters=None, page=None, limit=None):
"""
@summary: 兼容按照任务状态分类的扩展
@param group_by:
@param filters:
@param page:
@param limit:
@return:
"""
# 获得所有类型的dict列表
category_dict = dict(TASK_CATEGORY)
if filters is None:
filters = {}
tasktmpl_inst_regex = re.compile(r'^name|creator_name|editor_name|'
r'create_time|edit_time|edit_finish_time|finish_time')
prefix_filters = {}
for cond in filters:
# 如果conditions内容为空或为空字符，不可加入查询条件中
if filters[cond] == 'None' or filters[cond] == '' or cond == 'component_code':
continue
if tasktmpl_inst_regex.match(cond):
filter_cond = 'pipeline_template__%s' % cond
# 时间需要大于小于
if cond == 'create_time':
filter_cond = '%s__gte' % filter_cond
prefix_filters.update({filter_cond: timestamp_to_datetime(filters[cond])})
continue
elif cond == 'finish_time':
filter_cond = 'pipeline_template__create_time__lt'
prefix_filters.update(
{filter_cond: timestamp_to_datetime(filters[cond]) + datetime.timedelta(days=1)})
continue
# 编辑时间与创建时间需要分开查询
elif cond == 'edit_time':
filter_cond = '%s__gte' % filter_cond
prefix_filters.update({filter_cond: timestamp_to_datetime(filters[cond])})
continue
elif cond == 'edit_finish_time':
filter_cond = 'pipeline_template__edit_time__lt'
prefix_filters.update(
{filter_cond: timestamp_to_datetime(filters[cond]) + datetime.timedelta(days=1)})
continue
else:
filter_cond = cond
prefix_filters.update({filter_cond: filters[cond]})
# 获得原子dict列表
component_dict = ComponentModel.objects.get_component_dict()
if group_by == AE.state:
try:
tasktmpl = self.filter(**prefix_filters)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
total = tasktmpl.count()
groups = [
{
'code': 'CREATED',
'name': _(u"未执行"),
'value': tasktmpl.filter(pipeline_template__is_started=False).count()
},
{
'code': 'EXECUTING',
'name': _(u"执行中"),
'value': tasktmpl.filter(pipeline_template__is_started=True,
pipeline_template__is_finished=False).count()
},
{
'code': 'FINISHED',
'name': _(u"已完成"),
'value': tasktmpl.filter(pipeline_template__is_finished=True).count()
}
]
elif group_by == AE.business__cc_id:
cc_name = AE.business__cc_name
try:
tasktmpl = self.filter(**prefix_filters)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
# 获取所有数据
total = tasktmpl.count()
queryset = tasktmpl.values(group_by, cc_name).annotate(value=Count(group_by)).order_by("-value")
groups = []
for data in queryset:
groups.append({
'code': data.get(group_by),
'name': data.get(cc_name),
'value': data.get('value', 0)
})
elif group_by == AE.atom_cite:
# 这里没有其他原子节点的内容
try:
# 需要获得符合的查询的对应 template_id 列表
template_list = self.filter(**prefix_filters).values_list("pipeline_template__template_id")
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
component_list = ComponentModel.objects.filter(status=True).values("code")
# 用 template_id 列表获取所有符合条件的总数
other_component_list = ComponentInTemplate.objects.filter(template_id__in=template_list).values(
"component_code").annotate(
value=Count("component_code")).order_by()
components_dict = {}
total = 0
for component in other_component_list:
value = component["value"]
components_dict[component["component_code"]] = value
# 总数不能通过查询获得，需要通过循环计数
total += value
groups = []
# 循环聚合信息
for data in component_list:
code = data.get("code")
value = components_dict.get(code, 0)
groups.append({
'code': code,
'name': component_dict.get(code, None),
'value': value
})
elif group_by == AE.atom_template:
# 按起始时间、业务（可选）、类型（可选）、原子查询被引用的流程模板列表(dataTable)
try:
# 需要获得符合的查询的对应 template_id 列表
template_list = self.filter(**prefix_filters).filter()
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
# 获取原子code
component_code = filters.get("component_code")
# 获取到组件code对应的template_id_list
if component_code:
template_id_list = ComponentInTemplate.objects.filter(component_code=component_code).values_list(
"template_id")
else:
template_id_list = ComponentInTemplate.objects.all().values_list("template_id")
total = template_id_list.count()
template_list = template_list.filter(pipeline_template__template_id__in=template_id_list).values(
"id",
"business__cc_id",
"business__cc_name",
"pipeline_template__name",
"category",
"pipeline_template__edit_time",
"pipeline_template__editor"
)[(page - 1) * limit:page * limit]
groups = []
# 循环聚合信息
for data in template_list:
groups.append({
'templateId': data.get("id"),
'businessId': data.get("business__cc_id"),
'businessName': data.get("business__cc_name"),
'templateName': data.get("pipeline_template__name"),
'category': category_dict[data.get("category")], # 需要将code转为名称
"editTime": format_datetime(data.get("pipeline_template__edit_time")),
"editor": data.get("pipeline_template__editor")
})
elif group_by == AE.atom_execute:
try:
# 需要获得符合的查询的对应 template_id 列表
template_list = self.filter(**prefix_filters).filter()
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
# 获取原子code
component_code = filters.get("component_code")
# 获取到组件code对应的template_id
if component_code:
template_id_list = ComponentInTemplate.objects.filter(component_code=component_code).values_list(
"template_id")
else:
template_id_list = ComponentInTemplate.objects.all().values_list("template_id")
total = template_id_list.count()
template_list = template_list.filter(pipeline_template__template_id__in=template_id_list).values(
"id",
"business__cc_name",
"business__cc_id",
"pipeline_template__name",
"category",
"pipeline_template__edit_time",
"pipeline_template__editor")[(page - 1) * limit:page * limit]
groups = []
# 循环聚合信息
for data in template_list:
groups.append({
'templateId': data.get("id"),
'businessId': data.get("business__cc_id"),
'businessName': data.get("business__cc_name"),
'templateName': data.get("pipeline_template__name"),
'category': category_dict[data.get("category")],
"editTime": data.get("pipeline_template__edit_time").strftime("%Y-%m-%d %H:%M:%S"),
"editor": data.get("pipeline_template__editor")
})
elif group_by == AE.template_cite:
# 按起始时间、业务（可选）、类型（可选）查询各流程模板被引用为子流程个数、创建轻应用个数、创建任务实例个数
try:
# 需要获得符合的查询的对应 template_id 列表
template_list = self.filter(**prefix_filters).filter()
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
total = template_list.count()
template_list = template_list[(page - 1) * limit:page * limit]
# 获取分页的数据
id_list = list(template_list.values_list("id"))
t_id_list = list(template_list.values_list("pipeline_template__template_id"))
n_id_list = []
for x in t_id_list:
n_id_list.append(x[0])
# 轻应用的数据已分页
appmaker_list = template_list.values("id", "appmaker").annotate(value=Count("appmaker"))
taskflow_list = PipelineInstance.objects.filter(template_id__in=id_list).values("template_id").annotate(
value=Count("template_id")).order_by()
relationship_list = TemplateRelationship.objects.filter(descendant_template_id__in=n_id_list).values(
"descendant_template_id").annotate(
value=Count("descendant_template_id")).order_by()
appmaker_dict = {}
for appmaker in appmaker_list:
appmaker_dict[appmaker["id"]] = appmaker["value"]
relationship_dict = {}
for relationship in relationship_list:
relationship_dict[relationship["descendant_template_id"]] = relationship["value"]
taskflow_dict = {}
for taskflow in taskflow_list:
taskflow_dict[taskflow["template_id"]] = taskflow["value"]
groups = []
# todo 缺少子流程部分
for template in template_list:
id = template.id
groups.append({
'id': template.id,
'templateName': template.name,
'appmakerTotal': appmaker_dict[id],
'relationshipTotal': relationship_dict.get(template.pipeline_template.template_id, 0),
'instanceTotal': taskflow_dict.get(id, 0)
})
elif group_by == AE.template_node:
# 按起始时间、业务（可选）、类型（可选）查询各流程模板原子节点个数、子流程节点个数、网关节点数(dataTable)
try:
# 需要获得符合的查询的对应 template_id 列表
template_list = self.filter(**prefix_filters).filter()
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
# 总数
total = template_list.count()
groups = []
# 需要循环执行计算相关节点
for template in template_list[(page - 1) * limit:page * limit]:
atom_total = 0
subprocess_total = 0
pipeline_tree = template.pipeline_tree
tree_activities = pipeline_tree["activities"]
gateways_total = len(pipeline_tree["gateways"])
for activity in tree_activities:
activity_type = tree_activities[activity]["type"]
if activity_type == "ServiceActivity":
atom_total += 1
elif activity_type == "SubProcess":
subprocess_total += 1
pipeline_template = template.pipeline_template
# 插入信息
groups.append({
'templateId': template.id,
'businessId': template.business.cc_id,
'businessName': template.business.cc_name,
'templateName': pipeline_template.name,
'category': category_dict[template.category],
"editTime": format_datetime(pipeline_template.edit_time),
"creator": pipeline_template.creator,
"atomTotal": atom_total,
"subprocessTotal": subprocess_total,
"gatewaysTotal": gateways_total
})
elif group_by in [AE.category, AE.create_method, AE.flow_type]:
try:
total, groups = self.classified_count(prefix_filters, group_by)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
else:
total, groups = 0, []
data = {'total': total, 'groups': groups}
return True, data
def extend_classified_count(self,
group_by,
filters=None,
page=None,
limit=None):
"""
@summary: 兼容按照任务状态分类的扩展
@param group_by:
@param filters:
@param page:
@param limit:
@return:
"""
# 获得所有类型的dict列表
category_dict = dict(TASK_CATEGORY)
if filters is None:
filters = {}
prefix_filters = {}
for cond, value in filters.items():
# 如果conditions内容为空或为空字符，不可加入查询条件中
if value in ['None', ''
] or cond in ['component_code', 'order_by', 'type']:
continue
if PIPELINE_REGEX.match(cond):
filter_cond = 'pipeline_instance__%s' % cond
# 时间需要大于小于
if cond == 'create_time':
filter_cond = '%s__gte' % filter_cond
prefix_filters.update(
{filter_cond: timestamp_to_datetime(value)})
continue
# 结束时间由创建时间来决定
if cond == 'finish_time':
filter_cond = 'pipeline_instance__create_time__lt'
prefix_filters.update({
filter_cond:
timestamp_to_datetime(value) +
datetime.timedelta(days=1)
})
continue
else:
filter_cond = cond
prefix_filters.update({filter_cond: value})
try:
taskflow = self.filter(**prefix_filters)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (
filters, e)
return False, message
if group_by == AE.state:
total = taskflow.count()
groups = [{
'code':
'CREATED',
'name':
_(u"未执行"),
'value':
taskflow.filter(pipeline_instance__is_started=False).count()
}, {
'code':
'EXECUTING',
'name':
_(u"执行中"),
'value':
taskflow.filter(pipeline_instance__is_started=True,
pipeline_instance__is_finished=False).count()
}, {
'code':
'FINISHED',
'name':
_(u"已完成"),
'value':
taskflow.filter(pipeline_instance__is_finished=True).count()
}]
elif group_by == AE.business__cc_id:
# 获取所有数据
total = taskflow.count()
taskflow_list = taskflow.values(
AE.business__cc_id, AE.business__cc_name).annotate(
value=Count(group_by)).order_by()
groups = []
for data in taskflow_list:
groups.append({
'code': data.get(AE.business__cc_id),
'name': data.get(AE.business__cc_name),
'value': data.get('value', 0)
})
elif group_by == AE.appmaker_instance:
taskflow_values = taskflow.values("create_info")
order_by = filters.get("order_by", "-templateId")
business_id = filters["business__cc_id"]
category = filters["category"]
started_time = timestamp_to_datetime(filters["create_time"])
end_time = timestamp_to_datetime(
filters["finish_time"]) + datetime.timedelta(days=1)
appmaker_data = AppMaker.objects.filter(
is_deleted=False,
create_time__gte=started_time,
create_time__lte=end_time)
if business_id != '':
appmaker_data = appmaker_data.filter(
business__cc_id=business_id)
if category != '':
appmaker_data = appmaker_data.filter(
task_template__category=category)
# 获取所有轻应用数据数量
total = appmaker_data.count()
# 获得每一个轻应用的实例数量并变为 dict 字典数据进行查询
total_dict = {
appmaker['create_info']: appmaker['instance_total']
for appmaker in taskflow_values.annotate(
instance_total=Count("create_info")).order_by()
}
id_list = appmaker_data.values_list("id")[:]
id_list = sorted(
id_list,
key=lambda tuples_id: -total_dict.get(str(tuples_id[0]), 0))
id_list = id_list[(page - 1) * limit:page * limit]
app_id_list = [tuples[0] for tuples in id_list]
# 获得轻应用对象对应的模板和轻应用名称
appmaker_data = appmaker_data.filter(id__in=app_id_list).values(
"id", "task_template_id", "name", "create_time", "edit_time",
"editor", "business__cc_id", "business__cc_name",
"task_template__category")
groups = []
for data in appmaker_data:
code = data.get('task_template_id')
appmaker_id = data.get("id")
groups.append({
'templateId':
code,
'createTime':
format_datetime(data.get('create_time')),
'editTime':
format_datetime(data.get('edit_time')),
'editor':
data.get('editor'),
'templateName':
data.get('name'),
'businessId':
data.get('business__cc_id'),
'businessName':
data.get('business__cc_name'),
'category':
category_dict[data.get('task_template__category')],
# 需要将 code 转为字符型
'instanceTotal':
total_dict.get(str(appmaker_id), 0)
})
if order_by[0] == "-":
# 需要去除负号
order_by = order_by[1:]
groups = sorted(groups, key=lambda group: -group.get(order_by))
else:
groups = sorted(groups, key=lambda group: group.get(order_by))
elif group_by == AE.atom_execute:
# 查询各原子被执行次数、失败率、重试次数、平均耗时（不计算子流程）
instance_id_list = taskflow.values_list(
"pipeline_instance__instance_id")
# 获得原子
component = ComponentExecuteData.objects.filter(
instance_id__in=instance_id_list, is_sub=False)
component_data = component.values('component_code').annotate(
execute_times=Count('component_code'),
avg_execute_time=Avg('elapsed_time')).order_by(
'component_code')
# 统计次数
total = component_data.count()
component_success_data = component.filter(
is_retry=False).values('component_code').annotate(
success_times=Count('component_code')).order_by(
'component_code')
# 用于计算所有原子的成功列表
success_component_dict = {}
for data in component_success_data:
success_component_dict[
data['component_code']] = data['success_times']
groups = []
component_dict = {}
for bundle in ComponentModel.objects.all():
name = bundle.name.split('-')
group_name = _(name[0])
name = _(name[1])
component_dict[bundle.code] = '%s-%s' % (group_name, name)
for data in component_data:
code = data.get('component_code')
execute_times = data.get('execute_times')
failed_times = execute_times - success_component_dict.get(
code, 0)
failed_times_percent = '%.2f %%' % (failed_times / 1.0 /
execute_times * 100)
groups.append({
'componentName':
component_dict[code],
'executeTimes':
execute_times,
'avgExecuteTime':
'%.2f' % data.get('avg_execute_time', 0),
'failedTimes':
failed_times,
'failedTimesPercent':
failed_times_percent
})
elif group_by == AE.atom_instance:
# 被引用的任务实例列表
# 获得参数中的原子code
component_code = filters.get("component_code")
# 获取到组件code对应的instance_id_list
instance_id_list = ComponentExecuteData.objects.filter(
is_sub=False)
# 对code进行二次查找
instance_id_list = instance_id_list.filter(
component_code=component_code).distinct().values_list(
"instance_id")
taskflow_list = taskflow.filter(
pipeline_instance__instance_id__in=instance_id_list).values(
'id', 'business__cc_id', 'business__cc_name',
'pipeline_instance__name', 'category',
'pipeline_instance__create_time',
'pipeline_instance__creator')
# 获得总数
total = taskflow_list.count()
taskflow_list = taskflow_list[(page - 1) * limit:page * limit]
groups = []
# 循环信息
for data in taskflow_list:
groups.append({
'instanceId':
data.get("id"),
'businessId':
data.get("business__cc_id"),
'businessName':
data.get("business__cc_name"),
'instanceName':
data.get("pipeline_instance__name"),
'category':
category_dict[data.get("category")], # 需要将code转为名称
"createTime":
format_datetime(
data.get("pipeline_instance__create_time")),
"creator":
data.get("pipeline_instance__creator")
})
elif group_by == AE.instance_node:
# 各任务实例执行的原子节点个数、子流程节点个数、网关节点数
groups = []
# 排序
instance_id_list = taskflow.values(
"pipeline_instance__instance_id")
instance_pipeline_data = InstanceInPipeline.objects.filter(
instance_id__in=instance_id_list)
# 总数
total = instance_pipeline_data.count()
order_by = filters.get("order_by", "-instanceId")
# 使用驼峰转下划线进行转换order_by
camel_order_by = camel_case_to_underscore_naming(order_by)
# 排列获取分页后的数据
pipeline_data = instance_pipeline_data.order_by(
camel_order_by)[(page - 1) * limit:page * limit]
instance_id_list = [tuples.instance_id for tuples in pipeline_data]
taskflow = taskflow.filter(
pipeline_instance__instance_id__in=instance_id_list)
pipeline_dict = {}
for pipeline in pipeline_data:
pipeline_dict[pipeline.instance_id] = {
"atom_total": pipeline.atom_total,
"subprocess_total": pipeline.subprocess_total,
"gateways_total": pipeline.gateways_total
}
# 需要循环执行计算相关节点
for flow in taskflow:
pipeline_instance = flow.pipeline_instance
instance_id = flow.id
pipeline_instance_id = pipeline_instance.instance_id
# 插入信息
groups.append({
'instanceId':
instance_id,
'businessId':
flow.business.cc_id,
'businessName':
flow.business.cc_name,
'instanceName':
pipeline_instance.name,
'category':
category_dict[flow.category],
"createTime":
format_datetime(pipeline_instance.create_time),
"creator":
pipeline_instance.creator,
"atomTotal":
pipeline_dict[pipeline_instance_id]["atom_total"],
"subprocessTotal":
pipeline_dict[pipeline_instance_id]["subprocess_total"],
"gatewaysTotal":
pipeline_dict[pipeline_instance_id]["gateways_total"]
})
if order_by[0] == "-":
# 需要去除负号
order_by = order_by[1:]
groups = sorted(groups, key=lambda group: -group.get(order_by))
else:
groups = sorted(groups, key=lambda group: group.get(order_by))
elif group_by == AE.instance_details:
# 各任务执行耗时
started_time = prefix_filters[
'pipeline_instance__create_time__gte']
archived_time = prefix_filters[
'pipeline_instance__create_time__lt']
prefix_filters.update(
pipeline_instance__start_time__gte=prefix_filters.pop(
'pipeline_instance__create_time__gte'),
pipeline_instance__start_time__lt=prefix_filters.pop(
'pipeline_instance__create_time__lt'),
pipeline_instance__is_finished=True)
taskflow = self.filter(**prefix_filters)
# 需要distinct去除重复的
instance_id_list = ComponentExecuteData.objects.filter(
started_time__gte=started_time,
archived_time__lte=archived_time).values_list(
"instance_id").distinct().order_by()
total = instance_id_list.count()
# 排序
order_by = filters.get("order_by", "instanceId")
component_list = instance_id_list.annotate(
execute_time=Sum('elapsed_time')).order_by()
# 使用驼峰转下划线进行转换order_by
component_list = component_list.order_by(
camel_case_to_underscore_naming(order_by))
# 分页
component_list = component_list[(page - 1) * limit:page * limit]
component_dict = {}
instance_list = [tuples[0] for tuples in component_list]
for component in component_list:
component_dict[component[0]] = component[1]
taskflow_list = taskflow.filter(
pipeline_instance__instance_id__in=instance_list).values(
'id', 'pipeline_instance__instance_id', 'business__cc_id',
'business__cc_name', 'pipeline_instance__name', 'category',
'pipeline_instance__create_time',
'pipeline_instance__creator')
groups = []
for data in taskflow_list:
instance_id = data.get("pipeline_instance__instance_id")
groups.append({
'instanceId':
data.get("id"),
'businessId':
data.get("business__cc_id"),
'businessName':
data.get("business__cc_name"),
'instanceName':
data.get("pipeline_instance__name"),
'category':
category_dict[data.get("category")], # 需要将code转为名称
"createTime":
format_datetime(
data.get("pipeline_instance__create_time")),
"creator":
data.get("pipeline_instance__creator"),
"executeTime":
component_dict[instance_id]
})
if order_by[0] == "-":
# 需要去除负号
order_by = order_by[1:]
groups = sorted(groups, key=lambda group: -group.get(order_by))
else:
groups = sorted(groups, key=lambda group: group.get(order_by))
elif group_by == AE.instance_time:
# 按起始时间、业务（可选）、类型（可选）、图表类型（日视图，月视图），查询每一天或每一月的执行数量
instance_create_time_list = taskflow.values(
'pipeline_instance__create_time')
total = instance_create_time_list.count()
group_type = filters.get('type', 'day')
create_time = timestamp_to_datetime(filters['create_time'])
end_time = timestamp_to_datetime(
filters['finish_time']) + datetime.timedelta(days=1)
groups = []
date_list = []
for instance in instance_create_time_list:
instance_time = instance['pipeline_instance__create_time']
date_key = ''
# 添加在一个list中 之后使用count方法获取对应的数量
if group_type == 'day':
date_key = instance_time.strftime('%Y-%m-%d')
elif group_type == 'month':
date_key = instance_time.strftime('%Y-%m')
date_list.append(date_key)
if group_type == 'day':
# 日视图
for d in gen_day_dates(create_time,
(end_time - create_time).days + 1):
date_key = d.strftime('%Y-%m-%d')
groups.append({
'time': date_key,
'value': date_list.count(date_key)
})
elif group_type == 'month':
# 月视图
# 直接拿到对应的（年-月），不需要在字符串拼接
for date_key in get_month_dates(create_time, end_time):
groups.append({
'time': date_key,
'value': date_list.count(date_key)
})
elif group_by in [AE.category, AE.create_method, AE.flow_type]:
try:
total, groups = self.classified_count(prefix_filters, group_by)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (
filters, e)
return False, message
else:
total, groups = 0, []
data = {'total': total, 'groups': groups}
return True, data
def extend_classified_count(self, group_by, filters=None, page=None, limit=None):
"""
@summary: 兼容按照任务状态分类的扩展
@param group_by:
@param filters:
@param page:
@param limit:
@return:
"""
# 获得所有类型的dict列表
category_dict = dict(TASK_CATEGORY)
if filters is None:
filters = {}
pipeline_inst_regex = re.compile(r'^name|create_time|creator|create_time|executor|'
r'start_time|finish_time|is_started|is_finished')
prefix_filters = {}
for cond in filters:
# 如果conditions内容为空或为空字符，不可加入查询条件中
if filters[cond] == 'None' or filters[cond] == '' or cond == 'component_code':
continue
if pipeline_inst_regex.match(cond):
filter_cond = 'pipeline_instance__%s' % cond
# 时间需要大于小于
if cond == 'create_time':
filter_cond = '%s__gte' % filter_cond
prefix_filters.update(
{filter_cond: timestamp_to_datetime(filters[cond])})
continue
# 结束时间由创建时间来决定
if cond == 'finish_time':
filter_cond = 'pipeline_instance__create_time__lt'
prefix_filters.update(
{filter_cond: timestamp_to_datetime(filters[cond]) + datetime.timedelta(days=1)})
continue
else:
filter_cond = cond
prefix_filters.update({filter_cond: filters[cond]})
if group_by == AE.state:
try:
taskflow = self.filter(**prefix_filters)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
total = taskflow.count()
groups = [
{
'code': 'CREATED',
'name': _(u"未执行"),
'value': taskflow.filter(pipeline_instance__is_started=False).count()
},
{
'code': 'EXECUTING',
'name': _(u"执行中"),
'value': taskflow.filter(pipeline_instance__is_started=True,
pipeline_instance__is_finished=False).count()
},
{
'code': 'FINISHED',
'name': _(u"已完成"),
'value': taskflow.filter(pipeline_instance__is_finished=True).count()
}
]
elif group_by == AE.business__cc_id:
cc_name = AE.business__cc_name
try:
taskflow = self.filter(**prefix_filters)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
# 获取所有数据
total = taskflow.count()
taskflow_list = taskflow.values(group_by, cc_name).annotate(value=Count(group_by)).order_by()
groups = []
for data in taskflow_list:
groups.append({
'code': data.get(group_by),
'name': data.get(cc_name),
'value': data.get('value', 0)
})
elif group_by == AE.atom_execute:
# 查询各原子被执行次数、失败率、重试次数、平均耗时（不计算子流程）
try:
instance_id_list = self.filter(**prefix_filters).values_list("pipeline_instance__instance_id")
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
# 获得原子
component = ComponentExecuteData.objects.filter(instance_id__in=instance_id_list, is_sub=False)
component_data = component.values('component_code').annotate(
execute_times=Count('component_code'),
avg_execute_time=Avg('elapsed_time')).order_by('component_code')
# 统计次数
total = component_data.count()
component_success_data = component.filter(is_retry=False).values('component_code').annotate(
success_times=Count('component_code')
).order_by('component_code')
# 用于计算所有原子的成功列表
success_component_dict = {}
for data in component_success_data:
success_component_dict[data['component_code']] = data['success_times']
groups = []
component_dict = ComponentModel.objects.get_component_dict()
for data in component_data:
code = data.get('component_code')
execute_times = data.get('execute_times')
failed_times = execute_times - success_component_dict.get(code, 0)
failed_times_percent = '%.2f %%' % (failed_times / 1.0 / execute_times * 100)
groups.append({
'componentName': component_dict[code],
'executeTimes': execute_times,
'avgExecuteTime': '%.2f' % data.get('avg_execute_time', 0),
'failedTimes': failed_times,
'failedTimesPercent': failed_times_percent
})
elif group_by == AE.atom_instance:
# 被引用的任务实例列表
try:
taskflow_list = self.filter(**prefix_filters)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
# 获取原子code
# 获得参数中的原子code
component_code = filters.get("component_code")
# 获取到组件code对应的instance_id_list
instance_id_list = ComponentExecuteData.objects.filter(is_sub=False)
# 参数携带了原子code话，针对查询，不携带查询全部
if component_code:
instance_id_list = instance_id_list.filter(component_code=component_code).values_list(
"instance_id")
taskflow_list = taskflow_list.filter(pipeline_instance__instance_id__in=instance_id_list).values(
'id',
'business__cc_id',
'business__cc_name',
'pipeline_instance__name',
'category',
'pipeline_instance__create_time',
'pipeline_instance__creator'
)
# 获得总数
total = taskflow_list.count()
taskflow_list = taskflow_list[(page - 1) * limit:page * limit]
groups = []
# 循环信息
for data in taskflow_list:
groups.append({
'instanceId': data.get("id"),
'businessId': data.get("business__cc_id"),
'businessName': data.get("business__cc_name"),
'instanceName': data.get("pipeline_instance__name"),
'category': category_dict[data.get("category")], # 需要将code转为名称
"createTime": format_datetime(data.get("pipeline_instance__create_time")),
"creator": data.get("pipeline_instance__creator")
})
elif group_by == AE.instance_node:
# 各任务实例执行的原子节点个数、子流程节点个数、网关节点数
try:
taskflow_list = self.filter(**prefix_filters)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
# 总数
total = taskflow_list.count()
groups = []
for taskflow in taskflow_list[(page - 1) * limit:page * limit]:
atom_total = 0
subprocess_total = 0
pipeline_tree = taskflow.pipeline_tree
tree_activities = pipeline_tree["activities"]
gateways_total = len(pipeline_tree["gateways"])
for activity in tree_activities:
activity_type = tree_activities[activity]["type"]
if activity_type == "ServiceActivity":
atom_total += 1
elif activity_type == "SubProcess":
subprocess_total += 1
pipeline_instance = taskflow.pipeline_instance
# 插入信息
groups.append({
'instanceId': taskflow.id,
'businessId': taskflow.business.cc_id,
'businessName': taskflow.business.cc_name,
'instanceName': pipeline_instance.name,
'category': category_dict[taskflow.category],
"createTime": format_datetime(pipeline_instance.create_time),
"creator": pipeline_instance.creator,
"atomTotal": atom_total,
"subprocessTotal": subprocess_total,
"gatewaysTotal": gateways_total
})
pass
elif group_by == AE.instance_details:
# 各任务实例详情和执行耗时
try:
taskflow_list = self.filter(**prefix_filters)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
started_time = timestamp_to_datetime(filters["create_time"])
end_time = timestamp_to_datetime(filters["finish_time"])
# 需要去除重复的
instance_id_list = ComponentExecuteData.objects.filter(
started_time__gte=started_time,
started_time__lte=end_time).values_list(
"instance_id").distinct().order_by()
# 总数
total = instance_id_list.count()
# 获得所有的任务实例的执行耗时
component_list = instance_id_list.annotate(execute_times=Sum('elapsed_time'))
component_dict = {}
for component in component_list:
component_dict[component[0]] = component[1]
taskflow_list = taskflow_list.filter(pipeline_instance__instance_id__in=instance_id_list).values(
'id',
'pipeline_instance__instance_id',
'business__cc_id',
'business__cc_name',
'pipeline_instance__name',
'category',
'pipeline_instance__create_time',
'pipeline_instance__creator'
)[(page - 1) * limit:page * limit]
groups = []
for data in taskflow_list:
instance_id = data.get("pipeline_instance__instance_id")
groups.append({
'instanceId': data.get("id"),
'businessId': data.get("business__cc_id"),
'businessName': data.get("business__cc_name"),
'instanceName': data.get("pipeline_instance__name"),
'category': category_dict[data.get("category")], # 需要将code转为名称
"createTime": format_datetime(data.get("pipeline_instance__create_time")),
"creator": data.get("pipeline_instance__creator"),
"executeTime": component_dict[instance_id]
})
elif group_by in [AE.category, AE.create_method, AE.flow_type]:
try:
total, groups = self.classified_count(prefix_filters, group_by)
except Exception as e:
message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e)
return False, message
else:
total, groups = 0, []
data = {'total': total, 'groups': groups}
return True, data
