def extend_classified_count(self, group_by, filters=None): """ @summary: 兼容按照任务状态分类的扩展 @param group_by: @param filters: @return: """ if filters is None: filters = {} prefix_filters = {} for cond, value in filters.items(): if value in ['None', ''] or cond == 'type': continue if cond == 'create_time': filter_cond = '%s__gte' % cond prefix_filters.update( {filter_cond: timestamp_to_datetime(value)}) continue elif cond == 'finish_time': filter_cond = 'create_time__lt' prefix_filters.update({ filter_cond: timestamp_to_datetime(value) + datetime.timedelta(days=1) }) continue if APPMAKER_REGEX.match(cond): filter_cond = 'task_template__%s' % cond else: filter_cond = cond prefix_filters.update({filter_cond: value}) try: appmaker = self.filter(**prefix_filters) except Exception as e: message = u"query_appmaker params conditions[%s] have invalid key or value: %s" % ( filters, e) return False, message if group_by == AE.business__cc_id: # 按起始时间、业务(可选)查询各类型轻应用个数和占比√(echarts) total = appmaker.count() appmaker_list = appmaker.values( AE.business__cc_id, AE.business__cc_name).annotate( value=Count(group_by)).order_by() groups = [] for data in appmaker_list: groups.append({ 'code': data.get(AE.business__cc_id), 'name': data.get(AE.business__cc_name), 'value': data.get('value', 0) }) elif group_by == AE.category: # 按起始时间、类型(可选)查询各业务下新增轻应用个数(排序) # field 用于外键查询对应的类型内容 field = "task_template__%s" % group_by total = appmaker.count() # 获取choices字段 choices = TaskTemplate.objects.get_choices(group_by) appmaker_list = appmaker.values(field).annotate( value=Count(field)).order_by() values = {item[field]: item['value'] for item in appmaker_list} groups = [] for code, name in choices: groups.append({ 'code': code, 'name': name, 'value': values.get(code, 0) }) else: total, groups = 0, [] data = {'total': total, 'groups': groups} return True, data
def extend_classified_count(self, group_by, filters=None, page=None, limit=None): """ @summary: 兼容按照任务状态分类的扩展 @param group_by: @param filters: @param page: @param limit: @return: """ # 获得所有类型的dict列表 category_dict = dict(TASK_CATEGORY) if filters is None: filters = {} prefix_filters = {} for cond, value in filters.items(): # component_code不加入查询条件中 if value in ['None', '' ] or cond in ['component_code', 'order_by', 'type']: continue if TEMPLATE_REGEX.match(cond): filter_cond = 'pipeline_template__%s' % cond # 时间需要大于小于 if cond == 'create_time': filter_cond = '%s__gte' % filter_cond prefix_filters.update( {filter_cond: timestamp_to_datetime(value)}) continue elif cond == 'finish_time': filter_cond = 'pipeline_template__create_time__lt' prefix_filters.update({ filter_cond: timestamp_to_datetime(value) + datetime.timedelta(days=1) }) continue else: filter_cond = cond prefix_filters.update({filter_cond: value}) # 获得标准插件dict列表 component_dict = ComponentModel.objects.get_component_dict() try: tasktmpl = self.filter(**prefix_filters) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % ( filters, e) return False, message if group_by == AE.state: total = tasktmpl.count() groups = [{ 'code': 'CREATED', 'name': _(u"未执行"), 'value': tasktmpl.filter(pipeline_template__is_started=False).count() }, { 'code': 'EXECUTING', 'name': _(u"执行中"), 'value': tasktmpl.filter(pipeline_template__is_started=True, pipeline_template__is_finished=False).count() }, { 'code': 'FINISHED', 'name': _(u"已完成"), 'value': tasktmpl.filter(pipeline_template__is_finished=True).count() }] elif group_by == AE.business__cc_id: total = tasktmpl.count() template_list = tasktmpl.values( AE.business__cc_id, AE.business__cc_name).annotate( value=Count(group_by)).order_by("value") groups = [] for data in template_list: groups.append({ 'code': data.get(AE.business__cc_id), 'name': data.get(AE.business__cc_name), 'value': data.get('value', 0) }) elif group_by == AE.atom_cite: template_list = tasktmpl.values_list( "pipeline_template__template_id") component_list = ComponentModel.objects.filter( status=True).values("code") # 用 template_id 列表获取所有符合条件的总数 other_component_list = ComponentInTemplate.objects.filter( template_id__in=template_list).values( "component_code").annotate( value=Count("component_code")).order_by() components_dict = {} total = 0 for component in other_component_list: value = component["value"] components_dict[component["component_code"]] = value # 总数不能通过查询获得,需要通过循环计数 total += value groups = [] # 循环聚合信息 for data in component_list: code = data.get("code") groups.append({ 'code': code, 'name': component_dict.get(code, None), 'value': components_dict.get(code, 0) }) elif group_by == AE.atom_template: # 按起始时间、业务(可选)、类型(可选)、标准插件查询被引用的流程模板列表(dataTable) # 获取标准插件code component_code = filters.get("component_code") # 获取到组件code对应的template_id_list if component_code: template_id_list = ComponentInTemplate.objects.filter( component_code=component_code).distinct().values_list( "template_id") else: template_id_list = ComponentInTemplate.objects.all( ).values_list("template_id") template_list = tasktmpl.filter( pipeline_template__template_id__in=template_id_list) total = template_list.count() template_list = template_list.values( "id", "business__cc_id", "business__cc_name", "pipeline_template__name", "category", "pipeline_template__edit_time", "pipeline_template__editor")[(page - 1) * limit:page * limit] groups = [] # 循环聚合信息 for data in template_list: groups.append({ 'templateId': data.get("id"), 'businessId': data.get("business__cc_id"), 'businessName': data.get("business__cc_name"), 'templateName': data.get("pipeline_template__name"), 'category': category_dict[data.get("category")], # 需要将code转为名称 "editTime": format_datetime(data.get("pipeline_template__edit_time")), "editor": data.get("pipeline_template__editor") }) elif group_by == AE.atom_execute: # 需要获得符合的查询的对应 template_id 列表 # 获取标准插件code component_code = filters.get("component_code") # 获取到组件code对应的template_id template_id_list = ComponentInTemplate.objects.filter( component_code=component_code).values_list("template_id") total = template_id_list.count() template_list = tasktmpl.filter( pipeline_template__template_id__in=template_id_list).values( "id", "business__cc_name", "business__cc_id", "pipeline_template__name", "category", "pipeline_template__edit_time", "pipeline_template__editor")[(page - 1) * limit:page * limit] groups = [] # 循环聚合信息 for data in template_list: groups.append({ 'templateId': data.get("id"), 'businessId': data.get("business__cc_id"), 'businessName': data.get("business__cc_name"), 'templateName': data.get("pipeline_template__name"), 'category': category_dict[data.get("category")], "editTime": data.get("pipeline_template__edit_time").strftime( "%Y-%m-%d %H:%M:%S"), "editor": data.get("pipeline_template__editor") }) elif group_by == AE.template_cite: # 按起始时间、业务(可选)、类型(可选)查询各流程模板被引用为子流程个数、创建轻应用个数、创建任务实例个数 template_list = tasktmpl id_list = template_list.values( "pipeline_template__id", "id")[(page - 1) * limit:page * limit] template_id_map = { template['pipeline_template__id']: template['id'] for template in id_list } t_id_list = [ x[0] for x in template_list.values_list( "pipeline_template__template_id")[(page - 1) * limit:page * limit] ] appmaker_list = template_list.values("id", "appmaker").annotate( appmaker_total=Count("appmaker")).order_by("-id") taskflow_list = PipelineInstance.objects.filter( template_id__in=template_id_map.keys()).values( "template_id").annotate( instance_total=Count("template_id")).order_by() relationship_list = TemplateRelationship.objects.filter( descendant_template_id__in=t_id_list ).values("descendant_template_id").annotate( relationship_total=Count("descendant_template_id")).order_by() # 排序 order_by = filters.get("order_by", "-templateId") # 使用驼峰转下划线进行转换order_by camel_order_by = camel_case_to_underscore_naming(order_by) if order_by == "appmakerTotal": appmaker_list = appmaker_list.order_by(camel_order_by) elif order_by == "instanceTotal": taskflow_list = taskflow_list.order_by(camel_order_by) elif order_by == "relationshipTotal": relationship_list = relationship_list.order_by(camel_order_by) appmaker_list = appmaker_list[(page - 1) * limit:page * limit] relationship_list = relationship_list # 获得对应的dict数据 appmaker_dict = {} for appmaker in appmaker_list: appmaker_dict[appmaker["id"]] = appmaker["appmaker_total"] taskflow_dict = {} for taskflow in taskflow_list: taskflow_dict[template_id_map[ taskflow["template_id"]]] = taskflow["instance_total"] relationship_dict = {} for relationship in relationship_list: relationship_dict[ relationship["descendant_template_id"]] = relationship[ "relationship_total"] groups = [] for template in template_list[(page - 1) * limit:page * limit]: template_id = template.id groups.append({ 'templateId': template.id, 'templateName': template.name, 'businessId': template.business.cc_id, 'appmakerTotal': appmaker_dict.get(template_id, 0), 'relationshipTotal': relationship_dict.get( template.pipeline_template.template_id, 0), 'instanceTotal': taskflow_dict.get(template_id, 0) }) total = template_list.count() if order_by[0] == "-": # 需要去除负号 order_by = order_by[1:] groups = sorted(groups, key=lambda group: -group.get(order_by)) else: groups = sorted(groups, key=lambda group: group.get(order_by)) elif group_by == AE.template_node: # 按起始时间、业务(可选)、类型(可选)查询各流程模板标准插件节点个数、子流程节点个数、网关节点数 total = tasktmpl.count() groups = [] # 排序 template_id_list = tasktmpl.values( "pipeline_template__template_id") template_pipeline_data = TemplateInPipeline.objects.filter( template_id__in=template_id_list) order_by = filters.get("order_by", "-templateId") # 使用驼峰转下划线进行转换order_by camel_order_by = camel_case_to_underscore_naming(order_by) # 排列获取分页后的数据 pipeline_data = template_pipeline_data.order_by( camel_order_by)[(page - 1) * limit:page * limit] template_id_list = [ template.template_id for template in pipeline_data ] tasktmpl = tasktmpl.filter( pipeline_template__template_id__in=template_id_list) pipeline_dict = {} for pipeline in pipeline_data: pipeline_dict[pipeline.template_id] = { "atom_total": pipeline.atom_total, "subprocess_total": pipeline.subprocess_total, "gateways_total": pipeline.gateways_total } # 需要循环执行计算相关节点 for template in tasktmpl: pipeline_template = template.pipeline_template template_id = template.id pipeline_template_id = pipeline_template.template_id # 插入信息 groups.append({ 'templateId': template_id, 'businessId': template.business.cc_id, 'businessName': template.business.cc_name, 'templateName': pipeline_template.name, 'category': category_dict[template.category], "createTime": format_datetime(pipeline_template.create_time), "creator": pipeline_template.creator, "atomTotal": pipeline_dict[pipeline_template_id]["atom_total"], "subprocessTotal": pipeline_dict[pipeline_template_id]["subprocess_total"], "gatewaysTotal": pipeline_dict[pipeline_template_id]["gateways_total"] }) if order_by[0] == "-": # 需要去除负号 order_by = order_by[1:] groups = sorted(groups, key=lambda group: -group.get(order_by)) else: groups = sorted(groups, key=lambda group: group.get(order_by)) elif group_by in [AE.category, AE.create_method, AE.flow_type]: try: total, groups = self.classified_count(prefix_filters, group_by) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % ( filters, e) return False, message else: total, groups = 0, [] data = {'total': total, 'groups': groups} return True, data
def extend_classified_count(self, group_by, filters=None, page=None, limit=None): """ @summary: 兼容按照任务状态分类的扩展 @param group_by: @param filters: @param page: @param limit: @return: """ # 获得所有类型的dict列表 category_dict = dict(TASK_CATEGORY) if filters is None: filters = {} prefix_filters = {} for cond, value in filters.items(): # 如果conditions内容为空或为空字符,不可加入查询条件中 if value in ['None', '' ] or cond in ['component_code', 'order_by', 'type']: continue if PIPELINE_REGEX.match(cond): filter_cond = 'pipeline_instance__%s' % cond # 时间需要大于小于 if cond == 'create_time': filter_cond = '%s__gte' % filter_cond prefix_filters.update( {filter_cond: timestamp_to_datetime(value)}) continue # 结束时间由创建时间来决定 if cond == 'finish_time': filter_cond = 'pipeline_instance__create_time__lt' prefix_filters.update({ filter_cond: timestamp_to_datetime(value) + datetime.timedelta(days=1) }) continue else: filter_cond = cond prefix_filters.update({filter_cond: value}) try: taskflow = self.filter(**prefix_filters) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % ( filters, e) return False, message if group_by == AE.state: total = taskflow.count() groups = [{ 'code': 'CREATED', 'name': _(u"未执行"), 'value': taskflow.filter(pipeline_instance__is_started=False).count() }, { 'code': 'EXECUTING', 'name': _(u"执行中"), 'value': taskflow.filter(pipeline_instance__is_started=True, pipeline_instance__is_finished=False).count() }, { 'code': 'FINISHED', 'name': _(u"已完成"), 'value': taskflow.filter(pipeline_instance__is_finished=True).count() }] elif group_by == AE.business__cc_id: # 获取所有数据 total = taskflow.count() taskflow_list = taskflow.values( AE.business__cc_id, AE.business__cc_name).annotate( value=Count(group_by)).order_by() groups = [] for data in taskflow_list: groups.append({ 'code': data.get(AE.business__cc_id), 'name': data.get(AE.business__cc_name), 'value': data.get('value', 0) }) elif group_by == AE.appmaker_instance: taskflow_values = taskflow.values("create_info") order_by = filters.get("order_by", "-templateId") business_id = filters["business__cc_id"] category = filters["category"] started_time = timestamp_to_datetime(filters["create_time"]) end_time = timestamp_to_datetime( filters["finish_time"]) + datetime.timedelta(days=1) appmaker_data = AppMaker.objects.filter( is_deleted=False, create_time__gte=started_time, create_time__lte=end_time) if business_id != '': appmaker_data = appmaker_data.filter( business__cc_id=business_id) if category != '': appmaker_data = appmaker_data.filter( task_template__category=category) # 获取所有轻应用数据数量 total = appmaker_data.count() # 获得每一个轻应用的实例数量并变为 dict 字典数据进行查询 total_dict = { appmaker['create_info']: appmaker['instance_total'] for appmaker in taskflow_values.annotate( instance_total=Count("create_info")).order_by() } id_list = appmaker_data.values_list("id")[:] id_list = sorted( id_list, key=lambda tuples_id: -total_dict.get(str(tuples_id[0]), 0)) id_list = id_list[(page - 1) * limit:page * limit] app_id_list = [tuples[0] for tuples in id_list] # 获得轻应用对象对应的模板和轻应用名称 appmaker_data = appmaker_data.filter(id__in=app_id_list).values( "id", "task_template_id", "name", "create_time", "edit_time", "editor", "business__cc_id", "business__cc_name", "task_template__category") groups = [] for data in appmaker_data: code = data.get('task_template_id') appmaker_id = data.get("id") groups.append({ 'templateId': code, 'createTime': format_datetime(data.get('create_time')), 'editTime': format_datetime(data.get('edit_time')), 'editor': data.get('editor'), 'templateName': data.get('name'), 'businessId': data.get('business__cc_id'), 'businessName': data.get('business__cc_name'), 'category': category_dict[data.get('task_template__category')], # 需要将 code 转为字符型 'instanceTotal': total_dict.get(str(appmaker_id), 0) }) if order_by[0] == "-": # 需要去除负号 order_by = order_by[1:] groups = sorted(groups, key=lambda group: -group.get(order_by)) else: groups = sorted(groups, key=lambda group: group.get(order_by)) elif group_by == AE.atom_execute: # 查询各原子被执行次数、失败率、重试次数、平均耗时(不计算子流程) instance_id_list = taskflow.values_list( "pipeline_instance__instance_id") # 获得原子 component = ComponentExecuteData.objects.filter( instance_id__in=instance_id_list, is_sub=False) component_data = component.values('component_code').annotate( execute_times=Count('component_code'), avg_execute_time=Avg('elapsed_time')).order_by( 'component_code') # 统计次数 total = component_data.count() component_success_data = component.filter( is_retry=False).values('component_code').annotate( success_times=Count('component_code')).order_by( 'component_code') # 用于计算所有原子的成功列表 success_component_dict = {} for data in component_success_data: success_component_dict[ data['component_code']] = data['success_times'] groups = [] component_dict = {} for bundle in ComponentModel.objects.all(): name = bundle.name.split('-') group_name = _(name[0]) name = _(name[1]) component_dict[bundle.code] = '%s-%s' % (group_name, name) for data in component_data: code = data.get('component_code') execute_times = data.get('execute_times') failed_times = execute_times - success_component_dict.get( code, 0) failed_times_percent = '%.2f %%' % (failed_times / 1.0 / execute_times * 100) groups.append({ 'componentName': component_dict[code], 'executeTimes': execute_times, 'avgExecuteTime': '%.2f' % data.get('avg_execute_time', 0), 'failedTimes': failed_times, 'failedTimesPercent': failed_times_percent }) elif group_by == AE.atom_instance: # 被引用的任务实例列表 # 获得参数中的原子code component_code = filters.get("component_code") # 获取到组件code对应的instance_id_list instance_id_list = ComponentExecuteData.objects.filter( is_sub=False) # 对code进行二次查找 instance_id_list = instance_id_list.filter( component_code=component_code).distinct().values_list( "instance_id") taskflow_list = taskflow.filter( pipeline_instance__instance_id__in=instance_id_list).values( 'id', 'business__cc_id', 'business__cc_name', 'pipeline_instance__name', 'category', 'pipeline_instance__create_time', 'pipeline_instance__creator') # 获得总数 total = taskflow_list.count() taskflow_list = taskflow_list[(page - 1) * limit:page * limit] groups = [] # 循环信息 for data in taskflow_list: groups.append({ 'instanceId': data.get("id"), 'businessId': data.get("business__cc_id"), 'businessName': data.get("business__cc_name"), 'instanceName': data.get("pipeline_instance__name"), 'category': category_dict[data.get("category")], # 需要将code转为名称 "createTime": format_datetime( data.get("pipeline_instance__create_time")), "creator": data.get("pipeline_instance__creator") }) elif group_by == AE.instance_node: # 各任务实例执行的原子节点个数、子流程节点个数、网关节点数 groups = [] # 排序 instance_id_list = taskflow.values( "pipeline_instance__instance_id") instance_pipeline_data = InstanceInPipeline.objects.filter( instance_id__in=instance_id_list) # 总数 total = instance_pipeline_data.count() order_by = filters.get("order_by", "-instanceId") # 使用驼峰转下划线进行转换order_by camel_order_by = camel_case_to_underscore_naming(order_by) # 排列获取分页后的数据 pipeline_data = instance_pipeline_data.order_by( camel_order_by)[(page - 1) * limit:page * limit] instance_id_list = [tuples.instance_id for tuples in pipeline_data] taskflow = taskflow.filter( pipeline_instance__instance_id__in=instance_id_list) pipeline_dict = {} for pipeline in pipeline_data: pipeline_dict[pipeline.instance_id] = { "atom_total": pipeline.atom_total, "subprocess_total": pipeline.subprocess_total, "gateways_total": pipeline.gateways_total } # 需要循环执行计算相关节点 for flow in taskflow: pipeline_instance = flow.pipeline_instance instance_id = flow.id pipeline_instance_id = pipeline_instance.instance_id # 插入信息 groups.append({ 'instanceId': instance_id, 'businessId': flow.business.cc_id, 'businessName': flow.business.cc_name, 'instanceName': pipeline_instance.name, 'category': category_dict[flow.category], "createTime": format_datetime(pipeline_instance.create_time), "creator": pipeline_instance.creator, "atomTotal": pipeline_dict[pipeline_instance_id]["atom_total"], "subprocessTotal": pipeline_dict[pipeline_instance_id]["subprocess_total"], "gatewaysTotal": pipeline_dict[pipeline_instance_id]["gateways_total"] }) if order_by[0] == "-": # 需要去除负号 order_by = order_by[1:] groups = sorted(groups, key=lambda group: -group.get(order_by)) else: groups = sorted(groups, key=lambda group: group.get(order_by)) elif group_by == AE.instance_details: # 各任务执行耗时 started_time = prefix_filters[ 'pipeline_instance__create_time__gte'] archived_time = prefix_filters[ 'pipeline_instance__create_time__lt'] prefix_filters.update( pipeline_instance__start_time__gte=prefix_filters.pop( 'pipeline_instance__create_time__gte'), pipeline_instance__start_time__lt=prefix_filters.pop( 'pipeline_instance__create_time__lt'), pipeline_instance__is_finished=True) taskflow = self.filter(**prefix_filters) # 需要distinct去除重复的 instance_id_list = ComponentExecuteData.objects.filter( started_time__gte=started_time, archived_time__lte=archived_time).values_list( "instance_id").distinct().order_by() total = instance_id_list.count() # 排序 order_by = filters.get("order_by", "instanceId") component_list = instance_id_list.annotate( execute_time=Sum('elapsed_time')).order_by() # 使用驼峰转下划线进行转换order_by component_list = component_list.order_by( camel_case_to_underscore_naming(order_by)) # 分页 component_list = component_list[(page - 1) * limit:page * limit] component_dict = {} instance_list = [tuples[0] for tuples in component_list] for component in component_list: component_dict[component[0]] = component[1] taskflow_list = taskflow.filter( pipeline_instance__instance_id__in=instance_list).values( 'id', 'pipeline_instance__instance_id', 'business__cc_id', 'business__cc_name', 'pipeline_instance__name', 'category', 'pipeline_instance__create_time', 'pipeline_instance__creator') groups = [] for data in taskflow_list: instance_id = data.get("pipeline_instance__instance_id") groups.append({ 'instanceId': data.get("id"), 'businessId': data.get("business__cc_id"), 'businessName': data.get("business__cc_name"), 'instanceName': data.get("pipeline_instance__name"), 'category': category_dict[data.get("category")], # 需要将code转为名称 "createTime": format_datetime( data.get("pipeline_instance__create_time")), "creator": data.get("pipeline_instance__creator"), "executeTime": component_dict[instance_id] }) if order_by[0] == "-": # 需要去除负号 order_by = order_by[1:] groups = sorted(groups, key=lambda group: -group.get(order_by)) else: groups = sorted(groups, key=lambda group: group.get(order_by)) elif group_by == AE.instance_time: # 按起始时间、业务(可选)、类型(可选)、图表类型(日视图,月视图),查询每一天或每一月的执行数量 instance_create_time_list = taskflow.values( 'pipeline_instance__create_time') total = instance_create_time_list.count() group_type = filters.get('type', 'day') create_time = timestamp_to_datetime(filters['create_time']) end_time = timestamp_to_datetime( filters['finish_time']) + datetime.timedelta(days=1) groups = [] date_list = [] for instance in instance_create_time_list: instance_time = instance['pipeline_instance__create_time'] date_key = '' # 添加在一个list中 之后使用count方法获取对应的数量 if group_type == 'day': date_key = instance_time.strftime('%Y-%m-%d') elif group_type == 'month': date_key = instance_time.strftime('%Y-%m') date_list.append(date_key) if group_type == 'day': # 日视图 for d in gen_day_dates(create_time, (end_time - create_time).days + 1): date_key = d.strftime('%Y-%m-%d') groups.append({ 'time': date_key, 'value': date_list.count(date_key) }) elif group_type == 'month': # 月视图 # 直接拿到对应的(年-月),不需要在字符串拼接 for date_key in get_month_dates(create_time, end_time): groups.append({ 'time': date_key, 'value': date_list.count(date_key) }) elif group_by in [AE.category, AE.create_method, AE.flow_type]: try: total, groups = self.classified_count(prefix_filters, group_by) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % ( filters, e) return False, message else: total, groups = 0, [] data = {'total': total, 'groups': groups} return True, data
def extend_classified_count(self, group_by, filters=None, page=None, limit=None): """ @summary: 兼容按照任务状态分类的扩展 @param group_by: @param filters: @param page: @param limit: @return: """ # 获得所有类型的dict列表 category_dict = dict(TASK_CATEGORY) if filters is None: filters = {} tasktmpl_inst_regex = re.compile(r'^name|creator_name|editor_name|' r'create_time|edit_time|edit_finish_time|finish_time') prefix_filters = {} for cond in filters: # 如果conditions内容为空或为空字符,不可加入查询条件中 if filters[cond] == 'None' or filters[cond] == '' or cond == 'component_code': continue if tasktmpl_inst_regex.match(cond): filter_cond = 'pipeline_template__%s' % cond # 时间需要大于小于 if cond == 'create_time': filter_cond = '%s__gte' % filter_cond prefix_filters.update({filter_cond: timestamp_to_datetime(filters[cond])}) continue elif cond == 'finish_time': filter_cond = 'pipeline_template__create_time__lt' prefix_filters.update( {filter_cond: timestamp_to_datetime(filters[cond]) + datetime.timedelta(days=1)}) continue # 编辑时间与创建时间需要分开查询 elif cond == 'edit_time': filter_cond = '%s__gte' % filter_cond prefix_filters.update({filter_cond: timestamp_to_datetime(filters[cond])}) continue elif cond == 'edit_finish_time': filter_cond = 'pipeline_template__edit_time__lt' prefix_filters.update( {filter_cond: timestamp_to_datetime(filters[cond]) + datetime.timedelta(days=1)}) continue else: filter_cond = cond prefix_filters.update({filter_cond: filters[cond]}) # 获得原子dict列表 component_dict = ComponentModel.objects.get_component_dict() if group_by == AE.state: try: tasktmpl = self.filter(**prefix_filters) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message total = tasktmpl.count() groups = [ { 'code': 'CREATED', 'name': _(u"未执行"), 'value': tasktmpl.filter(pipeline_template__is_started=False).count() }, { 'code': 'EXECUTING', 'name': _(u"执行中"), 'value': tasktmpl.filter(pipeline_template__is_started=True, pipeline_template__is_finished=False).count() }, { 'code': 'FINISHED', 'name': _(u"已完成"), 'value': tasktmpl.filter(pipeline_template__is_finished=True).count() } ] elif group_by == AE.business__cc_id: cc_name = AE.business__cc_name try: tasktmpl = self.filter(**prefix_filters) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message # 获取所有数据 total = tasktmpl.count() queryset = tasktmpl.values(group_by, cc_name).annotate(value=Count(group_by)).order_by("-value") groups = [] for data in queryset: groups.append({ 'code': data.get(group_by), 'name': data.get(cc_name), 'value': data.get('value', 0) }) elif group_by == AE.atom_cite: # 这里没有其他原子节点的内容 try: # 需要获得符合的查询的对应 template_id 列表 template_list = self.filter(**prefix_filters).values_list("pipeline_template__template_id") except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message component_list = ComponentModel.objects.filter(status=True).values("code") # 用 template_id 列表获取所有符合条件的总数 other_component_list = ComponentInTemplate.objects.filter(template_id__in=template_list).values( "component_code").annotate( value=Count("component_code")).order_by() components_dict = {} total = 0 for component in other_component_list: value = component["value"] components_dict[component["component_code"]] = value # 总数不能通过查询获得,需要通过循环计数 total += value groups = [] # 循环聚合信息 for data in component_list: code = data.get("code") value = components_dict.get(code, 0) groups.append({ 'code': code, 'name': component_dict.get(code, None), 'value': value }) elif group_by == AE.atom_template: # 按起始时间、业务(可选)、类型(可选)、原子查询被引用的流程模板列表(dataTable) try: # 需要获得符合的查询的对应 template_id 列表 template_list = self.filter(**prefix_filters).filter() except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message # 获取原子code component_code = filters.get("component_code") # 获取到组件code对应的template_id_list if component_code: template_id_list = ComponentInTemplate.objects.filter(component_code=component_code).values_list( "template_id") else: template_id_list = ComponentInTemplate.objects.all().values_list("template_id") total = template_id_list.count() template_list = template_list.filter(pipeline_template__template_id__in=template_id_list).values( "id", "business__cc_id", "business__cc_name", "pipeline_template__name", "category", "pipeline_template__edit_time", "pipeline_template__editor" )[(page - 1) * limit:page * limit] groups = [] # 循环聚合信息 for data in template_list: groups.append({ 'templateId': data.get("id"), 'businessId': data.get("business__cc_id"), 'businessName': data.get("business__cc_name"), 'templateName': data.get("pipeline_template__name"), 'category': category_dict[data.get("category")], # 需要将code转为名称 "editTime": format_datetime(data.get("pipeline_template__edit_time")), "editor": data.get("pipeline_template__editor") }) elif group_by == AE.atom_execute: try: # 需要获得符合的查询的对应 template_id 列表 template_list = self.filter(**prefix_filters).filter() except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message # 获取原子code component_code = filters.get("component_code") # 获取到组件code对应的template_id if component_code: template_id_list = ComponentInTemplate.objects.filter(component_code=component_code).values_list( "template_id") else: template_id_list = ComponentInTemplate.objects.all().values_list("template_id") total = template_id_list.count() template_list = template_list.filter(pipeline_template__template_id__in=template_id_list).values( "id", "business__cc_name", "business__cc_id", "pipeline_template__name", "category", "pipeline_template__edit_time", "pipeline_template__editor")[(page - 1) * limit:page * limit] groups = [] # 循环聚合信息 for data in template_list: groups.append({ 'templateId': data.get("id"), 'businessId': data.get("business__cc_id"), 'businessName': data.get("business__cc_name"), 'templateName': data.get("pipeline_template__name"), 'category': category_dict[data.get("category")], "editTime": data.get("pipeline_template__edit_time").strftime("%Y-%m-%d %H:%M:%S"), "editor": data.get("pipeline_template__editor") }) elif group_by == AE.template_cite: # 按起始时间、业务(可选)、类型(可选)查询各流程模板被引用为子流程个数、创建轻应用个数、创建任务实例个数 try: # 需要获得符合的查询的对应 template_id 列表 template_list = self.filter(**prefix_filters).filter() except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message total = template_list.count() template_list = template_list[(page - 1) * limit:page * limit] # 获取分页的数据 id_list = list(template_list.values_list("id")) t_id_list = list(template_list.values_list("pipeline_template__template_id")) n_id_list = [] for x in t_id_list: n_id_list.append(x[0]) # 轻应用的数据已分页 appmaker_list = template_list.values("id", "appmaker").annotate(value=Count("appmaker")) taskflow_list = PipelineInstance.objects.filter(template_id__in=id_list).values("template_id").annotate( value=Count("template_id")).order_by() relationship_list = TemplateRelationship.objects.filter(descendant_template_id__in=n_id_list).values( "descendant_template_id").annotate( value=Count("descendant_template_id")).order_by() appmaker_dict = {} for appmaker in appmaker_list: appmaker_dict[appmaker["id"]] = appmaker["value"] relationship_dict = {} for relationship in relationship_list: relationship_dict[relationship["descendant_template_id"]] = relationship["value"] taskflow_dict = {} for taskflow in taskflow_list: taskflow_dict[taskflow["template_id"]] = taskflow["value"] groups = [] # todo 缺少子流程部分 for template in template_list: id = template.id groups.append({ 'id': template.id, 'templateName': template.name, 'appmakerTotal': appmaker_dict[id], 'relationshipTotal': relationship_dict.get(template.pipeline_template.template_id, 0), 'instanceTotal': taskflow_dict.get(id, 0) }) elif group_by == AE.template_node: # 按起始时间、业务(可选)、类型(可选)查询各流程模板原子节点个数、子流程节点个数、网关节点数(dataTable) try: # 需要获得符合的查询的对应 template_id 列表 template_list = self.filter(**prefix_filters).filter() except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message # 总数 total = template_list.count() groups = [] # 需要循环执行计算相关节点 for template in template_list[(page - 1) * limit:page * limit]: atom_total = 0 subprocess_total = 0 pipeline_tree = template.pipeline_tree tree_activities = pipeline_tree["activities"] gateways_total = len(pipeline_tree["gateways"]) for activity in tree_activities: activity_type = tree_activities[activity]["type"] if activity_type == "ServiceActivity": atom_total += 1 elif activity_type == "SubProcess": subprocess_total += 1 pipeline_template = template.pipeline_template # 插入信息 groups.append({ 'templateId': template.id, 'businessId': template.business.cc_id, 'businessName': template.business.cc_name, 'templateName': pipeline_template.name, 'category': category_dict[template.category], "editTime": format_datetime(pipeline_template.edit_time), "creator": pipeline_template.creator, "atomTotal": atom_total, "subprocessTotal": subprocess_total, "gatewaysTotal": gateways_total }) elif group_by in [AE.category, AE.create_method, AE.flow_type]: try: total, groups = self.classified_count(prefix_filters, group_by) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message else: total, groups = 0, [] data = {'total': total, 'groups': groups} return True, data
def extend_classified_count(self, group_by, filters=None, page=None, limit=None): """ @summary: 兼容按照任务状态分类的扩展 @param group_by: @param filters: @param page: @param limit: @return: """ # 获得所有类型的dict列表 category_dict = dict(TASK_CATEGORY) if filters is None: filters = {} pipeline_inst_regex = re.compile(r'^name|create_time|creator|create_time|executor|' r'start_time|finish_time|is_started|is_finished') prefix_filters = {} for cond in filters: # 如果conditions内容为空或为空字符,不可加入查询条件中 if filters[cond] == 'None' or filters[cond] == '' or cond == 'component_code': continue if pipeline_inst_regex.match(cond): filter_cond = 'pipeline_instance__%s' % cond # 时间需要大于小于 if cond == 'create_time': filter_cond = '%s__gte' % filter_cond prefix_filters.update( {filter_cond: timestamp_to_datetime(filters[cond])}) continue # 结束时间由创建时间来决定 if cond == 'finish_time': filter_cond = 'pipeline_instance__create_time__lt' prefix_filters.update( {filter_cond: timestamp_to_datetime(filters[cond]) + datetime.timedelta(days=1)}) continue else: filter_cond = cond prefix_filters.update({filter_cond: filters[cond]}) if group_by == AE.state: try: taskflow = self.filter(**prefix_filters) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message total = taskflow.count() groups = [ { 'code': 'CREATED', 'name': _(u"未执行"), 'value': taskflow.filter(pipeline_instance__is_started=False).count() }, { 'code': 'EXECUTING', 'name': _(u"执行中"), 'value': taskflow.filter(pipeline_instance__is_started=True, pipeline_instance__is_finished=False).count() }, { 'code': 'FINISHED', 'name': _(u"已完成"), 'value': taskflow.filter(pipeline_instance__is_finished=True).count() } ] elif group_by == AE.business__cc_id: cc_name = AE.business__cc_name try: taskflow = self.filter(**prefix_filters) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message # 获取所有数据 total = taskflow.count() taskflow_list = taskflow.values(group_by, cc_name).annotate(value=Count(group_by)).order_by() groups = [] for data in taskflow_list: groups.append({ 'code': data.get(group_by), 'name': data.get(cc_name), 'value': data.get('value', 0) }) elif group_by == AE.atom_execute: # 查询各原子被执行次数、失败率、重试次数、平均耗时(不计算子流程) try: instance_id_list = self.filter(**prefix_filters).values_list("pipeline_instance__instance_id") except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message # 获得原子 component = ComponentExecuteData.objects.filter(instance_id__in=instance_id_list, is_sub=False) component_data = component.values('component_code').annotate( execute_times=Count('component_code'), avg_execute_time=Avg('elapsed_time')).order_by('component_code') # 统计次数 total = component_data.count() component_success_data = component.filter(is_retry=False).values('component_code').annotate( success_times=Count('component_code') ).order_by('component_code') # 用于计算所有原子的成功列表 success_component_dict = {} for data in component_success_data: success_component_dict[data['component_code']] = data['success_times'] groups = [] component_dict = ComponentModel.objects.get_component_dict() for data in component_data: code = data.get('component_code') execute_times = data.get('execute_times') failed_times = execute_times - success_component_dict.get(code, 0) failed_times_percent = '%.2f %%' % (failed_times / 1.0 / execute_times * 100) groups.append({ 'componentName': component_dict[code], 'executeTimes': execute_times, 'avgExecuteTime': '%.2f' % data.get('avg_execute_time', 0), 'failedTimes': failed_times, 'failedTimesPercent': failed_times_percent }) elif group_by == AE.atom_instance: # 被引用的任务实例列表 try: taskflow_list = self.filter(**prefix_filters) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message # 获取原子code # 获得参数中的原子code component_code = filters.get("component_code") # 获取到组件code对应的instance_id_list instance_id_list = ComponentExecuteData.objects.filter(is_sub=False) # 参数携带了原子code话,针对查询,不携带查询全部 if component_code: instance_id_list = instance_id_list.filter(component_code=component_code).values_list( "instance_id") taskflow_list = taskflow_list.filter(pipeline_instance__instance_id__in=instance_id_list).values( 'id', 'business__cc_id', 'business__cc_name', 'pipeline_instance__name', 'category', 'pipeline_instance__create_time', 'pipeline_instance__creator' ) # 获得总数 total = taskflow_list.count() taskflow_list = taskflow_list[(page - 1) * limit:page * limit] groups = [] # 循环信息 for data in taskflow_list: groups.append({ 'instanceId': data.get("id"), 'businessId': data.get("business__cc_id"), 'businessName': data.get("business__cc_name"), 'instanceName': data.get("pipeline_instance__name"), 'category': category_dict[data.get("category")], # 需要将code转为名称 "createTime": format_datetime(data.get("pipeline_instance__create_time")), "creator": data.get("pipeline_instance__creator") }) elif group_by == AE.instance_node: # 各任务实例执行的原子节点个数、子流程节点个数、网关节点数 try: taskflow_list = self.filter(**prefix_filters) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message # 总数 total = taskflow_list.count() groups = [] for taskflow in taskflow_list[(page - 1) * limit:page * limit]: atom_total = 0 subprocess_total = 0 pipeline_tree = taskflow.pipeline_tree tree_activities = pipeline_tree["activities"] gateways_total = len(pipeline_tree["gateways"]) for activity in tree_activities: activity_type = tree_activities[activity]["type"] if activity_type == "ServiceActivity": atom_total += 1 elif activity_type == "SubProcess": subprocess_total += 1 pipeline_instance = taskflow.pipeline_instance # 插入信息 groups.append({ 'instanceId': taskflow.id, 'businessId': taskflow.business.cc_id, 'businessName': taskflow.business.cc_name, 'instanceName': pipeline_instance.name, 'category': category_dict[taskflow.category], "createTime": format_datetime(pipeline_instance.create_time), "creator": pipeline_instance.creator, "atomTotal": atom_total, "subprocessTotal": subprocess_total, "gatewaysTotal": gateways_total }) pass elif group_by == AE.instance_details: # 各任务实例详情和执行耗时 try: taskflow_list = self.filter(**prefix_filters) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message started_time = timestamp_to_datetime(filters["create_time"]) end_time = timestamp_to_datetime(filters["finish_time"]) # 需要去除重复的 instance_id_list = ComponentExecuteData.objects.filter( started_time__gte=started_time, started_time__lte=end_time).values_list( "instance_id").distinct().order_by() # 总数 total = instance_id_list.count() # 获得所有的任务实例的执行耗时 component_list = instance_id_list.annotate(execute_times=Sum('elapsed_time')) component_dict = {} for component in component_list: component_dict[component[0]] = component[1] taskflow_list = taskflow_list.filter(pipeline_instance__instance_id__in=instance_id_list).values( 'id', 'pipeline_instance__instance_id', 'business__cc_id', 'business__cc_name', 'pipeline_instance__name', 'category', 'pipeline_instance__create_time', 'pipeline_instance__creator' )[(page - 1) * limit:page * limit] groups = [] for data in taskflow_list: instance_id = data.get("pipeline_instance__instance_id") groups.append({ 'instanceId': data.get("id"), 'businessId': data.get("business__cc_id"), 'businessName': data.get("business__cc_name"), 'instanceName': data.get("pipeline_instance__name"), 'category': category_dict[data.get("category")], # 需要将code转为名称 "createTime": format_datetime(data.get("pipeline_instance__create_time")), "creator": data.get("pipeline_instance__creator"), "executeTime": component_dict[instance_id] }) elif group_by in [AE.category, AE.create_method, AE.flow_type]: try: total, groups = self.classified_count(prefix_filters, group_by) except Exception as e: message = u"query_task_list params conditions[%s] have invalid key or value: %s" % (filters, e) return False, message else: total, groups = 0, [] data = {'total': total, 'groups': groups} return True, data