def get_interface_graph(interface_id, graph_type): """获取任务流拓扑结构""" # 任务流中任务依赖 result = [] if graph_type == 1: data = InterfaceModel.get_interface_graph(db.etl_db, interface_id) result = job_nodes_graph(data, interface_id) # 局部-任务流依赖 elif graph_type == 2: # 任务流详情 detail = InterfaceModel.get_interface_detail( db.etl_db, interface_id) # 前后置依赖 parent = InterfaceModel.get_interface_parent_all(db.etl_db) child = InterfaceModel.get_interface_child_all(db.etl_db) result = interface_local_graph(detail, parent, child) # 全局-任务流依赖 elif graph_type == 3: # 任务流详情 detail = InterfaceModel.get_interface_detail( db.etl_db, interface_id) # 所有前后置依赖 parent = InterfaceModel.get_interface_parent_all(db.etl_db) child = InterfaceModel.get_interface_child_all(db.etl_db) result = interface_global_graph(detail, parent, child) return Response(result=result)
def generate_interface_dag_by_dispatch(dispatch_id, is_after=1): """ 生成执行任务流前后依赖关系 0.预处理: 任务流详情和全部依赖关系, id统一为字符串 1.构造节点: 节点获取一层父节点, 递归子节点 2.计算节点层级: 找出开始节点(无入度), 入度和当前节点最大层级+1为当前节点层级, 队列中添加出度 :param dispatch_id: 调度id :param is_after: 是否触发后置任务流 :return: 任务流依赖关系 """ def get_context_node(node_id, after=1, before=1): """获取上下文节点""" parent_node = [i for i in child if i['child_id'] == node_id] child_node = [i for i in parent if i['parent_id'] == node_id] # 父节点(局部拓扑情况下, 所有节点的父节点无需递归, 初始节点不添加父节点) if before: for parent_item in parent_node: # 添加入度 nodes[node_id]['in'].add(parent_item['interface_id']) if parent_item['interface_id'] not in nodes: # 添加节点 nodes[parent_item['interface_id']] = { 'id': parent_item['interface_id'], 'name': parent_item['interface_name'], 'in': set(), 'out': {node_id}, 'level': 0 } else: nodes[parent_item['interface_id']]['out'].add(node_id) # 子节点 if after: for child_item in child_node: # 添加出度 nodes[node_id]['out'].add(child_item['interface_id']) if child_item['interface_id'] not in nodes: # 添加节点 nodes[child_item['interface_id']] = { 'id': child_item['interface_id'], 'name': child_item['interface_name'], 'in': {node_id}, 'out': set(), 'level': 0 } else: nodes[child_item['interface_id']]['in'].add(node_id) # 递归节点 get_context_node(child_item['interface_id']) # 任务流详情 detail = InterfaceModel.get_interface_detail_by_dispatch_id(db.etl_db, dispatch_id) # 所有任务流前后置依赖 parent = InterfaceModel.get_interface_parent_all(db.etl_db) child = InterfaceModel.get_interface_child_all(db.etl_db) nodes = {} # 0.预处理: id统一为字符串 # 父节点 for item in parent: item['interface_id'] = str(item['interface_id']) item['parent_id'] = str(item['parent_id']) if item['parent_id'] else None # 当前节点 detail['interface_id'] = str(detail['interface_id']) # 子节点 for item in child: item['interface_id'] = str(item['interface_id']) item['child_id'] = str(item['child_id']) if item['child_id'] else None # 1.构造节点 # 当前节点 nodes[detail['interface_id']] = { 'id': detail['interface_id'], 'name': detail['interface_name'], 'is_start': True, 'in': set(), 'out': set(), 'level': 0 } # 节点上下文递归 get_context_node(detail['interface_id'], after=is_after, before=0) # 2.计算节点层级 node_queue = [] # 找出开始节点 for _, node in nodes.items(): node_queue.append(node) if not node['in'] else None # 计算层级 index = 0 while index < len(node_queue): node = node_queue[index] if node['in']: level = 0 for key in node['in']: level = max(level, nodes[key]['level']) node['level'] = level + 1 # 添加队列 for out_id in node['out']: if out_id not in map(lambda x: x['id'], node_queue): node_queue.append(nodes[out_id]) index += 1 return nodes
def generate_interface_tree_by_dispatch(dispatch_id): """ 生成执行任务流树形关系 0.预处理: 任务流详情和全部依赖关系, id统一为字符串 1.构造节点: 节点递归子节点 2.计算节点层级: 找出开始节点(无入度), 入度和当前节点最大层级+1为当前节点层级, 队列中添加出度 :param dispatch_id: 调度id :return: 任务流树形关系 """ def get_child_node(node_id): """获取子节点""" child_node = [i for i in parent if i['parent_id'] == node_id] # 子节点 for node_item in child_node: # 添加出度 nodes[node_id]['out'].add(node_item['interface_id']) if node_item['interface_id'] not in nodes: # 添加节点 nodes[node_item['interface_id']] = { 'id': node_item['interface_id'], 'name': node_item['interface_name'], 'in': {node_id}, 'out': set(), 'level': 0 } # 递归节点 get_child_node(node_item['interface_id']) # 任务流详情 detail = InterfaceModel.get_interface_detail_by_dispatch_id(db.etl_db, dispatch_id) # 所有任务流前后置依赖 parent = InterfaceModel.get_interface_parent_all(db.etl_db) child = InterfaceModel.get_interface_child_all(db.etl_db) nodes = {} # 0.预处理: id统一为字符串 # 父节点 for item in parent: item['interface_id'] = str(item['interface_id']) item['parent_id'] = str(item['parent_id']) if item['parent_id'] else None # 当前节点 detail['interface_id'] = str(detail['interface_id']) # 子节点 for item in child: item['interface_id'] = str(item['interface_id']) item['child_id'] = str(item['child_id']) if item['child_id'] else None # 1.构造节点 # 当前节点 nodes[detail['interface_id']] = { 'id': detail['interface_id'], 'name': detail['interface_name'], 'is_start': True, 'in': set(), 'out': set(), 'level': 0 } # 节点的子节点递归 get_child_node(detail['interface_id']) # 2.计算节点层级 node_queue = [] # 找出开始节点 for _, node in nodes.items(): node_queue.append(node) if not node['in'] else None # 计算层级 index = 0 while index < len(node_queue): node = node_queue[index] if node['in']: level = 0 for key in node['in']: level = max(level, nodes[key]['level']) node['level'] = level + 1 # 添加队列 for out_id in node['out']: if out_id not in map(lambda x: x['id'], node_queue): node_queue.append(nodes[out_id]) index += 1 return nodes