def get_interface_graph(interface_id, graph_type):
"""获取任务流拓扑结构"""
# 任务流中任务依赖
result = []
if graph_type == 1:
data = InterfaceModel.get_interface_graph(db.etl_db, interface_id)
result = job_nodes_graph(data, interface_id)
# 局部-任务流依赖
elif graph_type == 2:
# 任务流详情
detail = InterfaceModel.get_interface_detail(
db.etl_db, interface_id)
# 前后置依赖
parent = InterfaceModel.get_interface_parent_all(db.etl_db)
child = InterfaceModel.get_interface_child_all(db.etl_db)
result = interface_local_graph(detail, parent, child)
# 全局-任务流依赖
elif graph_type == 3:
# 任务流详情
detail = InterfaceModel.get_interface_detail(
db.etl_db, interface_id)
# 所有前后置依赖
parent = InterfaceModel.get_interface_parent_all(db.etl_db)
child = InterfaceModel.get_interface_child_all(db.etl_db)
result = interface_global_graph(detail, parent, child)
return Response(result=result)
def generate_interface_dag_by_dispatch(dispatch_id, is_after=1):
"""
生成执行任务流前后依赖关系
0.预处理: 任务流详情和全部依赖关系, id统一为字符串
1.构造节点: 节点获取一层父节点, 递归子节点
2.计算节点层级: 找出开始节点(无入度), 入度和当前节点最大层级+1为当前节点层级, 队列中添加出度
:param dispatch_id: 调度id
:param is_after: 是否触发后置任务流
:return: 任务流依赖关系
"""
def get_context_node(node_id, after=1, before=1):
"""获取上下文节点"""
parent_node = [i for i in child if i['child_id'] == node_id]
child_node = [i for i in parent if i['parent_id'] == node_id]
# 父节点(局部拓扑情况下, 所有节点的父节点无需递归, 初始节点不添加父节点)
if before:
for parent_item in parent_node:
# 添加入度
nodes[node_id]['in'].add(parent_item['interface_id'])
if parent_item['interface_id'] not in nodes:
# 添加节点
nodes[parent_item['interface_id']] = {
'id': parent_item['interface_id'],
'name': parent_item['interface_name'],
'in': set(),
'out': {node_id},
'level': 0
}
else:
nodes[parent_item['interface_id']]['out'].add(node_id)
# 子节点
if after:
for child_item in child_node:
# 添加出度
nodes[node_id]['out'].add(child_item['interface_id'])
if child_item['interface_id'] not in nodes:
# 添加节点
nodes[child_item['interface_id']] = {
'id': child_item['interface_id'],
'name': child_item['interface_name'],
'in': {node_id},
'out': set(),
'level': 0
}
else:
nodes[child_item['interface_id']]['in'].add(node_id)
# 递归节点
get_context_node(child_item['interface_id'])
# 任务流详情
detail = InterfaceModel.get_interface_detail_by_dispatch_id(db.etl_db, dispatch_id)
# 所有任务流前后置依赖
parent = InterfaceModel.get_interface_parent_all(db.etl_db)
child = InterfaceModel.get_interface_child_all(db.etl_db)
nodes = {}
# 0.预处理: id统一为字符串
# 父节点
for item in parent:
item['interface_id'] = str(item['interface_id'])
item['parent_id'] = str(item['parent_id']) if item['parent_id'] else None
# 当前节点
detail['interface_id'] = str(detail['interface_id'])
# 子节点
for item in child:
item['interface_id'] = str(item['interface_id'])
item['child_id'] = str(item['child_id']) if item['child_id'] else None
# 1.构造节点
# 当前节点
nodes[detail['interface_id']] = {
'id': detail['interface_id'],
'name': detail['interface_name'],
'is_start': True,
'in': set(),
'out': set(),
'level': 0
}
# 节点上下文递归
get_context_node(detail['interface_id'], after=is_after, before=0)
# 2.计算节点层级
node_queue = []
# 找出开始节点
for _, node in nodes.items():
node_queue.append(node) if not node['in'] else None
# 计算层级
index = 0
while index < len(node_queue):
node = node_queue[index]
if node['in']:
level = 0
for key in node['in']:
level = max(level, nodes[key]['level'])
node['level'] = level + 1
# 添加队列
for out_id in node['out']:
if out_id not in map(lambda x: x['id'], node_queue):
node_queue.append(nodes[out_id])
index += 1
return nodes
def generate_interface_tree_by_dispatch(dispatch_id):
"""
生成执行任务流树形关系
0.预处理: 任务流详情和全部依赖关系, id统一为字符串
1.构造节点: 节点递归子节点
2.计算节点层级: 找出开始节点(无入度), 入度和当前节点最大层级+1为当前节点层级, 队列中添加出度
:param dispatch_id: 调度id
:return: 任务流树形关系
"""
def get_child_node(node_id):
"""获取子节点"""
child_node = [i for i in parent if i['parent_id'] == node_id]
# 子节点
for node_item in child_node:
# 添加出度
nodes[node_id]['out'].add(node_item['interface_id'])
if node_item['interface_id'] not in nodes:
# 添加节点
nodes[node_item['interface_id']] = {
'id': node_item['interface_id'],
'name': node_item['interface_name'],
'in': {node_id},
'out': set(),
'level': 0
}
# 递归节点
get_child_node(node_item['interface_id'])
# 任务流详情
detail = InterfaceModel.get_interface_detail_by_dispatch_id(db.etl_db, dispatch_id)
# 所有任务流前后置依赖
parent = InterfaceModel.get_interface_parent_all(db.etl_db)
child = InterfaceModel.get_interface_child_all(db.etl_db)
nodes = {}
# 0.预处理: id统一为字符串
# 父节点
for item in parent:
item['interface_id'] = str(item['interface_id'])
item['parent_id'] = str(item['parent_id']) if item['parent_id'] else None
# 当前节点
detail['interface_id'] = str(detail['interface_id'])
# 子节点
for item in child:
item['interface_id'] = str(item['interface_id'])
item['child_id'] = str(item['child_id']) if item['child_id'] else None
# 1.构造节点
# 当前节点
nodes[detail['interface_id']] = {
'id': detail['interface_id'],
'name': detail['interface_name'],
'is_start': True,
'in': set(),
'out': set(),
'level': 0
}
# 节点的子节点递归
get_child_node(detail['interface_id'])
# 2.计算节点层级
node_queue = []
# 找出开始节点
for _, node in nodes.items():
node_queue.append(node) if not node['in'] else None
# 计算层级
index = 0
while index < len(node_queue):
node = node_queue[index]
if node['in']:
level = 0
for key in node['in']:
level = max(level, nodes[key]['level'])
node['level'] = level + 1
# 添加队列
for out_id in node['out']:
if out_id not in map(lambda x: x['id'], node_queue):
node_queue.append(nodes[out_id])
index += 1
return nodes