def add_interface(interface_name, interface_desc, interface_index,
parent_interface, run_time, retry, user_id):
"""新增任务流"""
# 任务流名称查重
if InterfaceModel.get_interface_detail_by_name(db.etl_db,
interface_name):
abort(400, **make_result(status=400, msg='任务流名称重复, 已存在数据库中'))
# 新增任务流
if not run_time:
run_time = (date.today() + timedelta(days=-1)).strftime('%Y-%m-%d')
interface_id = InterfaceModel.add_interface(db.etl_db, interface_name,
interface_desc,
interface_index, run_time,
retry, user_id)
# 新增任务流前置
parent_arr = []
for item in parent_interface:
parent_arr.append({
'interface_id': interface_id,
'parent_id': item,
'insert_time': int(time.time()),
'update_time': int(time.time()),
'creator_id': user_id,
'updater_id': user_id
})
InterfaceModel.add_interface_parent(db.etl_db,
parent_arr) if parent_arr else None
return Response(interface_id=interface_id)
def get_interface_list(interface_name, interface_index, start_time,
end_time, is_deleted, page, limit):
"""获取任务流列表"""
condition = []
if interface_name:
condition.append('interface_name LIKE "%%%%%s%%%%"' %
interface_name)
if interface_index:
condition.append('interface_index IN (%s)' %
','.join('"%s"' % item
for item in interface_index))
if start_time:
condition.append('insert_time >= %s' % start_time)
if end_time:
condition.append('insert_time <= %s' % end_time)
if is_deleted == 1:
condition.append('is_deleted = 0')
elif is_deleted == 2:
condition.append('is_deleted = 1')
condition = 'WHERE ' + ' AND '.join(condition) if condition else ''
result = InterfaceModel.get_interface_list(db.etl_db, condition, page,
limit)
for item in result:
item['run_time'] = item['run_time'].strftime(
'%Y-%m-%d') if item['run_time'] else ''
total = InterfaceModel.get_interface_count(db.etl_db, condition)
return Response(result=result, total=total)
def update_interface_detail(interface_id, interface_name, interface_desc,
retry, user_id, is_deleted):
"""修改接口详情"""
InterfaceModel.update_interface_detail(db.etl_db, interface_id,
interface_name, interface_desc,
retry, user_id, is_deleted)
return Response(interface_id=interface_id)
def delete_interface_many(flow_id_arr, user_id):
"""批量删除任务流"""
err_msg = []
for interface_id in flow_id_arr:
# 查询是否在调度内
if InterfaceModel.get_schedule_detail(db.etl_db, interface_id):
err_msg.append('任务流ID: [%s], 调度运行中, 请停止调度任务后删除' % interface_id)
# 任务流前后置依赖
parent = InterfaceModel.get_interface_parent(
db.etl_db, interface_id)
child = InterfaceModel.get_interface_child(db.etl_db, interface_id)
if parent or child:
abort(400, **make_result(status=400, msg='任务流存在前/后置依赖, 不可删除'))
# # 查询是否有任务依赖
# ids = InterfaceModel.get_job_prep_by_interface(db.etl_db, interface_id)
# job_ids = []
# out_ids = []
# for items in ids:
# if items['job_id']:
# job_ids.append(items['job_id'])
# if items['out_id']:
# out_ids.append(items['out_id'])
# if set(out_ids) - set(job_ids):
# abort(400, **make_result(status=400, msg='任务流中任务作为其他任务依赖, 请停止依赖任务后删除'))
# 删除任务流
if not err_msg:
condition = '(%s)' % ','.join(str(item) for item in flow_id_arr)
InterfaceModel.delete_interface_many(db.etl_db, condition, user_id)
return Response(msg=err_msg)
def get_interface_detail(interface_id):
"""获取任务流详情"""
# 任务流详情
detail = InterfaceModel.get_interface_detail(db.etl_db, interface_id)
detail['run_time'] = detail['run_time'].strftime(
'%Y-%m-%d') if detail['run_time'] else ''
# 任务流前置依赖
parent = InterfaceModel.get_interface_parent(db.etl_db, interface_id)
return Response(detail=detail, parent=parent)
def post(self, id_1, id_2):
banner_top = InterfaceModel.find_by_id(id_1)
banner_down = InterfaceModel.find_by_id(id_2)
if not banner_top:
return {'message': "Banner with ['id': {}] not found".format(id_1)}, 404
if not banner_down:
return {'message': "Banner with ['id': {}] not found".format(id_2)}, 404
order_aux = banner_top.order
banner_top.order = banner_down.order
banner_top.save_to_db()
banner_down.order = order_aux
banner_down.save_to_db()
return {'message': "Banners ID changed"}, 200
def get_interface_graph(interface_id, graph_type):
"""获取任务流拓扑结构"""
# 任务流中任务依赖
result = []
if graph_type == 1:
data = InterfaceModel.get_interface_graph(db.etl_db, interface_id)
result = job_nodes_graph(data, interface_id)
# 局部-任务流依赖
elif graph_type == 2:
# 任务流详情
detail = InterfaceModel.get_interface_detail(
db.etl_db, interface_id)
# 前后置依赖
parent = InterfaceModel.get_interface_parent_all(db.etl_db)
child = InterfaceModel.get_interface_child_all(db.etl_db)
result = interface_local_graph(detail, parent, child)
# 全局-任务流依赖
elif graph_type == 3:
# 任务流详情
detail = InterfaceModel.get_interface_detail(
db.etl_db, interface_id)
# 所有前后置依赖
parent = InterfaceModel.get_interface_parent_all(db.etl_db)
child = InterfaceModel.get_interface_child_all(db.etl_db)
result = interface_global_graph(detail, parent, child)
return Response(result=result)
def delete(self, id_interface, id_book):
interface = InterfaceModel.find_by_id(id_interface)
if not interface:
return {'message': "Interface with ['id': {}] not found".format(id_interface)}, 404
book = BookModel.find_by_id(id_book)
if not book:
return {'message': "Book with ['id': {}] not found".format(id_book)}, 404
interface.books.remove(book)
interface.save_to_db()
return {'message': "Book with ['id': {}] has successfully been deleted".format(id_book)}, 200
def delete(self, idd):
exists = InterfaceModel.find_by_id(idd)
if not exists:
return {'message': "Interface with ['id': {}] not found".format(idd)}, 404
exists.delete_from_db()
interfaces = sorted([i for i in db.session.query(InterfaceModel).all()], key=lambda x: x.order)
# file deepcode ignore C0200: <comment the reason here>
for i in range(len(interfaces)):
interfaces[i].order = i + 1
interfaces[i].save_to_db()
return {'message': "Interface with ['id': {}] has successfully been deleted".format(idd)}, 200
def delete_interface(interface_id, user_id):
"""删除接口"""
# 查询是否在调度内
if InterfaceModel.get_schedule_detail(db.etl_db, interface_id):
abort(400, **make_result(status=400, msg='调度任务运行中, 请停止调度任务后删除'))
# 查询是否有任务依赖
ids = InterfaceModel.get_job_prep_by_interface(db.etl_db, interface_id)
job_ids = []
out_ids = []
for items in ids:
if items['job_id']:
job_ids.append(items['job_id'])
if items['out_id']:
out_ids.append(items['out_id'])
if set(out_ids) - set(job_ids):
abort(400,
**make_result(status=400, msg='接口中任务作为其他任务依赖, 请停止依赖任务后删除'))
# 删除接口
InterfaceModel.delete_interface(db.etl_db, interface_id, user_id)
return Response(interface_id=interface_id)
def update_interface_detail(interface_id, interface_name, interface_desc,
interface_index, old_parent, parent_interface,
run_time, retry, user_id, is_deleted):
"""修改任务流详情"""
# 任务流名称查重
interface_detail = InterfaceModel.get_interface_detail_by_name(
db.etl_db, interface_name)
if interface_detail and interface_detail[
'interface_id'] != interface_id:
abort(400, **make_result(status=400, msg='任务流名称重复, 已存在数据库中'))
# 调度查重
if is_deleted == 1:
if InterfaceModel.get_schedule_detail(db.etl_db, interface_id):
abort(400, **make_result(status=400, msg='任务流在调度任务中, 不能设置失效'))
# 修改任务流
if not run_time:
run_time = (date.today() + timedelta(days=-1)).strftime('%Y-%m-%d')
InterfaceModel.update_interface_detail(db.etl_db, interface_id,
interface_name, interface_desc,
interface_index, run_time,
retry, user_id, is_deleted)
# 修改任务流前置
old_parent = set() if not old_parent else set(old_parent)
parent_interface = set() if not parent_interface else set(
parent_interface)
# 删
del_data = []
for parent_id in old_parent - parent_interface:
del_data.append({
'interface_id': interface_id,
'parent_id': parent_id,
'user_id': user_id,
'update_time': int(time.time())
})
InterfaceModel.delete_job_parent(db.etl_db,
del_data) if del_data else None
# 增
add_data = []
for parent_id in parent_interface - old_parent:
add_data.append({
'interface_id': interface_id,
'parent_id': parent_id,
'user_id': user_id,
'insert_time': int(time.time()),
'update_time': int(time.time())
})
InterfaceModel.add_job_parent(db.etl_db,
add_data) if add_data else None
return Response(interface_id=interface_id)
def get_interface_list(interface_name, start_time, end_time,
interface_type, is_deleted, page, limit):
"""获取接口列表"""
condition = []
if interface_name:
condition.append('interface_name LIKE "%%%%%s%%%%"' %
interface_name)
if start_time:
condition.append('insert_time >= %s' % start_time)
if end_time:
condition.append('insert_time <= %s' % end_time)
if interface_type:
condition.append('interface_type = %s' % interface_type)
if is_deleted == 1:
condition.append('is_deleted = 0')
elif is_deleted == 2:
condition.append('is_deleted = 1')
condition = 'WHERE ' + ' AND '.join(condition) if condition else ''
result = InterfaceModel.get_interface_list(db.etl_db, condition, page,
limit)
total = InterfaceModel.get_interface_count(db.etl_db, condition)
return Response(result=result, total=total)
def delete_interface(interface_id, user_id):
"""删除任务流"""
# 查询是否在调度内
if InterfaceModel.get_schedule_detail(db.etl_db, interface_id):
abort(400, **make_result(status=400, msg='调度任务运行中, 请停止调度任务后删除'))
# 任务流前后置依赖
parent = InterfaceModel.get_interface_parent(db.etl_db, interface_id)
child = InterfaceModel.get_interface_child(db.etl_db, interface_id)
if parent or child:
abort(400, **make_result(status=400, msg='任务流存在前/后置依赖, 不可删除'))
# # 查询是否有任务依赖
# ids = InterfaceModel.get_job_prep_by_interface(db.etl_db, interface_id)
# job_ids = []
# out_ids = []
# for items in ids:
# if items['job_id']:
# job_ids.append(items['job_id'])
# if items['out_id']:
# out_ids.append(items['out_id'])
# if set(out_ids) - set(job_ids):
# abort(400, **make_result(status=400, msg='任务流中任务作为其他任务依赖, 请停止依赖任务后删除'))
# 删除任务流
InterfaceModel.delete_interface(db.etl_db, interface_id, user_id)
return Response(interface_id=interface_id)
def post(self):
data = self.__parse_request__()
interface = InterfaceModel(data.get('front_type'), data.get('t2BookMode'), data.get('t1BackgndURL'),
data.get('t1BackgndCOL'), data.get('t1LinkTo'), data.get('t1Tit'),
data.get('t1Separator'), data.get('t1Sub'), data.get('t1Small'),
data.get('t2RowTitle'), data.get('t2RowNumber'), data.get('t1TxtColor'))
if data.get('t2Books'):
interface.books = [BookModel.find_by_id(int(i)) for i in data.get('t2Books').split(",")]
interface.save_to_db()
return interface.json(), 200
def loadData(task: Task) -> Result:
cwd = os.getcwd()
data = task.run(name=f"{task.host.name} dataModel",
task=load_yaml,
file=f"{cwd}/data/{task.host.name}/interface.yaml")
interfaces = data.result
ifaces = []
for interface in interfaces['interfaces']:
iface = InterfaceModel(
name=interface['name'],
description=interface['description'],
ipv4=interface['ipv4']['address'],
enabled=interface['enabled'] if 'enabled' in interface else False)
ifaces.append(iface)
interfaces = InterfacesModel(interfaces=ifaces)
controller = vendorFabric(task=task, model=interfaces).controller
return controller.testConfig()
def put(self, idd):
data = self.__parse_request__()
exists = InterfaceModel.find_by_id(idd)
if not exists:
return {'message': "Interface with ['id': {}] not found".format(idd)}, 404
exists.front_type = data.get('front_type')
exists.t2BookMode = data.get('t2BookMode')
exists.t1BackgndURL = data.get('t1BackgndURL')
exists.t1BackgndCOL = data.get('t1BackgndCOL')
exists.t1LinkTo = data.get('t1LinkTo')
exists.t1Tit = data.get('t1Tit')
exists.t1Separator = data.get('t1Separator')
exists.t1Sub = data.get('t1Sub')
exists.t1Small = data.get('t1Small')
exists.t2RowTitle = data.get('t2RowTitle')
exists.t2RowNumber = data.get('t2RowNumber')
exists.t1TxtColor = data.get('t1TxtColor')
if data.get('t2Books'):
exists.books = [BookModel.find_by_id(int(i)) for i in data.get('t2Books').split(",")]
exists.save_to_db()
return exists.json(), 200
def generate_interface_tree_by_dispatch(dispatch_id):
"""
生成执行任务流树形关系
0.预处理: 任务流详情和全部依赖关系, id统一为字符串
1.构造节点: 节点递归子节点
2.计算节点层级: 找出开始节点(无入度), 入度和当前节点最大层级+1为当前节点层级, 队列中添加出度
:param dispatch_id: 调度id
:return: 任务流树形关系
"""
def get_child_node(node_id):
"""获取子节点"""
child_node = [i for i in parent if i['parent_id'] == node_id]
# 子节点
for node_item in child_node:
# 添加出度
nodes[node_id]['out'].add(node_item['interface_id'])
if node_item['interface_id'] not in nodes:
# 添加节点
nodes[node_item['interface_id']] = {
'id': node_item['interface_id'],
'name': node_item['interface_name'],
'in': {node_id},
'out': set(),
'level': 0
}
# 递归节点
get_child_node(node_item['interface_id'])
# 任务流详情
detail = InterfaceModel.get_interface_detail_by_dispatch_id(db.etl_db, dispatch_id)
# 所有任务流前后置依赖
parent = InterfaceModel.get_interface_parent_all(db.etl_db)
child = InterfaceModel.get_interface_child_all(db.etl_db)
nodes = {}
# 0.预处理: id统一为字符串
# 父节点
for item in parent:
item['interface_id'] = str(item['interface_id'])
item['parent_id'] = str(item['parent_id']) if item['parent_id'] else None
# 当前节点
detail['interface_id'] = str(detail['interface_id'])
# 子节点
for item in child:
item['interface_id'] = str(item['interface_id'])
item['child_id'] = str(item['child_id']) if item['child_id'] else None
# 1.构造节点
# 当前节点
nodes[detail['interface_id']] = {
'id': detail['interface_id'],
'name': detail['interface_name'],
'is_start': True,
'in': set(),
'out': set(),
'level': 0
}
# 节点的子节点递归
get_child_node(detail['interface_id'])
# 2.计算节点层级
node_queue = []
# 找出开始节点
for _, node in nodes.items():
node_queue.append(node) if not node['in'] else None
# 计算层级
index = 0
while index < len(node_queue):
node = node_queue[index]
if node['in']:
level = 0
for key in node['in']:
level = max(level, nodes[key]['level'])
node['level'] = level + 1
# 添加队列
for out_id in node['out']:
if out_id not in map(lambda x: x['id'], node_queue):
node_queue.append(nodes[out_id])
index += 1
return nodes
def JobUpload():
"""上传任务配置文件"""
# 文件路径
file_dir = './uploads'
# 文件类型
file_type = {'xlsx', 'xls', 'csv'}
# 异常类型
err_type = {
0: '第%s行[序号]参数不为整型',
1: '第%s行[所属任务流id]参数不为整型',
2: '第%s行[任务名称]参数不为字符串类型',
3: '第%s行[任务描述]参数不为字符串类型',
4: '第%s行[服务器id]参数不为整型',
5: '第%s行[任务目录]参数不为字符串类型',
6: '第%s行[脚本目录]参数不为字符串类型',
7: '第%s行[脚本命令]参数不为字符串类型',
8: '第%s行[依赖任务序号(本次新建任务)]参数不为整型或没按逗号分隔',
9: '第%s行[依赖任务id(已有任务)]参数不为空或整型或没按逗号分隔',
10: '第%s行[任务参数]不为空或整型或没按逗号分隔',
11: '第%s行[返回值]参数不为整型'
}
file = request.files['file']
# 获取文件名
file_name = file.filename
# 获取文件后缀
file_suffix = '.' in file_name and file_name.rsplit('.', 1)[1]
if file and file_suffix in file_type:
# 保存文件到upload目录
file_path = os.path.join(file_dir, file_name)
file.save(file_path)
# excel文件
if file_suffix in {'xlsx', 'xls'}:
data = xlrd.open_workbook(file_path)
# 只取第一个sheet
sheet = data.sheet_by_name(data.sheet_names()[0])
# 从第2行开始读取
data = []
for i in range(1, sheet.nrows):
# 取前12列
data.append(sheet.row_values(i)[:12])
# csv文件
else:
data = []
with open(file_path, "r") as csv_file:
reader = csv.reader(csv_file)
for index, line in enumerate(reader):
if index > 0:
# 取前12列
data.append(line[:12])
# 异常原因
err_msg = []
# 任务流id列表
interface_result = InterfaceModel.get_interface_id_list(db.etl_db)
interface_ids = [i['interface_id'] for i in interface_result]
# 服务器id列表
exec_host_result = ExecHostModel.get_exec_host_all(db.etl_db)
exec_host_ids = [i['server_id'] for i in exec_host_result]
# 依赖任务id(已有任务)列表
job_result = JobModel.get_job_list_all(db.etl_db)
job_ids = [i['job_id'] for i in job_result]
# 任务参数列表
job_params = ParamsModel.get_params_all(db.etl_db)
job_params_ids = [i['param_id'] for i in job_params]
# 文件为空
if not data:
err_msg.append('文件为空')
# [依赖任务序号(本次新建任务)]列表
curr_job_num = []
for index, row in enumerate(data):
# Excel行号
row_num = index + 2
# 校验列数
if len(row) < 10:
err_msg.append('第%s行参数个数小于10个' % row_num)
else:
# 校验并处理每列参数
for i, param in enumerate(row):
try:
# int类型参数
if i in [0, 1, 4, 11]:
row[i] = int(param)
# 添加[依赖任务序号(本次新建任务)]
if i == 0:
curr_job_num.append(row[i])
# 字符串类型参数
elif i in [2, 3, 5, 6, 7]:
row[i] = str(param)
# 依赖任务
else:
if isinstance(param, str):
if param != '':
row[i] = [int(i) for i in str(param).split(',')]
else:
row[i] = []
else:
row[i] = [int(param)]
except:
err_msg.append(err_type[i] % row_num)
for i, param in enumerate(row):
# [所属任务流id]判空
if i == 1 and isinstance(param, int):
if param not in interface_ids:
err_msg.append('第%s行[所属任务流id]不存在' % row_num)
# [任务名称]判空
if i == 2 and param == '':
err_msg.append('第%s行[任务名称]参数不得为空' % row_num)
# [任务名称]数据库查重
if i == 2 and param != '':
if JobModel.get_job_detail_by_name(db.etl_db, param):
err_msg.append('第%s行[任务名称]参数已存在数据库中' % row_num)
# [服务器id]判空
if i == 4 and isinstance(param, int):
if param not in exec_host_ids:
err_msg.append('第%s行[服务器id]不存在' % row_num)
# [任务目录]判空和","字符串
if i == 5:
if param == '':
err_msg.append('第%s行[任务目录]参数不得为空' % row_num)
elif re.findall(',', param):
err_msg.append('第%s行[任务目录]参数不得出现逗号字符","' % row_num)
# [脚本目录]判空
if i == 6 and param == '':
err_msg.append('第%s行[脚本目录]参数不得为空' % row_num)
# [脚本命令]判空
if i == 7 and param == '':
err_msg.append('第%s行[脚本命令]参数不得为空' % row_num)
# [依赖任务序号(本次新建任务)]数据库判空
if i == 8 and isinstance(param, list):
for job_num in param:
if job_num not in curr_job_num:
err_msg.append('第%s行[依赖任务序号(本次新建任务)][%s]不存在' % (row_num, job_num))
# [依赖任务id(已有任务)]数据库判空
if i == 9 and isinstance(param, list):
for job_id in param:
if job_id not in job_ids:
err_msg.append('第%s行[依赖任务id(已有任务)][%s]不存在' % (row_num, job_id))
# [任务参数]数据库判空
if i == 10 and isinstance(param, list):
for job_param in param:
if job_param not in job_params_ids:
err_msg.append('第%s行[任务参数][%s]不存在' % (row_num, job_param))
# [序号]是否重复
serial_num = [row[0] for row in data]
if len(serial_num) != len(set(serial_num)):
err_msg.append('该文件中[序号]存在重复')
# 返回异常信息
if err_msg:
# 删除文件
os.remove(file_path)
return jsonify({'status': 401, 'msg': '文件类型错误', 'data': {'err_msg': err_msg}})
# 写入数据库
else:
# 用户id
user_info = curr_session.get_info()
user_id = user_info['id']
# 依赖任务序号 -> 任务id映射
job_map = {}
for line in data:
# 新增任务详情
job_id = JobModel.add_job_detail(db.etl_db, line[2], line[1], line[3], line[5], line[4], line[6],
line[7], line[11], user_id)
# 添加映射
job_map[line[0]] = job_id
# 表格数据中新增任务id字段
line.append(job_id)
# 新增任务依赖
prep_data = []
for line in data:
prep_job = []
# 本次新增任务序号与任务id映射
if line[8]:
prep_job = [job_map[i] for i in line[8]]
if line[9]:
# 合并数组
prep_job += line[9]
for prep_id in prep_job:
prep_data.append({
# 最后一位为新增的任务id字段
'job_id': line[len(line) - 1],
'prep_id': prep_id,
'user_id': user_id,
'insert_time': int(time.time()),
'update_time': int(time.time())
})
JobModel.add_job_prep(db.etl_db, prep_data) if prep_data else None
# 新增任务参数
job_params = []
for line in data:
for param_id in line[10]:
job_params.append({
# 最后一位为新增的任务id字段
'job_id': line[len(line) - 1],
'param_id': param_id,
'insert_time': int(time.time()),
'update_time': int(time.time()),
'user_id': user_id
})
JobModel.add_job_param(db.etl_db, job_params) if job_params else None
# 删除文件
os.remove(file_path)
return jsonify({'status': 200, 'msg': '成功', 'data': {}})
else:
return jsonify({'status': 400, 'msg': '文件类型错误', 'data': {}})
def InterfaceUpload():
"""上传任务流配置文件"""
# 文件路径
file_dir = './uploads'
# 文件类型
file_type = {'xlsx', 'xls', 'csv'}
# 异常类型
err_type = {
0: '第%s行[序号]参数不为整型',
1: '第%s行[任务流名称]参数不为字符串类型',
2: '第%s行[任务流描述]参数不为字符串类型',
3: '第%s行[任务流目录]参数不为字符串类型',
5: '第%s行[重试次数]参数不为整型',
6: '第%s行[依赖任务流序号(本次新建任务流)]参数不为整型或没按逗号分隔',
7: '第%s行[依赖任务流id(已有任务流)]参数不为空或整型或没按逗号分隔'
}
file = request.files['file']
# 获取文件名
file_name = file.filename
# 获取文件后缀
file_suffix = '.' in file_name and file_name.rsplit('.', 1)[1]
if file and file_suffix in file_type:
# 保存文件到upload目录
file_path = os.path.join(file_dir, file_name)
file.save(file_path)
# 异常原因
err_msg = []
# excel文件
if file_suffix in {'xlsx', 'xls'}:
data = xlrd.open_workbook(file_path)
# 只取第一个sheet
sheet = data.sheet_by_name(data.sheet_names()[0])
# 从第2行开始读取
data = []
for i in range(1, sheet.nrows):
# 取前8列
item = sheet.row_values(i)[:8]
try:
# 日期格式转换
if item[4]:
item[4] = xlrd.xldate_as_tuple(item[4], 0)
item[4] = datetime.date(*item[4][:3]).strftime('%Y-%m-%d')
else:
item[4] = None
except:
item[4] = None
err_msg.append('第%s行[数据日期]参数格式错误' % (i + 1))
data.append(item)
# csv文件
else:
data = []
with open(file_path, "r") as csv_file:
reader = csv.reader(csv_file)
for index, line in enumerate(reader):
if index > 0:
# 取前8列
item = line[:8]
try:
# 日期格式转换
date_value = re.findall(r'(\d{4}).(\d{1,2}).(\d{1,2})', item[4])
if date_value:
item[4] = '%04d-%02d-%02d' % tuple(int(item) for item in date_value[0])
else:
item[4] = (date.today() + timedelta(days=-1)).strftime('%Y-%m-%d')
except:
item[4] = None
err_msg.append('第%s行[数据日期]参数格式错误' % (index + 1))
data.append(item)
# 任务流id列表
interface_result = InterfaceModel.get_interface_id_list(db.etl_db)
interface_ids = [i['interface_id'] for i in interface_result]
# 文件为空
if not data:
err_msg.append('文件为空')
# [依赖任务流序号(本次新建任务流)]列表
curr_interface_num = []
for index, row in enumerate(data):
# Excel行号
row_num = index + 2
# 校验列数
if len(row) < 8:
err_msg.append('第%s行参数个数小于8个' % row_num)
else:
# 校验并处理每列参数
for i, param in enumerate(row):
try:
# int类型参数
if i in [0, 5]:
row[i] = int(param)
# 添加[依赖任务流序号(本次新建任务流)]
if i == 0:
curr_interface_num.append(row[i])
# 字符串类型参数
elif i in [1, 2, 3]:
row[i] = str(param)
# 依赖任务流
elif i in [6, 7]:
if isinstance(param, str):
if param != '':
row[i] = [int(i) for i in str(param).split(',')]
else:
row[i] = []
else:
row[i] = [int(param)]
except:
err_msg.append(err_type[i] % row_num)
for i, param in enumerate(row):
# [任务流名称]判空
if i == 1 and param == '':
err_msg.append('第%s行[任务流名称]参数不得为空' % row_num)
# [任务流名称]数据库查重
if i == 1 and param != '':
if InterfaceModel.get_interface_detail_by_name(db.etl_db, param):
err_msg.append('第%s行[任务流名称]参数已存在数据库中' % row_num)
# [任务流目录]判空
if i == 3 and param == '':
err_msg.append('第%s行[任务流目录]参数不得为空' % row_num)
# [重试次数]次数限制
if i == 5 and isinstance(param, int) and (param < 0 or param > 10):
err_msg.append('第%s行[重试次数]参数请限制在0-10之内' % row_num)
# [依赖任务流序号(本次新建任务流)]判空
if i == 6 and isinstance(param, list):
for item in param:
if item not in curr_interface_num:
err_msg.append('第%s行[依赖任务流序号(本次新建任务流)][%s]不存在' % (row_num, item))
# [依赖任务流id(已有任务流)]数据库判空
if i == 7 and isinstance(param, list):
for item in param:
if item not in interface_ids:
err_msg.append('第%s行[依赖任务流id(已有任务流)][%s]不存在' % (row_num, item))
# [序号]是否重复
serial_num = [row[0] for row in data]
if len(serial_num) != len(set(serial_num)):
err_msg.append('该文件中[序号]存在重复')
# [任务流名称]文件内查重
serial_name = [row[1] for row in data]
if len(serial_name) != len(set(serial_name)):
err_msg.append('该文件中[任务流名称]存在重复')
# 返回异常信息
if err_msg:
# 删除文件
os.remove(file_path)
return jsonify({'status': 401, 'msg': '文件参数错误', 'data': {'err_msg': err_msg}})
# 写入数据库
else:
# 用户id
user_info = curr_session.get_info()
user_id = user_info['id']
# 依赖任务流序号 -> 任务流id映射
interface_map = {}
for item in data:
# 新增任务流
interface_id = InterfaceModel.add_interface(db.etl_db, item[1], item[2], item[3], item[4], item[5],
user_id)
# 添加映射
interface_map[item[0]] = interface_id
# 表格数据中新增任务流id字段
item.append(interface_id)
# 新增任务流依赖
parent_arr = []
for line in data:
parent_interface = []
# [依赖任务流序号(本次新建任务流)]映射
if line[6]:
parent_interface = [interface_map[i] for i in line[6]]
# [依赖任务流id(已有任务流)]
if line[7]:
# 合并数组
parent_interface += line[7]
for parent_id in parent_interface:
parent_arr.append({
# 最后一位为新增的任务id字段
'interface_id': line[len(line) - 1],
'parent_id': parent_id,
'insert_time': int(time.time()),
'update_time': int(time.time()),
'creator_id': user_id,
'updater_id': user_id
})
InterfaceModel.add_interface_parent(db.etl_db, parent_arr) if parent_arr else None
# 删除文件
os.remove(file_path)
return jsonify({'status': 200, 'msg': '成功', 'data': {}})
else:
return jsonify({'status': 400, 'msg': '文件类型错误', 'data': {}})
def get_interface_id_list():
"""获取接口id列表"""
result = InterfaceModel.get_interface_id_list(db.etl_db)
return Response(result=result)
def get(self, idd):
interface = InterfaceModel.find_by_id(idd)
if interface:
return interface.json(), 200
return {'message': "Interface with ['id': {}] not found".format(idd)}, 404
def get_interface_detail(interface_id):
"""获取接口详情"""
# 接口详情
detail = InterfaceModel.get_interface_detail(db.etl_db, interface_id)
return Response(detail=detail)
def get_interface_graph(interface_id):
"""获取接口拓扑结构"""
# 接口任务依赖
job_nodes = InterfaceModel.get_interface_graph(db.etl_db, interface_id)
return Response(job_nodes=job_nodes)
def get(self, id_interface):
interface = InterfaceModel.find_by_id(id_interface)
if not interface:
return {'message': "Interface with ['id': {}] not found".format(id_interface)}, 404
return {'books': [i.json() for i in interface.books]}, 200
def generate_interface_dag_by_dispatch(dispatch_id, is_after=1):
"""
生成执行任务流前后依赖关系
0.预处理: 任务流详情和全部依赖关系, id统一为字符串
1.构造节点: 节点获取一层父节点, 递归子节点
2.计算节点层级: 找出开始节点(无入度), 入度和当前节点最大层级+1为当前节点层级, 队列中添加出度
:param dispatch_id: 调度id
:param is_after: 是否触发后置任务流
:return: 任务流依赖关系
"""
def get_context_node(node_id, after=1, before=1):
"""获取上下文节点"""
parent_node = [i for i in child if i['child_id'] == node_id]
child_node = [i for i in parent if i['parent_id'] == node_id]
# 父节点(局部拓扑情况下, 所有节点的父节点无需递归, 初始节点不添加父节点)
if before:
for parent_item in parent_node:
# 添加入度
nodes[node_id]['in'].add(parent_item['interface_id'])
if parent_item['interface_id'] not in nodes:
# 添加节点
nodes[parent_item['interface_id']] = {
'id': parent_item['interface_id'],
'name': parent_item['interface_name'],
'in': set(),
'out': {node_id},
'level': 0
}
else:
nodes[parent_item['interface_id']]['out'].add(node_id)
# 子节点
if after:
for child_item in child_node:
# 添加出度
nodes[node_id]['out'].add(child_item['interface_id'])
if child_item['interface_id'] not in nodes:
# 添加节点
nodes[child_item['interface_id']] = {
'id': child_item['interface_id'],
'name': child_item['interface_name'],
'in': {node_id},
'out': set(),
'level': 0
}
else:
nodes[child_item['interface_id']]['in'].add(node_id)
# 递归节点
get_context_node(child_item['interface_id'])
# 任务流详情
detail = InterfaceModel.get_interface_detail_by_dispatch_id(db.etl_db, dispatch_id)
# 所有任务流前后置依赖
parent = InterfaceModel.get_interface_parent_all(db.etl_db)
child = InterfaceModel.get_interface_child_all(db.etl_db)
nodes = {}
# 0.预处理: id统一为字符串
# 父节点
for item in parent:
item['interface_id'] = str(item['interface_id'])
item['parent_id'] = str(item['parent_id']) if item['parent_id'] else None
# 当前节点
detail['interface_id'] = str(detail['interface_id'])
# 子节点
for item in child:
item['interface_id'] = str(item['interface_id'])
item['child_id'] = str(item['child_id']) if item['child_id'] else None
# 1.构造节点
# 当前节点
nodes[detail['interface_id']] = {
'id': detail['interface_id'],
'name': detail['interface_name'],
'is_start': True,
'in': set(),
'out': set(),
'level': 0
}
# 节点上下文递归
get_context_node(detail['interface_id'], after=is_after, before=0)
# 2.计算节点层级
node_queue = []
# 找出开始节点
for _, node in nodes.items():
node_queue.append(node) if not node['in'] else None
# 计算层级
index = 0
while index < len(node_queue):
node = node_queue[index]
if node['in']:
level = 0
for key in node['in']:
level = max(level, nodes[key]['level'])
node['level'] = level + 1
# 添加队列
for out_id in node['out']:
if out_id not in map(lambda x: x['id'], node_queue):
node_queue.append(nodes[out_id])
index += 1
return nodes
def add_interface(interface_name, interface_desc, retry, user_id):
"""新增接口"""
interface_id = InterfaceModel.add_interface(db.etl_db, interface_name,
interface_desc, retry,
user_id)
return Response(interface_id=interface_id)
def continue_execute_interface_all(exec_id,
result=None,
exec_type=1,
run_date=''):
"""
获取可执行任务流
1.如果所有执行任务流都完成, 修改执行主表状态[成功]
2.所有任务流都完成, 修改执行主表状态[成功], 返回退出
3.获取当前执行id下的任务流, 遍历任务流
3.自动调度下(exec_type=1)当前节点出度的所有入度成功, 出度的所有入度数据日期>=出度的数据日期, 节点出度的状态为待运行;
手动调度下(exec_type=2)默认所有出度成功.
4.获取可执行任务流下初始任务, 存在空任务流, 修改执行任务流状态[成功], 修改任务流数据日期, 递归本方法
5.否则修改执行任务流状态[运行中], 返回结果集
:param result: 结果集
:param exec_id: 执行id
:param exec_type: 执行类型: 1.自动, 2.手动
:param run_date: 数据日期
:return:
"""
if not run_date:
run_date = time.strftime('%Y-%m-%d', time.localtime())
# 可执行任务流id
if result is None:
result = {}
next_interface = []
# {可执行任务流id: {'job_id': [可执行任务id], 'nodes': {'job_id': {任务详情}}}}
# 推进流程
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
interface_dict = get_interface_dag_by_exec_id(exec_id)
# 已完成任务流
complete_interface = [
_ for _, item in interface_dict.items() if item['status'] == 0
]
# 所有任务流都完成
if len(complete_interface) == len(interface_dict):
# 修改执行主表状态[成功]
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
ExecuteModel.update_execute_status(db.etl_db, exec_id, 0)
return
# 遍历所有节点
for interface_id in interface_dict:
# 自动调度下, 检查出度的入度数据日期和状态是否成功
if exec_type == 1:
# 出度任务流的执行详情
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
current_detail = InterfaceModel.get_interface_detail_last_execute(
db.etl_db, interface_id)
for out_id in interface_dict[interface_id]['out_degree']:
flag = True
for in_id in interface_dict[out_id]['in_degree']:
# 获取出度的入度任务流详情
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
in_detail = InterfaceModel.get_interface_detail_last_execute(
db.etl_db, in_id)
# 1.出度的入度本次执行状态不成功, 2.出度的入度没有数据日期, 3.出度的入度数据日期小于出度的数据日期, 4.如果存在出度的上一次执行记录, 上一次执行记录不成功
if in_detail['status'] != 0 or not in_detail['run_time'] \
or in_detail['run_time'] < current_detail['run_time']:
# or (current_detail['last_status'] and current_detail['last_status'] != 0):
flag = False
break
if flag and interface_dict[out_id]['status'] == 3:
next_interface.append(out_id)
# 手动调度下, 直接通过
else:
for out_id in interface_dict[interface_id]['out_degree']:
flag = True
for in_id in interface_dict[out_id]['in_degree']:
# 获取出度的入度详情
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
in_detail = InterfaceModel.get_interface_detail_last_execute(
db.etl_db, in_id)
# 1.出度的入度本次执行状态不成功
if in_detail['status'] != 0:
flag = False
break
if flag and interface_dict[out_id]['status'] == 3:
next_interface.append(out_id)
# 获取所有层级可执行任务
for next_interface_id in set(next_interface):
nodes = get_job_dag_by_exec_id(exec_id, next_interface_id)
# 可执行任务流设置默认可执行任务
result.setdefault(next_interface_id, {
'nodes': nodes,
'job_id': []
})
# 遍历所有节点
for job_id in nodes:
# 初始节点
if nodes[job_id]['level'] == 0 and nodes[job_id]['status'] in (
'preparing', 'ready'):
result[next_interface_id]['job_id'].append(job_id)
# 出度任务流中符合条件的任务为空, 寻找下一个可执行任务流
flag = False
result_deep = deepcopy(result)
for interface_id, item in result_deep.items():
# 修改执行任务流状态[成功]
if not item['job_id']:
flag = True
result.pop(interface_id)
log.info('任务流中任务为空: 执行id: %s, 任务流id: %s' % (exec_id, interface_id))
# 数据日期改成当天日期, 手动调度时可以再优化
new_date = time.strftime('%Y-%m-%d', time.localtime())
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
ExecuteModel.update_interface_run_time(db.etl_db, interface_id,
new_date)
ExecuteModel.update_exec_interface_status(
db.etl_db, exec_id, interface_id, 0)
# 修改执行任务流状态[运行中]
else:
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % exec_id):
ExecuteModel.update_exec_interface_status(
db.etl_db, exec_id, interface_id, 1)
# 存在空任务流
if flag:
return continue_execute_interface_all(exec_id, result, exec_type,
run_date)
else:
return result
def get_interface_index():
"""获取所有任务流目录"""
result = InterfaceModel.get_interface_index(db.etl_db)
return Response(result=result)