def get(self, request):
"""获取指定日期的验证数据"""
date = request.GET.get('date', None)
if date is None:
return Response(BaseResponse(code=400, message="日期必填!").context())
return Response(
BaseResponse(
data=smh().filter_schedule_result_by_date(date)).context())
def __init__(self, case_id):
self.db2_handler = Db2Handler()
self.model_handler = smh()
self.case_id = case_id
self.is_all_schedule_flag = True if case_id == "" or case_id is None else False
self.need_run_data = None
self.dispatch_no = None
self.schedule_config_id = None
self.need_assert_data = None
def get(self, request):
"""获取指定日期的验证数据"""
dispatch_no = request.GET.get('dispatch_no', None)
if dispatch_no is None or dispatch_no == '':
return Response(
BaseResponse(code=400, message="任务号必填!!!").context())
return Response(
BaseResponse(data=smh().filter_report_by_dispatch_no(
dispatch_no)).context())
def assert_data(self, assert_table):
logger.info(f"assert_data >>>>>> 开始进行数据验证")
self.need_assert_data = smh().get_all_no_asserted_dispatch({
"dispatch_no":
self.dispatch_no,
"is_assert":
False,
"schedule_config_id":
self.schedule_config_id,
"is_clean":
True,
"is_ready":
True
})
if len(self.need_assert_data) == 0:
return False, "无对应未完成的验证任务"
try:
# 根据设定ID取对应的配置项
for need_assert_data in json.loads(assert_table):
result = self.db2_handler.get_assert_result(
assert_table_name=need_assert_data['name'],
assert_tag=need_assert_data['tag'],
order=need_assert_data['order'],
eliminate=need_assert_data['eliminate'])
status = True
for i in json.loads(result):
if not i["assert_flag"]:
status = False
serializer = ScheduleResultSerializer(
data={
"dispatch_no": self.dispatch_no,
"schedule_config_id": self.schedule_config_id,
"result_info": result,
"status": status
})
if serializer.is_valid():
serializer.save()
# 回写任务明细表验证状态
self.model_handler.set_dispath_detail_flag(
self.dispatch_no, self.schedule_config_id, "is_assert")
except Exception as e:
logger.error(repr(e))
return False, f"{repr(e)}"
else:
return True, "执行成功"
def set_dispath_list_is_done(self):
# 处理完单个任务再捞一遍所有的待assert任务
need_assert_data = smh().get_all_no_asserted_dispatch({
"dispatch_no":
self.dispatch_no,
"is_assert":
False,
"is_clean":
True,
"is_ready":
True
})
# 若assert任务为0即为全部处理完毕
if len(need_assert_data) == 0:
self.model_handler.set_dispath_list_flag()
return True, "执行成功"
def post(self, request):
"""
新增设定
"""
logger.info(f"增加case >>>>>> 新增设定,入参：{request.data}")
format_request_data = {
'batch_group_name': request.data.get('batch_group_name', None),
"batch_group_desc": request.data.get('batch_group_desc', None),
'job': request.data.get('job', None),
}
model_handler = smh()
old_seq_no = model_handler.get_seq_no()
config_data = []
try:
for jobs_info in format_request_data['job']:
data = {
"case_id":
str(old_seq_no + 1).zfill(7),
"batch_group_name":
format_request_data['batch_group_name'],
"batch_group_desc":
format_request_data['batch_group_desc'],
"pacakge_name":
jobs_info['pacakge_name'],
"batch_job_name":
jobs_info['batch_job_name'],
"batch_job_desc":
jobs_info['batch_job_desc'],
"batch_job_param":
jobs_info['batch_job_param'],
"priority":
jobs_info['priority'],
"clean_table":
json.dumps(jobs_info['batch_data']['clean']['table']),
"ready_table":
json.dumps(jobs_info['batch_data']['ready']['table']),
"assert_table":
json.dumps(jobs_info['batch_data']['assert']['table']),
}
schedule_config_serializer = ScheduleConfigSerializer(
data=data)
if schedule_config_serializer.is_valid():
config_data.append(
ScheduleConfig(**schedule_config_serializer.data))
else:
logger.error(
f"增加case >>>>>> 序列化失败 {schedule_config_serializer.errors}"
)
return Response(
BaseResponse(
code=400,
message=
f"增加case >>>>>> 序列化失败 {schedule_config_serializer.errors}"
).context())
except Exception as e:
logger.error(repr(e))
return Response(BaseResponse(code=400, message=repr(e)).context())
try:
# 批量写入表中
ScheduleConfig.objects.bulk_create(config_data)
except Exception as e:
logger.error(repr(e))
return Response(
BaseResponse(
code=400,
message=f"增加case >>>>>> 批量写入表失败 {repr(e)}").context())
else:
if model_handler.set_seq_no(old_seq_no):
return Response(
BaseResponse(message="增加case >>>>>> 新增设定成功").context())
def post(self, request):
"""对已clean和已ready的任务进行验证数据"""
batch_job_name = request.POST.get('batch_job_name', None)
logger.info(f"batch任务 >>>>>> 数据验证接口,入参：{batch_job_name}")
model_handler = smh()
# 验证全部任务
if batch_job_name == '' or batch_job_name is None:
logger.info(f"batch任务 >>>>>> 验证批量任务")
# 任务调度增加一条主任务
no_asserted_dispatch = model_handler.get_all_no_asserted_dispatch({
"is_assert":
False,
"is_clean":
True,
"is_ready":
True
})
else:
logger.info(f"batch任务 >>>>>> 验证单个任务")
# 任务调度增加一条主任务
config_data = model_handler.get_schedule_config(
{"batch_job_name": batch_job_name})
no_asserted_dispatch = smh().get_all_no_asserted_dispatch({
"is_assert":
False,
"schedule_config_id":
config_data.id,
"is_clean":
True,
"is_ready":
True
})
logger.info(
f"batch任务 >>>>>> 数据验证, 总共需要验证 {len(no_asserted_dispatch)} 个任务")
if len(no_asserted_dispatch) == 0:
return Response(
BaseResponse(code=400, message="无对应未完成的验证任务").context())
db2_handler = Db2Handler()
try:
# 先取任务明细
for dispatch_detail in no_asserted_dispatch:
schedule_config_data = model_handler.get_all_schedule_config_by_pk(
dispatch_detail.schedule_config_id)
# 根据设定ID取对应的配置项
for need_assert_data in json.loads(
schedule_config_data.assert_table):
result = db2_handler.get_assert_result(
assert_table_name=need_assert_data['name'],
assert_tag=need_assert_data['tag'],
order=need_assert_data['order'],
eliminate=need_assert_data['eliminate'])
logger.info(
f"验证设定 {schedule_config_data.batch_job_name} 完成")
serializer = ScheduleResultSerializer(
data={
"dispatch_no": dispatch_detail.dispatch_no,
"schedule_config_id":
dispatch_detail.schedule_config_id,
"result_info": result
})
if serializer.is_valid():
serializer.save()
# 回写任务明细表验证状态
model_handler.set_dispath_detail_flag(
dispatch_detail.dispatch_no,
dispatch_detail.schedule_config_id, "is_assert")
except Exception as e:
db2_handler.close_conn()
logger.error(repr(e))
return Response(BaseResponse(code=400, message=repr(e)).context())
else:
db2_handler.close_conn()
no_asserted_dispatch = smh().get_all_no_asserted_dispatch({
"is_assert":
False,
"is_clean":
True,
"is_ready":
True
})
if len(no_asserted_dispatch) == 0:
model_handler.set_dispath_list_flag()
return Response(BaseResponse(message="数据验证成功").context())
def post(self, request):
"""任务调度"""
batch_job_name = request.POST.get('batch_job_name', None)
logger.info(f"batch任务 >>>>>> 任务调度接口,入参：{batch_job_name}")
model_handler = smh()
if ModelUtil(sdl_model).check_data_exist(is_done=False)[0]:
return Response(
BaseResponse(code=400, message="有任务运行中,请稍后重试").context())
is_all_schedule_flag = False
# 跑全部任务
if batch_job_name == '' or batch_job_name is None:
logger.info(f"batch任务 >>>>>> 调度批量任务")
# 任务调度增加一条主任务
dispatch_no = CommonUtil().generate_dispatch_no()
is_all_schedule_flag = True
else:
logger.info(f"batch任务 >>>>>> 调度单个任务")
# 任务调度增加一条主任务
dispatch_no = CommonUtil().generate_dispatch_no(batch_job_name)
sdls_serializer = ScheduleDispatchListSerializer(
data={"dispatch_no": dispatch_no})
if sdls_serializer.is_valid():
sdls_serializer.save()
else:
return Response(
BaseResponse(code=400,
message=sdls_serializer.errors).context())
# 任务明细增加对应明细任务
# 查询对应的设定
if is_all_schedule_flag:
config_data = model_handler.get_all_schedule_config()
else:
config_data = model_handler.filter_schedule_config(
{"batch_job_name": batch_job_name})
dispatch_detail = []
for data in config_data:
pks = {"dispatch_no": dispatch_no, "schedule_config_id": data.id}
dispatch_detail.append(ScheduleDispatchDetail(**pks))
try:
# 批量写入任务明细表中
ScheduleDispatchDetail.objects.bulk_create(dispatch_detail)
except Exception as e:
logger.error(repr(e))
return Response(BaseResponse(code=400, message=repr(e)).context())
# 开始进行数据清理
# 创建db2链接
db2_handler = Db2Handler()
for get_config in config_data:
# 循环清理表数据
for table_name in json.loads(get_config.clean_table):
db2_handler.clean_data(table_name)
# 写表记录完成状态
model_handler.set_dispath_detail_flag(dispatch_no, get_config.id,
"is_clean")
# 循环准备表数据
for config_dict in json.loads(get_config.ready_table):
db2_handler.ready_data(config_dict['name'], config_dict['tag'])
model_handler.set_dispath_detail_flag(dispatch_no, get_config.id,
"is_ready")
db2_handler.close_conn()
logger.info("batch任务 >>>>>> 任务调度成功")
return Response(BaseResponse(message="任务调度成功").context())