def __exec_step09_task(self, target_classify):
try:
daily_start_date = get_current_time()
exec_temp_api_bill_sql_by_target(target_classify=target_classify,
is_log=False)
process_status = 'sucess'
daily_end_date = get_current_time()
step_number = '9'
operate_desc = f'成功执行聚合临时表API表的目标分类为{target_classify}的SQL'
orgin_source = 'kudu分析表/落地表'
destin_source = '汇总API中间表'
importdate = get_yyyymmdd_date()
insert_finance_data_process(process_status, target_classify,
daily_start_date, daily_end_date,
step_number, operate_desc,
orgin_source, destin_source,
importdate)
except Exception as e:
log.error(f'* 第9步，执行SQL报错')
print(e)
process_status = 'false'
daily_end_date = get_current_time()
step_number = '9'
operate_desc = ''
orgin_source = 'kudu分析表/落地表'
destin_source = '汇总API中间表'
importdate = get_yyyymmdd_date()
insert_finance_data_process(process_status, target_classify,
daily_start_date, daily_end_date,
step_number, operate_desc,
orgin_source, destin_source,
importdate)
def __exec_step07_task(self, records, cost_project):
daily_start_date = get_current_time()
for record in records:
unusual_shell = str(record[0])
isalgorithm = str(record[1])
unusual_id = str(record[2])
try:
# time.sleep(1)
if isalgorithm == '1':
### 执行 SQL ###
prod_execute_sql(conn_type=CONN_TYPE,
sqltype='insert',
sql=unusual_shell)
log.info(
f'* 第7步，成功执行 cost_project为 {cost_project}, unusual_id为 {unusual_id} 的SQL'
)
elif isalgorithm == '2':
### 执行算法 python 脚本 ###
exec(unusual_shell, globals())
log.info(
f'* 第7步，成功执行 cost_project为 {cost_project}, unusual_id为 {unusual_id} 的Python Shell'
)
process_status = 'sucess'
daily_end_date = get_current_time()
step_number = '7'
operate_desc = f'成功执行目标分类为 {cost_project} 的SQL'
orgin_source = 'kudu分析表'
destin_source = 'kudu落地表'
importdate = get_yyyymmdd_date()
insert_finance_data_process(process_status, cost_project,
daily_start_date, daily_end_date,
step_number, operate_desc,
orgin_source, destin_source,
importdate)
except Exception as e:
daily_source = 'SQL' if isalgorithm == '1' else 'Python Shell'
log.error(
f'* 执行第7步，目标分类为 {cost_project} ,unusual_id为 {unusual_id} 的 {daily_source} 报错'
)
print(e)
process_status = 'false'
daily_end_date = get_current_time()
step_number = '7'
operate_desc = str(e)
orgin_source = 'kudu分析表'
destin_source = 'kudu落地表'
importdate = get_yyyymmdd_date()
insert_finance_data_process(process_status, cost_project,
daily_start_date, daily_end_date,
step_number, operate_desc,
orgin_source, destin_source,
importdate)
def __exec_step08_task(self, records, target_classify):
daily_start_date = get_current_time()
try:
for record in records:
sql = str(record[0])
order_number = str(record[1])
prod_execute_sql(conn_type=CONN_TYPE,
sqltype='insert',
sql=sql)
log.info(
f'* 第8步，成功执行序号为 {order_number} ,target_classify为 {target_classify} 的SQL'
)
process_status = 'sucess'
daily_end_date = get_current_time()
step_number = '8'
operate_desc = f'成功执行绩效接口API表的目标分类为{target_classify}的SQL'
orgin_source = 'kudu分析表/落地表'
destin_source = '绩效API中间表'
importdate = get_yyyymmdd_date()
insert_finance_data_process(process_status, target_classify,
daily_start_date, daily_end_date,
step_number, operate_desc,
orgin_source, destin_source,
importdate)
except Exception as e:
log.error(
f'* 执行第8步，序号为 {order_number} 的SQL报错,target_classify为 {target_classify} 的SQL'
)
# print(sql)
print(e)
process_status = 'false'
daily_end_date = get_current_time()
step_number = '8'
operate_desc = str(e)
orgin_source = 'kudu分析表/落地表'
destin_source = '绩效API中间表'
importdate = get_yyyymmdd_date()
insert_finance_data_process(process_status, target_classify,
daily_start_date, daily_end_date,
step_number, operate_desc,
orgin_source, destin_source,
importdate)
def __exec_step06_task(self, records, target_classify):
try:
daily_start_date = get_current_time()
for idx, record in enumerate(records):
sql = str(record[0])
receipt_id = str(record[1])
prod_execute_sql(conn_type=CONN_TYPE,
sqltype='insert',
sql=sql)
log.info(
f'* 第6步，成功执行 target_classify为 {target_classify},receipt_id为 {receipt_id} 的SQL'
)
process_status = 'sucess'
daily_end_date = get_current_time()
step_number = '6'
operate_desc = f'成功执行目标分类为{target_classify}的SQL'
orgin_source = '发票信息hive表'
destin_source = 'kudu分析表'
importdate = get_yyyymmdd_date()
insert_finance_data_process(process_status, target_classify,
daily_start_date, daily_end_date,
step_number, operate_desc,
orgin_source, destin_source,
importdate)
except Exception as e:
log.error(f'* 执行第6步，目标分类为{target_classify} ，序号为{receipt_id}的SQL报错')
log.info(sql)
print(e)
process_status = 'false'
daily_end_date = get_current_time()
step_number = '6'
operate_desc = str(e)
orgin_source = '发票信息hive表'
destin_source = 'kudu分析表'
importdate = get_yyyymmdd_date()
insert_finance_data_process(process_status, target_classify,
daily_start_date, daily_end_date,
step_number, operate_desc,
orgin_source, destin_source,
importdate)
def exec_step07(self):
"""
执行第7步
7、稽查点sql将数据写到kudu落地表（脚本）
:return:
"""
travel_fee = '差旅费'
meeting_fee = '会议费'
office_fee = '办公费'
car_fee = '车辆使用费'
daily_start_date = get_current_time()
try:
travel_records = query_finance_unusual(cost_project=travel_fee)
except Exception as e:
log.error(f'* 执行第7步，查询目标分类为 {travel_fee} 的SQL报错')
print(e)
process_status = 'false'
daily_end_date = get_current_time()
step_number = '7'
operate_desc = str(e)
orgin_source = 'kudu分析表'
destin_source = 'kudu落地表'
importdate = get_yyyymmdd_date()
insert_finance_data_process(process_status, travel_fee,
daily_start_date, daily_end_date,
step_number, operate_desc,
orgin_source, destin_source,
importdate)
try:
meeting_records = query_finance_unusual(cost_project=meeting_fee)
except Exception as e:
log.error(f'* 执行第7步，查询目标分类为 {travel_fee} 的SQL报错')
print(e)
process_status = 'false'
daily_end_date = get_current_time()
step_number = '7'
operate_desc = str(e)
orgin_source = 'kudu分析表'
destin_source = 'kudu落地表'
importdate = get_yyyymmdd_date()
insert_finance_data_process(process_status, travel_fee,
daily_start_date, daily_end_date,
step_number, operate_desc,
orgin_source, destin_source,
importdate)
try:
office_records = query_finance_unusual(cost_project=office_fee)
except Exception as e:
log.error(f'* 执行第7步，查询目标分类为 {travel_fee} 的SQL报错')
print(e)
process_status = 'false'
daily_end_date = get_current_time()
step_number = '7'
operate_desc = str(e)
orgin_source = 'kudu分析表'
destin_source = 'kudu落地表'
importdate = get_yyyymmdd_date()
insert_finance_data_process(process_status, travel_fee,
daily_start_date, daily_end_date,
step_number, operate_desc,
orgin_source, destin_source,
importdate)
try:
car_records = query_finance_unusual(cost_project=car_fee)
except Exception as e:
log.error(f'* 执行第7步，查询目标分类为 {travel_fee} 的SQL报错')
print(e)
process_status = 'false'
daily_end_date = get_current_time()
step_number = '7'
operate_desc = str(e)
orgin_source = 'kudu分析表'
destin_source = 'kudu落地表'
importdate = get_yyyymmdd_date()
insert_finance_data_process(process_status, travel_fee,
daily_start_date, daily_end_date,
step_number, operate_desc,
orgin_source, destin_source,
importdate)
threadPool = ThreadPoolExecutor(max_workers=4,
thread_name_prefix="thr")
all_task = []
task1 = threadPool.submit(self.__exec_step07_task, travel_records,
travel_fee)
task2 = threadPool.submit(self.__exec_step07_task, meeting_records,
meeting_fee)
task3 = threadPool.submit(self.__exec_step07_task, office_records,
office_fee)
task4 = threadPool.submit(self.__exec_step07_task, car_records,
car_fee)
all_task.append(task1)
all_task.append(task2)
all_task.append(task3)
all_task.append(task4)
wait(all_task, return_when=ALL_COMPLETED)
threadPool.shutdown(wait=True)