def monitor_task5():
tasks = getStepStatus('step5')
print('-0-', tasks)
for param, values in tasks.items():
if len(values) == 0: continue
task_names = zip(*values)[0]
print('-1-', task_names)
tag = str(task_names[0].rsplit('_', 1)[-1])
print('-2-', tag)
tasks_status = from_tag_get_tasks_status(tag, True)
finaled_date = max(a[-1] for a in tasks_status)
all_finaled_data = [a for a in tasks_status if a[-1] == finaled_date]
print('-3-', tasks_status)
if len(tasks_status) < len(task_names):
print('-4-', '完蛋')
continue
status_list_len = 0
for (_0, _1, _2, l_done, l_failed, _5, l_all, d_done, d_failed, d_all,
i_done, i_failed, i_all, _13) in all_finaled_data:
#规则 1 完成任务数 + 失败任务数 = 任务总数
#规则 2 失败任务数 == 任务总数 发邮件报警
if not (l_done + l_failed == l_all and d_done + d_failed == d_all
and i_done + i_failed
== i_all) or l_failed == l_all or d_failed == d_all:
if _1 in ('Qyer', 'Daodao'): continue
print('-5-', '不行')
break
else:
print('-6-', '可以', _1)
status_list_len += 1
if status_list_len == len(task_names) - 2:
update_step_report('', param, 1, 0, 5)
modify_status('step5', param, flag=False)
print('=== %s === 任务完成' % tag)
def monitor_task4_bak():
print('running===============0')
tasks = getStepStatus('step4')
tasks_list = list(tasks.items())
if len(tasks_list) == 0:
return
collection_name = tasks_list[0][1][0]
print('0==========', collection_name)
client = pymongo.MongoClient(host='10.10.231.105')
collection = client['MongoTask'][collection_name]
total_count = collection.find({}).count()
print('01==========', total_count)
conn = pymysql.connect(**OpCity_config)
cursor = conn.cursor()
for param, (_, task_name) in tasks.items():
print(param, (_, task_name))
if total_count == 0:
update_step_report('', param, -1, 0, 4)
return
select_sql = "select step4 from city_order where id=%s"
print('1==========', select_sql, param)
cursor.execute(select_sql, (param))
status_id = cursor.fetchone()[0]
print('2==========', status_id)
if int(status_id) == 2:
results = collection.find(
{
'$and': [{
'finished': 1
}, {
'useds_times': {
'$lt': 7
}
}, {
'task_name': task_name
}]
},
hint=[('task_name', 1), ('finished', 1), ('used_times', 1)])
not_finish_num = results.count()
print('3==========', not_finish_num)
if int(not_finish_num) / int(total_count) <= 0:
update_step_report('', param, 1, 0, 4)
modify_status('step4', param, flag=False)
print('running===============1')
def monitor_step_six():
tasks = getStepStatus('step6')
tasks_list = list(tasks.items())
if len(tasks_list) == 0:
return
param = tasks.keys()[0]
os.system("rsync -avI 10.10.150.16::opcity/{0} {1}".format(
param, base_step_six_path))
step_six_path = ''.join([base_step_six_path, str(param)])
file_list = os.listdir(step_six_path)
for file_name in file_list:
if 'hotel_report' in file_name:
save_path = '/'.join([str(param), file_name])
update_step_report(save_path, param, 1)
modify_status(step, param, flag=False)
def monitor_task4():
tasks = getStepStatus('step4')
print('-0-', tasks)
for param, values in tasks.items():
if len(values) == 0: continue
task_name = values[-1]
print('-1-', task_name)
tasks_status = from_tag_get_tasks_status(task_name)
print('-2-', tasks_status)
line = tasks_status[0]
print('-3-', line)
t_all, t_done, t_failed = line[3], line[4], line[5]
if t_all == t_done + t_failed:
update_step_report('', param, 1, 0, 4)
modify_status('step4', param, flag=False)
print('-4-', '完成')
def monitor_step3(stepa):
step = 'step' + stepa
tasks = getStepStatus(step)
temp_config['db'] = 'Cityupline'
if len(tasks) == 0: return
for param, collection_names in tasks.items():
collection_name, task_name = collection_names
total_count = get_total_count(collection_name)
if int(total_count) == 0:
return '0%'
client = pymongo.MongoClient(host='10.10.231.105')
collection = client['MongoTask'][collection_name]
success_results = collection.find(
{
'finished': 1,
'used_times': {
'$lt': 7
},
'task_name': task_name
},
hint=[('task_name', 1), ('finished', 1), ('used_times', 1)])
success_finish_num = success_results.count()
failed_results = collection.find(
{
'finished': 0,
'used_times': 7,
'task_name': task_name
},
hint=[('task_name', 1), ('finished', 1), ('used_times', 1)])
failed_finish_num = failed_results.count()
logger.info(
'{0}, collections: {1} total: {2} success: {3} failed: {4}'.
format(step, collection_name, total_count, success_finish_num,
failed_finish_num))
if failed_finish_num > 0 and failed_finish_num + success_finish_num == total_count:
logger.info('{0}, {1} 失败'.format(step, collection_name))
if success_finish_num == total_count:
from_ota_get_city(temp_config, param)
modify_status(step, param, flag=False)
logger.info('{0}, {1} 成功'.format(step, collection_name))
return format(success_finish_num / total_count, '.0%')
def monitor_step3_bark():
tasks = getStepStatus('step3')
tasks_list = list(tasks.items())
if len(tasks_list) == 0:
return
conn = pymysql.connect(**OpCity_config)
cursor = conn.cursor()
save_result = []
for param, collection_names in tasks.items():
select_sql = "select step3 from city_order where id=%s"
cursor.execute(select_sql, (param))
status_id = cursor.fetchone()[0]
for collection_name, task_name in collection_names:
total_count = get_total_count(collection_name)
if int(total_count) == 0:
update_step_report('', param, -1, 0)
return
client = pymongo.MongoClient(host='10.10.231.105')
collection = client['MongoTask'][collection_name]
for collection_name, task_name in collection_names:
if int(status_id) == 2:
results = collection.find(
{
'$and': [{
'finished': 1
}, {
'useds_times': {
'$lt': 7
}
}, {
'task_name': task_name
}]
},
hint=[('task_name', 1), ('finished', 1),
('used_times', 1)])
not_finish_num = results.count()
if int(not_finish_num) / int(total_count) <= 0:
save_result.append(int(not_finish_num) / int(total_count))
if max(save_result) <= 0:
update_step_report('', param, 1, 0)
job = scheduler.get_job('step3')
job.remove()
def monitor_task_summary(step):
stepa = 'step' + step
logger.info('================= ' + stepa + ' ================= 开始')
csvpath = ''
tasks = getStepStatus(stepa)
for param, values in tasks.items():
if len(values) == 0: return
if type(values[0]) is list:
task_naems = list(zip(*values))[1]
else:
task_naems = [values[1]]
the_progress_of = 0
for task_name in task_naems:
logger.info('{}, {}'.format(stepa, task_name))
tasks_status = from_tag_get_tasks_status(task_name)
logger.info('{}, {}'.format(stepa, tasks_status))
line = tasks_status[0]
t_all, t_done, t_failed = line[3], line[4], line[5]
if t_all == t_done + t_failed:
the_progress_of += 1
if the_progress_of == len(task_naems):
if step == '7':
tag = task_name.rsplit('_')[-1]
step7_detection(tag)
# if not get_file(param, 'poireport.csv'):
elif step == '8':
if not make_image_content_report(t_all, t_done, t_failed,
param):
return
csvpath = '{}/merge_image_and_content.txt'.format(param)
if step in ('4', '9'):
update_step_report(csvpath, param, 4, 0, int(step))
elif step not in ('6', '7'):
update_step_report(csvpath, param, 1, 0, int(step))
modify_status(stepa, param, flag=False)
logger.info('================= ' + stepa +
' ================= 完成')
logger.info('================= ' + stepa + ' ================= 1')
def monitor_report(step):
stepa = 'step' + step
logger.info('================= ' + stepa + ' ================= 开始')
tasks = getStepStatus(stepa)
for param, values in tasks.items():
if len(values) == 0: continue
task_names = zip(*values).__next__()
logger.info('{}, {}'.format(stepa, task_names))
tag = str(task_names[0].rsplit('_', 1)[-1])
logger.info('{}, {}'.format(stepa, tag))
tasks_status = from_tag_get_tasks_status(tag, True)
finaled_date = max(a[-1] for a in tasks_status)
all_finaled_data = [a for a in tasks_status if a[-1] == finaled_date]
logger.info('{}, {}'.format(stepa, tasks_status))
if len(tasks_status) < len(task_names):
logger.info('{}, {}'.format(stepa, '任务状态数 小于 任务已发任务数'))
continue
status_list_len = 0
for (_0, source, _2, l_done, l_failed, _5, l_all, d_done, d_failed,
d_all, i_done, i_failed, i_all, _13) in all_finaled_data:
#规则 1 完成任务数 + 失败任务数 = 任务总数
#规则 2 失败任务数 == 任务总数 发邮件报警
if not (l_done + l_failed == l_all and d_done + d_failed == d_all
and i_done + i_failed
== i_all) or l_failed == l_all or d_failed == d_all:
if source in ('Qyer', 'Daodao'): continue
logger.info('{}, {}: {}'.format(stepa, '未完成', source))
break
else:
logger.info('{}, {}: {}'.format(stepa, '已完成', source))
status_list_len += 1
if status_list_len >= len(task_names):
modify_status(stepa, param, flag=False)
logger.info('{}, 开始生成报表'.format(stepa))
csv_file = make_poi_and_hotel_report(all_finaled_data, param)
update_step_report(csv_file, param, 1, 0, int(step))
logger.info('================= ' + stepa + ' ================= 完成')
logger.info('================= ' + stepa + ' ================= 1')