def monitor_task5():
tasks = getStepStatus('step5')
print('-0-', tasks)
for param, values in tasks.items():
if len(values) == 0: continue
task_names = zip(*values)[0]
print('-1-', task_names)
tag = str(task_names[0].rsplit('_', 1)[-1])
print('-2-', tag)
tasks_status = from_tag_get_tasks_status(tag, True)
finaled_date = max(a[-1] for a in tasks_status)
all_finaled_data = [a for a in tasks_status if a[-1] == finaled_date]
print('-3-', tasks_status)
if len(tasks_status) < len(task_names):
print('-4-', '完蛋')
continue
status_list_len = 0
for (_0, _1, _2, l_done, l_failed, _5, l_all, d_done, d_failed, d_all,
i_done, i_failed, i_all, _13) in all_finaled_data:
#规则 1 完成任务数 + 失败任务数 = 任务总数
#规则 2 失败任务数 == 任务总数 发邮件报警
if not (l_done + l_failed == l_all and d_done + d_failed == d_all
and i_done + i_failed
== i_all) or l_failed == l_all or d_failed == d_all:
if _1 in ('Qyer', 'Daodao'): continue
print('-5-', '不行')
break
else:
print('-6-', '可以', _1)
status_list_len += 1
if status_list_len == len(task_names) - 2:
update_step_report('', param, 1, 0, 5)
modify_status('step5', param, flag=False)
print('=== %s === 任务完成' % tag)
def monitor_task4_bak():
print('running===============0')
tasks = getStepStatus('step4')
tasks_list = list(tasks.items())
if len(tasks_list) == 0:
return
collection_name = tasks_list[0][1][0]
print('0==========', collection_name)
client = pymongo.MongoClient(host='10.10.231.105')
collection = client['MongoTask'][collection_name]
total_count = collection.find({}).count()
print('01==========', total_count)
conn = pymysql.connect(**OpCity_config)
cursor = conn.cursor()
for param, (_, task_name) in tasks.items():
print(param, (_, task_name))
if total_count == 0:
update_step_report('', param, -1, 0, 4)
return
select_sql = "select step4 from city_order where id=%s"
print('1==========', select_sql, param)
cursor.execute(select_sql, (param))
status_id = cursor.fetchone()[0]
print('2==========', status_id)
if int(status_id) == 2:
results = collection.find(
{
'$and': [{
'finished': 1
}, {
'useds_times': {
'$lt': 7
}
}, {
'task_name': task_name
}]
},
hint=[('task_name', 1), ('finished', 1), ('used_times', 1)])
not_finish_num = results.count()
print('3==========', not_finish_num)
if int(not_finish_num) / int(total_count) <= 0:
update_step_report('', param, 1, 0, 4)
modify_status('step4', param, flag=False)
print('running===============1')
def monitor_step_six():
tasks = getStepStatus('step6')
tasks_list = list(tasks.items())
if len(tasks_list) == 0:
return
param = tasks.keys()[0]
os.system("rsync -avI 10.10.150.16::opcity/{0} {1}".format(
param, base_step_six_path))
step_six_path = ''.join([base_step_six_path, str(param)])
file_list = os.listdir(step_six_path)
for file_name in file_list:
if 'hotel_report' in file_name:
save_path = '/'.join([str(param), file_name])
update_step_report(save_path, param, 1)
modify_status(step, param, flag=False)
def start_task():
logger.info('[step8][%s]======== start =======' % (param, ))
try:
return_result = defaultdict(dict)
return_result['data'] = {}
return_result['error']['error_id'] = 0
return_result['error']['error_str'] = ''
task_name = 'merge_hotel_image_' + datetime.datetime.now().strftime(
'%Y%m%d_') + param
with InsertTask(worker='proj.total_tasks.hotel_img_merge_task',
queue='merge_task',
routine_key='merge_task',
task_name=task_name,
source='Any',
_type='HotelImgMerge',
priority=11) as it:
for uid in get_tasks():
args = {
'uid': uid,
'min_pixels': '200000',
'target_table': 'hotel'
}
it.insert_task(args)
save_collection_names = it.generate_collection_name(), task_name
tasks = modify_status('step8', param, save_collection_names)
logger.info('[step8][%s] tasks: %s' % (param, str(tasks)))
# update_step_report('', param, 1, 0)
logger.info('[step8][%s]======== success =======' % (param, ))
except Exception as e:
return_result['error']['error_id'] = 1
return_result['error']['error_str'] = traceback.format_exc()
return_result = json.dumps(return_result)
logger.info('[step8][%s]======== failed =======' % (return_result, ))
def monitor_task4():
tasks = getStepStatus('step4')
print('-0-', tasks)
for param, values in tasks.items():
if len(values) == 0: continue
task_name = values[-1]
print('-1-', task_name)
tasks_status = from_tag_get_tasks_status(task_name)
print('-2-', tasks_status)
line = tasks_status[0]
print('-3-', line)
t_all, t_done, t_failed = line[3], line[4], line[5]
if t_all == t_done + t_failed:
update_step_report('', param, 1, 0, 4)
modify_status('step4', param, flag=False)
print('-4-', '完成')
def monitor_step3(stepa):
step = 'step' + stepa
tasks = getStepStatus(step)
temp_config['db'] = 'Cityupline'
if len(tasks) == 0: return
for param, collection_names in tasks.items():
collection_name, task_name = collection_names
total_count = get_total_count(collection_name)
if int(total_count) == 0:
return '0%'
client = pymongo.MongoClient(host='10.10.231.105')
collection = client['MongoTask'][collection_name]
success_results = collection.find(
{
'finished': 1,
'used_times': {
'$lt': 7
},
'task_name': task_name
},
hint=[('task_name', 1), ('finished', 1), ('used_times', 1)])
success_finish_num = success_results.count()
failed_results = collection.find(
{
'finished': 0,
'used_times': 7,
'task_name': task_name
},
hint=[('task_name', 1), ('finished', 1), ('used_times', 1)])
failed_finish_num = failed_results.count()
logger.info(
'{0}, collections: {1} total: {2} success: {3} failed: {4}'.
format(step, collection_name, total_count, success_finish_num,
failed_finish_num))
if failed_finish_num > 0 and failed_finish_num + success_finish_num == total_count:
logger.info('{0}, {1} 失败'.format(step, collection_name))
if success_finish_num == total_count:
from_ota_get_city(temp_config, param)
modify_status(step, param, flag=False)
logger.info('{0}, {1} 成功'.format(step, collection_name))
return format(success_finish_num / total_count, '.0%')
def task_start():
logger.info('[step5][%s]======== start =======' % (param, ))
try:
sources = ['agoda', 'ctrip', 'elong', 'hotels', 'expedia', 'booking']
return_result = defaultdict(dict)
return_result['data'] = {}
return_result['error']['error_id'] = 0
return_result['error']['error_str'] = ''
save_cityId = []
database_name = ''.join(['add_city_', param])
temp_config = config
temp_config['db'] = database_name
path = ''.join([base_path, param, '/', 'city_id.csv'])
with open(path, 'r+') as city:
reader = csv.DictReader(city)
for row in reader:
save_cityId.append(row['city_id'])
logger.info('[step5][%s, %s, %s] 启动发 daodao 任务' %
(save_cityId, param, temp_config))
daodao_collection_name, daodao_task_name = daodao_city(
save_cityId, param, temp_config)
logger.info('[step5] 发 daodao 任务完成 [%s, %s]' %
(daodao_collection_name, daodao_task_name))
logger.info('[step5][%s, %s, %s] 启动发 qyer 任务' %
(save_cityId, param, temp_config))
qyer_collection_name, qyer_task_name = qyer_city(
save_cityId, param, temp_config)
logger.info('[step5] 发 qyer 任务完成 [%s, %s]' %
(qyer_collection_name, qyer_task_name))
logger.info('[step5][%s, %s, %s] 启动发 hotel 任务' %
(save_cityId, param, temp_config))
hotel_collections_name = hotel_city(save_cityId, param, sources,
temp_config)
logger.info('[step5] 发 hotel 任务完成 [%s]' % (hotel_collections_name))
save_collection_names = []
for collection_name in hotel_collections_name:
save_collection_names.append(collection_name)
save_collection_names.append(
(daodao_collection_name, daodao_task_name))
save_collection_names.append((qyer_collection_name, qyer_task_name))
tasks = modify_status('step5', param, save_collection_names)
logger.info('[step5] 发 hotel 任务完成 [%s]' % (tasks))
return_result = json.dumps(return_result)
logger.info('[step5][%s]======== successed ======= \n%s' %
(param, return_result))
except Exception as e:
return_result['error']['error_id'] = 1
return_result['error']['error_str'] = traceback.format_exc()
return_result = json.dumps(return_result)
update_step_report('', param, -1, 0)
logger.info('[step5] [result][{0}]'.format(return_result))
def upload_file():
param = request.form.get('param')
param = str(get_param())
if not param:
return jsonify({'status': False, 'message': '任务批次号为空'})
file_dir = os.path.join(basedir, app.config['UPLOAD_FOLDER'], param)
if not os.path.exists(file_dir):
os.makedirs(file_dir)
f = request.files['myfile'] # 从表单的file字段获取文件，myfile为该表单的name值
if f and f.filename.endswith('.zip'): # 判断是否是允许上传的文件类型
fname = secure_filename(f.filename)
logger.info('[upload][%s] tasks: %s' % (param, fname))
ext = fname.rsplit('.', 1)[1] # 获取文件后缀
unix_time = int(time.time())
new_filename = str(unix_time) + '.' + ext # 修改了上传的文件名
# zip_file_name = os.path.join(file_dir, new_filename)
zip_file_name = os.path.join(file_dir, fname)
f.save(zip_file_name) # 保存文件到upload目录
print(zip_file_name)
token = base64.b64encode(new_filename.encode('utf8'))
logger.info('[upload][%s] token: %s' % (param, token))
zip_path = os.path.join(file_dir, fname)
zipfiles = zipfile.ZipFile(zip_path)
zipfile_status = subprocess.Popen('unzip ' + fname,
stdout=subprocess.PIPE,
shell=True,
cwd=file_dir)
if zipfile_status.returncode:
return jsonify({'status': False, 'message': '解压文件失败'})
city_file = ''
zip_files_list = zipfiles.filelist
logger.info('[upload][%s] zipfiles: %s' % (param, zip_files_list))
for child_file in zip_files_list:
real_file_name = child_file.filename.encode('cp437').decode('utf8')
if '新增城市' in real_file_name:
city_file = os.path.join(file_dir, real_file_name)
break
if not city_file:
return jsonify({'status': False, 'message': '无新增城市配置文件'})
truncate_table()
collection_name, task_name = create_task(city_file, file_dir,
'BaseDataFinal', param)
tasks = modify_status('step3', param, [collection_name, task_name])
logger.info('[upload][%s] tasks: %s' % (param, str(tasks)))
return jsonify({"code": 0, "errmsg": "上传成功"})
else:
return jsonify({"code": 1001, "errmsg": "上传失败"})
def monitor_task_summary(step):
stepa = 'step' + step
logger.info('================= ' + stepa + ' ================= 开始')
csvpath = ''
tasks = getStepStatus(stepa)
for param, values in tasks.items():
if len(values) == 0: return
if type(values[0]) is list:
task_naems = list(zip(*values))[1]
else:
task_naems = [values[1]]
the_progress_of = 0
for task_name in task_naems:
logger.info('{}, {}'.format(stepa, task_name))
tasks_status = from_tag_get_tasks_status(task_name)
logger.info('{}, {}'.format(stepa, tasks_status))
line = tasks_status[0]
t_all, t_done, t_failed = line[3], line[4], line[5]
if t_all == t_done + t_failed:
the_progress_of += 1
if the_progress_of == len(task_naems):
if step == '7':
tag = task_name.rsplit('_')[-1]
step7_detection(tag)
# if not get_file(param, 'poireport.csv'):
elif step == '8':
if not make_image_content_report(t_all, t_done, t_failed,
param):
return
csvpath = '{}/merge_image_and_content.txt'.format(param)
if step in ('4', '9'):
update_step_report(csvpath, param, 4, 0, int(step))
elif step not in ('6', '7'):
update_step_report(csvpath, param, 1, 0, int(step))
modify_status(stepa, param, flag=False)
logger.info('================= ' + stepa +
' ================= 完成')
logger.info('================= ' + stepa + ' ================= 1')
def monitor_report(step):
stepa = 'step' + step
logger.info('================= ' + stepa + ' ================= 开始')
tasks = getStepStatus(stepa)
for param, values in tasks.items():
if len(values) == 0: continue
task_names = zip(*values).__next__()
logger.info('{}, {}'.format(stepa, task_names))
tag = str(task_names[0].rsplit('_', 1)[-1])
logger.info('{}, {}'.format(stepa, tag))
tasks_status = from_tag_get_tasks_status(tag, True)
finaled_date = max(a[-1] for a in tasks_status)
all_finaled_data = [a for a in tasks_status if a[-1] == finaled_date]
logger.info('{}, {}'.format(stepa, tasks_status))
if len(tasks_status) < len(task_names):
logger.info('{}, {}'.format(stepa, '任务状态数 小于 任务已发任务数'))
continue
status_list_len = 0
for (_0, source, _2, l_done, l_failed, _5, l_all, d_done, d_failed,
d_all, i_done, i_failed, i_all, _13) in all_finaled_data:
#规则 1 完成任务数 + 失败任务数 = 任务总数
#规则 2 失败任务数 == 任务总数 发邮件报警
if not (l_done + l_failed == l_all and d_done + d_failed == d_all
and i_done + i_failed
== i_all) or l_failed == l_all or d_failed == d_all:
if source in ('Qyer', 'Daodao'): continue
logger.info('{}, {}: {}'.format(stepa, '未完成', source))
break
else:
logger.info('{}, {}: {}'.format(stepa, '已完成', source))
status_list_len += 1
if status_list_len >= len(task_names):
modify_status(stepa, param, flag=False)
logger.info('{}, 开始生成报表'.format(stepa))
csv_file = make_poi_and_hotel_report(all_finaled_data, param)
update_step_report(csv_file, param, 1, 0, int(step))
logger.info('================= ' + stepa + ' ================= 完成')
logger.info('================= ' + stepa + ' ================= 1')
def task_start():
logger.info('[step7][%s]======== start =======' % (param,))
try:
return_result = defaultdict(dict)
return_result['data'] = {}
return_result['error']['error_id'] = 0
return_result['error']['error_str'] = ''
return_result = json.dumps(return_result)
logger.info('[step7][%s] 汇总数据到BaseDataFinal 开始' % (param,))
tag = selectServicePlatform2BaseDataFinal()
logger.info('[step7][%s] 汇总数据到BaseDataFinal 完成' % (param,))
logger.info('[step7][%s] mapping daodao 开始' % (param,))
mapping_daodao_by_sid_and_sourcecityid(tag)
logger.info('[step7][%s] mapping daodao 开始' % (param,))
logger.info('[step7][%s] mapping qyer 开始' % (param,))
mapping_daodao_by_othersinfoscityid_and_sourcecityid(tag)
logger.info('[step7][%s] mapping qyer 开始' % (param,))
logger.info('[step7][%s] 检查数据 开始' % (param,))
_1, qyer_tasks_data, _2, daodao_tasks_data = check_POI_data(tag)
logger.info('[step7][%s] 检查数据 完成 \n%s\n%s' % (param, str(qyer_tasks_data), str(daodao_tasks_data)))
tasks_names = []
if qyer_tasks_data:
logger.info('[step7][%s] qyer补充mapinfo任务 开始' % (param,))
collection_name, task_name = send_tasks(qyer_tasks_data, tag)
tasks_names.append([collection_name, task_name])
logger.info('[step7][%s] qyer补充mapinfo任务 完成' % (param,))
if daodao_tasks_data:
logger.info('[step7][%s] daodao补充mapinfo任务 完成' % (param,))
collection_name, task_name = send_tasks(daodao_tasks_data, tag)
tasks_names.append([collection_name, task_name])
logger.info('[step7][%s] daodao补充mapinfo任务 完成' % (param,))
# logger.info('[step7][%s] 导出数据 开始' % (param,))
# data_path = dumps_sql(tag)
# logger.info('[step7][%s] 导出数据 完成' % (param,))
tasks = modify_status('step7', param, tasks_names)
logger.info('[step7][%s] tasks: %s' % (param, str(tasks)))
# update_step_report('', param, 1, 0)
logger.info('[step7][%s]======== success =======' % (param,))
except Exception as e:
return_result['error']['error_id'] = 1
return_result['error']['error_str'] = traceback.format_exc()
return_result = json.dumps(return_result)
send_email('城市上线酒店融合' + '第 %s 批次' % param,
"""POI融合前检查失败""", SEND_TO[:1])
update_step_report('', param, -1, 0)
logger.info('[step7][%s]======== failed =======' % (return_result,))
def task_start():
logger.info('[step9][%s]======== start =======' % (param, ))
try:
return_result = defaultdict(dict)
return_result['data'] = {}
return_result['error']['error_id'] = 0
return_result['error']['error_str'] = ''
data_name = ''.join(['add_city_', str(param)])
config['db'] = data_name
save_cityId = []
path = ''.join([base_path, param, '/', 'city_id.csv'])
with open(path, 'r+') as city:
reader = csv.DictReader(city)
for row in reader:
save_cityId.append(row['city_id'])
#获取google url
logger.info('[step9]获取google_url开始')
inner_city(cid_list=save_cityId, config=config)
logger.info('[step9]获取google_url完成')
file_list = os.listdir(urlsfile_path)
if '.DS_Store' in file_list:
file_list.remove('.DS_Store')
save_urls = []
for city_id in save_cityId:
for child_path in file_list:
if city_id in child_path:
with open(urlsfile_path + '{0}'.format(child_path),
'r+') as city:
urls = city.readlines()
save_urls.extend(urls)
logger.info('[step9] 开启城市内任务')
collection_name, task_name = city_inter_google_driver(save_urls, param)
logger.info('[step9] 开启城市内任务完成 %s %s' % (collection_name, task_name))
tasks = modify_status('step9', param, [collection_name, task_name])
return_result = json.dumps(return_result)
logger.info('[step9][%s]======== success ======= %s' %
(param, return_result))
except Exception as e:
return_result['error']['error_id'] = 1
return_result['error']['error_str'] = traceback.format_exc()
return_result = json.dumps(return_result)
logger.info('[step9][%s]======== failed =======' % (return_result, ))
update_step_report('', param, -1, 0)
def task_start():
logger.info('[step4][%s]======== start =======' % [param])
try:
return_result = defaultdict(dict)
return_result['data'] = {}
return_result['error']['error_id'] = 0
return_result['error']['error_str'] = ''
save_cityId = []
data_name = ''.join(['add_city_', str(param)])
config['db'] = data_name
path = ''.join([base_path, param, '/', 'city_id.csv'])
with open(path, 'r+') as city:
reader = csv.DictReader(city)
for row in reader:
save_cityId.append(row['city_id'])
# save_cityId = ['10001','10003','10005']
logger.info('[step4][%s] 启动发任务' % [param])
collection_name, task_name = google_driver(save_cityId, param, config)
logger.info('[step4][%s] 任务已发完[%s][%s]' % (
param,
collection_name,
task_name,
))
tasks = modify_status('step4', param, [collection_name, task_name])
logger.info('[step4][%s] tasks: %s' % (param, str(tasks)))
# job = backgroudscheduler.add_job(monitor_google_driver,trigger='cron',minute='*/2',hour='*',id='step4',kwargs={'collection_name':collection_name,'param':param, 'task_name': task_name})
# backgroudscheduler.start()
return_result = json.dumps(return_result)
print('[result][{0}]'.format(return_result))
logger.info('[step4][%s]======== successed =======' % [param])
except Exception as e:
return_result['error']['error_id'] = 1
return_result['error']['error_str'] = traceback.format_exc()
return_result = json.dumps(return_result)
print('[result][{0}]'.format(return_result))
logger.info('[step4][%s]======== failed ======= \n %s' % (
param,
traceback.format_exc(),
))
update_step_report('', param, -1, 0)
def task_start():
logger.info('[step6][%s]======== start =======' % (param, ))
try:
return_result = defaultdict(dict)
return_result['data'] = {}
return_result['error']['error_id'] = 0
return_result['error']['error_str'] = ''
logger.info('[step6][%s] 汇总数据到BaseDataFinal 开始' % (param, ))
tag = selectServicePlatform2BaseDataFinal()
logger.info('[step6][%s] 汇总数据到BaseDataFinal 完成' % (param, ))
logger.info('[step6][%s] 检查数据 开始' % (param, ))
check_result = check_hotel_data(tag)
logger.info('[step6][%s] 检查数据 完成' % (param, ))
logger.info('[step6][%s] 导出数据 开始' % (param, ))
data_path = dumps_sql(tag)
logger.info('[step6][%s] 导出数据 完成' % (param, ))
return_result = json.dumps(return_result)
send_email(
'城市上线酒店融合' + '第 %s 批次' % param, """
大家好:
脚本检查结果：
{check_result}
数据地址：
{data_path}
融合类型：增量融合
""".format(check_result=check_result[:-2], data_path=data_path),
SEND_TO)
# update_step_report('', param, 1, 0)
tasks = modify_status(
'step6',
param,
)
logger.info('[step6][%s]======== success =======' % (param, ))
except Exception as e:
return_result['error']['error_id'] = 1
return_result['error']['error_str'] = traceback.format_exc()
send_email('城市上线酒店融合' + '第 %s 批次' % param, """酒店融合前检查失败""",
SEND_TO[:1])
return_result = json.dumps(return_result)
update_step_report('', param, -1, 0)
logger.info('[step6][%s]======== failed =======' % (return_result, ))
