def monitor_task5(): tasks = getStepStatus('step5') print('-0-', tasks) for param, values in tasks.items(): if len(values) == 0: continue task_names = zip(*values)[0] print('-1-', task_names) tag = str(task_names[0].rsplit('_', 1)[-1]) print('-2-', tag) tasks_status = from_tag_get_tasks_status(tag, True) finaled_date = max(a[-1] for a in tasks_status) all_finaled_data = [a for a in tasks_status if a[-1] == finaled_date] print('-3-', tasks_status) if len(tasks_status) < len(task_names): print('-4-', '完蛋') continue status_list_len = 0 for (_0, _1, _2, l_done, l_failed, _5, l_all, d_done, d_failed, d_all, i_done, i_failed, i_all, _13) in all_finaled_data: #规则 1 完成任务数 + 失败任务数 = 任务总数 #规则 2 失败任务数 == 任务总数 发邮件报警 if not (l_done + l_failed == l_all and d_done + d_failed == d_all and i_done + i_failed == i_all) or l_failed == l_all or d_failed == d_all: if _1 in ('Qyer', 'Daodao'): continue print('-5-', '不行') break else: print('-6-', '可以', _1) status_list_len += 1 if status_list_len == len(task_names) - 2: update_step_report('', param, 1, 0, 5) modify_status('step5', param, flag=False) print('=== %s === 任务完成' % tag)
def monitor_task4_bak(): print('running===============0') tasks = getStepStatus('step4') tasks_list = list(tasks.items()) if len(tasks_list) == 0: return collection_name = tasks_list[0][1][0] print('0==========', collection_name) client = pymongo.MongoClient(host='10.10.231.105') collection = client['MongoTask'][collection_name] total_count = collection.find({}).count() print('01==========', total_count) conn = pymysql.connect(**OpCity_config) cursor = conn.cursor() for param, (_, task_name) in tasks.items(): print(param, (_, task_name)) if total_count == 0: update_step_report('', param, -1, 0, 4) return select_sql = "select step4 from city_order where id=%s" print('1==========', select_sql, param) cursor.execute(select_sql, (param)) status_id = cursor.fetchone()[0] print('2==========', status_id) if int(status_id) == 2: results = collection.find( { '$and': [{ 'finished': 1 }, { 'useds_times': { '$lt': 7 } }, { 'task_name': task_name }] }, hint=[('task_name', 1), ('finished', 1), ('used_times', 1)]) not_finish_num = results.count() print('3==========', not_finish_num) if int(not_finish_num) / int(total_count) <= 0: update_step_report('', param, 1, 0, 4) modify_status('step4', param, flag=False) print('running===============1')
def monitor_step_six(): tasks = getStepStatus('step6') tasks_list = list(tasks.items()) if len(tasks_list) == 0: return param = tasks.keys()[0] os.system("rsync -avI 10.10.150.16::opcity/{0} {1}".format( param, base_step_six_path)) step_six_path = ''.join([base_step_six_path, str(param)]) file_list = os.listdir(step_six_path) for file_name in file_list: if 'hotel_report' in file_name: save_path = '/'.join([str(param), file_name]) update_step_report(save_path, param, 1) modify_status(step, param, flag=False)
def start_task(): logger.info('[step8][%s]======== start =======' % (param, )) try: return_result = defaultdict(dict) return_result['data'] = {} return_result['error']['error_id'] = 0 return_result['error']['error_str'] = '' task_name = 'merge_hotel_image_' + datetime.datetime.now().strftime( '%Y%m%d_') + param with InsertTask(worker='proj.total_tasks.hotel_img_merge_task', queue='merge_task', routine_key='merge_task', task_name=task_name, source='Any', _type='HotelImgMerge', priority=11) as it: for uid in get_tasks(): args = { 'uid': uid, 'min_pixels': '200000', 'target_table': 'hotel' } it.insert_task(args) save_collection_names = it.generate_collection_name(), task_name tasks = modify_status('step8', param, save_collection_names) logger.info('[step8][%s] tasks: %s' % (param, str(tasks))) # update_step_report('', param, 1, 0) logger.info('[step8][%s]======== success =======' % (param, )) except Exception as e: return_result['error']['error_id'] = 1 return_result['error']['error_str'] = traceback.format_exc() return_result = json.dumps(return_result) logger.info('[step8][%s]======== failed =======' % (return_result, ))
def monitor_task4(): tasks = getStepStatus('step4') print('-0-', tasks) for param, values in tasks.items(): if len(values) == 0: continue task_name = values[-1] print('-1-', task_name) tasks_status = from_tag_get_tasks_status(task_name) print('-2-', tasks_status) line = tasks_status[0] print('-3-', line) t_all, t_done, t_failed = line[3], line[4], line[5] if t_all == t_done + t_failed: update_step_report('', param, 1, 0, 4) modify_status('step4', param, flag=False) print('-4-', '完成')
def monitor_step3(stepa): step = 'step' + stepa tasks = getStepStatus(step) temp_config['db'] = 'Cityupline' if len(tasks) == 0: return for param, collection_names in tasks.items(): collection_name, task_name = collection_names total_count = get_total_count(collection_name) if int(total_count) == 0: return '0%' client = pymongo.MongoClient(host='10.10.231.105') collection = client['MongoTask'][collection_name] success_results = collection.find( { 'finished': 1, 'used_times': { '$lt': 7 }, 'task_name': task_name }, hint=[('task_name', 1), ('finished', 1), ('used_times', 1)]) success_finish_num = success_results.count() failed_results = collection.find( { 'finished': 0, 'used_times': 7, 'task_name': task_name }, hint=[('task_name', 1), ('finished', 1), ('used_times', 1)]) failed_finish_num = failed_results.count() logger.info( '{0}, collections: {1} total: {2} success: {3} failed: {4}'. format(step, collection_name, total_count, success_finish_num, failed_finish_num)) if failed_finish_num > 0 and failed_finish_num + success_finish_num == total_count: logger.info('{0}, {1} 失败'.format(step, collection_name)) if success_finish_num == total_count: from_ota_get_city(temp_config, param) modify_status(step, param, flag=False) logger.info('{0}, {1} 成功'.format(step, collection_name)) return format(success_finish_num / total_count, '.0%')
def task_start(): logger.info('[step5][%s]======== start =======' % (param, )) try: sources = ['agoda', 'ctrip', 'elong', 'hotels', 'expedia', 'booking'] return_result = defaultdict(dict) return_result['data'] = {} return_result['error']['error_id'] = 0 return_result['error']['error_str'] = '' save_cityId = [] database_name = ''.join(['add_city_', param]) temp_config = config temp_config['db'] = database_name path = ''.join([base_path, param, '/', 'city_id.csv']) with open(path, 'r+') as city: reader = csv.DictReader(city) for row in reader: save_cityId.append(row['city_id']) logger.info('[step5][%s, %s, %s] 启动发 daodao 任务' % (save_cityId, param, temp_config)) daodao_collection_name, daodao_task_name = daodao_city( save_cityId, param, temp_config) logger.info('[step5] 发 daodao 任务完成 [%s, %s]' % (daodao_collection_name, daodao_task_name)) logger.info('[step5][%s, %s, %s] 启动发 qyer 任务' % (save_cityId, param, temp_config)) qyer_collection_name, qyer_task_name = qyer_city( save_cityId, param, temp_config) logger.info('[step5] 发 qyer 任务完成 [%s, %s]' % (qyer_collection_name, qyer_task_name)) logger.info('[step5][%s, %s, %s] 启动发 hotel 任务' % (save_cityId, param, temp_config)) hotel_collections_name = hotel_city(save_cityId, param, sources, temp_config) logger.info('[step5] 发 hotel 任务完成 [%s]' % (hotel_collections_name)) save_collection_names = [] for collection_name in hotel_collections_name: save_collection_names.append(collection_name) save_collection_names.append( (daodao_collection_name, daodao_task_name)) save_collection_names.append((qyer_collection_name, qyer_task_name)) tasks = modify_status('step5', param, save_collection_names) logger.info('[step5] 发 hotel 任务完成 [%s]' % (tasks)) return_result = json.dumps(return_result) logger.info('[step5][%s]======== successed ======= \n%s' % (param, return_result)) except Exception as e: return_result['error']['error_id'] = 1 return_result['error']['error_str'] = traceback.format_exc() return_result = json.dumps(return_result) update_step_report('', param, -1, 0) logger.info('[step5] [result][{0}]'.format(return_result))
def upload_file(): param = request.form.get('param') param = str(get_param()) if not param: return jsonify({'status': False, 'message': '任务批次号为空'}) file_dir = os.path.join(basedir, app.config['UPLOAD_FOLDER'], param) if not os.path.exists(file_dir): os.makedirs(file_dir) f = request.files['myfile'] # 从表单的file字段获取文件,myfile为该表单的name值 if f and f.filename.endswith('.zip'): # 判断是否是允许上传的文件类型 fname = secure_filename(f.filename) logger.info('[upload][%s] tasks: %s' % (param, fname)) ext = fname.rsplit('.', 1)[1] # 获取文件后缀 unix_time = int(time.time()) new_filename = str(unix_time) + '.' + ext # 修改了上传的文件名 # zip_file_name = os.path.join(file_dir, new_filename) zip_file_name = os.path.join(file_dir, fname) f.save(zip_file_name) # 保存文件到upload目录 print(zip_file_name) token = base64.b64encode(new_filename.encode('utf8')) logger.info('[upload][%s] token: %s' % (param, token)) zip_path = os.path.join(file_dir, fname) zipfiles = zipfile.ZipFile(zip_path) zipfile_status = subprocess.Popen('unzip ' + fname, stdout=subprocess.PIPE, shell=True, cwd=file_dir) if zipfile_status.returncode: return jsonify({'status': False, 'message': '解压文件失败'}) city_file = '' zip_files_list = zipfiles.filelist logger.info('[upload][%s] zipfiles: %s' % (param, zip_files_list)) for child_file in zip_files_list: real_file_name = child_file.filename.encode('cp437').decode('utf8') if '新增城市' in real_file_name: city_file = os.path.join(file_dir, real_file_name) break if not city_file: return jsonify({'status': False, 'message': '无新增城市配置文件'}) truncate_table() collection_name, task_name = create_task(city_file, file_dir, 'BaseDataFinal', param) tasks = modify_status('step3', param, [collection_name, task_name]) logger.info('[upload][%s] tasks: %s' % (param, str(tasks))) return jsonify({"code": 0, "errmsg": "上传成功"}) else: return jsonify({"code": 1001, "errmsg": "上传失败"})
def monitor_task_summary(step): stepa = 'step' + step logger.info('================= ' + stepa + ' ================= 开始') csvpath = '' tasks = getStepStatus(stepa) for param, values in tasks.items(): if len(values) == 0: return if type(values[0]) is list: task_naems = list(zip(*values))[1] else: task_naems = [values[1]] the_progress_of = 0 for task_name in task_naems: logger.info('{}, {}'.format(stepa, task_name)) tasks_status = from_tag_get_tasks_status(task_name) logger.info('{}, {}'.format(stepa, tasks_status)) line = tasks_status[0] t_all, t_done, t_failed = line[3], line[4], line[5] if t_all == t_done + t_failed: the_progress_of += 1 if the_progress_of == len(task_naems): if step == '7': tag = task_name.rsplit('_')[-1] step7_detection(tag) # if not get_file(param, 'poireport.csv'): elif step == '8': if not make_image_content_report(t_all, t_done, t_failed, param): return csvpath = '{}/merge_image_and_content.txt'.format(param) if step in ('4', '9'): update_step_report(csvpath, param, 4, 0, int(step)) elif step not in ('6', '7'): update_step_report(csvpath, param, 1, 0, int(step)) modify_status(stepa, param, flag=False) logger.info('================= ' + stepa + ' ================= 完成') logger.info('================= ' + stepa + ' ================= 1')
def monitor_report(step): stepa = 'step' + step logger.info('================= ' + stepa + ' ================= 开始') tasks = getStepStatus(stepa) for param, values in tasks.items(): if len(values) == 0: continue task_names = zip(*values).__next__() logger.info('{}, {}'.format(stepa, task_names)) tag = str(task_names[0].rsplit('_', 1)[-1]) logger.info('{}, {}'.format(stepa, tag)) tasks_status = from_tag_get_tasks_status(tag, True) finaled_date = max(a[-1] for a in tasks_status) all_finaled_data = [a for a in tasks_status if a[-1] == finaled_date] logger.info('{}, {}'.format(stepa, tasks_status)) if len(tasks_status) < len(task_names): logger.info('{}, {}'.format(stepa, '任务状态数 小于 任务已发任务数')) continue status_list_len = 0 for (_0, source, _2, l_done, l_failed, _5, l_all, d_done, d_failed, d_all, i_done, i_failed, i_all, _13) in all_finaled_data: #规则 1 完成任务数 + 失败任务数 = 任务总数 #规则 2 失败任务数 == 任务总数 发邮件报警 if not (l_done + l_failed == l_all and d_done + d_failed == d_all and i_done + i_failed == i_all) or l_failed == l_all or d_failed == d_all: if source in ('Qyer', 'Daodao'): continue logger.info('{}, {}: {}'.format(stepa, '未完成', source)) break else: logger.info('{}, {}: {}'.format(stepa, '已完成', source)) status_list_len += 1 if status_list_len >= len(task_names): modify_status(stepa, param, flag=False) logger.info('{}, 开始生成报表'.format(stepa)) csv_file = make_poi_and_hotel_report(all_finaled_data, param) update_step_report(csv_file, param, 1, 0, int(step)) logger.info('================= ' + stepa + ' ================= 完成') logger.info('================= ' + stepa + ' ================= 1')
def task_start(): logger.info('[step7][%s]======== start =======' % (param,)) try: return_result = defaultdict(dict) return_result['data'] = {} return_result['error']['error_id'] = 0 return_result['error']['error_str'] = '' return_result = json.dumps(return_result) logger.info('[step7][%s] 汇总数据到BaseDataFinal 开始' % (param,)) tag = selectServicePlatform2BaseDataFinal() logger.info('[step7][%s] 汇总数据到BaseDataFinal 完成' % (param,)) logger.info('[step7][%s] mapping daodao 开始' % (param,)) mapping_daodao_by_sid_and_sourcecityid(tag) logger.info('[step7][%s] mapping daodao 开始' % (param,)) logger.info('[step7][%s] mapping qyer 开始' % (param,)) mapping_daodao_by_othersinfoscityid_and_sourcecityid(tag) logger.info('[step7][%s] mapping qyer 开始' % (param,)) logger.info('[step7][%s] 检查数据 开始' % (param,)) _1, qyer_tasks_data, _2, daodao_tasks_data = check_POI_data(tag) logger.info('[step7][%s] 检查数据 完成 \n%s\n%s' % (param, str(qyer_tasks_data), str(daodao_tasks_data))) tasks_names = [] if qyer_tasks_data: logger.info('[step7][%s] qyer补充mapinfo任务 开始' % (param,)) collection_name, task_name = send_tasks(qyer_tasks_data, tag) tasks_names.append([collection_name, task_name]) logger.info('[step7][%s] qyer补充mapinfo任务 完成' % (param,)) if daodao_tasks_data: logger.info('[step7][%s] daodao补充mapinfo任务 完成' % (param,)) collection_name, task_name = send_tasks(daodao_tasks_data, tag) tasks_names.append([collection_name, task_name]) logger.info('[step7][%s] daodao补充mapinfo任务 完成' % (param,)) # logger.info('[step7][%s] 导出数据 开始' % (param,)) # data_path = dumps_sql(tag) # logger.info('[step7][%s] 导出数据 完成' % (param,)) tasks = modify_status('step7', param, tasks_names) logger.info('[step7][%s] tasks: %s' % (param, str(tasks))) # update_step_report('', param, 1, 0) logger.info('[step7][%s]======== success =======' % (param,)) except Exception as e: return_result['error']['error_id'] = 1 return_result['error']['error_str'] = traceback.format_exc() return_result = json.dumps(return_result) send_email('城市上线酒店融合' + '第 %s 批次' % param, """POI融合前检查失败""", SEND_TO[:1]) update_step_report('', param, -1, 0) logger.info('[step7][%s]======== failed =======' % (return_result,))
def task_start(): logger.info('[step9][%s]======== start =======' % (param, )) try: return_result = defaultdict(dict) return_result['data'] = {} return_result['error']['error_id'] = 0 return_result['error']['error_str'] = '' data_name = ''.join(['add_city_', str(param)]) config['db'] = data_name save_cityId = [] path = ''.join([base_path, param, '/', 'city_id.csv']) with open(path, 'r+') as city: reader = csv.DictReader(city) for row in reader: save_cityId.append(row['city_id']) #获取google url logger.info('[step9]获取google_url开始') inner_city(cid_list=save_cityId, config=config) logger.info('[step9]获取google_url完成') file_list = os.listdir(urlsfile_path) if '.DS_Store' in file_list: file_list.remove('.DS_Store') save_urls = [] for city_id in save_cityId: for child_path in file_list: if city_id in child_path: with open(urlsfile_path + '{0}'.format(child_path), 'r+') as city: urls = city.readlines() save_urls.extend(urls) logger.info('[step9] 开启城市内任务') collection_name, task_name = city_inter_google_driver(save_urls, param) logger.info('[step9] 开启城市内任务完成 %s %s' % (collection_name, task_name)) tasks = modify_status('step9', param, [collection_name, task_name]) return_result = json.dumps(return_result) logger.info('[step9][%s]======== success ======= %s' % (param, return_result)) except Exception as e: return_result['error']['error_id'] = 1 return_result['error']['error_str'] = traceback.format_exc() return_result = json.dumps(return_result) logger.info('[step9][%s]======== failed =======' % (return_result, )) update_step_report('', param, -1, 0)
def task_start(): logger.info('[step4][%s]======== start =======' % [param]) try: return_result = defaultdict(dict) return_result['data'] = {} return_result['error']['error_id'] = 0 return_result['error']['error_str'] = '' save_cityId = [] data_name = ''.join(['add_city_', str(param)]) config['db'] = data_name path = ''.join([base_path, param, '/', 'city_id.csv']) with open(path, 'r+') as city: reader = csv.DictReader(city) for row in reader: save_cityId.append(row['city_id']) # save_cityId = ['10001','10003','10005'] logger.info('[step4][%s] 启动发任务' % [param]) collection_name, task_name = google_driver(save_cityId, param, config) logger.info('[step4][%s] 任务已发完[%s][%s]' % ( param, collection_name, task_name, )) tasks = modify_status('step4', param, [collection_name, task_name]) logger.info('[step4][%s] tasks: %s' % (param, str(tasks))) # job = backgroudscheduler.add_job(monitor_google_driver,trigger='cron',minute='*/2',hour='*',id='step4',kwargs={'collection_name':collection_name,'param':param, 'task_name': task_name}) # backgroudscheduler.start() return_result = json.dumps(return_result) print('[result][{0}]'.format(return_result)) logger.info('[step4][%s]======== successed =======' % [param]) except Exception as e: return_result['error']['error_id'] = 1 return_result['error']['error_str'] = traceback.format_exc() return_result = json.dumps(return_result) print('[result][{0}]'.format(return_result)) logger.info('[step4][%s]======== failed ======= \n %s' % ( param, traceback.format_exc(), )) update_step_report('', param, -1, 0)
def task_start(): logger.info('[step6][%s]======== start =======' % (param, )) try: return_result = defaultdict(dict) return_result['data'] = {} return_result['error']['error_id'] = 0 return_result['error']['error_str'] = '' logger.info('[step6][%s] 汇总数据到BaseDataFinal 开始' % (param, )) tag = selectServicePlatform2BaseDataFinal() logger.info('[step6][%s] 汇总数据到BaseDataFinal 完成' % (param, )) logger.info('[step6][%s] 检查数据 开始' % (param, )) check_result = check_hotel_data(tag) logger.info('[step6][%s] 检查数据 完成' % (param, )) logger.info('[step6][%s] 导出数据 开始' % (param, )) data_path = dumps_sql(tag) logger.info('[step6][%s] 导出数据 完成' % (param, )) return_result = json.dumps(return_result) send_email( '城市上线酒店融合' + '第 %s 批次' % param, """ 大家好: 脚本检查结果: {check_result} 数据地址: {data_path} 融合类型:增量融合 """.format(check_result=check_result[:-2], data_path=data_path), SEND_TO) # update_step_report('', param, 1, 0) tasks = modify_status( 'step6', param, ) logger.info('[step6][%s]======== success =======' % (param, )) except Exception as e: return_result['error']['error_id'] = 1 return_result['error']['error_str'] = traceback.format_exc() send_email('城市上线酒店融合' + '第 %s 批次' % param, """酒店融合前检查失败""", SEND_TO[:1]) return_result = json.dumps(return_result) update_step_report('', param, -1, 0) logger.info('[step6][%s]======== failed =======' % (return_result, ))