async def process_callback_kb1btn1(callback_query: types.CallbackQuery): code = callback_query.data chat_id = callback_query.message.chat.id member_id = callback_query.from_user.id user_title = callback_query.from_user.mention debug_id = f'{callback_query.message.chat.username}-({user_title})' chat_title = callback_query.message.chat.title log.info(f'{debug_id}: {code}') _id = f'{callback_query.message.message_id}-{callback_query.message.chat.id}' try: pass_item = data_store.get_captcha(_id) except KeyError: log.error(f'Something gone wrong: {data_store} {_id}') await bot.answer_callback_query(callback_query.id, text=s('something_gone_wrong_warn', {'lang': 'ru'})) return if not pass_item.user_check(callback_query.from_user.id): await bot.answer_callback_query(callback_query.id, text=s('not_for_you_warn', {'lang': 'ru'})) return text = pass_item.new_char(code[-1:]) result = pass_item.check() if result is CAPTCHA_STATE.INPUTING: await bot.answer_callback_query(callback_query.id, text=text) elif result is CAPTCHA_STATE.SUCSSES: data_store.remove_captcha(_id) log.info(f'{debug_id}: SUCSSES') await bot.answer_callback_query(callback_query.id, text='SUCSSES') try: pass_item.messages.pop(MESSAGE_TYPES.LOGIN) except KeyError: pass await clear(pass_item) await bot.send_message( callback_query.message.chat.id, s('success_msg', { 'lang': 'ru', 'user_title': user_title, 'chat_title': chat_title })) unmute = ChatPermissions(can_send_messages=True, can_send_media_messages=True, can_add_web_page_previews=True, can_send_other_messages=True, can_send_polls=True) await bot.restrict_chat_member(chat_id, member_id, permissions=unmute) else: data_store.remove_captcha(_id) delay = datetime.now() + timedelta(minutes=1) log.info(f'{debug_id}: FAIL, ban until {delay}') # workaround to send new event. await bot.kick_chat_member(chat_id, member_id, delay) await bot.answer_callback_query(callback_query.id, text=s('fail_msg', {'lang': 'ru'})) await clear(pass_item) # await bot.unban_chat_member(chat_id, member_id) data_store.sync()
def send_dingtalk(alert_type, params, is_at_all=False): """ 发送钉钉消息 @webhook: "https://oapi.dingtalk.com/robot/send?access_token=bd92817749b11207820971bab315e0b46b59d1cf422f33e976495acf4aa89ef7" @msg: "发送消息" """ exec_id = params['exec_id'] dispatch_id = params['dispatch_id'] dispatch_name = params['dispatch_name'] update_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(params['update_time'])) webhook = params['param_config'] if alert_type == 1: sub = '调度id: %s执行成功' % dispatch_id else: sub = '调度id: %s执行失败' % dispatch_id content = '%s\n调度id: %s\n调度名称: %s\n执行id: %s\n执行时间: %s' % ( sub, dispatch_id, dispatch_name, exec_id, update_time) try: bot = DingtalkChatbot(webhook) bot.send_text(msg=content, is_at_all=is_at_all) log.info("params: %s, 发送钉钉成功" % str({ 'exec_id': exec_id, 'alert_type': alert_type })) except Exception as e: log.error("发送钉钉消息失败[ERROR: %s]" % e, exc_info=True)
def mqtt_control(command): if command == "start": try: if global_context[ 'mqtt_conn_status'] == "offline" or global_context[ 'mqtt_conn_status'] == "reconnecting": if mqtt.connect(conf["mqtt"]["host"], int(conf["mqtt"]["port"])): mqtt.start() status = "Connected to broker." else: status = "Can't connect to broker." else: log.info("The system is already connected to mqtt broker.") status = "Start command was skiped . The system is already connected to mqtt broker." except Exception as ex: log.error("Can't connect to server because of error:") log.error(ex) status = "Can't connect to broker" elif command == "stop": mqtt.stop() status = "Disconnected from broker" # dev = json.dumps({"success":True}) # return Response(response=dev, mimetype='application/json' ) return render_template('mqtt_status.html', status=status, global_context=global_context)
def request(exec_id, interface_id, job_id, status): """封装requests请求""" url = 'http://%(host)s:%(port)s/execute/callback/' % { 'host': config.master.host, 'port': config.master.port } params = { 'exec_id': exec_id, 'interface_id': interface_id, 'job_id': job_id, 'status': status } for _ in range(5): try: response = requests.get(url=url, params=params) log.debug('回调web服务响应结果:[status_code: %d][text: %s]' % (response.status_code, response.text)) if response.status_code == 200: return True else: return False except Exception as e: log.error('回调web服务失败: [params: %s][error: %s]' % (str(params), e), exc_info=True) return False
def delete(): """登出""" try: session.login_out() return make_result(200, msg='成功'), 200 except Exception as e: log.error('用户登出出错 [ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='无用户登陆信息'))
def load_captcha(self): for item in self.captcha_store.all(): captcha_id = item.pop('item_id') log.error(item) try: self.current_captcha[captcha_id] = PassStorage(**item) except TypeError as e: log.error(f'Load problem {e!r}')
def run_ftp_event(ftp_event_id, run_date, date_format): """立即执行调度""" for item in ftp_event_id: try: get_event_job(item, 2, run_date, date_format) except Exception as e: log.error('立即执行调度异常 [ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='立即执行调度异常')) return Response(ftp_event_id=ftp_event_id)
def run_dispatch(dispatch_id): """立即执行调度""" try: run_id = 'scheduler_%s' % dispatch_id scheduler_handler.run_job(run_id) return Response(dispatch_id=dispatch_id) except Exception as e: log.error('立即执行调度异常 [ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='立即执行调度异常'))
def run_dispatch(dispatch_id, run_date, date_format, is_after): """立即执行调度""" for item in dispatch_id: try: get_dispatch_job(item, 2, run_date, date_format, is_after) except Exception as e: log.error('立即执行调度异常 [ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='立即执行调度异常')) return Response(dispatch_id=dispatch_id)
def stop_execute_job(exec_id): """ 中止执行任务 1.修改调度主表状态为[中断] 2.获取正在执行任务 3.rpc分发-停止任务 4.修改执行详情表为[失败], 修改执行任务流状态[中断] """ msg = [] for item in exec_id: # 修改数据库, 分布式锁 with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item): # 修改调度主表状态为[中断] stop_num = ExecuteModel.update_execute_stop(db.etl_db, item, 2) # 是否成功中断判断 if not stop_num: msg.append('执行ID: [%s]状态为非执行中, 中断失败' % item) continue # 获取正在执行任务, 分布式锁 with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item): result = ExecuteModel.get_execute_detail_by_status( db.etl_db, item, 'running') # 去重 result = {item['job_id']: item for item in result} for _, execute in result.items(): try: # 获取进程id if execute['pid']: # rpc分发-停止任务 client = Connection(execute['server_host'], config.exec.port) client.rpc.stop(exec_id=item, job_id=execute['job_id'], pid=execute['pid']) client.disconnect() log.info('rpc分发-停止任务: 执行id: %s, 任务id: %s' % (item, execute['job_id'])) # 修改数据库, 分布式锁 with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item): # 修改执行详情表为[失败] ScheduleModel.update_exec_job_status( db.etl_db, item, execute['interface_id'], execute['job_id'], 'failed') # 修改执行任务流状态[中断] ExecuteModel.update_exec_interface_status( db.etl_db, item, execute['interface_id'], 2) log.info( '修改执行详情表为[失败]: [%s, %s, %s]' % (item, execute['interface_id'], execute['job_id'])) except: err_msg = 'rpc分发-停止任务异常: host: %s, port: %s, 执行id: %s, 任务id: %s' % ( execute['server_host'], config.exec.port, item, execute['job_id']) log.error(err_msg, exc_info=True) msg.append(err_msg) return Response(msg=msg)
def rpc_push_job(exec_id, interface_id, job_id, server_host, port, params_value, server_dir, server_script, return_code, status, date_format='%Y%m%d', run_date=''): """ RPC分发任务 1.替换$date变量 :param exec_id: 执行id :param interface_id: 任务流id :param job_id: 任务id :param server_host: RPC执行服务器域名 :param port: RPC执行服务器端口 :param params_value: 参数值字符串 :param server_dir: 脚本目录 :param server_script: 运行脚本 :param return_code: 状态返回码 :param status: 任务状态 :param date_format: 日期格式 :param run_date: 数据日期 :return: """"" try: # rpc分发任务 client = Connection(server_host, port) # 任务参数中数据日期变量为T-1 if not run_date: run_time = (date.today() + timedelta(days=-1)).strftime(date_format) else: run_time = run_date params = params_value.split(',') if params_value else [] client.rpc.event_execute( exec_id=exec_id, interface_id=interface_id, job_id=job_id, server_dir=server_dir, server_script=server_script, return_code=return_code, params=[item if item != '$date' else run_time for item in params], status=status ) client.disconnect() return '' except: err_msg = 'rpc连接异常: host: %s, port: %s' % (server_host, port) # 添加执行任务详情日志 EventModel.add_event_exec_detail_job(db.etl_db, exec_id, interface_id, job_id, 'ERROR', server_dir, server_script, err_msg, 3) # 修改数据库, 分布式锁 with MysqlLock(config.mysql.etl, 'event_lock_%s' % exec_id): # 修改执行详情表状态[失败] EventModel.update_event_exec_job_status(db.etl_db, exec_id, interface_id, job_id, 'failed') # 修改执行任务流状态[失败] EventModel.update_event_exec_interface_status(db.etl_db, exec_id, interface_id, -1) # 修改执行主表状态[失败] EventModel.update_event_execute_status(db.etl_db, exec_id, -1) log.error(err_msg, exc_info=True) return err_msg
def delete_dispatch_detail(dispatch_id, user_id): """删除调度详情""" try: run_id = 'scheduler_%s' % dispatch_id DispatchModel.update_dispatch_status(db.etl_db, dispatch_id, 0, user_id) scheduler_handler.remove_job(run_id) return Response(dispatch_id=dispatch_id) except Exception as e: log.error('删除调度异常[ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='删除调度异常'))
def send_mail(alert_type, params): """ 发送邮件 @alert_type: 1: 成功, 2: 失败 @exec_id: 执行id @host: "smtp.exmail.qq.com" @port: 465 @send_user: "******" / "*****@*****.**" @to_list: "*****@*****.**" @sub: python群发邮件测试 @content: "<a href='http://blog.csdn.net/Marksinoberg/article/details/51501377'>测试链接</a>" password: "******" """ exec_id = params['exec_id'] dispatch_id = params['dispatch_id'] dispatch_name = params['dispatch_name'] update_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(params['update_time'])) host = params['param_host'] port = params['param_port'] send_user = params['param_config'] to_list = params['send_mail'] password = params['param_pass'] if alert_type == 1: sub = '调度id: %s执行成功' % dispatch_id else: sub = '调度id: %s执行失败' % dispatch_id content = ''' %s</br></br> 调度id: %s</br> 调度名称: %s</br> 执行id: %s</br> 执行时间: %s ''' % (sub, dispatch_id, dispatch_name, exec_id, update_time) msg = MIMEText(content, _subtype='html', _charset='utf-8') msg['Subject'] = sub msg['From'] = send_user msg['To'] = to_list try: # 连接smtp服务器 s = smtplib.SMTP_SSL() s.connect(host=host, port=port) s.login(send_user, password) s.sendmail(send_user, to_list, msg.as_string()) s.close() log.info("params: %s, 发送邮件成功" % str({ 'exec_id': exec_id, 'alert_type': alert_type })) except Exception as e: log.error("发送邮件失败[ERROR: %s]" % e, exc_info=True)
def s(name: str, params: Dict) -> str: if 'lang' not in params: log.error(f'Missing required lang on {params}') return '' lang = params['lang'] if lang not in lang_cache: with open(f'templates/{lang}.json', 'r') as template: lang_cache[lang] = json.loads(template.read()) # TODO: check if it is a list template = lang_cache[lang][name] return Template(template).render(**params)
def get_db_data_one(source_type, source_host, source_port, source_user, source_password, source_database, auth_type, param_value): """ 连接数据源获取数据 :param source_type: 数据库类型: 1.mysql, 2.mongo, 3.mssql, 4.hive, 5.impala :param source_host: 数据库ip或域名 :param source_port: 数据库端口 :param source_user: 用户名 :param source_password: 密码 :param source_database: 数据库库名 :param auth_type: 认证方式(仅hive和impala使用):0.无,1.NOSASL,2.PLAIN,3.GSSAPI,4.LDAP :param param_value: SQL查询 """ # 用户名 if not source_user: source_user = None # 密码 if not source_password: source_password = None # 数据库库名 if not source_database: source_database = None try: # mysql if source_type == 1: cursor = MysqlConn(source_host, source_port, source_user, source_password, source_database) result = cursor.query_one(param_value) # mongo # elif source_type == 2: # MongoLinks(source_host, source_port, source_database, source_user, source_password) # mssql elif source_type == 3: cursor = MssqlConn(source_host, source_port, source_user, source_password, source_database) result = cursor.query_one(param_value) result = str(result.values()[0]) if result else '' # hive / impala else: cursor = ImpalaLink(source_host, source_port, source_user, source_password, source_database, auth_type) result = cursor.query_one(param_value) if isinstance(result, tuple): result = str(result[0]) elif isinstance(result, dict): result = str( [i for i in result.values()][0] if result.values() else '') else: result = '' return {'result': result, 'msg': '成功', 'flag': 0} except Exception as e: log.error('测试数据源连接异常: [error: %s]' % e, exc_info=1) return {'result': '', 'msg': traceback.format_exc(), 'flag': 1}
def delete_dispatch_detail(dispatch_id): """删除调度详情""" for item in dispatch_id: try: run_id = 'scheduler_%s' % item detail = DispatchModel.get_dispatch_detail(db.etl_db, item) DispatchModel.delete_dispatch_detail(db.etl_db, item) if detail['status'] != 0: SchedulerHandler.remove_job(run_id) except Exception as e: log.error('删除调度异常[ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='删除调度错误, 该调度不存在')) return Response(dispatch_id=dispatch_id)
def delete_ftp_event(ftp_event_id): """删除调度详情""" for item in ftp_event_id: try: run_id = 'ftp_event_%s' % item detail = FtpEventModel.get_ftp_event_detail(db.etl_db, item) FtpEventModel.delete_ftp_event_detail(db.etl_db, item) FtpEventModel.delete_file_event_interface(db.etl_db, item) if detail['status'] != 0: EventHandler.remove_job(run_id) except Exception as e: log.error('删除调度异常[ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='删除调度错误, 该调度不存在')) return Response(ftp_event_id=ftp_event_id)
def CrontabNextTime(sched, timeFormat='%Y-%m-%d %H:%M:%S', queryTimes=10): """计算定时任务下次运行时间""" try: now = datetime.datetime.now() # 以当前时间为基准开始计算 cron = croniter.croniter(sched, now) result = [ cron.get_next(datetime.datetime).strftime(timeFormat) for _ in range(queryTimes) ] except Exception as e: log.error('计算cron表达式异常 [ERROR: %s]' % e, exc_info=True) result = ['计算cron表达式异常'] return Response(run_times=result)
async def process_callback_backspace(callback_query: types.CallbackQuery): debug_id = f'{callback_query.message.chat.username}-({callback_query.from_user.mention})' log.info(f'{debug_id}: backspace') _id = f'{callback_query.message.message_id}-{callback_query.message.chat.id}' try: pass_item = data_store.get_captcha(_id) except KeyError: log.error(f'Something gone wrong: {data_store} {_id}') await bot.answer_callback_query(callback_query.id, text=s('something_gone_wrong_warn', {'lang': 'ru'})) return if not pass_item.user_check(callback_query.from_user.id): await bot.answer_callback_query(callback_query.id, text=s('not_for_you_warn', {'lang': 'ru'})) return text = pass_item.backspace() await bot.answer_callback_query(callback_query.id, text=text)
def exposed_stop(exec_id, job_id, pid): """停止正在进行中的调度任务""" # kill -9 进程id if os.name == 'nt': # Windows系统 cmd = 'taskkill /pid %s /f' % pid os.system(cmd) elif os.name == 'posix': # Linux系统 cmd = 'kill -9 %s' % pid os.system(cmd) else: raise Exception('Undefined os.name') log.error('停止调度, 执行id: %s, 任务id: %s, 任务状态: %s' % (exec_id, job_id, 'failed')) return {'status': True, 'msg': '任务已停止'}
def add_dispatch_detail(interface_id, dispatch_name, dispatch_desc, minute, hour, day, month, week, user_id): """新增调度""" try: # 新增调度表 dispatch_id = DispatchModel.add_dispatch_detail( db.etl_db, interface_id, dispatch_name, dispatch_desc, minute, hour, day, month, week, user_id) # 添加调度任务 run_id = 'scheduler_%s' % dispatch_id SchedulerHandler.add_job(run_id, dispatch_id, minute, hour, day, month, week) return Response(dispatch_id=dispatch_id) except pymysql.err.IntegrityError as e: log.error('请勿重复为任务流添加调度 [ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='请勿重复为任务流添加调度'))
async def cleaner(): while True: try: await asyncio.sleep(60) now = datetime.now() for _id, item in data_store.list_captcha(): if item.is_expired(now): log.info(f'{item.debug_id}: Timeout, kick and clean') data_store.remove_captcha(_id) chat_id = item.chat_id member_id = item.user_id await clear(item) await bot.kick_chat_member(chat_id, member_id) await bot.unban_chat_member(chat_id, member_id) except Exception as e: log.error(f'Uncaught exception {e}')
def test_exec_host(server_host): """测试执行服务器""" # 服务器状态 host_item = ExecHostModel.get_exec_host_status_by_host( db.etl_db, server_host) try: # rpc调用 client = Connection(server_host, config.exec.port) result = json.loads(client.rpc.test()) # 不存在跳过 if not host_item: pass # 新增服务器状态 elif host_item['server_id'] and not host_item['status_id']: now_time = int(time.time()) ExecHostModel.add_exec_host_status_by_host_success( db.etl_db, host_item['server_id'], result['cpu'], result['system'], result['disk']['used'], result['disk']['total'], result['memory']['used'], result['memory']['total'], now_time, 0) # 修改服务器状态 elif host_item['server_id'] and host_item['status_id']: now_time = int(time.time()) ExecHostModel.update_exec_host_status_by_host_success( db.etl_db, host_item['server_id'], result['cpu'], result['system'], result['disk']['used'], result['disk']['total'], result['memory']['used'], result['memory']['total'], now_time, 0) return Response(result=result) except Exception as e: # 不存在跳过 if not host_item: pass # 新增服务器状态 elif host_item['server_id'] and not host_item['status_id']: now_time = int(time.time()) ExecHostModel.add_exec_host_status_by_host_failed( db.etl_db, host_item['server_id'], now_time, 1) # 修改服务器状态 elif host_item['server_id'] and host_item['status_id']: now_time = int(time.time()) ExecHostModel.update_exec_host_status_by_host_failed( db.etl_db, host_item['server_id'], now_time, 1) log.error('执行服务器连通失败, [server_host: %s]' % server_host, exc_info=1) abort(403, **make_result(status=403, msg='服务器连通失败: [ERROR: %s]' % e))
def action_dispatch(dispatch_id, action, user_id): """暂停/恢复调度任务""" try: run_id = 'scheduler_%s' % dispatch_id # 暂停调度任务 if action == 1: DispatchModel.update_dispatch_status(db.etl_db, dispatch_id, 2, user_id) scheduler_handler.pause_job(run_id) # 恢复调度任务 elif action == 2: DispatchModel.update_dispatch_status(db.etl_db, dispatch_id, 1, user_id) scheduler_handler.resume_job(run_id) return Response(dispatch_id=dispatch_id) except Exception as e: log.error('暂停/恢复调度异常 [ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='暂停/恢复调度任务异常'))
def __exit__(self, exc_type, exc_value, traceback): """释放锁""" try: sql = "SELECT RELEASE_LOCK('%s') AS 'release_lock'" % self.lock_name cursor = self.execute(sql) result = cursor.fetchone() if result['release_lock'] == 0: raise BaseException('锁[%s]未被释放, 该锁并非由此数据库连接创建.' % self.lock_name) elif result['release_lock'] == 1: log.debug("锁[%s]释放成功." % self.lock_name) else: raise BaseException("锁[%s]不存在." % self.lock_name) except Exception as e: log.error(e, exc_info=True) raise e finally: # 一定要释放连接, 不然会死锁 self.conn and self.conn.close()
def action_ftp_event(ftp_event_id, action, user_id): """暂停/恢复调度事件""" for item in ftp_event_id: try: run_id = 'ftp_event_%s' % item # 暂停调度任务 if action == 1: FtpEventModel.update_ftp_event_status( db.etl_db, item, 2, user_id) EventHandler.pause_job(run_id) # 恢复调度任务 elif action == 2: FtpEventModel.update_ftp_event_status( db.etl_db, item, 1, user_id) EventHandler.resume_job(run_id) except Exception as e: log.error('暂停/恢复调度异常 [ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='暂停/恢复调度任务异常')) return Response(ftp_event_id=ftp_event_id)
def update_dispatch_detail(dispatch_id, interface_id, dispatch_name, dispatch_desc, minute, hour, day, month, week, old_status, new_status, user_id): """修改调度详情""" try: # 修改调度表 DispatchModel.update_dispatch_detail(db.etl_db, dispatch_id, interface_id, dispatch_name, dispatch_desc, minute, hour, day, month, week, new_status, user_id) # 修改调度状态 run_id = 'scheduler_%s' % dispatch_id # 仍删除 if (old_status == 0 and new_status == 0): pass # 新增 or 先新增后暂停 elif (old_status == 0 and new_status == 1 or old_status == 0 and new_status == 2): scheduler_handler.add_job(run_id, dispatch_id, minute, hour, day, month, week) # 修改 else: scheduler_handler.modify_job(run_id, dispatch_id, minute, hour, day, month, week) # 先新增后暂停 if (old_status == 0 and new_status == 2): scheduler_handler.pause_job(run_id) # 暂停 elif (old_status == 1 and new_status == 2): scheduler_handler.pause_job(run_id) # 删除 elif (old_status == 1 and new_status == 0 or old_status == 2 and new_status == 0): scheduler_handler.remove_job(run_id) # 恢复 elif (old_status == 2 and new_status == 1): scheduler_handler.resume_job(run_id) return Response(dispatch_id=dispatch_id) except pymysql.err.IntegrityError as e: log.error('请勿重复为接口添加调度 [ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='请勿重复为接口添加调度'))
def test_db_conn(source_type, auth_type, source_host, source_port, source_database, source_user, source_password): """ 测试数据源连接 :param source_type: 数据库类型: 1.mysql, 2.mongo, 3.mssql, 4.hive, 5.impala :param auth_type: 认证方式(仅hive和impala使用):0.无,1.NOSASL,2.PLAIN,3.GSSAPI,4.LDAP :param source_host: 数据库ip或域名 :param source_port: 数据库端口 :param source_database: 数据库库名 :param source_user: 用户名 :param source_password: 密码 :return: BOOLEAN """ # 用户名 if not source_user: source_user = None # 密码 if not source_password: source_password = None # 数据库库名 if not source_database: source_database = None try: # mysql if source_type == 1: MysqlConn(source_host, source_port, source_user, source_password, source_database) # mongo # elif source_type == 2: # MongoLinks(source_host, source_port, source_database, source_user, source_password) # mssql elif source_type == 3: MssqlConn(source_host, source_port, source_user, source_password, source_database) # hive / impala else: ImpalaLink(source_host, source_port, source_user, source_password, source_database, auth_type) return {'tag': True, 'msg': '成功'} except Exception as e: log.error('测试数据源连接异常: [error: %s]' % e, exc_info=1) return {'tag': False, 'msg': traceback.format_exc()}
def __enter__(self): """获取锁""" try: # 获取锁 sql = "SELECT GET_LOCK('%s', %s) AS 'lock'" % (self.lock_name, self.timeout) cursor = self.execute(sql) if cursor.rowcount != 1: raise BaseException('锁[%s]在lock函数中返回多行.') result = cursor.fetchone() if result['lock'] == 0: raise BaseException('锁[%s]已被其他客户端锁定.' % self.lock_name) elif result['lock'] == 1: log.debug('获取锁[%s]成功.' % self.lock_name) return else: raise BaseException('mysql lock函数其他异常.') except Exception as e: log.error(e, exc_info=True) raise e
def get_dispatch_job(dispatch_id): """获取调度任务""" result = generate_dag_by_dispatch_id(dispatch_id) source = result['source'] # 添加相关信息至数据库 exec_id = add_exec_record(dispatch_id, source) # rpc分发任务 for job in source: if job['level'] == 0 and job['position'] == 1: client = Connection(job['server_host'], config.exec.port) try: client.rpc.execute(exec_id=exec_id, job_id=job['id'], server_dir=job['server_dir'], server_script=job['server_script'], status=job['status']) log.info('分发任务: 执行id: %s, 任务id: %s' % (exec_id, job['id'])) except: log.error('rpc连接异常: host: %s, port: %s' % (job['server_host'], config.exec.port), exc_info=True)