async def process_callback_kb1btn1(callback_query: types.CallbackQuery):
code = callback_query.data
chat_id = callback_query.message.chat.id
member_id = callback_query.from_user.id
user_title = callback_query.from_user.mention
debug_id = f'{callback_query.message.chat.username}-({user_title})'
chat_title = callback_query.message.chat.title
log.info(f'{debug_id}: {code}')
_id = f'{callback_query.message.message_id}-{callback_query.message.chat.id}'
try:
pass_item = data_store.get_captcha(_id)
except KeyError:
log.error(f'Something gone wrong: {data_store} {_id}')
await bot.answer_callback_query(callback_query.id,
text=s('something_gone_wrong_warn',
{'lang': 'ru'}))
return
if not pass_item.user_check(callback_query.from_user.id):
await bot.answer_callback_query(callback_query.id,
text=s('not_for_you_warn',
{'lang': 'ru'}))
return
text = pass_item.new_char(code[-1:])
result = pass_item.check()
if result is CAPTCHA_STATE.INPUTING:
await bot.answer_callback_query(callback_query.id, text=text)
elif result is CAPTCHA_STATE.SUCSSES:
data_store.remove_captcha(_id)
log.info(f'{debug_id}: SUCSSES')
await bot.answer_callback_query(callback_query.id, text='SUCSSES')
try:
pass_item.messages.pop(MESSAGE_TYPES.LOGIN)
except KeyError:
pass
await clear(pass_item)
await bot.send_message(
callback_query.message.chat.id,
s('success_msg', {
'lang': 'ru',
'user_title': user_title,
'chat_title': chat_title
}))
unmute = ChatPermissions(can_send_messages=True,
can_send_media_messages=True,
can_add_web_page_previews=True,
can_send_other_messages=True,
can_send_polls=True)
await bot.restrict_chat_member(chat_id, member_id, permissions=unmute)
else:
data_store.remove_captcha(_id)
delay = datetime.now() + timedelta(minutes=1)
log.info(f'{debug_id}: FAIL, ban until {delay}')
# workaround to send new event.
await bot.kick_chat_member(chat_id, member_id, delay)
await bot.answer_callback_query(callback_query.id,
text=s('fail_msg', {'lang': 'ru'}))
await clear(pass_item)
# await bot.unban_chat_member(chat_id, member_id)
data_store.sync()
def send_dingtalk(alert_type, params, is_at_all=False):
"""
发送钉钉消息
@webhook: "https://oapi.dingtalk.com/robot/send?access_token=bd92817749b11207820971bab315e0b46b59d1cf422f33e976495acf4aa89ef7"
@msg: "发送消息"
"""
exec_id = params['exec_id']
dispatch_id = params['dispatch_id']
dispatch_name = params['dispatch_name']
update_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(params['update_time']))
webhook = params['param_config']
if alert_type == 1:
sub = '调度id: %s执行成功' % dispatch_id
else:
sub = '调度id: %s执行失败' % dispatch_id
content = '%s\n调度id: %s\n调度名称: %s\n执行id: %s\n执行时间: %s' % (
sub, dispatch_id, dispatch_name, exec_id, update_time)
try:
bot = DingtalkChatbot(webhook)
bot.send_text(msg=content, is_at_all=is_at_all)
log.info("params: %s, 发送钉钉成功" % str({
'exec_id': exec_id,
'alert_type': alert_type
}))
except Exception as e:
log.error("发送钉钉消息失败[ERROR: %s]" % e, exc_info=True)
def mqtt_control(command):
if command == "start":
try:
if global_context[
'mqtt_conn_status'] == "offline" or global_context[
'mqtt_conn_status'] == "reconnecting":
if mqtt.connect(conf["mqtt"]["host"],
int(conf["mqtt"]["port"])):
mqtt.start()
status = "Connected to broker."
else:
status = "Can't connect to broker."
else:
log.info("The system is already connected to mqtt broker.")
status = "Start command was skiped . The system is already connected to mqtt broker."
except Exception as ex:
log.error("Can't connect to server because of error:")
log.error(ex)
status = "Can't connect to broker"
elif command == "stop":
mqtt.stop()
status = "Disconnected from broker"
# dev = json.dumps({"success":True})
# return Response(response=dev, mimetype='application/json' )
return render_template('mqtt_status.html',
status=status,
global_context=global_context)
def request(exec_id, interface_id, job_id, status):
"""封装requests请求"""
url = 'http://%(host)s:%(port)s/execute/callback/' % {
'host': config.master.host,
'port': config.master.port
}
params = {
'exec_id': exec_id,
'interface_id': interface_id,
'job_id': job_id,
'status': status
}
for _ in range(5):
try:
response = requests.get(url=url, params=params)
log.debug('回调web服务响应结果：[status_code: %d][text: %s]' %
(response.status_code, response.text))
if response.status_code == 200:
return True
else:
return False
except Exception as e:
log.error('回调web服务失败: [params: %s][error: %s]' % (str(params), e),
exc_info=True)
return False
def delete():
"""登出"""
try:
session.login_out()
return make_result(200, msg='成功'), 200
except Exception as e:
log.error('用户登出出错 [ERROR: %s]' % e, exc_info=True)
abort(400, **make_result(status=400, msg='无用户登陆信息'))
def load_captcha(self):
for item in self.captcha_store.all():
captcha_id = item.pop('item_id')
log.error(item)
try:
self.current_captcha[captcha_id] = PassStorage(**item)
except TypeError as e:
log.error(f'Load problem {e!r}')
def run_ftp_event(ftp_event_id, run_date, date_format):
"""立即执行调度"""
for item in ftp_event_id:
try:
get_event_job(item, 2, run_date, date_format)
except Exception as e:
log.error('立即执行调度异常 [ERROR: %s]' % e, exc_info=True)
abort(400, **make_result(status=400, msg='立即执行调度异常'))
return Response(ftp_event_id=ftp_event_id)
def run_dispatch(dispatch_id):
"""立即执行调度"""
try:
run_id = 'scheduler_%s' % dispatch_id
scheduler_handler.run_job(run_id)
return Response(dispatch_id=dispatch_id)
except Exception as e:
log.error('立即执行调度异常 [ERROR: %s]' % e, exc_info=True)
abort(400, **make_result(status=400, msg='立即执行调度异常'))
def run_dispatch(dispatch_id, run_date, date_format, is_after):
"""立即执行调度"""
for item in dispatch_id:
try:
get_dispatch_job(item, 2, run_date, date_format, is_after)
except Exception as e:
log.error('立即执行调度异常 [ERROR: %s]' % e, exc_info=True)
abort(400, **make_result(status=400, msg='立即执行调度异常'))
return Response(dispatch_id=dispatch_id)
def stop_execute_job(exec_id):
"""
中止执行任务
1.修改调度主表状态为[中断]
2.获取正在执行任务
3.rpc分发-停止任务
4.修改执行详情表为[失败], 修改执行任务流状态[中断]
"""
msg = []
for item in exec_id:
# 修改数据库, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
# 修改调度主表状态为[中断]
stop_num = ExecuteModel.update_execute_stop(db.etl_db, item, 2)
# 是否成功中断判断
if not stop_num:
msg.append('执行ID: [%s]状态为非执行中, 中断失败' % item)
continue
# 获取正在执行任务, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
result = ExecuteModel.get_execute_detail_by_status(
db.etl_db, item, 'running')
# 去重
result = {item['job_id']: item for item in result}
for _, execute in result.items():
try:
# 获取进程id
if execute['pid']:
# rpc分发-停止任务
client = Connection(execute['server_host'],
config.exec.port)
client.rpc.stop(exec_id=item,
job_id=execute['job_id'],
pid=execute['pid'])
client.disconnect()
log.info('rpc分发-停止任务: 执行id: %s, 任务id: %s' %
(item, execute['job_id']))
# 修改数据库, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
# 修改执行详情表为[失败]
ScheduleModel.update_exec_job_status(
db.etl_db, item, execute['interface_id'],
execute['job_id'], 'failed')
# 修改执行任务流状态[中断]
ExecuteModel.update_exec_interface_status(
db.etl_db, item, execute['interface_id'], 2)
log.info(
'修改执行详情表为[失败]: [%s, %s, %s]' %
(item, execute['interface_id'], execute['job_id']))
except:
err_msg = 'rpc分发-停止任务异常: host: %s, port: %s, 执行id: %s, 任务id: %s' % (
execute['server_host'], config.exec.port, item,
execute['job_id'])
log.error(err_msg, exc_info=True)
msg.append(err_msg)
return Response(msg=msg)
def rpc_push_job(exec_id, interface_id, job_id, server_host, port, params_value, server_dir, server_script, return_code,
status, date_format='%Y%m%d', run_date=''):
"""
RPC分发任务
1.替换$date变量
:param exec_id: 执行id
:param interface_id: 任务流id
:param job_id: 任务id
:param server_host: RPC执行服务器域名
:param port: RPC执行服务器端口
:param params_value: 参数值字符串
:param server_dir: 脚本目录
:param server_script: 运行脚本
:param return_code: 状态返回码
:param status: 任务状态
:param date_format: 日期格式
:param run_date: 数据日期
:return:
"""""
try:
# rpc分发任务
client = Connection(server_host, port)
# 任务参数中数据日期变量为T-1
if not run_date:
run_time = (date.today() + timedelta(days=-1)).strftime(date_format)
else:
run_time = run_date
params = params_value.split(',') if params_value else []
client.rpc.event_execute(
exec_id=exec_id,
interface_id=interface_id,
job_id=job_id,
server_dir=server_dir,
server_script=server_script,
return_code=return_code,
params=[item if item != '$date' else run_time for item in params],
status=status
)
client.disconnect()
return ''
except:
err_msg = 'rpc连接异常: host: %s, port: %s' % (server_host, port)
# 添加执行任务详情日志
EventModel.add_event_exec_detail_job(db.etl_db, exec_id, interface_id, job_id, 'ERROR', server_dir,
server_script, err_msg, 3)
# 修改数据库, 分布式锁
with MysqlLock(config.mysql.etl, 'event_lock_%s' % exec_id):
# 修改执行详情表状态[失败]
EventModel.update_event_exec_job_status(db.etl_db, exec_id, interface_id, job_id, 'failed')
# 修改执行任务流状态[失败]
EventModel.update_event_exec_interface_status(db.etl_db, exec_id, interface_id, -1)
# 修改执行主表状态[失败]
EventModel.update_event_execute_status(db.etl_db, exec_id, -1)
log.error(err_msg, exc_info=True)
return err_msg
def delete_dispatch_detail(dispatch_id, user_id):
"""删除调度详情"""
try:
run_id = 'scheduler_%s' % dispatch_id
DispatchModel.update_dispatch_status(db.etl_db, dispatch_id, 0,
user_id)
scheduler_handler.remove_job(run_id)
return Response(dispatch_id=dispatch_id)
except Exception as e:
log.error('删除调度异常[ERROR: %s]' % e, exc_info=True)
abort(400, **make_result(status=400, msg='删除调度异常'))
def send_mail(alert_type, params):
"""
发送邮件
@alert_type: 1: 成功, 2: 失败
@exec_id: 执行id
@host: "smtp.exmail.qq.com"
@port: 465
@send_user: "******" / "*****@*****.**"
@to_list: "*****@*****.**"
@sub: python群发邮件测试
@content: "<a href='http://blog.csdn.net/Marksinoberg/article/details/51501377'>测试链接</a>"
password: "******"
"""
exec_id = params['exec_id']
dispatch_id = params['dispatch_id']
dispatch_name = params['dispatch_name']
update_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(params['update_time']))
host = params['param_host']
port = params['param_port']
send_user = params['param_config']
to_list = params['send_mail']
password = params['param_pass']
if alert_type == 1:
sub = '调度id: %s执行成功' % dispatch_id
else:
sub = '调度id: %s执行失败' % dispatch_id
content = '''
%s</br></br>
调度id: %s</br>
调度名称: %s</br>
执行id: %s</br>
执行时间: %s
''' % (sub, dispatch_id, dispatch_name, exec_id, update_time)
msg = MIMEText(content, _subtype='html', _charset='utf-8')
msg['Subject'] = sub
msg['From'] = send_user
msg['To'] = to_list
try:
# 连接smtp服务器
s = smtplib.SMTP_SSL()
s.connect(host=host, port=port)
s.login(send_user, password)
s.sendmail(send_user, to_list, msg.as_string())
s.close()
log.info("params: %s, 发送邮件成功" % str({
'exec_id': exec_id,
'alert_type': alert_type
}))
except Exception as e:
log.error("发送邮件失败[ERROR: %s]" % e, exc_info=True)
def s(name: str, params: Dict) -> str:
if 'lang' not in params:
log.error(f'Missing required lang on {params}')
return ''
lang = params['lang']
if lang not in lang_cache:
with open(f'templates/{lang}.json', 'r') as template:
lang_cache[lang] = json.loads(template.read())
# TODO: check if it is a list
template = lang_cache[lang][name]
return Template(template).render(**params)
def get_db_data_one(source_type, source_host, source_port, source_user,
source_password, source_database, auth_type, param_value):
"""
连接数据源获取数据
:param source_type: 数据库类型: 1.mysql, 2.mongo, 3.mssql, 4.hive, 5.impala
:param source_host: 数据库ip或域名
:param source_port: 数据库端口
:param source_user: 用户名
:param source_password: 密码
:param source_database: 数据库库名
:param auth_type: 认证方式(仅hive和impala使用):0.无,1.NOSASL,2.PLAIN,3.GSSAPI,4.LDAP
:param param_value: SQL查询
"""
# 用户名
if not source_user:
source_user = None
# 密码
if not source_password:
source_password = None
# 数据库库名
if not source_database:
source_database = None
try:
# mysql
if source_type == 1:
cursor = MysqlConn(source_host, source_port, source_user,
source_password, source_database)
result = cursor.query_one(param_value)
# mongo
# elif source_type == 2:
# MongoLinks(source_host, source_port, source_database, source_user, source_password)
# mssql
elif source_type == 3:
cursor = MssqlConn(source_host, source_port, source_user,
source_password, source_database)
result = cursor.query_one(param_value)
result = str(result.values()[0]) if result else ''
# hive / impala
else:
cursor = ImpalaLink(source_host, source_port, source_user,
source_password, source_database, auth_type)
result = cursor.query_one(param_value)
if isinstance(result, tuple):
result = str(result[0])
elif isinstance(result, dict):
result = str(
[i for i in result.values()][0] if result.values() else '')
else:
result = ''
return {'result': result, 'msg': '成功', 'flag': 0}
except Exception as e:
log.error('测试数据源连接异常: [error: %s]' % e, exc_info=1)
return {'result': '', 'msg': traceback.format_exc(), 'flag': 1}
def delete_dispatch_detail(dispatch_id):
"""删除调度详情"""
for item in dispatch_id:
try:
run_id = 'scheduler_%s' % item
detail = DispatchModel.get_dispatch_detail(db.etl_db, item)
DispatchModel.delete_dispatch_detail(db.etl_db, item)
if detail['status'] != 0:
SchedulerHandler.remove_job(run_id)
except Exception as e:
log.error('删除调度异常[ERROR: %s]' % e, exc_info=True)
abort(400, **make_result(status=400, msg='删除调度错误, 该调度不存在'))
return Response(dispatch_id=dispatch_id)
def delete_ftp_event(ftp_event_id):
"""删除调度详情"""
for item in ftp_event_id:
try:
run_id = 'ftp_event_%s' % item
detail = FtpEventModel.get_ftp_event_detail(db.etl_db, item)
FtpEventModel.delete_ftp_event_detail(db.etl_db, item)
FtpEventModel.delete_file_event_interface(db.etl_db, item)
if detail['status'] != 0:
EventHandler.remove_job(run_id)
except Exception as e:
log.error('删除调度异常[ERROR: %s]' % e, exc_info=True)
abort(400, **make_result(status=400, msg='删除调度错误, 该调度不存在'))
return Response(ftp_event_id=ftp_event_id)
def CrontabNextTime(sched, timeFormat='%Y-%m-%d %H:%M:%S', queryTimes=10):
"""计算定时任务下次运行时间"""
try:
now = datetime.datetime.now()
# 以当前时间为基准开始计算
cron = croniter.croniter(sched, now)
result = [
cron.get_next(datetime.datetime).strftime(timeFormat)
for _ in range(queryTimes)
]
except Exception as e:
log.error('计算cron表达式异常 [ERROR: %s]' % e, exc_info=True)
result = ['计算cron表达式异常']
return Response(run_times=result)
async def process_callback_backspace(callback_query: types.CallbackQuery):
debug_id = f'{callback_query.message.chat.username}-({callback_query.from_user.mention})'
log.info(f'{debug_id}: backspace')
_id = f'{callback_query.message.message_id}-{callback_query.message.chat.id}'
try:
pass_item = data_store.get_captcha(_id)
except KeyError:
log.error(f'Something gone wrong: {data_store} {_id}')
await bot.answer_callback_query(callback_query.id, text=s('something_gone_wrong_warn', {'lang': 'ru'}))
return
if not pass_item.user_check(callback_query.from_user.id):
await bot.answer_callback_query(callback_query.id, text=s('not_for_you_warn', {'lang': 'ru'}))
return
text = pass_item.backspace()
await bot.answer_callback_query(callback_query.id, text=text)
def exposed_stop(exec_id, job_id, pid):
"""停止正在进行中的调度任务"""
# kill -9 进程id
if os.name == 'nt':
# Windows系统
cmd = 'taskkill /pid %s /f' % pid
os.system(cmd)
elif os.name == 'posix':
# Linux系统
cmd = 'kill -9 %s' % pid
os.system(cmd)
else:
raise Exception('Undefined os.name')
log.error('停止调度, 执行id: %s, 任务id: %s, 任务状态: %s' % (exec_id, job_id, 'failed'))
return {'status': True, 'msg': '任务已停止'}
def add_dispatch_detail(interface_id, dispatch_name, dispatch_desc, minute,
hour, day, month, week, user_id):
"""新增调度"""
try:
# 新增调度表
dispatch_id = DispatchModel.add_dispatch_detail(
db.etl_db, interface_id, dispatch_name, dispatch_desc, minute,
hour, day, month, week, user_id)
# 添加调度任务
run_id = 'scheduler_%s' % dispatch_id
SchedulerHandler.add_job(run_id, dispatch_id, minute, hour, day,
month, week)
return Response(dispatch_id=dispatch_id)
except pymysql.err.IntegrityError as e:
log.error('请勿重复为任务流添加调度 [ERROR: %s]' % e, exc_info=True)
abort(400, **make_result(status=400, msg='请勿重复为任务流添加调度'))
async def cleaner():
while True:
try:
await asyncio.sleep(60)
now = datetime.now()
for _id, item in data_store.list_captcha():
if item.is_expired(now):
log.info(f'{item.debug_id}: Timeout, kick and clean')
data_store.remove_captcha(_id)
chat_id = item.chat_id
member_id = item.user_id
await clear(item)
await bot.kick_chat_member(chat_id, member_id)
await bot.unban_chat_member(chat_id, member_id)
except Exception as e:
log.error(f'Uncaught exception {e}')
def test_exec_host(server_host):
"""测试执行服务器"""
# 服务器状态
host_item = ExecHostModel.get_exec_host_status_by_host(
db.etl_db, server_host)
try:
# rpc调用
client = Connection(server_host, config.exec.port)
result = json.loads(client.rpc.test())
# 不存在跳过
if not host_item:
pass
# 新增服务器状态
elif host_item['server_id'] and not host_item['status_id']:
now_time = int(time.time())
ExecHostModel.add_exec_host_status_by_host_success(
db.etl_db, host_item['server_id'], result['cpu'],
result['system'], result['disk']['used'],
result['disk']['total'], result['memory']['used'],
result['memory']['total'], now_time, 0)
# 修改服务器状态
elif host_item['server_id'] and host_item['status_id']:
now_time = int(time.time())
ExecHostModel.update_exec_host_status_by_host_success(
db.etl_db, host_item['server_id'], result['cpu'],
result['system'], result['disk']['used'],
result['disk']['total'], result['memory']['used'],
result['memory']['total'], now_time, 0)
return Response(result=result)
except Exception as e:
# 不存在跳过
if not host_item:
pass
# 新增服务器状态
elif host_item['server_id'] and not host_item['status_id']:
now_time = int(time.time())
ExecHostModel.add_exec_host_status_by_host_failed(
db.etl_db, host_item['server_id'], now_time, 1)
# 修改服务器状态
elif host_item['server_id'] and host_item['status_id']:
now_time = int(time.time())
ExecHostModel.update_exec_host_status_by_host_failed(
db.etl_db, host_item['server_id'], now_time, 1)
log.error('执行服务器连通失败, [server_host: %s]' % server_host, exc_info=1)
abort(403, **make_result(status=403,
msg='服务器连通失败: [ERROR: %s]' % e))
def action_dispatch(dispatch_id, action, user_id):
"""暂停/恢复调度任务"""
try:
run_id = 'scheduler_%s' % dispatch_id
# 暂停调度任务
if action == 1:
DispatchModel.update_dispatch_status(db.etl_db, dispatch_id, 2,
user_id)
scheduler_handler.pause_job(run_id)
# 恢复调度任务
elif action == 2:
DispatchModel.update_dispatch_status(db.etl_db, dispatch_id, 1,
user_id)
scheduler_handler.resume_job(run_id)
return Response(dispatch_id=dispatch_id)
except Exception as e:
log.error('暂停/恢复调度异常 [ERROR: %s]' % e, exc_info=True)
abort(400, **make_result(status=400, msg='暂停/恢复调度任务异常'))
def __exit__(self, exc_type, exc_value, traceback):
"""释放锁"""
try:
sql = "SELECT RELEASE_LOCK('%s') AS 'release_lock'" % self.lock_name
cursor = self.execute(sql)
result = cursor.fetchone()
if result['release_lock'] == 0:
raise BaseException('锁[%s]未被释放, 该锁并非由此数据库连接创建.' %
self.lock_name)
elif result['release_lock'] == 1:
log.debug("锁[%s]释放成功." % self.lock_name)
else:
raise BaseException("锁[%s]不存在." % self.lock_name)
except Exception as e:
log.error(e, exc_info=True)
raise e
finally:
# 一定要释放连接, 不然会死锁
self.conn and self.conn.close()
def action_ftp_event(ftp_event_id, action, user_id):
"""暂停/恢复调度事件"""
for item in ftp_event_id:
try:
run_id = 'ftp_event_%s' % item
# 暂停调度任务
if action == 1:
FtpEventModel.update_ftp_event_status(
db.etl_db, item, 2, user_id)
EventHandler.pause_job(run_id)
# 恢复调度任务
elif action == 2:
FtpEventModel.update_ftp_event_status(
db.etl_db, item, 1, user_id)
EventHandler.resume_job(run_id)
except Exception as e:
log.error('暂停/恢复调度异常 [ERROR: %s]' % e, exc_info=True)
abort(400, **make_result(status=400, msg='暂停/恢复调度任务异常'))
return Response(ftp_event_id=ftp_event_id)
def update_dispatch_detail(dispatch_id, interface_id, dispatch_name,
dispatch_desc, minute, hour, day, month, week,
old_status, new_status, user_id):
"""修改调度详情"""
try:
# 修改调度表
DispatchModel.update_dispatch_detail(db.etl_db, dispatch_id,
interface_id, dispatch_name,
dispatch_desc, minute, hour,
day, month, week, new_status,
user_id)
# 修改调度状态
run_id = 'scheduler_%s' % dispatch_id
# 仍删除
if (old_status == 0 and new_status == 0):
pass
# 新增 or 先新增后暂停
elif (old_status == 0 and new_status == 1
or old_status == 0 and new_status == 2):
scheduler_handler.add_job(run_id, dispatch_id, minute, hour,
day, month, week)
# 修改
else:
scheduler_handler.modify_job(run_id, dispatch_id, minute, hour,
day, month, week)
# 先新增后暂停
if (old_status == 0 and new_status == 2):
scheduler_handler.pause_job(run_id)
# 暂停
elif (old_status == 1 and new_status == 2):
scheduler_handler.pause_job(run_id)
# 删除
elif (old_status == 1 and new_status == 0
or old_status == 2 and new_status == 0):
scheduler_handler.remove_job(run_id)
# 恢复
elif (old_status == 2 and new_status == 1):
scheduler_handler.resume_job(run_id)
return Response(dispatch_id=dispatch_id)
except pymysql.err.IntegrityError as e:
log.error('请勿重复为接口添加调度 [ERROR: %s]' % e, exc_info=True)
abort(400, **make_result(status=400, msg='请勿重复为接口添加调度'))
def test_db_conn(source_type, auth_type, source_host, source_port,
source_database, source_user, source_password):
"""
测试数据源连接
:param source_type: 数据库类型: 1.mysql, 2.mongo, 3.mssql, 4.hive, 5.impala
:param auth_type: 认证方式(仅hive和impala使用):0.无,1.NOSASL,2.PLAIN,3.GSSAPI,4.LDAP
:param source_host: 数据库ip或域名
:param source_port: 数据库端口
:param source_database: 数据库库名
:param source_user: 用户名
:param source_password: 密码
:return: BOOLEAN
"""
# 用户名
if not source_user:
source_user = None
# 密码
if not source_password:
source_password = None
# 数据库库名
if not source_database:
source_database = None
try:
# mysql
if source_type == 1:
MysqlConn(source_host, source_port, source_user, source_password,
source_database)
# mongo
# elif source_type == 2:
# MongoLinks(source_host, source_port, source_database, source_user, source_password)
# mssql
elif source_type == 3:
MssqlConn(source_host, source_port, source_user, source_password,
source_database)
# hive / impala
else:
ImpalaLink(source_host, source_port, source_user, source_password,
source_database, auth_type)
return {'tag': True, 'msg': '成功'}
except Exception as e:
log.error('测试数据源连接异常: [error: %s]' % e, exc_info=1)
return {'tag': False, 'msg': traceback.format_exc()}
def __enter__(self):
"""获取锁"""
try:
# 获取锁
sql = "SELECT GET_LOCK('%s', %s) AS 'lock'" % (self.lock_name,
self.timeout)
cursor = self.execute(sql)
if cursor.rowcount != 1:
raise BaseException('锁[%s]在lock函数中返回多行.')
result = cursor.fetchone()
if result['lock'] == 0:
raise BaseException('锁[%s]已被其他客户端锁定.' % self.lock_name)
elif result['lock'] == 1:
log.debug('获取锁[%s]成功.' % self.lock_name)
return
else:
raise BaseException('mysql lock函数其他异常.')
except Exception as e:
log.error(e, exc_info=True)
raise e
def get_dispatch_job(dispatch_id):
"""获取调度任务"""
result = generate_dag_by_dispatch_id(dispatch_id)
source = result['source']
# 添加相关信息至数据库
exec_id = add_exec_record(dispatch_id, source)
# rpc分发任务
for job in source:
if job['level'] == 0 and job['position'] == 1:
client = Connection(job['server_host'], config.exec.port)
try:
client.rpc.execute(exec_id=exec_id,
job_id=job['id'],
server_dir=job['server_dir'],
server_script=job['server_script'],
status=job['status'])
log.info('分发任务: 执行id: %s, 任务id: %s' % (exec_id, job['id']))
except:
log.error('rpc连接异常: host: %s, port: %s' %
(job['server_host'], config.exec.port),
exc_info=True)
