def settings_ui():
if request.method == 'POST':
conf["system"]["sid"] = request.form["system_sid"]
conf["mqtt"]["host"] = request.form["mqtt_host"]
conf["mqtt"]["port"] = request.form["mqtt_port"]
conf["mqtt"]["client_id"] = request.form["mqtt_client_id"]
conf["mqtt"]["username"] = request.form["mqtt_username"]
conf["mqtt"]["password"] = request.form["mqtt_password"]
conf["mqtt"]["global_topic_prefix"] = request.form[
"mqtt_global_topic_prefix"]
conf["mqtt"]["enable_sys"] = convert_bool(
request.form["enable_sys"])
mqtt.set_mqtt_params(request.form["mqtt_client_id"],
request.form["mqtt_username"],
request.form["mqtt_password"],
request.form["mqtt_global_topic_prefix"],
conf["mqtt"]["enable_sys"])
utils.save_config(conf_path, conf)
log.info("Global config was successfully updated")
log.info("New values are mqtt host = " +
request.form["mqtt_host"] + " port = " +
request.form["mqtt_port"] + " root topic = " +
" client id=" + request.form["mqtt_client_id"])
return render_template('settings.html',
cfg=conf,
global_context=global_context)
def send_dingtalk(alert_type, params, is_at_all=False):
"""
发送钉钉消息
@webhook: "https://oapi.dingtalk.com/robot/send?access_token=bd92817749b11207820971bab315e0b46b59d1cf422f33e976495acf4aa89ef7"
@msg: "发送消息"
"""
exec_id = params['exec_id']
dispatch_id = params['dispatch_id']
dispatch_name = params['dispatch_name']
update_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(params['update_time']))
webhook = params['param_config']
if alert_type == 1:
sub = '调度id: %s执行成功' % dispatch_id
else:
sub = '调度id: %s执行失败' % dispatch_id
content = '%s\n调度id: %s\n调度名称: %s\n执行id: %s\n执行时间: %s' % (
sub, dispatch_id, dispatch_name, exec_id, update_time)
try:
bot = DingtalkChatbot(webhook)
bot.send_text(msg=content, is_at_all=is_at_all)
log.info("params: %s, 发送钉钉成功" % str({
'exec_id': exec_id,
'alert_type': alert_type
}))
except Exception as e:
log.error("发送钉钉消息失败[ERROR: %s]" % e, exc_info=True)
def check(self) -> CAPTCHA_STATE:
if len(self.input_num) < len(self.items):
return CAPTCHA_STATE.INPUTING
log.info(f'{self.input_num} {self.items}')
if self.input_num == self.items:
return CAPTCHA_STATE.SUCSSES
return CAPTCHA_STATE.FAIL
def get_connection(self, command_name, *keys, **options):
"""
Get a connection, blocking for ``self.timeout`` until a connection
is available from the pool.
If the connection returned is ``None`` then creates a new connection.
Because we use a last-in first-out queue, the existing connections
(having been returned to the pool after the initial ``None`` values
were added) will be returned before ``None`` values. This means we only
create new connections when we need to, i.e.: the actual number of
connections will only increase in response to demand.
"""
# Make sure we haven't changed process.
self._checkpid()
# Try and get a connection from the pool. If one isn't available within
# self.timeout then raise a ``ConnectionError``.
connection = None
try:
connection = self.pool.get(block=True, timeout=self.timeout)
except queue.Empty:
# Note that this is not caught by the redis client and will be
# raised unless handled by application code. If you want never to
raise ConnectionError("No connection available.")
# If the ``connection`` is actually ``None`` then that's a cue to make
# a new connection to add to the pool.
if connection is None:
connection = self.make_connection()
log.info('获取链接: %s' % id(connection))
return connection
def mqtt_client():
status = ""
address = ""
if request.method == "GET":
payload = {
"origin": {
"@id": "blackfly",
"@type": "app"
},
"uuid": str(uuid.uuid4()),
"creation_time": int(time.time() * 1000),
"command": {
"default": {
"value": "__fill_me__"
},
"subtype": "__fill_me__",
"@type": "__fill_me__"
},
"spid": "SP1",
}
payload = json.dumps(payload, indent=True)
return render_template('mqtt_client.html',
global_context=global_context,
address=address,
payload=payload,
status=status)
if request.method == "POST":
address = request.form["address"]
payload = request.form["payload"]
log.info(type(payload))
log.info("Payload" + str(payload))
mqtt.publish(address, str(payload), 1)
return Response(response="{}", mimetype='application/json')
def mqtt_control(command):
if command == "start":
try:
if global_context[
'mqtt_conn_status'] == "offline" or global_context[
'mqtt_conn_status'] == "reconnecting":
if mqtt.connect(conf["mqtt"]["host"],
int(conf["mqtt"]["port"])):
mqtt.start()
status = "Connected to broker."
else:
status = "Can't connect to broker."
else:
log.info("The system is already connected to mqtt broker.")
status = "Start command was skiped . The system is already connected to mqtt broker."
except Exception as ex:
log.error("Can't connect to server because of error:")
log.error(ex)
status = "Can't connect to broker"
elif command == "stop":
mqtt.stop()
status = "Disconnected from broker"
# dev = json.dumps({"success":True})
# return Response(response=dev, mimetype='application/json' )
return render_template('mqtt_status.html',
status=status,
global_context=global_context)
def patch():
"""暂停/恢复调度任务"""
payload = get_payload()
params = Response(dispatch_id=payload.get('dispatch_id', []),
action=int(payload.get('action', 0)))
log.info('暂停/恢复调度任务[params: %s]' % str(params))
return params
def tools():
tools = Tools()
output = ""
services = []
logs = []
try:
if request.method == "POST":
action = request.form["action"]
log.info("Tools request.action=" + action)
if action == "start_service":
service_name = request.form["service_name"]
output = tools.start_service(service_name)
elif action == "stop_service":
service_name = request.form["service_name"]
output = tools.stop_service(service_name)
elif action == "query_status":
service_name = request.form["service_name"]
output = tools.process_status(service_name)
elif action == "kill_process":
service_name = request.form["service_name"]
output = tools.kill_process(service_name)
logs = tools.get_logfiles()
services = tools.get_services()
except Exception as ex:
output = str(ex)
return render_template('tools.html',
output=output,
global_context=global_context,
logs=logs,
services=services,
autoescape=False)
def exposed_test():
"""测试连接"""
# CPU逻辑内核数
cpu_count = psutil.cpu_count()
# 系统版本
system = platform.platform()
# 磁盘使用
disk_result = {
'total': 0,
'used': 0,
'free': 0
}
disk_total = [psutil.disk_usage(i.mountpoint) for i in psutil.disk_partitions()]
for item in disk_total:
disk_result['total'] += item.total
disk_result['used'] += item.used
disk_result['free'] += item.free
disk_result['total'] = '%0.2fGB' % (disk_result['total'] / 2 ** 30)
disk_result['used'] = '%0.2fGB' % (disk_result['used'] / 2 ** 30)
disk_result['free'] = '%0.2fGB' % (disk_result['free'] / 2 ** 30)
# 内存使用
memory_total = psutil.virtual_memory()
memory_result = {
'total': '%0.2fGB' % (memory_total.total / 2 ** 30),
'used': '%0.2fGB' % (memory_total.used / 2 ** 30),
'free': '%0.2fGB' % (memory_total.free / 2 ** 30)
}
log.info('获取测试连接请求')
return json.dumps({'cpu': cpu_count, 'system': system, 'disk': disk_result, 'memory': memory_result})
def post():
"""新增执行服务器"""
payload = get_payload()
params = Response(server_host=payload.get('server_host', ''),
server_name=payload.get('server_name', ''))
log.info('新增执行服务器[params: %s]' % str(params))
return params
def exposed_event_execute(exec_id, interface_id, job_id, server_dir, server_script, return_code, params, status):
"""接受事件任务"""
# 添加至调度器
run_id = 'event_%s_%s_%s' % (exec_id, interface_id, job_id)
kwargs = {
'exec_id': exec_id,
'interface_id': interface_id,
'job_id': job_id,
'server_dir': server_dir,
'server_script': server_script,
'return_code': return_code,
'params': params,
'status': status
}
log.info('接收任务: %s, run_id: %s' % (str(kwargs), run_id))
# 添加调度任务
next_run_time = datetime.datetime.now() + datetime.timedelta(seconds=2)
try:
# 查询任务是否存在
run_job = SchedulerModel.get_scheduler_by_id(db.etl_db, run_id, config.exec.table_name)
if run_job:
return {'status': False, 'msg': '该任务已在运行中'}
# scheduler.add_job(id=run_id, func=start_job,
# args=(exec_id, job_id, server_dir, server_script, status), trigger='interval', seconds=3)
scheduler.add_job(id=run_id, func=start_job,
args=(
exec_id, interface_id, job_id, server_dir, server_script, return_code, params,
status),
next_run_time=next_run_time)
return {'status': True, 'msg': '任务已开始运行'}
except Exception as e:
return {'status': False, 'msg': e}
def get():
"""获取该执行下的任务流列表"""
params = Response(
exec_id=int(get_arg('exec_id', 0))
)
log.info('获取该执行下的任务流列表[params: %s]' % str(params))
return params
def get():
"""执行服务任务回调"""
params = Response(
exec_id=int(get_arg('exec_id', 0)),
status=get_arg('status', '')
)
log.info('获取执行服务任务回调[params: %s]' % str(params))
return params
def get():
"""获取执行日志"""
params = Response(
exec_id=int(get_arg('exec_id', 0)),
job_id=int(get_arg('job_id', 0))
)
log.info('获取执行日志[params: %s]' % str(params))
return params
def post():
"""新增接口"""
payload = get_payload()
params = Response(interface_name=payload.get('interface_name', ''),
interface_desc=payload.get('interface_desc', ''),
retry=int(payload.get('retry', 0)))
log.info('新增接口[params: %s]' % str(params))
return params
def get():
"""获取执行拓扑结构"""
params = Response(
exec_id=int(get_arg('exec_id', 0)),
interface_id=int(get_arg('interface_id', 0))
)
log.info('获取执行拓扑结构[params: %s]' % str(params))
return params
def post():
"""断点续跑"""
payload = get_payload()
params = Response(
exec_id=payload.get('exec_id', [])
)
log.info('执行任务断点续跑[params: %s]' % str(params))
return params
def delete():
"""批量删除参数"""
payload = get_payload()
params = Response(
param_id_arr=payload.get('param_id_arr', [])
)
log.info('批量删除任务流[params: %s]' % str(params))
return params
def get():
"""获取调度时间"""
params = Response(sched=get_arg('sched'),
timeFormat=get_arg('timeFormat',
'%Y-%m-%d %H:%M:%S'),
queryTimes=int(get_arg('queryTimes', 10)))
log.info('获取调度时间[params: %s]' % str(params))
return params
def post():
"""立即执行任务"""
payload = get_payload()
params = Response(job_id=int(payload.get('job_id', 0)),
run_date=payload.get('run_date', ''),
date_format=payload.get('date_format', ''))
log.info('立即执行任务[params: %s]' % str(params))
return params
def put(server_id):
"""修改执行服务器详情"""
payload = get_payload()
params = Response(server_id=server_id,
server_host=payload.get('server_host', ''),
server_name=payload.get('server_name', ''),
is_deleted=int(payload.get('is_deleted', 0)))
log.info('修改执行服务器详情[params: %s]' % str(params))
return params
def get():
"""获取执行服务器列表"""
params = Response(server_name=get_arg('server_name', ''),
server_host=get_arg('server_host', ''),
is_deleted=int(get_arg('is_deleted', 0)),
page=int(get_arg('page', 1)),
limit=int(get_arg('limit', 10)))
log.info('获取执行服务器列表[params: %s]' % str(params))
return params
def get():
"""获取预警配置列表"""
params = Response(alert_channel=int(get_arg('alert_channel', 0)),
conf_name=get_arg('conf_name', ''),
is_deleted=int(get_arg('is_deleted', 0)),
page=int(get_arg('page', 1)),
limit=int(get_arg('limit', 10)))
log.info('获取预警配置列表[params: %s]' % str(params))
return params
def post():
"""登入"""
payload = get_payload()
username = payload.get('username', '')
password = payload.get('password', '')
params = Response(username=username, password=password)
log.info('用户登录[params: %s]' % str(params))
return params
def post():
"""测试SQL参数"""
payload = get_payload()
params = Response(
source_id=int(payload.get('source_id', 0)),
param_value=payload.get('param_value', '')
)
log.info('SQL参数测试[params: %s]' % params)
return params
def post():
"""立即执行调度任务"""
payload = get_payload()
params = Response(dispatch_id=payload.get('dispatch_id', []),
run_date=payload.get('run_date', ''),
date_format=payload.get('date_format', ''),
is_after=int(payload.get('is_after', 1)))
log.info('立即执行调度任务[params: %s]' % str(params))
return params
def patch():
"""暂停/恢复调度事件"""
payload = get_payload()
params = Response(
ftp_event_id=payload.get('ftp_event_id', []),
action=int(payload.get('action', 0))
)
log.info('暂停/恢复调度事件[params: %s]' % str(params))
return params
def get():
"""获取调度列表"""
params = Response(interface_id=int(get_arg('interface_id', 0)),
dispatch_name=get_arg('dispatch_name', ''),
status=int(get_arg('status', 0)),
page=int(get_arg('page', 1)),
limit=int(get_arg('limit', 10)))
log.info('获取调度列表[params: %s]' % str(params))
return params
def stop_execute_job(exec_id):
"""
中止执行任务
1.修改调度主表状态为[中断]
2.获取正在执行任务
3.rpc分发-停止任务
4.修改执行详情表为[失败], 修改执行任务流状态[中断]
"""
msg = []
for item in exec_id:
# 修改数据库, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
# 修改调度主表状态为[中断]
stop_num = ExecuteModel.update_execute_stop(db.etl_db, item, 2)
# 是否成功中断判断
if not stop_num:
msg.append('执行ID: [%s]状态为非执行中, 中断失败' % item)
continue
# 获取正在执行任务, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
result = ExecuteModel.get_execute_detail_by_status(
db.etl_db, item, 'running')
# 去重
result = {item['job_id']: item for item in result}
for _, execute in result.items():
try:
# 获取进程id
if execute['pid']:
# rpc分发-停止任务
client = Connection(execute['server_host'],
config.exec.port)
client.rpc.stop(exec_id=item,
job_id=execute['job_id'],
pid=execute['pid'])
client.disconnect()
log.info('rpc分发-停止任务: 执行id: %s, 任务id: %s' %
(item, execute['job_id']))
# 修改数据库, 分布式锁
with MysqlLock(config.mysql.etl, 'exec_lock_%s' % item):
# 修改执行详情表为[失败]
ScheduleModel.update_exec_job_status(
db.etl_db, item, execute['interface_id'],
execute['job_id'], 'failed')
# 修改执行任务流状态[中断]
ExecuteModel.update_exec_interface_status(
db.etl_db, item, execute['interface_id'], 2)
log.info(
'修改执行详情表为[失败]: [%s, %s, %s]' %
(item, execute['interface_id'], execute['job_id']))
except:
err_msg = 'rpc分发-停止任务异常: host: %s, port: %s, 执行id: %s, 任务id: %s' % (
execute['server_host'], config.exec.port, item,
execute['job_id'])
log.error(err_msg, exc_info=True)
msg.append(err_msg)
return Response(msg=msg)
def get():
"""获取任务列表"""
params = Response(job_name=get_arg('job_name', ''),
job_index=get_arg('job_index', ''),
interface_id=int(get_arg('interface_id', 0)),
is_deleted=int(get_arg('is_deleted', 0)),
page=int(get_arg('page', 1)),
limit=int(get_arg('limit', 10)))
log.info('获取任务列表[params: %s]' % str(params))
return params