def queue_result_forever():
while True:
num = myredis.mlen(RedisSql.result_queue_name)
if num > 0:
for n in range(num):
try:
data = json.loads(myredis.mpop(RedisSql.result_queue_name))
print(f'处理短信数据：{data}', 0, '处理短信结果')
channel_type = data.pop('channel_type')
with channel.ChannelDB() as db:
channel_id, need_report = db.get_channel_info(
channel_type=channel_type)
if data['status']:
resp = handle_success(data['to'], data['message_id'])
else:
resp = handle_failed(data['to'],
data['yzm'],
channel_id,
data['message'],
data['message_id'],
is_update=True)
if not resp:
if data.get('count', 0) < 3:
data['count'] = data.get('count', 0) + 1
myredis.mpush(json.dumps(data),
RedisSql.result_queue_name)
else:
print(f'更新短信结果失败, 当前数据[{data}]', 2, '更新短信结果失败')
except Exception as e:
print(f'处理短信结果失败, 失败原因[{e}]', 2, '处理短信结果错误')
time.sleep(60)
def queue_pending_forever():
while True:
num = myredis.mlen(RedisSql.pending)
if num > 0:
for n in range(num):
try:
data = json.loads(myredis.mpop(RedisSql.pending))
print(f'处理pending数据：{data}', 0, '处理pending数据')
channel_type = data.pop('channel_type')
with channel.ChannelDB() as db:
channel_id, need_report = db.get_channel_info(
channel_type=channel_type)
if data['status']:
if need_report:
handle_pending(data['to'], data['yzm'], channel_id,
data['message_id'])
else:
handle_success(data['to'],
data['message_id'],
data['yzm'],
channel_id,
is_update=False)
else:
handle_failed(data['to'], data['yzm'], channel_id,
data['message'], data['message_id'])
except Exception as e:
print(f'处理pending短信失败, 失败原因[{e}]', 2, '处理pending短信错误')
def connection_machine():
# 连接服务器
while True:
try:
if mlen(RedisSql.cm_queue_name) > 0:
machine_id = mpop(RedisSql.cm_queue_name)
with Machine() as db:
data = db.get_machine_by_id(machine_id)
for i in range(3):
try:
ssh = SSHConnection(data['ip'], data['usr'],
data['pwd'], data['ssh_port'])
ssh.connect()
result1 = ssh.cmd(Remote.remote_configuration_comm)
print(
f'远程服务器[{data["ip"]}执行初始化命令成功, 执行结果[{result1}]',
0, '启动远程服务')
ssh.upload(Remote.local_py_path,
Remote.remote_py_path)
ssh.upload(Remote.local_service_path,
Remote.remote_service_path)
print(f'拷贝文件到远程服务器[{data["ip"]}]成功', 0, '启动远程服务')
result2 = ssh.cmd(
Remote.remote_change_port_comm.format(
port=data['http_port']))
result3 = ssh.cmd(Remote.remote_reload_comm)
result4 = ssh.cmd(Remote.remote_start_comm)
result5 = ssh.cmd(Remote.remote_enable_comm)
print(
f'远程服务器[{data["ip"]}执行启动命令成功, 执行结果[{result5}]',
0, '启动远程服务')
ssh.close()
db.change_status_by_id(machine_id, '连接正常', 1)
with UnhandledAlert() as db1:
alert_id = db1.is_have_alert(
machine_id, Alert.alert_type[0])
if alert_id != -1:
db.update_alert(alert_id, '已处理')
break
except Exception as e:
if i == 2:
mpush(machine_id, RedisSql.cm_queue_name)
db.change_status_by_id(machine_id, '连接失败', 0)
with UnhandledAlert() as db1:
db1.create_alert(Alert.alert_type[0],
'连接失败, 请检查远程主机的网络连通性!',
machine_id)
print(f'第{i+1}次连接远程服务器错误, 错误原因[{e}]', 2, '连接主机错误')
time.sleep(60)
time.sleep(60 * 5)
else:
time.sleep(60 * 10)
except Exception as e:
# traceback.print_exc()
print(f'连接服务器进程错误, 错误原因[{e}]', 2, '连接主机错误')
def get_one():
# 从redis里弹出一条
"""
:return: (user, to_number, text) or (None, None, None)
"""
try:
r = myredis.mpop(RedisSql.sending)
return json.loads(r) if r else None
except Exception as e:
print(f'从redis拿出短信失败: {e}', 2, 'redis错误')
return None
def handle_machine_info():
while True:
try:
num = mlen(RedisSql.machine_info_queue_name)
if num > 0:
with Machine() as db:
for n in range(num):
data = json.loads(
mpop(RedisSql.machine_info_queue_name))
machine_info = db.get_alarm_by_ip(data['ip'])
print(
f'处理远程主机[{data["ip"]}:{data["type"]}:{data["status"]}]数据',
0, '处理主机信息')
with UnhandledAlert() as db1:
if data['status']:
db.create_jk_by_machine_id(
data['type'], data['data'],
machine_info['id'], data['time'])
alert_id = db1.is_have_alert(
machine_info['id'], Alert.alert_type[1])
if alert_id != -1:
db.change_status_by_id(
machine_info['id'], '连接正常', 1)
db1.update_alert(alert_id, '已处理')
if data['type'] == 'mem':
with ProcessDB() as db2:
db2.create_process(
data['data']['process'],
machine_info['id'], 'mem')
if data['data'][
'mem_percent'] > machine_info[
'mem_alarm']:
print(
f'远程主机[{data["ip"]}]内存使用量过高,当前已使用{data["data"]["mem_used"]}G!',
1, '主机警报信息')
db1.create_alert(
Alert.alert_type[2],
f'内存使用量过高,当前已使用{data["data"]["mem_used"]}G!',
machine_info['id'])
else:
alert_id = db1.is_have_alert(
machine_info['id'],
Alert.alert_type[2])
if alert_id != -1:
print(
f'远程主机[{data["ip"]}]的内存警报已解除',
0, '主机警报信息')
db1.update_alert(alert_id, '已处理')
elif data['type'] == 'cpu':
with ProcessDB() as db2:
db2.create_process(
data['data']['process'],
machine_info['id'], 'cpu')
if data['data'][
'cpu_percent'] > machine_info[
'cpu_alarm']:
print(
f'远程主机[{data["ip"]}]cpu使用量过高,当前已使用{data["data"]["cpu_percent"]}%!',
1, '主机警报信息')
db1.create_alert(
Alert.alert_type[3],
f'cpu使用量过高,当前已使用{data["data"]["cpu_percent"]}%!',
machine_info['id'])
else:
alert_id = db1.is_have_alert(
machine_info['id'],
Alert.alert_type[3])
if alert_id != -1:
print(
f'远程主机[{data["ip"]}]的cpu警报已解除',
0, '主机警报信息')
db1.update_alert(alert_id, '已处理')
else:
db1.create_alert(
Alert.alert_type[1],
f'与远程主机{data["ip"]}连接失败,请检查远程主机{data["http_port"]}端口是否开放!',
machine_info['id'])
db.change_status_by_id(machine_info['id'],
'断开连接', 1)
time.sleep(60)
else:
time.sleep(60)
except Exception as e:
# traceback.print_exc()
print(f'处理远程主机信息失败, 失败原因[{e}]', 2, '处理主机信息错误')