def disaster_alert_listen(task_id=None):
mysql_conn = db_mysql.MyPymysqlPool()
# 告警统一聚合和过滤，如果有灾难级别报警，则发出电话告警
sql = "select * from %s where current_state = 'ALARM' and priority = 1 limit 1" % (
alert_conf.table_name)
print(sql)
#sql = "select * from %s where current_state = 'ALARM' limit 1" %(alert_conf.table_name)
try:
result = mysql_conn.select(sql)
except:
result = False
if result: # 检测到处于alarm状态的灾难告警
# 检查已记录的告警状态，确认是否已经告警过
key = 'cannon_voice_call_status'
try:
result = db_redis.RedisApi().check_exist_of_key(key)
except:
result = 1 # 查询不到，则默认存在，不告警，交给下次查询
if result == 0: # key不存在
ret = voice_call()
if ret:
ttl = 1800
value = '{}'
db_redis.RedisApi().set_key_with_ttl(key, value, ttl)
mysql_conn.dispose()
return True
def call_voice():
# 检查已记录的告警状态，确认是否已经告警过
key = 'cannon_voice_call_status'
status = False # 默认告警失败
try:
result = db_redis.RedisApi().check_exist_of_key(key)
except:
result = 1 # 查询不到，则默认存在，不告警，交给下次查询
if result == 0: # key不存在
ret = voice_function()
if ret:
status = True
ttl = 1800
value = '{}'
db_redis.RedisApi().set_key_with_ttl(key, value, ttl)
return status
def set_redis_key(key, ttl, value):
status = False # 默认设置key不成功
try:
result = db_redis.RedisApi().set_key_with_ttl(key, value, ttl)
except:
print('except')
else:
status = True
return status
def query_redis_key_exist(key):
status = True # 默认key存在，不做任何处理
try:
result = db_redis.RedisApi().check_exist_of_key(key)
except:
print('except') # 查询不到，则默认存在，不告警，交给下次查询
else:
status = True
if result == 0: # key不存在，可以写入
status = False
return status # true表示key存在，false表示key不存在
def check_redis_key_content(value, key):
status = False # 默认内容不一致
list_in_redis = json.loads(db_redis.RedisApi().get_value_of_string(key))['alert_id_list']
print(value, list_in_redis)
if len(value) == len(list_in_redis):
if value == list_in_redis:
status = True
print('数据一致，跳过')
return status
def query_redis_key_exist(key):
status = True # 默认key存在，不做任何处理
# noinspection PyBroadException
try:
result = db_redis.RedisApi().check_exist_of_key(key)
except Exception as e:
message = "查询锁失败，原因: %s" % e
print(message) # 查询不到，则默认存在，不告警，交给下次查询
record_info_log(task_log_file, message)
else:
if result == 0: # key不存在，可以写入
status = False
return status # true表示key存在，false表示key不存在
def scheduler_main():
count = 1 # 计数，控制时间用
task_startup = False # 任务调度未启动，用于初次执行scheduler.start()
lock_status = False # 是否本机的锁
ttl_time = 300 # 加锁的ttl值，也就是双活切换的时间
global job_event_status_dict
job_event_status_dict = {}
query_redis_fail_count = 0 # 查询redis失败次数，次数范围内，可以重试；如果超过一定次数，则需要调度执行
while True:
# 查询锁，判断自己是否注册
# noinspection PyBroadException
try:
result = query_redis_key_exist(key)
except Exception as e:
if query_redis_fail_count < 5:
msg = "查询key失败，reason: %s, 暂时不调度!" % e
print_func(msg)
query_redis_fail_count += 1
time.sleep(loop_interval) # 这个时间不会影响任务调度的间隔
count += 1
continue
else:
result = False
msg = "查询key失败超过一定次数，开始任务调度!"
print_func(msg)
else:
query_redis_fail_count = 0 # 查询成功，数据清零
msg = 'query redis key exist: %s' % result
print_func(msg)
local_hostname = get_local_hostname() # 获取本机机器名
if result is True: # key存在, 检查内容和本机机器名是否一致
try:
hostname_in_redis = db_redis.RedisApi().get_value_of_string(
key)
except Exception as e:
msg = 'query hostname from redis fail, reason: %s' % e
print_func(msg)
print_func('local_hostname: %s, hostname_in_redis: %s' %
(local_hostname, hostname_in_redis))
if local_hostname == hostname_in_redis: # 是本机的锁
msg = '锁内容一致，是本机的锁'
print_func(msg)
record_debug_log(task_debug_file, msg)
lock_status = True
else: # 非本机的锁
msg = "当前锁名称: %s, 和本机名称: %s 不同! 本轮不调度，等待下个周期继续抢锁..." % (
hostname_in_redis, local_hostname)
print_func(msg)
record_info_log(task_log_file, msg)
elif result is False:
lock_status = True
msg = '%s 锁不存在，开始执行!' % datetime.datetime.now()
print_func(msg)
record_info_log(task_log_file, msg)
if lock_status: # 是本机的锁，或锁不存在，则抢锁(或更新锁时间)，并向下继续调度任务
msg = "开始加锁..."
print_func(msg)
record_debug_log(task_debug_file, msg)
result = db_redis.RedisApi().set_key_with_ttl(
key, local_hostname, ttl_time)
result = True
if result:
msg = "加锁成功!"
print_func(msg)
record_debug_log(task_debug_file, msg)
else:
msg = "加锁失败，进入下个循环!"
print_func(msg)
time.sleep(5)
continue
if task_startup is False:
try:
msg = '开始启动任务调度...'
print_func(msg)
record_info_log(task_log_file, msg)
scheduler.start()
except Exception as e:
msg = '启动任务调度失败!'
print_func(msg)
db_redis.RedisApi().delete_key(key)
exit_scheduler(e)
else:
task_startup = True
scheduler.print_jobs()
job_list = scheduler.get_jobs()
for job in job_list:
message = "任务状态展示: %s" % job
record_info_log(task_log_file, message)
if count >= 60: # 次数
query_task_status()
count = 1 # 控制展示状态一个小时展示一次
if job_event_status_dict:
status = flush_job_info_to_mysql()
msg = 'MySQL commit success!'
print_func(msg)
record_info_log(task_log_file, msg)
else: # 非本机的锁，本机任务进入下一个周期，继续判断锁
pass
time.sleep(loop_interval) # 这个时间不会影响任务调度的间隔
count += 1