def update_check_plan(app_config_dict, queue):
logger = logging.getLogger("task.generator")
pipe = REDIS_DB.pipeline()
change_list = pop_multi(pipe, '_app_change_list', 30)
logger.info(u'发现应用配置发生变动,应用列表:%s', change_list)
for appname in set(change_list):
# 获取此应用的最新配置
new_config = REDIS_DB.hgetall(appname + '_config')
# 1. 此应用已经被删除了
if new_config is None:
app_config_dict.pop(appname, None)
continue
item = app_config_dict.get(appname)
# 2. 此应用是新添加的应用
if item is None:
host_list = new_config['host_list']
for host in host_list.split(','):
task = CheckTask(appname, host,
int(new_config['check_interval']))
queue.put_task(task)
else:
old_host_list = item['host_list'].split(',')
new_host_list = new_config['host_list'].split(',')
for host in (set(new_host_list) - set(old_host_list)):
task = CheckTask(appname, host,
int(new_config['check_interval']))
queue.put_task(task)
# 更新配置
app_config_dict[appname] = new_config
def main():
# ----------删除原有的所有检查任务----------
REDIS_DB.delete('_check_task')
# 1. 获取应用配置
app_config_dict = get_config()
# 2. 初始化任务队列
queue = DelayQueue()
# 3. 装载任务
for appname in app_config_dict:
item = app_config_dict.get(appname)
if item:
host_list = item['host_list']
for host in host_list.split(','):
task = CheckTask(appname, host, int(item['check_interval']))
queue.put_task(task)
# 4. 启动消费者
for i in range(4):
t = Generator(queue, app_config_dict)
t.daemon = True
t.start()
# 5. 更新检查计划(包括添加新的检查任务)
# 间隔:30秒
t = LoopTimer(10, update_check_plan, [app_config_dict, queue])
t.start()
queue.join()
def clear_msg(sender, to_user):
msg = REDIS_DB.get(to_user)
if msg:
msg = json.loads(msg)
msg[sender] = 0
REDIS_DB.set(to_user, json.dumps(msg))
def get_reids_one(to_user, from_user):
to_user_msg = REDIS_DB.get(to_user)
if to_user_msg:
to_user_msg = json.loads(to_user_msg)
count = to_user_msg.get(from_user, 0)
to_user_msg[from_user] = 0
REDIS_DB.set(to_user, json.dumps(to_user_msg))
return count
else:
return 0
def set_redis(sender, to_user):
user_msg = REDIS_DB.get(to_user)
if not user_msg:
REDIS_DB.set(to_user, json.dumps({sender: 1}))
else:
user_msg = json.loads(user_msg)
if not user_msg.get(sender):
user_msg[sender] = 1
else:
user_msg[sender] += 1
REDIS_DB.set(to_user, json.dumps(user_msg))
def set_redis(to_user, from_user):
# toy:{app:2}
to_user_msg = REDIS_DB.get(to_user)
if to_user_msg:
to_user_msg = json.loads(to_user_msg)
if to_user_msg.get(from_user):
to_user_msg[from_user] += 1
else:
to_user_msg[from_user] = 1
else:
to_user_msg = {from_user: 1}
REDIS_DB.set(to_user, json.dumps(to_user_msg))
def set_redis(to_user, from_user):
# 如果app给玩具发消息,那么站在app的角度上to_user就是玩具,from_user就是app
# 这里的to_user也是玩具,所以get(to_user)就是get自己
self_redis = REDIS_DB.get(to_user) # 查看数据库中是否存在自己的空间,如果有就+1,没有就创建
if self_redis:
chat_dict = json.loads(self_redis) # 先到redis中loads,看看有没有数据
chat_dict.setdefault(from_user, 0)
chat_dict[from_user] += 1
REDIS_DB.set(to_user, json.dumps(chat_dict))
else:
# {'我':{'联系人1号':1,'联系人2号':3}} redis中没有字典类型,需要哈希存储能拿字典
# 所以我们可以换思路,把字典给json,变成字符串
chat = {from_user: 1} # 因为一次只能发一条消息,所以默认为1
REDIS_DB.set(to_user, json.dumps(chat))
def get_redis(to_user):
msg = REDIS_DB.get(to_user)
if msg:
msg = json.loads(msg)
msg["count"] = sum(msg.values())
return msg
def get_config():
# 1. 获取应用配置
app_list_str = REDIS_DB.get('_all_app_list')
app_list = []
if app_list_str:
app_list = app_list_str.split(',')
app_config_dict = {}
pipe = REDIS_DB.pipeline()
for appname in app_list:
pipe.hgetall(appname + '_config')
res_list = pipe.execute()
for i in range(len(res_list)):
app_config_dict[app_list[i]] = res_list[i]
return app_config_dict
def get_redis_random(toy_id):
toy_msg = REDIS_DB.get(toy_id)
if toy_msg:
toy_msg = json.loads(toy_msg)
for sender, count in toy_msg.items():
if count:
return sender, count
return None, None
def get_redis_one_toy(to_user, from_user):
to_user_msg = REDIS_DB.get(to_user)
if to_user_msg:
to_user_msg = json.loads(to_user_msg)
count = to_user_msg.get(from_user, 0)
if count == 0:
for key, value in to_user_msg.items():
if value:
count = to_user_msg.get(key)
from_user = key
to_user_msg[from_user] = 0
REDIS_DB.set(to_user, json.dumps(to_user_msg))
return count, from_user
else:
return 0
def get_redis_all(to_user):
self_redis = REDIS_DB.get(to_user)
chat_dict = {'count': 0}
if self_redis:
chat_dict = json.loads(self_redis) # {'联系人1号':1,'联系人2号':3}
# 添加一个未读数据的总和
chat_dict['count'] = sum(chat_dict.values())
return chat_dict
def get_connection(**kwargs):
_RQ_DEFAULT_CONNECTION = {
'HOST': REDIS_HOST,
'PORT': 6379,
'DB': REDIS_DB.get('tasks'),
'PASSWORD': '',
'DEFAULT_TIMEOUT': 360,
}
_RQ_DEFAULT_CONNECTION.update(kwargs)
return _RQ_DEFAULT_CONNECTION
def get_redis(to_user, from_user):
# 如果app给玩具发消息,那么站在app的角度上to_user就是玩具,from _user就是app
# 这里的to_user也是玩具,所以get(to_user)就是get自己
self_redis = REDIS_DB.get(to_user)
count = 0
if self_redis:
chat_dict = json.loads(self_redis) # 先到redis中loads,看看有没有数据
chat_dict.setdefault(from_user, 0)
count = chat_dict[from_user]
if count == 0:
key, value = re_get_redis(chat_dict)
if key:
from_user = key
count = value
chat_dict[from_user] = 0
REDIS_DB.set(to_user, json.dumps(chat_dict))
return from_user, count
def get_redis_all(to_user):
to_user_msg = REDIS_DB.get(to_user)
if to_user_msg:
to_user_msg = json.loads(to_user_msg)
#{"toy1":1,"toy2":1,"toy3":2}
to_user_msg['count'] = sum(to_user_msg.values())
return to_user_msg
else:
return {'count': 0}
def get_redis_count(sender, to_user):
msg = REDIS_DB.get(to_user)
if msg:
msg = json.loads(msg)
return msg[sender]
def record_app_status(task, status, info, check_latency, duration):
pipe = REDIS_DB.pipeline()
# 1. 获取应用的部分配置
pipe.hget(task.appname + '_config', 'max_check_attempts')
pipe.hget(task.appname + '_config', 'notify_interval')
pipe.hget(task.appname + '_config', 'check_interval')
pipe.hget(task.appname + '_config', 'parent_app_list')
res_list = pipe.execute()
max_check_attempts = int(res_list[0])
notify_interval = int(res_list[1])
check_interval = int(res_list[2])
parent_app_list = res_list[3]
# 2. 获取上一次状态信息
key = 'ah_' + task.appname + '_' + task.host
if not REDIS_DB.exists(key):
pipe.hset(key, 'current_status', 'OK')
pipe.hset(key, 'status_info', 'OK - ')
pipe.hset(key, 'current_attempt', 0)
pipe.hset(key, 'last_state_change',
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
pipe.hset(key, 'last_notification',
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
pipe.execute()
status_dict = REDIS_DB.hgetall(key)
current_status = status_dict.pop('current_status')
current_attempt = status_dict.pop('current_attempt')
current_attempt = int(current_attempt)
last_state_change = status_dict.pop('last_state_change')
last_state_change = datetime.strptime(last_state_change,
'%Y-%m-%d %H:%M:%S')
last_notification = status_dict.pop('last_notification')
last_notification = datetime.strptime(last_notification,
'%Y-%m-%d %H:%M:%S')
if current_status != status or status != 'OK':
current_attempt = current_attempt + 1
pipe.hset(key, 'current_attempt', current_attempt)
if current_attempt >= max_check_attempts:
current_status = status
pipe.hset(key, 'current_status', current_status)
pipe.hset(key, 'status_info', info)
pipe.hset(key, 'current_attempt', 0)
pipe.hset(key, 'last_state_change',
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
# recovery 事件不受notify_interval影响
if status == 'OK':
notify_app.apply_async(
(task.appname, task.host, 'RECOVERY', status, info))
margin = total_seconds(datetime.now() - last_notification)
if status != 'OK' and margin > notify_interval * 60:
pipe.hset(key, 'last_notification',
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
notify_app.apply_async(
(task.appname, task.host, 'PROBLEM', status, info))
else:
pipe.hset(key, 'current_attempt', 0)
pipe.hset(key, 'duration', duration)
pipe.hset(key, 'check_latency', check_latency)
next_scheduled_check = task.plan_time + timedelta(seconds=check_interval *
60)
next_scheduled_check = next_scheduled_check.strftime('%Y-%m-%d %H:%M:%S')
pipe.hset(key, 'next_scheduled_check', next_scheduled_check)
# -------- 其它特殊情况 (为了方便判断主机是否存活) ------------
if parent_app_list is not None:
if parent_app_list.find('HOST_STATUS') != -1:
key = 'ah_HOST_STATUS_' + task.host
pipe.hset(key, 'current_status', current_status)
if parent_app_list.find('SYS_PING') != -1:
key = 'ah_SYS_PING_' + task.host
pipe.hset(key, 'current_status', current_status)
pipe.execute()
def record_app_status(task, status, info, check_latency, duration):
pipe = REDIS_DB.pipeline()
# 1. 获取应用的部分配置
pipe.hget(task.appname + '_config', 'max_check_attempts')
pipe.hget(task.appname + '_config', 'notify_interval')
pipe.hget(task.appname + '_config', 'check_interval')
pipe.hget(task.appname + '_config', 'parent_app_list')
res_list = pipe.execute()
max_check_attempts = int(res_list[0])
notify_interval = int(res_list[1])
check_interval = int(res_list[2])
parent_app_list = res_list[3]
# 2. 获取上一次状态信息
key = 'ah_' + task.appname + '_' + task.host
if not REDIS_DB.exists(key):
pipe.hset(key, 'current_status', 'OK')
pipe.hset(key, 'status_info', 'OK - ')
pipe.hset(key, 'current_attempt', 0)
pipe.hset(key, 'last_state_change', datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
pipe.hset(key, 'last_notification', datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
pipe.execute()
status_dict = REDIS_DB.hgetall(key)
current_status = status_dict.pop('current_status')
current_attempt = status_dict.pop('current_attempt')
current_attempt = int(current_attempt)
last_state_change = status_dict.pop('last_state_change')
last_state_change = datetime.strptime(last_state_change, '%Y-%m-%d %H:%M:%S')
last_notification = status_dict.pop('last_notification')
last_notification = datetime.strptime(last_notification, '%Y-%m-%d %H:%M:%S')
if current_status != status or status != 'OK':
current_attempt = current_attempt + 1
pipe.hset(key, 'current_attempt', current_attempt)
if current_attempt >= max_check_attempts:
current_status = status
pipe.hset(key, 'current_status', current_status)
pipe.hset(key, 'status_info', info)
pipe.hset(key, 'current_attempt', 0)
pipe.hset(key, 'last_state_change', datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
# recovery 事件不受notify_interval影响
if status == 'OK':
notify_app.apply_async((task.appname, task.host, 'RECOVERY', status, info))
margin = total_seconds(datetime.now() - last_notification)
if status !='OK' and margin > notify_interval * 60:
pipe.hset(key, 'last_notification', datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
notify_app.apply_async((task.appname, task.host, 'PROBLEM', status, info))
else:
pipe.hset(key, 'current_attempt', 0)
pipe.hset(key, 'duration', duration)
pipe.hset(key, 'check_latency', check_latency)
next_scheduled_check = task.plan_time + timedelta(seconds=check_interval * 60)
next_scheduled_check = next_scheduled_check.strftime('%Y-%m-%d %H:%M:%S')
pipe.hset(key, 'next_scheduled_check', next_scheduled_check)
# -------- 其它特殊情况 (为了方便判断主机是否存活) ------------
if parent_app_list is not None:
if parent_app_list.find('HOST_STATUS') != -1:
key = 'ah_HOST_STATUS_' + task.host
pipe.hset(key, 'current_status', current_status)
if parent_app_list.find('SYS_PING') != -1:
key = 'ah_SYS_PING_' + task.host
pipe.hset(key, 'current_status', current_status)
pipe.execute()