class Communication(object):
INSTANCE = None
Lock = threading.RLock()
def __new__(cls, *args, **kwargs):
"""
单例模式的 double check 确保线程安全，增加缓冲标量
确保第一次初始化完全
:param args:
:param kwargs:
:return:
"""
_instance = None
if not cls.INSTANCE:
try:
cls.Lock.acquire()
if not cls.INSTANCE:
_instance = object.__new__(cls)
cls.INSTANCE = _instance
finally:
cls.Lock.release()
return cls.INSTANCE
def __init__(self, conf):
self.redis = RedisWrapper(conf['redis'])
self.etcd = EtcdWrapper(conf['etcd'])
self.etcd_interval_time = conf['etcd']['interval']
self.watchdog = WatchDog(conf)
self.node = conf['node']
self.ip = conf['local_ip']
self.app = conf['application']
self.paths = conf['paths']
self.data = dict()
self._name = 'communication'
self.log = list()
self.hash = None
# 重启刷新缓存
self.flush_data()
@property
def name(self):
return self._name
def work(self, *args):
"""
获取 event loop，并启动异步循环
:param args:
:return:
"""
loop = asyncio.SelectorEventLoop()
asyncio.set_event_loop(loop)
loop.call_soon_threadsafe(loop.create_task, self.handle())
loop.call_soon_threadsafe(loop.create_task, self.write_into_remote())
if platform.system() == "Windows":
loop.call_soon_threadsafe(loop.create_task, self.monitor())
loop.run_forever()
async def handle(self):
"""
异步方法，监听 redis 给出的命令
:return:
"""
while True:
# 获取外部命令，并处理
command = self.check_order()
if command == b'get_status':
self.write_into_local()
elif command == b'restart':
self.watchdog.restart = True
self.flush_data()
elif command == b'reload':
self.watchdog.reload = True
self.flush_data()
self.re_load()
elif command == b'upload':
for path in self.paths:
self.upload(path)
await asyncio.sleep(0.5)
def check_order(self):
"""
Get outside command
:return: None or order
"""
try:
if self.redis.get_len("order_name") > 0:
return self.redis.dequeue("order_name")
except Exception as e:
log.error("\n%s", e)
def write_into_local(self):
"""
Write the data to local redis
:return: None
"""
if self.app == 'ziyan':
status = {
'node': self.node,
'data': self.data,
'log': self.log,
'check_restart_time': self.watchdog.check_restart_num,
'handle_restart_time': self.watchdog.handle_restart_num,
'real_time_thread_name': self.watchdog.thread_real_time_names
}
else:
status = {
'node': self.node,
'data': self.data,
'log': self.log,
'transport_restart_time': self.watchdog.transport_restart_num,
'real_time_thread_name': self.watchdog.thread_real_time_names
}
try:
self.redis.sadd('status', status)
self.redis.expire('status', 60 * 5)
except Exception as e:
log.error("\n%s", e)
def upload(self, path):
"""
upload specific path file
:param path:
:return:
"""
key = "/nodes/" + self.node + "/" + self.app
if 'toml' in path:
key += '/conf'
elif 'py' in path:
key += '/code'
elif 'lua' in path:
key += '/lua'
try:
with open(path, 'rb') as f:
self.etcd.write(key, f.read())
except Exception as e:
log.error("\n%s", e)
def flush_data(self):
"""
Delete the existing key "status"
:return:
"""
try:
if self.redis.exists("status"):
self.redis.delete("status")
except Exception as e:
log.error("\n%s", e)
async def write_into_remote(self):
"""
异步方法，每10分钟向服务器 etcd 中注册当前状态
:return:
"""
while True:
if self.app == 'ziyan':
status = {
'node':
self.node,
'data':
self.data,
'log':
self.log,
'check_restart_time':
self.watchdog.check_restart_num,
'handle_restart_time':
self.watchdog.handle_restart_num,
'real_time_thread_name':
list(self.watchdog.thread_real_time_names)
}
else:
status = {
'node':
self.node,
'data':
self.data,
'log':
self.log,
'transport_restart_time':
self.watchdog.transport_restart_num,
'real_time_thread_name':
list(self.watchdog.thread_real_time_names)
}
key = "/nodes/" + self.node + "/" + self.app + "/status"
try:
self.etcd.write(key, json.dumps(status))
log.debug("\nkey: %s\ndata: %s\n", key, status)
except Exception as e:
log.error("\n%s", e)
await asyncio.sleep(self.etcd_interval_time)
def enqueue_log(self, msg):
"""
保存定长的历史 log 日志
:param msg: str, log
:return:
"""
if len(self.log) < 10:
self.log.append(msg)
else:
self.log.pop(0)
self.log.append(msg)
def re_load(self):
conf = get_conf()
self.node = conf['node']
self.ip = conf['local_ip']
self.app = conf['application']
self.paths = conf['paths']
self.etcd_interval_time = conf['etcd']['interval']
async def monitor(self):
while True:
data = ""
for file in self.paths:
data += str(os.stat(file).st_mtime)
sha = hashlib.sha1(data.encode()).hexdigest()
if not self.hash:
self.hash = sha
elif self.hash != sha:
self.hash = sha
self.redis.rpush("order_name", "reload")
await asyncio.sleep(30)
class Transport:
def __init__(self, conf, redis_address):
self.to_where = conf['send_to_where']
self.redis = RedisWrapper(redis_address)
self.data_original = None
self.name = None
self.communication = Communication(conf)
if self.to_where == 'influxdb':
self.db = InfluxdbWrapper(conf['influxdb'])
def work(self, *args):
while True:
raw_data = self.unpack()
data = self.pack(raw_data)
if data:
try:
self.send(data)
except Exception as e:
log.error("\n%s", e)
self.reque_data()
time.sleep(3)
def unpack(self):
"""
Get data from redis and unpack it
:return: dict
"""
data_len = self.redis.get_len("data_queue")
data = dict()
if data_len > 0:
self.data_original = self.redis.dequeue("data_queue")
try:
for k, v in msgpack.unpackb(self.data_original).items():
if v == True:
continue
else:
data[k.decode('utf-8')] = msgpack.unpackb(
v, encoding='utf-8')
except Exception as e:
log.error("\n%s", e)
else:
log.info('redis have no data')
time.sleep(5)
return data
def pack(self, data):
"""
Converts the data to the format required for the corresponding database
:param data: dict
:return: the format data, usually dict
"""
if data:
if self.to_where == 'influxdb':
try:
measurement = data['table_name']
tags = data['fields'].pop('tags')
unit = data['fields'].pop('unit')
fields = data['fields']
timestamp = data['time']
json_data = [{
'measurement': measurement,
'tags': tags,
'time': timestamp,
'fields': fields,
'unit': unit
}]
return json_data
except Exception as e:
log.error("\n%s", e)
else:
data['fields'].pop('tags')
data['fields'].pop('unit')
def send(self, data):
"""
Sent to the corresponding database
:param data: dict
:return: None
"""
if self.to_where == 'influxdb':
time_precision = data[0].pop('unit')
info = self.db.send(data, time_precision)
self.communication.data[data[0]["measurement"]] = data
if info:
log.info('send data to inflxudb.{}, {}'.format(
data[0]['measurement'], info))
else:
raise Exception("\nCan't connect influxdb")
def reque_data(self):
"""
return data to redis
:return:
"""
self.redis.queue_back('data_queue', self.data_original)
def re_load(self):
conf = get_conf()
self.to_where = conf['send_to_where']
self.data_original = None
self.name = None
if self.to_where == 'influxdb':
self.db = InfluxdbWrapper(conf['influxdb'])
return self
class Communication(object):
INSTANCE = None
Lock = threading.RLock()
def __new__(cls, *args, **kwargs):
"""
单例模式的 double check 确保线程安全，增加缓冲标量
确保第一次初始化完全
:param args:
:param kwargs:
:return:
"""
_instance = None
if not cls.INSTANCE:
try:
cls.Lock.acquire()
if not cls.INSTANCE:
_instance = object.__new__(cls)
cls.INSTANCE = _instance
finally:
cls.Lock.release()
return cls.INSTANCE
def __init__(self, conf):
self.redis = RedisWrapper(conf['redis'])
self.watchdog = WatchDog(conf)
self.app = conf['application']
self.data = dict()
self._name = 'communication'
self.log = list()
self.hash = None
# 20-12-14 zhy: windows启动异常，communication中没有self.paths参数
self.__init_paths()
# 重启刷新缓存
self.flush_data()
def __init_paths(self):
if self.app == "ziyan":
self.paths = [
'./conf/conf.toml', './lua/enque_script.lua',
'./plugins/your_plugin.py'
]
elif self.app == "chitu":
self.paths = ['./conf/conf.toml']
else:
self.paths = []
@property
def name(self):
return self._name
def work(self, *args):
"""
获取 event loop，并启动异步循环
:param args:
:return:
"""
if platform.system() == "Windows":
gevent.joinall(
[gevent.spawn(self.handle),
gevent.spawn(self.monitor)])
else:
gevent.joinall([
gevent.spawn(self.handle),
])
def handle(self):
"""
异步方法，监听 redis 给出的命令
:return:
"""
while True:
# 获取外部命令，并处理
command = self.check_order()
if command == b'get_status':
self.write_into_local()
elif command == b'restart':
self.watchdog.restart = True
self.flush_data()
elif command == b'reload':
self.watchdog.reload = True
self.flush_data()
self.re_load()
gevent.sleep(0.5)
def check_order(self):
"""
Get outside command
:return: None or order
"""
try:
if self.redis.get_len("order_name") > 0:
return self.redis.dequeue("order_name")
except Exception as err:
log.exception(err)
def write_into_local(self):
"""
Write the data to local redis
:return: None
"""
if self.app == 'ziyan':
status = {
'data': self.data,
'log': self.log,
'check_restart_time': self.watchdog.check_restart_num,
'handle_restart_time': self.watchdog.handle_restart_num,
'real_time_thread_name': self.watchdog.thread_real_time_names
}
else:
status = {
'data': self.data,
'log': self.log,
'transport_restart_time': self.watchdog.transport_restart_num,
'real_time_thread_name': self.watchdog.thread_real_time_names
}
try:
self.redis.sadd('status', status)
self.redis.expire('status', 60 * 5)
except Exception as err:
log.exception(err)
def flush_data(self):
"""
Delete the existing key "status"
:return:
"""
try:
if self.redis.exists("status"):
self.redis.delete("status")
except Exception as err:
log.exception(err)
def enqueue_log(self, msg):
"""
保存定长的历史 log 日志
:param msg: str, log
:return:
"""
if len(self.log) < 10:
self.log.append(msg)
else:
self.log.pop(0)
self.log.append(msg)
def re_load(self):
conf = get_conf()
self.app = conf['application']
def monitor(self):
while True:
data = ""
for file in self.paths:
data += str(os.stat(file).st_mtime)
sha = hashlib.sha1(data.encode()).hexdigest()
if not self.hash:
self.hash = sha
elif self.hash != sha:
self.hash = sha
self.redis.rpush("order_name", "reload")
gevent.sleep(30)