def set_pub():
name = 'Queue:alert:error:list'
dic = {
"tag": ["order", "play", "parse"],
"msg": "接收到开始播放事件",
"data": {
"trade_id": "YH20191210000063"
},
"datetime": "2019-12-10 17:51:20,979",
"host": "ed21fd9bb5ca",
"file_line": "event.py:35",
"state": "receive_start_event",
"level": "ERROR",
"source": "Cloud-PlayParse",
"alias": "order.info"
}
push_list = [json.dumps(dic) for _ in range(num)]
try:
logger.debug(
"=" * 10 +
"执行启动时间 {}".format(int(str(time.time()).replace('.', ''))) +
"=" * 10)
with redis_server.pipeline(transaction=False) as pipe:
pipe.lpush(name, *push_list)
pipe.execute()
except Exception as e:
logger.debug(e)
def parse(self):
"""解析数据"""
len_num = 10
if self.redis_queue_key is None:
raise ExceptionService('redis list key is None')
if self.parse_callback is None:
raise ExceptionService('parse callback is None')
# 获取队列
try:
with redis_server.pipeline(transaction=False) as pipe:
pipe.multi()
pipe.lrange(self.redis_queue_key, 0 - len_num, -1)
pipe.ltrim(self.redis_queue_key, 0, 0 - len_num)
data = pipe.execute()
except redis.exceptions.ResponseError:
pipe.reset()
raise ExceptionService(
'Redis get data error, redis key is not list. ||| %s' %
self.redis_queue_key)
except redis.exceptions.ConnectionError:
self.redis_state = False
return False
# 开始解析数据
if not data or len(data[0]) == 0:
time.sleep(self.parse_sleep)
return None
else:
logger.debug('ParseData: %s' % data)
# 解析 Json 数据
if data[1] == False:
logger.exception(
'json data parse error, Please check if it is json string. ||| %s'
% data)
return False
data = data[0]
try:
data = data if isinstance(data, list) else json.loads(data)
except:
logger.exception(
'json data parse error, Please check if it is json string. ||| %s'
% data)
return False
# 回调执行
try:
self.parser_pipe(data)
except ExceptionError as ex:
logger.error(ex)
except ExceptionWarning as ex:
logger.warning(ex)
except ExceptionInfo as ex:
logger.info(ex)
def parse(self):
"""解析数据"""
if self.redis_queue_key is None:
raise ExceptionService('redis list key is None')
if self.parse_callback is None:
raise ExceptionService('parse callback is None')
# 获取队列
try:
key = self.redis_queue_key
queue_len = 1000 if redis_server.llen(
key) > 1000 else redis_server.llen(key)
with redis_server.pipeline(transaction=False) as pipe:
for _ in range(queue_len):
pipe.rpop(key)
data = pipe.execute()
except redis.exceptions.ResponseError:
raise ExceptionService(
'Redis get data error, redis key is not list. ||| %s' %
self.redis_queue_key)
except redis.exceptions.ConnectionError:
self.redis_state = False
return False
# 开始解析数据
if not data:
time.sleep(self.parse_sleep)
return None
else:
logger.debug('ParseData: %s' % data)
# 解析 Json 数据
try:
data = json.loads(data) if self.data_to_json else data
except:
logger.exception(
'json data parse error, Please check if it is json string. ||| %s'
% data)
return False
# 回调执行
try:
return self.parse_callback(data)
except ExceptionError as ex:
logger.error(ex)
except ExceptionWarning as ex:
logger.warning(ex)
except ExceptionInfo as ex:
logger.info(ex)
def set_keys_expire(args=None):
app_options.set('service.id', args.service_id)
iter_count = 500
exec_count = 5000
if args.pattern and args.second:
from env.server import redis_server
pipe = redis_server.pipeline()
count = 0
for key in redis_server.scan_iter(match=args.pattern,
count=iter_count):
count += 1
pipe.expire(key, args.second)
if count == exec_count:
count = 0
pipe.execute()
def set_keys_expire(args=None):
app_options.set('service.id', args.service_id)
if args.keys and args.second:
from env.server import mysql_server, redis_server, mqtt_server
keys = {}
for i in range(100):
keys['test-key:' + str(i)] = 'value' + str(i)
redis_server.mset(keys)
pipe = redis_server.pipeline()
for key in redis_server.scan_iter(match=args.pattern, count=10):
pipe.expire(key, args.second)
results = pipe.execute()
print(results)
def set_pub():
name = 'Queue:mqtt:public'
dic = dict(
topic='test',
payload={
"topic_cmd": "switch",
"trade_id": "YH20191122000005",
"publish_type": "single",
"addr": 2,
"publish_time": 1574412759,
"type": "lock",
"control_cmd": "switch_on",
"gateway_id": "GAwg4hSuJ09r"
}
)
push_list = [json.dumps(dic) for _ in range(num)]
try:
logger.debug("="*10,"执行启动时间 {}".format(int(str(time.time()).replace('.', ''))),"="*10)
with redis_server.pipeline(transaction=False) as pipe:
pipe.lpush(name, *push_list)
pipe.execute()
except Exception as e:
logger.debug(e)