def set_pub(): name = 'Queue:alert:error:list' dic = { "tag": ["order", "play", "parse"], "msg": "接收到开始播放事件", "data": { "trade_id": "YH20191210000063" }, "datetime": "2019-12-10 17:51:20,979", "host": "ed21fd9bb5ca", "file_line": "event.py:35", "state": "receive_start_event", "level": "ERROR", "source": "Cloud-PlayParse", "alias": "order.info" } push_list = [json.dumps(dic) for _ in range(num)] try: logger.debug( "=" * 10 + "执行启动时间 {}".format(int(str(time.time()).replace('.', ''))) + "=" * 10) with redis_server.pipeline(transaction=False) as pipe: pipe.lpush(name, *push_list) pipe.execute() except Exception as e: logger.debug(e)
def parse(self): """解析数据""" len_num = 10 if self.redis_queue_key is None: raise ExceptionService('redis list key is None') if self.parse_callback is None: raise ExceptionService('parse callback is None') # 获取队列 try: with redis_server.pipeline(transaction=False) as pipe: pipe.multi() pipe.lrange(self.redis_queue_key, 0 - len_num, -1) pipe.ltrim(self.redis_queue_key, 0, 0 - len_num) data = pipe.execute() except redis.exceptions.ResponseError: pipe.reset() raise ExceptionService( 'Redis get data error, redis key is not list. ||| %s' % self.redis_queue_key) except redis.exceptions.ConnectionError: self.redis_state = False return False # 开始解析数据 if not data or len(data[0]) == 0: time.sleep(self.parse_sleep) return None else: logger.debug('ParseData: %s' % data) # 解析 Json 数据 if data[1] == False: logger.exception( 'json data parse error, Please check if it is json string. ||| %s' % data) return False data = data[0] try: data = data if isinstance(data, list) else json.loads(data) except: logger.exception( 'json data parse error, Please check if it is json string. ||| %s' % data) return False # 回调执行 try: self.parser_pipe(data) except ExceptionError as ex: logger.error(ex) except ExceptionWarning as ex: logger.warning(ex) except ExceptionInfo as ex: logger.info(ex)
def parse(self): """解析数据""" if self.redis_queue_key is None: raise ExceptionService('redis list key is None') if self.parse_callback is None: raise ExceptionService('parse callback is None') # 获取队列 try: key = self.redis_queue_key queue_len = 1000 if redis_server.llen( key) > 1000 else redis_server.llen(key) with redis_server.pipeline(transaction=False) as pipe: for _ in range(queue_len): pipe.rpop(key) data = pipe.execute() except redis.exceptions.ResponseError: raise ExceptionService( 'Redis get data error, redis key is not list. ||| %s' % self.redis_queue_key) except redis.exceptions.ConnectionError: self.redis_state = False return False # 开始解析数据 if not data: time.sleep(self.parse_sleep) return None else: logger.debug('ParseData: %s' % data) # 解析 Json 数据 try: data = json.loads(data) if self.data_to_json else data except: logger.exception( 'json data parse error, Please check if it is json string. ||| %s' % data) return False # 回调执行 try: return self.parse_callback(data) except ExceptionError as ex: logger.error(ex) except ExceptionWarning as ex: logger.warning(ex) except ExceptionInfo as ex: logger.info(ex)
def set_keys_expire(args=None): app_options.set('service.id', args.service_id) iter_count = 500 exec_count = 5000 if args.pattern and args.second: from env.server import redis_server pipe = redis_server.pipeline() count = 0 for key in redis_server.scan_iter(match=args.pattern, count=iter_count): count += 1 pipe.expire(key, args.second) if count == exec_count: count = 0 pipe.execute()
def set_keys_expire(args=None): app_options.set('service.id', args.service_id) if args.keys and args.second: from env.server import mysql_server, redis_server, mqtt_server keys = {} for i in range(100): keys['test-key:' + str(i)] = 'value' + str(i) redis_server.mset(keys) pipe = redis_server.pipeline() for key in redis_server.scan_iter(match=args.pattern, count=10): pipe.expire(key, args.second) results = pipe.execute() print(results)
def set_pub(): name = 'Queue:mqtt:public' dic = dict( topic='test', payload={ "topic_cmd": "switch", "trade_id": "YH20191122000005", "publish_type": "single", "addr": 2, "publish_time": 1574412759, "type": "lock", "control_cmd": "switch_on", "gateway_id": "GAwg4hSuJ09r" } ) push_list = [json.dumps(dic) for _ in range(num)] try: logger.debug("="*10,"执行启动时间 {}".format(int(str(time.time()).replace('.', ''))),"="*10) with redis_server.pipeline(transaction=False) as pipe: pipe.lpush(name, *push_list) pipe.execute() except Exception as e: logger.debug(e)