def load_raw_source(cls): conn = get_config_redis_client() name_keys_map = {} try: d = conn.hgetall(conf.REDIS_SOURCE_MAP) except redis.RedisError: logger.exception('load raw sources error') return for name, fields_str in d.items(): fields = json.loads(fields_str) fields.pop('name_show', None) keys = {} for key_name, key_type in fields.items(): # print('********' + key_name + '********\n') if key_type in {'string', 'str'}: key_type = 'str' try: type_ = getattr(builtins, key_type) except AttributeError: type_ = getattr(builtins, "str") keys[key_name] = type_ name_keys_map[name] = keys cls.name_keys_map = name_keys_map
def clear_test_data(): from clients import get_mongo_client, get_config_redis_client db = get_mongo_client() db.client.drop_database(db.name) client = get_config_redis_client() client.flushdb()
def load(self): name_sources_map = defaultdict(set) conn = get_config_redis_client() for name in conn.scan_iter(match='*', count=100): d = conn.hgetall(name) source_name = d['strategy_source'] keys = d['strategy_body'].split(',') name_sources_map[source_name].add(self(source_name, keys)) return name_sources_map
def load_strategys(self): uuid_strategy_map = {} conn = get_config_redis_client() logger.info('start load strategys from db, current strategy: %s', self.uuid_strategy_map.keys()) for strategy_cls in _used_strategies: try: for name in conn.scan_iter(match=strategy_cls.prefix): d = conn.hgetall(name) strategy = strategy_cls(d) uuid_strategy_map[strategy.uuid] = strategy except redis.RedisError: logger.error('load strategys occur redis conn error') return self.uuid_strategy_map = uuid_strategy_map logger.info('load strategys success, current strategy: %s', self.uuid_strategy_map.keys())
def load_rules(self): id_rule_map = {} conn = get_config_redis_client() logger.info('start load rules, current rule ids: %s', self.id_rule_map.keys()) try: for name in conn.scan_iter(match='rule:*'): d = conn.hgetall(name) if self._should_load(d): rule = Rule(d) id_rule_map[rule.id] = rule except redis.RedisError: logger.error('load rules occur redis conn error') return self.id_rule_map = id_rule_map logger.info('load rules success, current rule ids: %s', self.id_rule_map.keys())
def __init__(self, auto_persist=False): self.conn = get_config_redis_client() self.id_count_map = defaultdict(int) self.redis_hash_key_temp = 'access_count_{}' if auto_persist: gevent.spawn(self.auto_persist)