def load_raw_source(cls):
conn = get_config_redis_client()
name_keys_map = {}
try:
d = conn.hgetall(conf.REDIS_SOURCE_MAP)
except redis.RedisError:
logger.exception('load raw sources error')
return
for name, fields_str in d.items():
fields = json.loads(fields_str)
fields.pop('name_show', None)
keys = {}
for key_name, key_type in fields.items():
# print('********' + key_name + '********\n')
if key_type in {'string', 'str'}:
key_type = 'str'
try:
type_ = getattr(builtins, key_type)
except AttributeError:
type_ = getattr(builtins, "str")
keys[key_name] = type_
name_keys_map[name] = keys
cls.name_keys_map = name_keys_map
def clear_test_data():
from clients import get_mongo_client, get_config_redis_client
db = get_mongo_client()
db.client.drop_database(db.name)
client = get_config_redis_client()
client.flushdb()
def load(self):
name_sources_map = defaultdict(set)
conn = get_config_redis_client()
for name in conn.scan_iter(match='*', count=100):
d = conn.hgetall(name)
source_name = d['strategy_source']
keys = d['strategy_body'].split(',')
name_sources_map[source_name].add(self(source_name, keys))
return name_sources_map
def load_strategys(self):
uuid_strategy_map = {}
conn = get_config_redis_client()
logger.info('start load strategys from db, current strategy: %s',
self.uuid_strategy_map.keys())
for strategy_cls in _used_strategies:
try:
for name in conn.scan_iter(match=strategy_cls.prefix):
d = conn.hgetall(name)
strategy = strategy_cls(d)
uuid_strategy_map[strategy.uuid] = strategy
except redis.RedisError:
logger.error('load strategys occur redis conn error')
return
self.uuid_strategy_map = uuid_strategy_map
logger.info('load strategys success, current strategy: %s',
self.uuid_strategy_map.keys())
def load_rules(self):
id_rule_map = {}
conn = get_config_redis_client()
logger.info('start load rules, current rule ids: %s',
self.id_rule_map.keys())
try:
for name in conn.scan_iter(match='rule:*'):
d = conn.hgetall(name)
if self._should_load(d):
rule = Rule(d)
id_rule_map[rule.id] = rule
except redis.RedisError:
logger.error('load rules occur redis conn error')
return
self.id_rule_map = id_rule_map
logger.info('load rules success, current rule ids: %s',
self.id_rule_map.keys())
def __init__(self, auto_persist=False):
self.conn = get_config_redis_client()
self.id_count_map = defaultdict(int)
self.redis_hash_key_temp = 'access_count_{}'
if auto_persist:
gevent.spawn(self.auto_persist)
