def test_is_rate_limited_script(): now = int(time.time()) client = StrictRedis(db=9) # The item should not be rate limited by either key. assert map( bool, is_rate_limited(client, ('foo', 'bar'), (1, now + 60, 2, now + 120))) == [False, False] # The item should be rate limited by the first key (1). assert map( bool, is_rate_limited(client, ('foo', 'bar'), (1, now + 60, 2, now + 120))) == [True, False] # The item should still be rate limited by the first key (1), but *not* # rate limited by the second key (2) even though this is the third time # we've checked the quotas. This ensures items that are rejected by a lower # quota don't affect unrelated items that share a parent quota. assert map( bool, is_rate_limited(client, ('foo', 'bar'), (1, now + 60, 2, now + 120))) == [True, False] assert client.get('foo') == '1' assert 59 <= client.ttl('foo') <= 60 assert client.get('bar') == '1' assert 119 <= client.ttl('bar') <= 120
class RedisDataSource(AbstractDataSource): _r = None def __init__(self,config): if self._validateConfig(config): self._r = StrictRedis(host=config[REDIS_DATASOURCE_CONFIG][REDIS_DATASOURCE_CONFIG_HOST], port=config[REDIS_DATASOURCE_CONFIG][REDIS_DATASOURCE_CONFIG_PORT], db=config[REDIS_DATASOURCE_CONFIG][REDIS_DATASOURCE_CONFIG_DB]) logger.debug("Obtained internal redis handler" + str(self._r)) else: raise BaseException("Error validating config ") def update(self,item): self.store(item) def store(self,item): self._r.set(item.getHash(), item.getValue()) def get(self,item): return self._r.get(item.getHash()) def exists(self,item): return self.get(item) is not None def all(self): result = [] # Obtain all keys keys = self._r.keys() #For each key, get value for k in keys: value = self._r.get(k) result.append(BaseItem({"origin":"redis"},value)) #return result return result def _validateConfig(self,config): validator = MultipleConfigValidator( {VALIDATORS_LIST:[ContainsKeyConfigValidator({KEY_VALUE:REDIS_DATASOURCE_CONFIG})]}) if not validator.validate(config): raise BaseException("Config validation error : does not contain " + REDIS_DATASOURCE_CONFIG) # Validate redis datasource config validator = MultipleConfigValidator( {VALIDATORS_LIST:[ContainsKeysConfigValidator({KEYS_LIST:[REDIS_DATASOURCE_CONFIG_DB, REDIS_DATASOURCE_CONFIG_HOST, REDIS_DATASOURCE_CONFIG_PORT]})]}) if not validator.validate(config[REDIS_DATASOURCE_CONFIG]): raise BaseException("Config validation error : config not complete ") return True def delete(self,item): self._r.delete(item.getHash())
class Record(object): def __init__(self,host='127.0.0.1',port=6379): self.r=StrictRedis() def run(self): while True: value=self.r.rpop('alerts') if value: obj=json.loads(value) keyredis=obj['src_ip']+'_'+str(obj['src_port'])+'_'+ obj['dest_ip']+'_'+str(obj['dest_port']) entry=self.r.get(keyredis) if entry: restruct=json.loads(entry) else: restruct={} if not 'http' in restruct: restruct['http']=[] if not 'alerts' in restruct: restruct['alerts']=[] if not 'files' in restruct: restruct['files']=[] if 'alert' in obj: restruct['alerts'].append(obj['alert']['signature']) if 'fileinfo' in obj: restruct['files'].append(obj['fileinfo']) if 'http' in obj: restruct['http'].append(obj['http']) if len(restruct)>0: self.r.set(keyredis, json.dumps(restruct)) else: sleep(1)
def test_is_rate_limited_script(): now = int(time.time()) client = StrictRedis(db=9) # The item should not be rate limited by either key. assert map(bool, is_rate_limited(client, ('foo', 'bar'), (1, now + 60, 2, now + 120))) == [False, False] # The item should be rate limited by the first key (1). assert map(bool, is_rate_limited(client, ('foo', 'bar'), (1, now + 60, 2, now + 120))) == [True, False] # The item should still be rate limited by the first key (1), but *not* # rate limited by the second key (2) even though this is the third time # we've checked the quotas. This ensures items that are rejected by a lower # quota don't affect unrelated items that share a parent quota. assert map(bool, is_rate_limited(client, ('foo', 'bar'), (1, now + 60, 2, now + 120))) == [True, False] assert client.get('foo') == '1' assert 59 <= client.ttl('foo') <= 60 assert client.get('bar') == '1' assert 119 <= client.ttl('bar') <= 120
class RedisDataStorage(DataStorage): def __init__(self, host, port, level): DataStorage.__init__(self, level) self._storage = StrictRedis(host=host, port=port) def get_key(self, pid): return "{}_{}".format(self.storage_level(), pid) def get_data(self, pid, default=None): result = self._storage.get(name=self.get_key(pid)) if result is None: result = default return result def store_data(self, pid, data): self._storage.set(name=self.get_key(pid), value=data) def delete_data(self, pid): self._storage.delete(self.get_key(pid)) def exists(self, pid): return self._storage.exists(self.get_key(pid))
class RedisCache(CacheBase): def __init__(self, config, section): from redis.client import StrictRedis self.conn = StrictRedis( config.get(section, 'redis-server'), config.getint(section, 'redis-port'), config.getint(section, 'redis-db'), decode_responses=True ) def check_password(self, user, password): """Check the given user and password. Returns None on cache miss, True if password matches, False if not. """ cached = self.conn.get(self.prefix('%s-pass' % user)) if cached is None: return cached else: return cached == self.hash(password, cached) def set_password(self, user, password): self.conn.set(self.prefix('%s-pass' % user), self.hash(password, None), ex=self.expire) def in_groups(self, user, groups): key = self.prefix('%s-groups' % user) if not self.conn.exists(key): return None return not self.conn.smembers(key).isdisjoint(groups) def set_groups(self, user, groups): key = self.prefix('%s-groups' % user) pipe = self.conn.pipeline() pipe.sadd(key, *groups).expire(key, self.expire) pipe.execute()
class RedisCache(CacheBase): def __init__(self, config, section): from redis.client import StrictRedis self.conn = StrictRedis(config.get(section, 'redis-server'), config.getint(section, 'redis-port'), config.getint(section, 'redis-db'), decode_responses=True) def check_password(self, user, password): """Check the given user and password. Returns None on cache miss, True if password matches, False if not. """ cached = self.conn.get(self.prefix('%s-pass' % user)) if cached is None: return cached else: return cached == self.hash(password, cached) def set_password(self, user, password): self.conn.set(self.prefix('%s-pass' % user), self.hash(password, None), ex=self.expire) def in_groups(self, user, groups): key = self.prefix('%s-groups' % user) if not self.conn.exists(key): return None return not self.conn.smembers(key).isdisjoint(groups) def set_groups(self, user, groups): key = self.prefix('%s-groups' % user) pipe = self.conn.pipeline() pipe.sadd(key, *groups).expire(key, self.expire) pipe.execute()
class DistanceCalculator(object): _geoip4 = None _geoip6 = None def __init__(self): # Load the GeoIP databases into class attributes since they each need 20+ MB in memory if not self.__class__._geoip4: self.__class__._geoip4 = GeoIP(Config.GEOIP_PATH_V4, MEMORY_CACHE) if not self.__class__._geoip6: self.__class__._geoip6 = GeoIP(Config.GEOIP_PATH_V6, MEMORY_CACHE) self.redis = StrictRedis(Config.REDIS['HOST'], Config.REDIS['PORT'], Config.REDIS['DB']) @staticmethod def _haversine(lon1, lat1, lon2, lat2): """ Calculate the great circle distance between two points on the earth (specified in decimal degrees) """ # convert decimal degrees to radians lon1, lat1, lon2, lat2 = map(lambda v: radians(float(v)), [lon1, lat1, lon2, lat2]) # haversine formula dlon = lon2 - lon1 dlat = lat2 - lat1 a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2 c = 2 * asin(sqrt(a)) km = 6367 * c # convert to km return km def get_mirror_distances(self, address): last_update = self.redis.get(Config.KEY_LAST_UPDATE) key = Config.KEY_MIRROR.format(address, last_update) distances = OrderedDict(self.redis.zrange(key, 0, -1, withscores=True)) if not distances: if address.startswith("::ffff:"): address = address.replace("::ffff:", "") try: if ":" in address: record = self._geoip6.record_by_addr(address) else: record = self._geoip4.record_by_addr(address) except socket.error: raise GeoIPLookupError() if not record: raise GeoIPLookupError() lat = record['latitude'] lon = record['longitude'] distances = OrderedDict( sorted(((mirror.name, self._haversine(lon, lat, mirror.lon, mirror.lat)) for mirror in Mirror.objects.filter(age__lt=3601)), key=itemgetter(1))) if distances: self.redis.zadd(key, **distances) self.redis.expire(key, 60 * 10) # 10 min return distances def get_nearest_mirror(self, address): try: distances = self.get_mirror_distances(address) if distances: return next(distances.iteritems())[0] return Config.FALLBACK_MIRROR except GeoIPLookupError: return Config.FALLBACK_MIRROR
class RedisSessionObject(): implements(ISession) def __init__(self, request): self._options = _options self.rd = None self._master_rd = False self.request = request self._data = None self.id = None self._new_session = True self._changed = False cookie = self.request.headers.get('Cookie') if cookie is None: self.__create_id() else: c = SimpleCookie() c.load(cookie) session_cookie = c.get(self._options['_cookie_name']) if session_cookie is None: #new session! self.__create_id() else: self.id = session_cookie.value self._new_session = False def session_callback(request, response): exception = getattr(request, 'exception', None) commit = self._changed increase_expire_mod = _options['_increase_expire_mod'] if increase_expire_mod > 0: rnd = round(random.random() * 1000000) mod = rnd % increase_expire_mod if not mod: # print 'Saving due to increase_expire_mod' commit = True if exception is None and commit: self.__save() cookie = SimpleCookie() _cname = self._options['_cookie_name'] cookie[_cname] = self.id domain = self._options.get('cookie_domain') cookie[_cname]['path'] = _options['_path'] if domain is not None: cookie[_cname]['domain'] = domain if self._options['_secure']: cookie[_cname]['secure'] = True header = cookie[_cname].output(header='') # print 'Writing cookie header:',header response.headerlist.append(('Set-Cookie', header)) request.add_response_callback(session_callback) # private methods def __init_rd(self, master=False): if self.rd is None: if master: self.rd = StrictRedis(host=_redis_servers[0][0], port=_redis_servers[0][1], db=_redis_servers[0][2]) self._master_rd = True else: server = random.choice(_redis_servers) self.rd = StrictRedis(host=server[0], port=server[1], db=server[2]) self._master_rd = False elif master and not self._master_rd: self.rd = StrictRedis(host=_redis_servers[0][0], port=_redis_servers[0][1], db=_redis_servers[0][2]) self._master_rd = True def __key(self): return 'rd:ses:%s' % self.id def __load(self): if self._data is None: self.__init_rd() data = self.rd.get(self.__key()) if data is not None: self._data = msgpack.unpackb(data, use_list=False, encoding='utf-8') else: self._data = {} def __save(self): if self._data is not None and len(self._data): self.__init_rd(master=True) self.rd.setex(self.__key(), self._options['_expire'], msgpack.packb(self._data, encoding='utf-8')) def __create_id(self): self.id = hashlib.sha1( hashlib.sha1("%f%s%f%s" % (time.time(), id({}), random.random(), getpid())).hexdigest(), ).hexdigest() def init_with_id(self, session_id): """ Init the session with custom id. the session data is no loaded immediately but loaded only when data is accessed :param session_id: :return: self """ self.id = session_id self._data = None return self def set_expire(self, expire): self._options['_expire'] = expire # ISession API def save(self): self._changed = True def invalidate(self): self.__init_rd(master=True) self.rd.delete(self.__key()) #todo: delete cookie def changed(self): self._changed = True def flash(self, msg, queue='', allow_duplicate=True): self.__load() key = '_flsh:%s_' % queue q = self.get(key, []) if not allow_duplicate: if msg not in q: q.append(msg) else: q.append(msg) self[key] = q def pop_flash(self, queue=''): self.__load() key = '_flsh:%s_' % queue q = self.get(key, []) if len(q): e = q.pop() self[key] = q return e return None def peek_flash(self, queue=''): self.__load() key = '_flsh:%s_' % queue q = self.get(key, []) if len(q): e = q[0] return e return None def new_csrf_token(self): token = os.urandom(20).encode('hex') self['_csrft_'] = token return token def get_csrf_token(self): token = self.get('_csrft_', None) if token is None: token = self.new_csrf_token() return token # mapping methods def __getitem__(self, key): self.__load() return self._data[key] def get(self, key, default=None): self.__load() return self._data.get(key, default) def __delitem__(self, key): self.__load() del self._data[key] self._changed = True def __setitem__(self, key, value): self.__load() self._data[key] = value self._changed = True def keys(self): self.__load() return self._data.keys() def values(self): self.__load() return self._data.values() def items(self): self.__load() return self._data.items() def iterkeys(self): self.__load() return iter(self._data.keys()) def itervalues(self): self.__load() return iter(self._data.values()) def iteritems(self): self.__load() return iter(self._data.items()) def clear(self): self.__load() self._data = {} self._changed = True def update(self, d): self.__load() for k in self._data.keys(): d[k] = self._data[k] def multi_set(self, d): # print '[update]', self.id self.__load() for k in d.keys(): self._data[k] = d[k] self._changed = True def setdefault(self, key, default=None): """D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D""" self.__load() if key in self._data: return self._data[key] else: self._data[key] = default self._changed = True return default def pop(self, k, *args): """remove specified key and return the corresponding value ``*args`` may contain a single default value, or may not be supplied. If key is not found, default is returned if given, otherwise ``KeyError`` is raised""" pass def popitem(self): """remove and return some (key, value) pair as a 2-tuple; but raise ``KeyError`` if mapping is empty""" pass def __len__(self): self.__load() return len(self._data) def __iter__(self): return self.iterkeys() def __contains__(self, key): self.__load() return key in self._data @property def new(self): return self._new_session
class MainWindow(QtWidgets.QMainWindow): def __init__(self): super().__init__() self.setWindowTitle("Redis Explorer") self.resize(QtWidgets.QDesktopWidget().availableGeometry(self).size() * 0.5) self.tree = QtWidgets.QTreeWidget() self.label = QtWidgets.QTextEdit() font = self.label.font() font.setPointSize(12) self.label.setFont(font) self.tree.setColumnCount(2) self.tree.header().setSectionResizeMode(0, QtWidgets.QHeaderView.ResizeToContents) self.tree.setHeaderHidden(True) splitter = QtWidgets.QSplitter() splitter.addWidget(self.tree) scroll_area = QtWidgets.QScrollArea() scroll_area.setWidgetResizable(True) scroll_area.setWidget(self.label) splitter.addWidget(scroll_area) splitter.setSizes([1, 1]) self.setCentralWidget(splitter) toolbar = self.addToolBar("") toolbar.setMovable(False) toolbar.setIconSize(QtCore.QSize(32, 32)) toolbar.addAction(QtGui.QIcon("resources/list-add.png"), "").triggered.connect(lambda: self.plus_font(1)) toolbar.addAction(QtGui.QIcon("resources/list-remove.png"), "").triggered.connect(lambda: self.plus_font(-1)) toolbar.addAction(QtGui.QIcon("resources/view-refresh.png"), "").triggered.connect(lambda: self.refresh()) self.redis = StrictRedis() def item_clicked(item: QtWidgets.QTreeWidgetItem): if item.parent() is None: return value = self.redis.get(item.text(0)) value = value.decode() text = json.dumps(json.loads(value), ensure_ascii=False, indent=4) text = text.replace(" ", " ").replace("\n", "<br/>").replace(":", "<font color=red>:</font>") text = re.sub(r'"(.*?)(?<!\\)"', r'<font color=green>"\g<1>"</font>', text) self.label.setHtml(text) self.tree.itemClicked.connect(item_clicked) def plus_font(self, number: int): font = self.label.font() font.setPointSize(font.pointSize() + number) self.label.setFont(font) def refresh(self): self.tree.clear() self.label.setPlainText("") keys = sorted(self.redis.keys("*")) keys = [x.decode() for x in keys] dct = dict() for key in keys: parts = key.split(":") prefix = parts[0] if prefix not in dct: dct[prefix] = [key] else: dct[prefix].append(key) for prefix, keys in dct.items(): item = QtWidgets.QTreeWidgetItem([prefix]) for key in keys: item.addChild(QtWidgets.QTreeWidgetItem([key])) self.tree.addTopLevelItem(item) def showEvent(self, event: QtGui.QShowEvent): super().showEvent(event) self.refresh()
class DistanceCalculator(object): _geoip4 = None _geoip6 = None def __init__(self): # Load the GeoIP databases into class attributes since they each need 20+ MB in memory if not self.__class__._geoip4: self.__class__._geoip4 = GeoIP(Config.GEOIP_PATH_V4, MEMORY_CACHE) if not self.__class__._geoip6: self.__class__._geoip6 = GeoIP(Config.GEOIP_PATH_V6, MEMORY_CACHE) self.redis = StrictRedis(Config.REDIS['HOST'], Config.REDIS['PORT'], Config.REDIS['DB']) @staticmethod def _haversine(lon1, lat1, lon2, lat2): """ Calculate the great circle distance between two points on the earth (specified in decimal degrees) """ # convert decimal degrees to radians lon1, lat1, lon2, lat2 = map(lambda v: radians(float(v)), [lon1, lat1, lon2, lat2]) # haversine formula dlon = lon2 - lon1 dlat = lat2 - lat1 a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2 c = 2 * asin(sqrt(a)) km = 6367 * c # convert to km return km def get_mirror_distances(self, address): last_update = self.redis.get(Config.KEY_LAST_UPDATE) key = Config.KEY_MIRROR.format(address, last_update) distances = OrderedDict(self.redis.zrange(key, 0, -1, withscores=True)) if not distances: if address.startswith("::ffff:"): address = address.replace("::ffff:", "") try: if ":" in address: record = self._geoip6.record_by_addr(address) else: record = self._geoip4.record_by_addr(address) except socket.error: raise GeoIPLookupError() if not record: raise GeoIPLookupError() lat = record['latitude'] lon = record['longitude'] distances = OrderedDict( sorted( ( (mirror.name, self._haversine(lon, lat, mirror.lon, mirror.lat)) for mirror in Mirror.objects.filter(age__lt=3601) ), key=itemgetter(1) ) ) if distances: self.redis.zadd(key, **distances) self.redis.expire(key, 60 * 10) # 10 min return distances def get_nearest_mirror(self, address): try: distances = self.get_mirror_distances(address) if distances: return next(distances.iteritems())[0] return Config.FALLBACK_MIRROR except GeoIPLookupError: return Config.FALLBACK_MIRROR
from redis.client import StrictRedis if __name__ == '__main__': strict_redis = StrictRedis(host='127.0.0.1', port=6379, db=0, decode_responses=True) strict_redis.set('aa', '111') strict_redis.set('bb', '222') strict_redis.set('cc', '333') aa = strict_redis.get('aa') bb = strict_redis.get('bb') cc = strict_redis.get('cc') dd = strict_redis.get('dd') print(aa, bb, cc) keys = strict_redis.keys() print(keys, type(keys))
class RedisStore(): def __init__(self, db_host, db_port, db_num, db_pw): self.pool = ConnectionPool(max_connections=2, db=db_num, host=db_host, port=db_port, password=db_pw, decode_responses=True) self.redis = StrictRedis(connection_pool=self.pool) self.redis.ping() self._object_map = WeakValueDictionary() def create_object(self, dbo_class, dbo_dict, update_timestamp=True): dbo_class = get_dbo_class(getattr(dbo_class, 'dbo_key_type', dbo_class)) if not dbo_class: return try: dbo_id = dbo_dict['dbo_id'] except KeyError: dbo_id, dbo_dict = dbo_dict, {} if dbo_id is None or dbo_id == '': warn("create_object called with empty dbo_id") return dbo_id = str(dbo_id).lower() if self.object_exists(dbo_class.dbo_key_type, dbo_id): raise ObjectExistsError(dbo_id) dbo = dbo_class() dbo.dbo_id = dbo_id dbo.hydrate(dbo_dict) dbo.db_created() if dbo.dbo_set_key: self.redis.sadd(dbo.dbo_set_key, dbo.dbo_id) self.save_object(dbo, update_timestamp) return dbo def save_object(self, dbo, update_timestamp=False, autosave=False): if update_timestamp: dbo.dbo_ts = int(time.time()) if dbo.dbo_indexes: self._update_indexes(dbo) self._clear_old_refs(dbo) save_root, new_refs = dbo.to_db_value() self.redis.set(dbo.dbo_key, json_encode(save_root)) if new_refs: self._set_new_refs(dbo, new_refs) debug("db object {} {}saved", dbo.dbo_key, "auto" if autosave else "") self._object_map[dbo.dbo_key] = dbo return dbo def save_raw(self, key, raw): self.redis.set(key, json_encode(raw)) def load_raw(self, key, default=None): json = self.redis.get(key) if json: return json_decode(json) return default def load_cached(self, dbo_key): return self._object_map.get(dbo_key) def load_object(self, dbo_key, key_type=None, silent=False): if key_type: try: key_type = key_type.dbo_key_type except AttributeError: pass try: dbo_key, dbo_id = ':'.join([key_type, dbo_key]), dbo_key except TypeError: if not silent: exception("Invalid dbo_key passed to load_object", stack_info=True) return else: key_type, _, dbo_id = dbo_key.partition(':') cached_dbo = self._object_map.get(dbo_key) if cached_dbo: return cached_dbo json_str = self.redis.get(dbo_key) if not json_str: if not silent: warn("Failed to find {} in database", dbo_key) return return self.load_from_json(json_str, key_type, dbo_id) def load_from_json(self, json_str, key_type, dbo_id): dbo_dict = json_decode(json_str) dbo = get_mixed_type(key_type, dbo_dict.get('mixins'))() dbo.dbo_id = dbo_id self._object_map[dbo.dbo_key] = dbo dbo.hydrate(dbo_dict) return dbo def object_exists(self, obj_type, obj_id): return self.redis.exists('{}:{}'.format(obj_type, obj_id)) def load_object_set(self, dbo_class, set_key=None): key_type = dbo_class.dbo_key_type if not set_key: set_key = dbo_class.dbo_set_key results = set() keys = deque() pipeline = self.redis.pipeline() for key in self.fetch_set_keys(set_key): dbo_key = ':'.join([key_type, key]) try: results.add(self._object_map[dbo_key]) except KeyError: keys.append(key) pipeline.get(dbo_key) for dbo_id, json_str in zip(keys, pipeline.execute()): if json_str: obj = self.load_from_json(json_str, key_type, dbo_id) if obj: results.add(obj) continue warn("Removing missing object from set {}", set_key) self.delete_set_key(set_key, dbo_id) return results def delete_object_set(self, dbo_class, set_key=None): if not set_key: set_key = dbo_class.dbo_set_key for dbo in self.load_object_set(dbo_class, set_key): self.delete_object(dbo) self.delete_key(set_key) def update_object(self, dbo, dbo_dict): dbo.hydrate(dbo_dict) return self.save_object(dbo, True) def delete_object(self, dbo): key = dbo.dbo_key dbo.db_deleted() self.delete_key(key) self._clear_old_refs(dbo) if dbo.dbo_set_key: self.redis.srem(dbo.dbo_set_key, dbo.dbo_id) for children_type in dbo.dbo_children_types: self.delete_object_set(get_dbo_class(children_type), "{}_{}s:{}".format(dbo.dbo_key_type, children_type, dbo.dbo_id)) for ix_name in dbo.dbo_indexes: ix_value = getattr(dbo, ix_name, None) if ix_value is not None and ix_value != '': self.delete_index('ix:{}:{}'.format(dbo.dbo_key_type, ix_name), ix_value) debug("object deleted: {}", key) self.evict_object(dbo) def reload_object(self, dbo_key): dbo = self._object_map.get(dbo_key) if dbo: json_str = self.redis.get(dbo_key) if not json_str: warn("Failed to find {} in database for reload", dbo_key) return None return self.update_object(dbo, json_decode(json_str)) return self.load_object(dbo_key) def evict_object(self, dbo): self._object_map.pop(dbo.dbo_key, None) def fetch_set_keys(self, set_key): return self.redis.smembers(set_key) def add_set_key(self, set_key, *values): self.redis.sadd(set_key, *values) def delete_set_key(self, set_key, value): self.redis.srem(set_key, value) def set_key_exists(self, set_key, value): return self.redis.sismember(set_key, value) def db_counter(self, counter_id, inc=1): return self.redis.incr("counter:{}".format(counter_id), inc) def delete_key(self, key): self.redis.delete(key) def set_index(self, index_name, key, value): return self.redis.hset(index_name, key, value) def get_index(self, index_name, key): return self.redis.hget(index_name, key) def get_full_index(self, index_name): return self.redis.hgetall(index_name) def delete_index(self, index_name, key): return self.redis.hdel(index_name, key) def get_all_hash(self, index_name): return {key: json_decode(value) for key, value in self.redis.hgetall(index_name).items()} def set_db_hash(self, hash_id, hash_key, value): return self.redis.hset(hash_id, hash_key, json_encode(value)) def get_db_hash(self, hash_id, hash_key): return json_decode(self.redis.hget(hash_id, hash_key)) def remove_db_hash(self, hash_id, hash_key): self.redis.hdel(hash_id, hash_key) def get_all_db_hash(self, hash_id): return [json_decode(value) for value in self.redis.hgetall(hash_id).values()] def get_db_list(self, list_id, start=0, end=-1): return [json_decode(value) for value in self.redis.lrange(list_id, start, end)] def add_db_list(self, list_id, value): self.redis.lpush(list_id, json_encode(value)) def trim_db_list(self, list_id, start, end): return self.redis.ltrim(list_id, start, end) def _update_indexes(self, dbo): try: old_dbo = json_decode(self.redis.get(dbo.dbo_key)) except TypeError: old_dbo = None for ix_name in dbo.dbo_indexes: new_val = getattr(dbo, ix_name, None) old_val = old_dbo.get(ix_name) if old_dbo else None if old_val == new_val: continue ix_key = 'ix:{}:{}'.format(dbo.dbo_key_type, ix_name) if old_val is not None: self.delete_index(ix_key, old_val) if new_val is not None and new_val != '': if self.get_index(ix_key, new_val): raise NonUniqueError(ix_key, new_val) self.set_index(ix_key, new_val, dbo.dbo_id) def _clear_old_refs(self, dbo): dbo_key = dbo.dbo_key ref_key = '{}:refs'.format(dbo_key) for ref_id in self.fetch_set_keys(ref_key): holder_key = '{}:holders'.format(ref_id) self.delete_set_key(holder_key, dbo_key) self.delete_key(ref_key) def _set_new_refs(self, dbo, new_refs): dbo_key = dbo.dbo_key self.add_set_key("{}:refs".format(dbo_key), *new_refs) for ref_id in new_refs: holder_key = '{}:holders'.format(ref_id) self.add_set_key(holder_key, dbo_key)
class RedisStore(): def __init__(self, dispatcher, db_host, db_port, db_num, db_pw): self.dispatcher = dispatcher pool = ConnectionPool(max_connections=2, db=db_num, host=db_host, port=db_port, password=db_pw) self.redis = StrictRedis(connection_pool=pool) self.encoder = JSONEncoder() self.decoder = JSONDecoder() self.class_map = {} self.object_map = {} def create_object(self, dbo, update_rev=False): self.save_object(dbo) dbo.on_loaded() def save_object(self, dbo, update_rev=False, autosave=False): if update_rev: dbo.dbo_rev = getattr(dbo, "dbo_rev", 0) + 1 json_obj = self.build_json(dbo) key = dbo.dbo_key self.redis.set(key, self.encoder.encode(json_obj)) if dbo.dbo_set_key: self.redis.sadd(dbo.dbo_set_key, key) self.dispatcher.dispatch("db_log{0}".format("_auto" if autosave else ""), "object saved: " + key) self.object_map[dbo.dbo_key] = dbo; def build_json(self, dbo): dbo.before_save() json_obj = {} if dbo.__class__ != dbo.dbo_base_class: json_obj["class_name"] = dbo.__module__ + "." + dbo.__class__.__name__ for field_name in dbo.dbo_fields: json_obj[field_name] = getattr(dbo, field_name, None) for dbo_col in dbo.dbo_collections: coll_list = list() for child_dbo in getattr(dbo, dbo_col.field_name): if dbo_col.key_type: coll_list.append(child_dbo.dbo_id) else: coll_list.append(self.build_json(child_dbo)) json_obj[dbo_col.field_name] = coll_list for dbo_ref in dbo.dbo_refs: ref = getattr(dbo, dbo_ref.field_name, None) if ref: json_obj[dbo_ref.field_name] = ref.dbo_id return json_obj def cache_object(self, dbo): self.object_map[dbo.dbo_key] def load_cached(self, key): return self.object_map.get(key) def evict(self, dbo): try: del self.object_map[dbo.dbo_key] except: self.dispatcher.dispatch("db_log", "Failed to evict " + dbo.dbo_key + " from db cache") def load_by_key(self, key_type, key, base_class=None): dbo_key = key_type + ":" + key cached_dbo = self.object_map.get(dbo_key) if cached_dbo: return cached_dbo json_str = self.redis.get(dbo_key) if not json_str: return None json_obj = self.decoder.decode(json_str) dbo = self.load_class(json_obj, base_class)(key) if dbo.dbo_key_type: self.object_map[dbo.dbo_key] = dbo self.load_json(dbo, json_obj) return dbo def load_class(self, json_obj, base_class): class_path = json_obj.get("class_name") if not class_path: return base_class clazz = self.class_map.get(class_path) if clazz: return clazz split_path = class_path.split(".") module_name = ".".join(split_path[:-1]) class_name = split_path[-1] module = __import__(module_name, globals(), locals(), [class_name]) clazz = getattr(module, class_name) self.class_map[class_path] = clazz return clazz def load_object(self, dbo_class, key): return self.load_by_key(dbo_class.dbo_key_type, key, dbo_class) def load_json(self, dbo, json_obj): for field_name in dbo.dbo_fields: try: setattr(dbo, field_name, json_obj[field_name]) except KeyError: self.dispatcher.dispatch("db_log", "db: Object " + dbo.dbo_key + " json missing field " + field_name) for dbo_col in dbo.dbo_collections: coll = getattr(dbo, dbo_col.field_name, []) try: for child_json in json_obj[dbo_col.field_name]: if dbo_col.key_type: child_dbo = self.load_by_key(dbo_col.key_type, child_json, dbo_col.base_class) else: child_dbo = self.load_class(child_json, dbo_col.base_class)() self.load_json(child_dbo, child_json) coll.append(child_dbo) except AttributeError: self.dispatcher.dispatch("db_log", "{0} json failed to load for coll {1} in {2}".format(child_json, dbo_col.field_name, dbo.dbo_id)) except KeyError: self.dispatcher.dispatch("db_log", "db: Object " + dbo.dbo_key + " json missing collection " + dbo_col.field_name) for dbo_ref in dbo.dbo_refs: try: ref_key = json_obj[dbo_ref.field_name] ref_obj = self.load_by_key(dbo_ref.key_type, ref_key, dbo_ref.base_class) setattr(dbo, dbo_ref.field_name, ref_obj) except: self.dispatcher.dispatch("db_log", "db: Object " + dbo.dbo_key + " json missing ref " + dbo_ref.field_name) dbo.on_loaded() return True def delete_object(self, dbo): key = dbo.dbo_key self.redis.delete(key) if dbo.dbo_set_key: self.redis.srem(dbo.dbo_set_key, key) for dbo_col in dbo.dbo_collections: if dbo_col.key_type: coll = getattr(dbo, dbo_col.field_name, set()) for child_dbo in coll: self.delete_object(child_dbo) self.dispatcher.dispatch("db_log", "object deleted: " + key) if self.object_map.get(dbo.dbo_key): del self.object_map[dbo.dbo_key] return True def fetch_set_keys(self, set_key): return self.redis.smembers(set_key)
class RedisStore: def __init__(self, db_host, db_port, db_num, db_pw): self.pool = ConnectionPool(max_connections=2, db=db_num, host=db_host, port=db_port, password=db_pw, decode_responses=True) self.redis = StrictRedis(connection_pool=self.pool) self.redis.ping() self._object_map = WeakValueDictionary() def create_object(self, dbo_class, dbo_dict, update_timestamp=True): dbo_class = get_dbo_class(getattr(dbo_class, 'dbo_key_type', dbo_class)) if not dbo_class: return try: dbo_id = dbo_dict['dbo_id'] except KeyError: dbo_id, dbo_dict = dbo_dict, {} if dbo_id is None or dbo_id == '': log.warn("create_object called with empty dbo_id") return dbo_id = str(dbo_id).lower() if self.object_exists(dbo_class.dbo_key_type, dbo_id): raise ObjectExistsError(dbo_id) dbo = dbo_class() dbo.dbo_id = dbo_id dbo.hydrate(dbo_dict) dbo.db_created() if dbo.dbo_set_key: self.redis.sadd(dbo.dbo_set_key, dbo.dbo_id) self.save_object(dbo, update_timestamp) return dbo def load_object(self, dbo_key, key_type=None, silent=False): if key_type: try: key_type = key_type.dbo_key_type except AttributeError: pass try: dbo_key, dbo_id = ':'.join((key_type, dbo_key)), dbo_key except TypeError: if not silent: log.exception("Invalid dbo_key passed to load_object", stack_info=True) return else: key_type, _, dbo_id = dbo_key.partition(':') cached_dbo = self._object_map.get(dbo_key) if cached_dbo: return cached_dbo json_str = self.redis.get(dbo_key) if not json_str: if not silent: log.warn("Failed to find {} in database", dbo_key) return return self._json_to_obj(json_str, key_type, dbo_id) def save_object(self, dbo, update_timestamp=False, autosave=False): if update_timestamp: dbo.dbo_ts = int(time.time()) if dbo.dbo_indexes: self._update_indexes(dbo) self._clear_old_refs(dbo) save_root, new_refs = dbo.to_db_value() self.redis.set(dbo.dbo_key, json_encode(save_root)) if new_refs: self._set_new_refs(dbo, new_refs) log.debug("db object {} {}saved", dbo.dbo_key, "auto" if autosave else "") self._object_map[dbo.dbo_key] = dbo return dbo def update_object(self, dbo, dbo_dict): dbo.hydrate(dbo_dict) return self.save_object(dbo, True) def delete_object(self, dbo): key = dbo.dbo_key dbo.db_deleted() self.delete_key(key) self._clear_old_refs(dbo) if dbo.dbo_set_key: self.redis.srem(dbo.dbo_set_key, dbo.dbo_id) for children_type in dbo.dbo_children_types: self.delete_object_set( get_dbo_class(children_type), "{}_{}s:{}".format(dbo.dbo_key_type, children_type, dbo.dbo_id)) for ix_name in dbo.dbo_indexes: ix_value = getattr(dbo, ix_name, None) if ix_value is not None and ix_value != '': self.delete_index('ix:{}:{}'.format(dbo.dbo_key_type, ix_name), ix_value) log.debug("object deleted: {}", key) self.evict_object(dbo) def load_cached(self, dbo_key): return self._object_map.get(dbo_key) def object_exists(self, obj_type, obj_id): return self.redis.exists('{}:{}'.format(obj_type, obj_id)) def load_object_set(self, dbo_class, set_key=None): dbo_class = get_dbo_class(getattr(dbo_class, 'dbo_key_type', dbo_class)) key_type = dbo_class.dbo_key_type if not set_key: set_key = dbo_class.dbo_set_key results = set() keys = deque() pipeline = self.redis.pipeline() for key in self.fetch_set_keys(set_key): dbo_key = ':'.join([key_type, key]) try: results.add(self._object_map[dbo_key]) except KeyError: keys.append(key) pipeline.get(dbo_key) for dbo_id, json_str in zip(keys, pipeline.execute()): if json_str: obj = self._json_to_obj(json_str, key_type, dbo_id) if obj: results.add(obj) continue log.warn("Removing missing object from set {}", set_key) self.delete_set_key(set_key, dbo_id) return results def delete_object_set(self, dbo_class, set_key=None): if not set_key: set_key = dbo_class.dbo_set_key for dbo in self.load_object_set(dbo_class, set_key): self.delete_object(dbo) self.delete_key(set_key) def reload_object(self, dbo_key): dbo = self._object_map.get(dbo_key) if dbo: json_str = self.redis.get(dbo_key) if not json_str: log.warn("Failed to find {} in database for reload", dbo_key) return None return self.update_object(dbo, json_decode(json_str)) return self.load_object(dbo_key) def evict_object(self, dbo): self._object_map.pop(dbo.dbo_key, None) def load_value(self, key, default=None): json = self.redis.get(key) if json: return json_decode(json) return default def save_value(self, key, value): self.redis.set(key, json_encode(value)) def fetch_set_keys(self, set_key): return self.redis.smembers(set_key) def add_set_key(self, set_key, *values): self.redis.sadd(set_key, *values) def delete_set_key(self, set_key, value): self.redis.srem(set_key, value) def set_key_exists(self, set_key, value): return self.redis.sismember(set_key, value) def db_counter(self, counter_id, inc=1): return self.redis.incr("counter:{}".format(counter_id), inc) def delete_key(self, key): self.redis.delete(key) def set_index(self, index_name, key, value): return self.redis.hset(index_name, key, value) def get_index(self, index_name, key): return self.redis.hget(index_name, key) def get_full_index(self, index_name): return self.redis.hgetall(index_name) def delete_index(self, index_name, key): return self.redis.hdel(index_name, key) def get_all_hash(self, index_name): return { key: json_decode(value) for key, value in self.redis.hgetall(index_name).items() } def get_hash_keys(self, hash_id): return self.redis.hkeys(hash_id) def set_db_hash(self, hash_id, hash_key, value): return self.redis.hset(hash_id, hash_key, json_encode(value)) def get_db_hash(self, hash_id, hash_key): return json_decode(self.redis.hget(hash_id, hash_key)) def remove_db_hash(self, hash_id, hash_key): self.redis.hdel(hash_id, hash_key) def get_all_db_hash(self, hash_id): return [ json_decode(value) for value in self.redis.hgetall(hash_id).values() ] def get_db_list(self, list_id, start=0, end=-1): return [ json_decode(value) for value in self.redis.lrange(list_id, start, end) ] def add_db_list(self, list_id, value): self.redis.lpush(list_id, json_encode(value)) def trim_db_list(self, list_id, start, end): return self.redis.ltrim(list_id, start, end) def dbo_holders(self, dbo_key, degrees=0): all_keys = set() def find(find_key, degree): holder_keys = self.fetch_set_keys('{}:holders'.format(find_key)) for new_key in holder_keys: if new_key != dbo_key and new_key not in all_keys: all_keys.add(new_key) if degree < degrees: find(new_key, degree + 1) find(dbo_key, 0) return all_keys def _json_to_obj(self, json_str, key_type, dbo_id): dbo_dict = json_decode(json_str) dbo = get_mixed_type(key_type, dbo_dict.get('mixins'))() dbo.dbo_id = dbo_id dbo.hydrate(dbo_dict) self._object_map[dbo.dbo_key] = dbo return dbo def _update_indexes(self, dbo): try: old_dbo = json_decode(self.redis.get(dbo.dbo_key)) except TypeError: old_dbo = None for ix_name in dbo.dbo_indexes: new_val = getattr(dbo, ix_name, None) old_val = old_dbo.get(ix_name) if old_dbo else None if old_val == new_val: continue ix_key = 'ix:{}:{}'.format(dbo.dbo_key_type, ix_name) if old_val is not None: self.delete_index(ix_key, old_val) if new_val is not None and new_val != '': if self.get_index(ix_key, new_val): raise NonUniqueError(ix_key, new_val) self.set_index(ix_key, new_val, dbo.dbo_id) def _clear_old_refs(self, dbo): dbo_key = dbo.dbo_key ref_key = '{}:refs'.format(dbo_key) for ref_id in self.fetch_set_keys(ref_key): holder_key = '{}:holders'.format(ref_id) self.delete_set_key(holder_key, dbo_key) self.delete_key(ref_key) def _set_new_refs(self, dbo, new_refs): dbo_key = dbo.dbo_key self.add_set_key("{}:refs".format(dbo_key), *new_refs) for ref_id in new_refs: holder_key = '{}:holders'.format(ref_id) self.add_set_key(holder_key, dbo_key)
# -*- coding: gbk -*- ''' Created on 2012-5-26 @author: Sky ''' from redis.client import StrictRedis Sr = StrictRedis(host='localhost', port=6379, db=0) Sr.set('foo', 'bar') #print(str(Sr.get('foo'), encoding = "utf-8") == 'bar') print(Sr.get('foo')) Sr.hset("MyHash", "field1", "ль¤У") print(Sr.hget("MyHash", "field11")) Sr.rpush("list", "one") Sr.rpush("list", "two") print(Sr.llen("list")) Sr.ltrim("list", 1, 0) print(Sr.llen("list")) Sr.hset("MyHash", "Key1", "Value1") Sr.hset("MyHash", "Key2", "Value2") for i in Sr.hkeys("MyHash"): print(i) print(Sr.hlen("PlayerHash")) print(Sr.get("XXX")) print(type(Sr.smembers("EnemyTemplate:16:LOOT"))) for i in Sr.smembers("EnemyTemplate:16:LOOT"): print(i)
class RedisStore(): def __init__(self, db_host, db_port, db_num, db_pw): pool = ConnectionPool(max_connections=2, db=db_num, host=db_host, port=db_port, password=db_pw) self.redis = StrictRedis(connection_pool=pool) self.class_map = {} self.object_map = {} def create_object(self, dbo, update_rev=False): self.save_object(dbo, update_rev) dbo.on_loaded() def save_object(self, dbo, update_rev=False, autosave=False): if update_rev: rev = getattr(dbo, "dbo_rev", None) dbo.dbo_rev = 1 if not rev else rev + 1 dbo.before_save() key = dbo.dbo_key self.redis.set(key, self.json_encode(dbo.save_json_obj)) if dbo.dbo_set_key: self.redis.sadd(dbo.dbo_set_key, key) self.dispatch("db_log{0}".format("_auto" if autosave else ""), "object saved: " + key) self.object_map[dbo.dbo_key] = dbo def load_cached(self, key): return self.object_map.get(key) def evict_object(self, dbo): try: del self.object_map[dbo.dbo_key] except: debug("Failed to evict " + dbo.dbo_key + " from db cache", self) @logged def load_by_key(self, key_type, key, base_class=None): dbo_key = unicode('{0}:{1}'.format(key_type, key)) cached_dbo = self.object_map.get(dbo_key) if cached_dbo: return cached_dbo json_str = self.redis.get(dbo_key) if not json_str: return None json_obj = self.json_decode(json_str) dbo = self._load_class(json_obj, base_class)(key) if dbo.dbo_key_type: self.object_map[dbo.dbo_key] = dbo self.load_json(dbo, json_obj) return dbo def object_exists(self, obj_type, obj_id): key = unicode('{0}:{1}'.format(obj_type, obj_id)) return key in self.redis.keys(key) def load_object(self, dbo_class, key): return self.load_by_key(dbo_class.dbo_key_type, key, dbo_class) def update_object(self, dbo, json_obj): self.load_json(dbo, json_obj) self.save_object(dbo, True) def delete_object(self, dbo): key = dbo.dbo_key self.redis.delete(key) if dbo.dbo_set_key: self.redis.srem(dbo.dbo_set_key, key) for dbo_col in dbo.dbo_collections: if dbo_col.key_type: coll = getattr(dbo, dbo_col.field_name, set()) for child_dbo in coll: self.delete_object(child_dbo) debug("object deleted: " + key, self) if self.object_map.get(dbo.dbo_key): del self.object_map[dbo.dbo_key] return True def fetch_set_keys(self, set_key): return self.redis.smembers(set_key) def set_index(self, index_name, key, value): return self.redis.hset(index_name, key, value) def get_index(self, index_name, key): return self.redis.hget(index_name, key) def delete_index(self, index_name, key): return self.redis.hdel(index_name, key) def _load_class(self, json_obj, base_class): class_path = json_obj.get("class_name") if not class_path: return self.cls_registry(base_class) clazz = self.class_map.get(class_path) if clazz: return clazz split_path = class_path.split(".") module_name = ".".join(split_path[:-1]) class_name = split_path[-1] module = __import__(module_name, globals(), locals(), [class_name]) clazz = getattr(module, class_name) clazz = self.cls_registry(clazz) self.class_map[class_path] = clazz return clazz def load_json(self, dbo, json_obj): for field_name in dbo.dbo_fields: try: setattr(dbo, field_name, json_obj[field_name]) except KeyError: pass for dbo_col in dbo.dbo_collections: coll = getattr(dbo, dbo_col.field_name, []) try: for child_json in json_obj[dbo_col.field_name]: try: if dbo_col.key_type: child_dbo = self.load_by_key(dbo_col.key_type, child_json, dbo_col.base_class) else: child_dbo = self._load_class(child_json, dbo_col.base_class)() self.load_json(child_dbo, child_json) coll.append(child_dbo) except AttributeError: warn("{0} json failed to load for coll {1} in {2}".format(child_json, dbo_col.field_name, unicode(dbo.dbo_id)), self) except KeyError: if dbo.dbo_key_type: trace("db: Object " + unicode(dbo.dbo_debug_key) + " json missing collection " + dbo_col.field_name, self) for dbo_ref in dbo.dbo_refs: try: ref_key = json_obj[dbo_ref.field_name] ref_obj = self.load_by_key(dbo_ref.key_type, ref_key, dbo_ref.base_class) setattr(dbo, dbo_ref.field_name, ref_obj) except: if dbo.dbo_key_type: trace("db: Object " + unicode(dbo.dbo_debug_key) + " json missing ref " + dbo_ref.field_name, self) dbo.on_loaded() return True
class RedisDataSource(AbstractDataSource): _r = None def __init__(self, config): if self._validateConfig(config): self._r = StrictRedis( host=config[REDIS_DATASOURCE_CONFIG] [REDIS_DATASOURCE_CONFIG_HOST], port=config[REDIS_DATASOURCE_CONFIG] [REDIS_DATASOURCE_CONFIG_PORT], db=config[REDIS_DATASOURCE_CONFIG][REDIS_DATASOURCE_CONFIG_DB]) logger.debug("Obtained internal redis handler" + str(self._r)) else: raise BaseException("Error validating config ") def update(self, item): self.store(item) def store(self, item): self._r.set(item.getHash(), item.getValue()) def get(self, item): return self._r.get(item.getHash()) def exists(self, item): return self.get(item) is not None def all(self): result = [] # Obtain all keys keys = self._r.keys() #For each key, get value for k in keys: value = self._r.get(k) result.append(BaseItem({"origin": "redis"}, value)) #return result return result def _validateConfig(self, config): validator = MultipleConfigValidator({ VALIDATORS_LIST: [ContainsKeyConfigValidator({KEY_VALUE: REDIS_DATASOURCE_CONFIG})] }) if not validator.validate(config): raise BaseException("Config validation error : does not contain " + REDIS_DATASOURCE_CONFIG) # Validate redis datasource config validator = MultipleConfigValidator({ VALIDATORS_LIST: [ ContainsKeysConfigValidator({ KEYS_LIST: [ REDIS_DATASOURCE_CONFIG_DB, REDIS_DATASOURCE_CONFIG_HOST, REDIS_DATASOURCE_CONFIG_PORT ] }) ] }) if not validator.validate(config[REDIS_DATASOURCE_CONFIG]): raise BaseException( "Config validation error : config not complete ") return True def delete(self, item): self._r.delete(item.getHash())
class RedisSessionObject(): implements(ISession) def __init__(self, request): self._options = _options self.rd = None self._master_rd = False self.request = request self._data = None self.id = None self._new_session = True self._changed = False cookie = self.request.headers.get('Cookie') if cookie is None: self.__create_id() else: c = SimpleCookie() c.load(cookie) session_cookie = c.get(self._options['_cookie_name']) if session_cookie is None: #new session! self.__create_id() else: self.id = session_cookie.value self._new_session = False def session_callback(request, response): exception = getattr(request, 'exception', None) commit = self._changed increase_expire_mod = _options['_increase_expire_mod'] if increase_expire_mod > 0: rnd = round(random.random() * 1000000) mod = rnd % increase_expire_mod if not mod: # print 'Saving due to increase_expire_mod' commit = True if exception is None and commit: self.__save() cookie = SimpleCookie() _cname = self._options['_cookie_name'] cookie[_cname] = self.id domain = self._options.get('cookie_domain') cookie[_cname]['path'] = _options['_path'] if domain is not None: cookie[_cname]['domain'] = domain if self._options['_secure']: cookie[_cname]['secure'] = True header = cookie[_cname].output(header='') # print 'Writing cookie header:',header response.headerlist.append(('Set-Cookie', header)) request.add_response_callback(session_callback) # private methods def __init_rd(self, master=False): if self.rd is None: if master: self.rd = StrictRedis(host=_redis_servers[0][0], port=_redis_servers[0][1], db=_redis_servers[0][2]) self._master_rd = True else: server = random.choice(_redis_servers) self.rd = StrictRedis(host=server[0], port=server[1], db=server[2]) self._master_rd = False elif master and not self._master_rd: self.rd = StrictRedis(host=_redis_servers[0][0], port=_redis_servers[0][1], db=_redis_servers[0][2]) self._master_rd = True def __key(self): return 'rd:ses:%s' % self.id def __load(self): if self._data is None: self.__init_rd() data = self.rd.get(self.__key()) if data is not None: self._data = msgpack.unpackb(data, use_list=True, encoding='utf-8') else: self._data = {} def __save(self): if self._data is not None and len(self._data): self.__init_rd(master=True) self.rd.setex(self.__key(), self._options['_expire'], msgpack.packb(self._data, encoding='utf-8')) def __create_id(self): self.id = hashlib.sha1(hashlib.sha1("%f%s%f%s" % (time.time(), id({}), random.random(), getpid())).hexdigest(), ).hexdigest() def init_with_id(self, session_id): """ Init the session with custom id. the session data is no loaded immediately but loaded only when data is accessed :param session_id: :return: self """ self.id = session_id self._data = None return self def set_expire(self, expire): self._options['_expire'] = expire # ISession API def save(self): self._changed = True def invalidate(self): self.__init_rd(master=True) self.rd.delete(self.__key()) #todo: delete cookie def changed(self): self._changed = True def flash(self, msg, queue='', allow_duplicate=True): self.__load() key = '_flsh:%s_' % queue q = self.get(key, []) if not allow_duplicate: if msg not in q: q.append(msg) else: q.append(msg) self[key] = q def pop_flash(self, queue=''): self.__load() key = '_flsh:%s_' % queue q = self.get(key, []) if len(q): e = q.pop() self[key] = q return e return None def peek_flash(self, queue=''): self.__load() key = '_flsh:%s_' % queue q = self.get(key, []) if len(q): e = q[0] return e return None def new_csrf_token(self): token = os.urandom(20).encode('hex') self['_csrft_'] = token return token def get_csrf_token(self): token = self.get('_csrft_', None) if token is None: token = self.new_csrf_token() return token # mapping methods def __getitem__(self, key): self.__load() return self._data[key] def get(self, key, default=None): self.__load() return self._data.get(key, default) def __delitem__(self, key): self.__load() del self._data[key] self._changed = True def __setitem__(self, key, value): self.__load() self._data[key] = value self._changed = True def keys(self): self.__load() return self._data.keys() def values(self): self.__load() return self._data.values() def items(self): self.__load() return self._data.items() def iterkeys(self): self.__load() return iter(self._data.keys()) def itervalues(self): self.__load() return iter(self._data.values()) def iteritems(self): self.__load() return iter(self._data.items()) def clear(self): self.__load() self._data = {} self._changed = True def update(self, d): self.__load() for k in self._data.keys(): d[k] = self._data[k] def multi_set(self, d): # print '[update]', self.id self.__load() for k in d.keys(): self._data[k] = d[k] self._changed = True def setdefault(self, key, default=None): """D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D""" pass def pop(self, k, *args): """remove specified key and return the corresponding value ``*args`` may contain a single default value, or may not be supplied. If key is not found, default is returned if given, otherwise ``KeyError`` is raised""" pass def popitem(self): """remove and return some (key, value) pair as a 2-tuple; but raise ``KeyError`` if mapping is empty""" pass def __len__(self): self.__load() return len(self._data) def __iter__(self): return self.iterkeys() def __contains__(self, key): self.__load() return key in self._data @property def new(self): return self._new_session