def __init__(self, server, params): """ Connect to Redis, and set up cache backend. """ super(RedisCache, self).__init__(params) self.sharder = CacheSharder() options = params.get('OPTIONS', {}) password = params.get('password', options.get('PASSWORD', None)) db = params.get('db', options.get('DB', 1)) try: db = int(db) except (ValueError, TypeError): db = 1 if isinstance(server, basestring): self._servers = server.split(';') else: self._servers = server or ['localhost'] for host in self._servers: port = 6379 if ':' in host: host, port = host.split(':') try: port = int(port) except (ValueError, TypeError): port = 6379 client = redis.Redis(host=host, port=port, db=db, password=password) self.sharder.add(client, ":".join([host, str(port)]))
class RedisCache(BaseCache): def __init__(self, server, params): """ Connect to Redis, and set up cache backend. """ super(RedisCache, self).__init__(params) self.sharder = CacheSharder() options = params.get('OPTIONS', {}) password = params.get('password', options.get('PASSWORD', None)) db = params.get('db', options.get('DB', 1)) try: db = int(db) except (ValueError, TypeError): db = 1 if isinstance(server, basestring): self._servers = server.split(';') else: self._servers = server or ['localhost'] for host in self._servers: port = 6379 if ':' in host: host, port = host.split(':') try: port = int(port) except (ValueError, TypeError): port = 6379 client = redis.Redis(host=host, port=port, db=db, password=password) self.sharder.add(client, ":".join([host, str(port)])) def get_cache(self, key): return self.sharder.get_node(key) def get_caches(self, keys, version=None): """ Returns a dict of keys that belong to a cache's keyspace. """ caches = defaultdict(list) keys = map(lambda key: self.make_key(key, version=version), keys) for key in keys: caches[self.get_cache(key)].append(key) return caches @property def caches(self): return self.sharder._nodes def make_key(self, key, version=None): if not isinstance(key, CacheKey): key = CacheKey(super(RedisCache, self).make_key(key, version)) return key def get(self, key, default=None, version=None): """ Retrieve a value from the cache. Returns unpickled value if key is found, the default if not. """ key = self.make_key(key, version=version) value = self.get_cache(key).get(key) if value is None: return default return self.unpickle(value) def get_many(self, keys, version=None): """ Retrieve many keys. """ recovered_data = SortedDict() new_keys = map(lambda key: self.make_key(key, version=version), keys) map_keys = dict(zip(new_keys, keys)) caches = self.get_caches(new_keys) for cache, keys in caches.items(): values = cache.mget(keys) for key, value in zip(keys, values): if value is None: continue value = self.unpickle(value) if isinstance(value, basestring): value = smart_unicode(value) recovered_data[map_keys[key]] = value return recovered_data def add(self, key, value, timeout=None, version=None): """ Add a value to the cache, failing if the key already exists. Returns ``True`` if the object was added, ``False`` if not. """ key = self.make_key(key, version=version) if self.get_cache(key).exists(key): return False return self.set(key, value, timeout) def set(self, key, value, timeout=None, version=None): """ Persist a value to the cache, and set an optional expiration time. """ key = self.make_key(key, version=version) result = self.get_cache(key).set(key, pickle.dumps(value)) self.expire(key, timeout, version=version) return result def set_many(self, data, timeout=None, version=None): """ Set a bunch of values in the cache at once from a dict of key/value pairs. This is much more efficient than calling set() multiple times. If timeout is given, that timeout will be used for the key; otherwise the default cache timeout will be used. """ caches = defaultdict(dict) for key, value in data.iteritems(): cache_key = self.make_key(key, version=version) caches[self.get_cache(cache_key)][cache_key] = pickle.dumps(value) if caches: for cache, safe_data in caches.items(): cache.mset(safe_data) map(self.expire, safe_data.keys(), [timeout]*len(safe_data)) def expire(self, key, timeout=None, version=None): """ Set content expiration, if necessary """ key = self.make_key(key, version=version) if timeout is None: timeout = self.default_timeout if timeout <= 0: # force the key to be non-volatile result = self.get_cache(key).get(key) self.get_cache(key).set(key, result) else: # If the expiration command returns false, we need to reset the key # with the new expiration if not self.get_cache(key).expire(key, timeout): value = self.get(key, version=version) self.set(key, value, timeout, version=version) def delete(self, key, version=None): """ Remove a key from the cache. """ self.get_cache(key).delete(self.make_key(key, version=version)) def delete_many(self, keys, version=None): """ Remove multiple keys at once. """ if keys: caches = self.get_caches(keys, version) for cache, keys in caches.iteritems(): cache.delete(*keys) def clear(self): """ Flush all cache keys. """ # TODO : potential data loss here, should we only delete keys based on the correct version ? for cache in self.caches: cache._node.flushdb() def unpickle(self, value): """ Unpickles the given value. """ old = value value = smart_str(value) try: return pickle.loads(value) except pickle.UnpicklingError: return old def incr_version(self, key, delta=1, version=None): """ Adds delta to the cache version for the supplied key. Returns the new version. Note: In Redis 2.0 you cannot rename a volitle key, so we have to move the value from the old key to the new key and maintain the ttl. """ if version is None: version = self.version old_key = self.make_key(key, version) value = self.get(old_key, version=version) ttl = self.get_cache(key).ttl(old_key) if value is None: raise ValueError("Key '%s' not found" % key) new_key = self.make_key(key, version=version+delta) # TODO: See if we can check the version of Redis, since 2.2 will be able # to rename volitile keys. self.set(new_key, value, timeout=ttl) self.delete(old_key) return version + delta def close(self, **kwargs): """ Disconnect from the cache. """ for cache in self.caches: node = cache._node if hasattr(node, 'connection_pool'): cache._node.connection_pool.disconnect() else: cache._node.connection.disconnect()