def __init__(self, server, params):
super(ShardedRedisCache, self).__init__(server, params)
self.sharder = HashRing()
for server in self.servers:
client = self.create_client(server)
self.clients[client.connection_pool.connection_identifier] = client
self.sharder.add(client.connection_pool.connection_identifier)
self.client_list = self.clients.values()
def test_make_key_distribution(self):
ring = HashRing()
nodes = set([str(node._node) for node in self.cache.sharder._nodes])
nodes = [
('127.0.0.1', 6379, 15, '/tmp/redis0.sock'),
('127.0.0.1', 6379, 15, '/tmp/redis1.sock'),
('127.0.0.1', 6379, 15, '/tmp/redis2.sock'),
]
for node in nodes:
ring.add(str(node))
n = 50000
counter = Counter(
[ring.get_node(str(i)) for i in range(n)]
)
self.assertLess(
((stddev(counter.values()) / n) * 100.0), 10, counter.values()
)
def __init__(self, server, params):
super(ShardedRedisCache, self).__init__(server, params)
self.sharder = HashRing()
for server in self.servers:
client = self.create_client(server)
self.clients[client.connection_pool.connection_identifier] = client
self.sharder.add(client.connection_pool.connection_identifier)
self.client_list = self.clients.values()
def __init__(self, server, params):
super(ShardedRedisCache, self).__init__(server, params)
self._params = params
self._server = server
self._pickle_version = None
self.__master_client = None
self.clients = {}
self.sharder = HashRing()
if not isinstance(server, (list, tuple)):
servers = [server]
else:
servers = server
for server in servers:
client = self.create_client(server)
self.clients[client.connection_pool.connection_identifier] = client
self.sharder.add(client.connection_pool.connection_identifier)
class ShardedRedisCache(BaseRedisCache):
def __init__(self, server, params):
super(ShardedRedisCache, self).__init__(server, params)
self.sharder = HashRing()
for server in self.servers:
client = self.create_client(server)
self.clients[client.connection_pool.connection_identifier] = client
self.sharder.add(client.connection_pool.connection_identifier)
self.client_list = self.clients.values()
def get_client(self, key, write=False):
node = self.sharder.get_node(key)
return self.clients[node]
def shard(self, keys, write=False, version=None):
"""
Returns a dict of keys that belong to a cache's keyspace.
"""
clients = defaultdict(list)
for key in keys:
clients[self.get_client(key,
write)].append(self.make_key(key, version))
return clients
####################
# Django cache api #
####################
def delete_many(self, keys, version=None):
"""
Remove multiple keys at once.
"""
clients = self.shard(keys, write=True, version=version)
for client, keys in clients.items():
self._delete_many(client, keys)
def clear(self, version=None):
"""
Flush cache keys.
If version is specified, all keys belonging the version's key
namespace will be deleted. Otherwise, all keys will be deleted.
"""
if version is None:
for client in self.clients.values():
self._clear(client)
else:
self.delete_pattern('*', version=version)
def get_many(self, keys, version=None):
data = {}
clients = self.shard(keys, version=version)
for client, versioned_keys in clients.items():
original_keys = [key._original_key for key in versioned_keys]
data.update(
self._get_many(client,
original_keys,
versioned_keys=versioned_keys))
return data
def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None):
"""
Set a bunch of values in the cache at once from a dict of key/value
pairs. This is much more efficient than calling set() multiple times.
If timeout is given, that timeout will be used for the key; otherwise
the default cache timeout will be used.
"""
timeout = self.get_timeout(timeout)
clients = self.shard(data.keys(), write=True, version=version)
if timeout is None:
for client, keys in clients.items():
subset = {}
for key in keys:
subset[key] = self.prep_value(data[key._original_key])
self._set_many(client, subset)
return
for client, keys in clients.items():
pipeline = client.pipeline()
for key in keys:
value = self.prep_value(data[key._original_key])
self._set(pipeline, key, value, timeout)
pipeline.execute()
def incr_version(self, key, delta=1, version=None):
"""
Adds delta to the cache version for the supplied key. Returns the
new version.
"""
if version is None:
version = self.version
client = self.get_client(key, write=True)
old = self.make_key(key, version=version)
new = self.make_key(key, version=version + delta)
return self._incr_version(client, old, new, delta, version)
#####################
# Extra api methods #
#####################
def delete_pattern(self, pattern, version=None):
pattern = self.make_key(pattern, version=version)
for client in self.clients.values():
self._delete_pattern(client, pattern)
def get_pattern(self, pattern, version=None):
pattern = self.make_key(pattern, version=version)
recovered_data = {}
for client in self.clients.values():
data = self._get_pattern(client, pattern)
if data:
recovered_data.update(data)
return recovered_data if recovered_data else None
def reinsert_keys(self):
"""
Reinsert cache entries using the current pickle protocol version.
"""
for client in self.clients.values():
self._reinsert_keys(client)
class ShardedRedisCache(BaseRedisCache):
def __init__(self, server, params):
super(ShardedRedisCache, self).__init__(server, params)
self.sharder = HashRing()
for server in self.servers:
client = self.create_client(server)
self.clients[client.connection_pool.connection_identifier] = client
self.sharder.add(client.connection_pool.connection_identifier)
self.client_list = self.clients.values()
def get_client(self, key, write=False):
node = self.sharder.get_node(key)
return self.clients[node]
def shard(self, keys, write=False, version=None):
"""
Returns a dict of keys that belong to a cache's keyspace.
"""
clients = defaultdict(list)
for key in keys:
clients[self.get_client(key, write)].append(
self.make_key(key, version)
)
return clients
####################
# Django cache api #
####################
def delete_many(self, keys, version=None):
"""
Remove multiple keys at once.
"""
clients = self.shard(keys, write=True, version=version)
for client, keys in clients.items():
self._delete_many(client, keys)
def clear(self, version=None):
"""
Flush cache keys.
If version is specified, all keys belonging the version's key
namespace will be deleted. Otherwise, all keys will be deleted.
"""
if version is None:
for client in self.clients.values():
self._clear(client)
else:
self.delete_pattern('*', version=version)
def get_many(self, keys, version=None):
data = {}
clients = self.shard(keys, version=version)
for client, versioned_keys in clients.items():
original_keys = [key._original_key for key in versioned_keys]
data.update(
self._get_many(
client,
original_keys,
versioned_keys=versioned_keys
)
)
return data
def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None):
"""
Set a bunch of values in the cache at once from a dict of key/value
pairs. This is much more efficient than calling set() multiple times.
If timeout is given, that timeout will be used for the key; otherwise
the default cache timeout will be used.
"""
timeout = self.get_timeout(timeout)
clients = self.shard(data.keys(), write=True, version=version)
if timeout is None:
for client, keys in clients.items():
subset = {}
for key in keys:
subset[key] = self.prep_value(data[key._original_key])
self._set_many(client, subset)
return
for client, keys in clients.items():
pipeline = client.pipeline()
for key in keys:
value = self.prep_value(data[key._original_key])
self._set(pipeline, key, value, timeout)
pipeline.execute()
def incr_version(self, key, delta=1, version=None):
"""
Adds delta to the cache version for the supplied key. Returns the
new version.
"""
if version is None:
version = self.version
client = self.get_client(key, write=True)
old = self.make_key(key, version=version)
new = self.make_key(key, version=version + delta)
return self._incr_version(client, old, new, delta, version)
#####################
# Extra api methods #
#####################
def delete_pattern(self, pattern, version=None):
pattern = self.make_key(pattern, version=version)
for client in self.clients.values():
self._delete_pattern(client, pattern)
def reinsert_keys(self):
"""
Reinsert cache entries using the current pickle protocol version.
"""
for client in self.clients.values():
self._reinsert_keys(client)
class ShardedRedisCache(BaseRedisCache):
def __init__(self, server, params):
super(ShardedRedisCache, self).__init__(server, params)
self._params = params
self._server = server
self._pickle_version = None
self.__master_client = None
self.clients = {}
self.sharder = HashRing()
if not isinstance(server, (list, tuple)):
servers = [server]
else:
servers = server
for server in servers:
client = self.create_client(server)
self.clients[client.connection_pool.connection_identifier] = client
self.sharder.add(client.connection_pool.connection_identifier)
@property
def master_client(self):
"""
Get the write server:port of the master cache
"""
if not hasattr(self, '_master_client') and self.__master_client is None:
cache = self.options.get('MASTER_CACHE', None)
if cache is None:
self._master_client = None
else:
self._master_client = None
try:
host, port = cache.split(":")
except ValueError:
raise ImproperlyConfigured("MASTER_CACHE must be in the form <host>:<port>")
for client in self.clients:
connection_kwargs = client.connection_pool.connection_kwargs
if connection_kwargs['host'] == host and connection_kwargs['port'] == int(port):
self._master_client = client
break
if self._master_client is None:
raise ImproperlyConfigured("%s is not in the list of available redis-server instances." % cache)
return self._master_client
def get_client(self, key, for_write=False):
if for_write and self.master_client is not None:
return self.master_client
node = self.sharder.get_node(unicode(key))
return self.clients[node]
def shard(self, keys, for_write=False, version=None):
"""
Returns a dict of keys that belong to a cache's keyspace.
"""
clients = defaultdict(list)
for key in keys:
clients[self.get_client(key, for_write)].append(self.make_key(key, version))
return clients
####################
# Django cache api #
####################
def add(self, key, value, timeout=None, version=None):
"""
Add a value to the cache, failing if the key already exists.
Returns ``True`` if the object was added, ``False`` if not.
"""
client = self.get_client(key)
key = self.make_key(key, version=version)
return self._add(client, key, value, timeout)
def get(self, key, default=None, version=None):
"""
Retrieve a value from the cache.
Returns unpickled value if key is found, the default if not.
"""
client = self.get_client(key)
key = self.make_key(key, version=version)
return self._get(client, key, default)
def set(self, key, value, timeout=None, version=None, client=None):
"""
Persist a value to the cache, and set an optional expiration time.
"""
if client is None:
client = self.get_client(key, for_write=True)
key = self.make_key(key, version=version)
return self._set(key, value, timeout, client=client)
def delete(self, key, version=None):
"""
Remove a key from the cache.
"""
client = self.get_client(key, for_write=True)
key = self.make_key(key, version=version)
return self._delete(client, key)
def delete_many(self, keys, version=None):
"""
Remove multiple keys at once.
"""
clients = self.shard(keys, for_write=True, version=version)
for client, keys in clients.items():
self._delete_many(client, keys)
def clear(self, version=None):
"""
Flush cache keys.
If version is specified, all keys belonging the version's key
namespace will be deleted. Otherwise, all keys will be deleted.
"""
if version is None:
if self.master_client is None:
for client in self.clients.itervalues():
self._clear(client)
else:
self._clear(self.master_client)
else:
self.delete_pattern('*', version=version)
def get_many(self, keys, version=None):
data = {}
clients = self.shard(keys, version=version)
for client, versioned_keys in clients.items():
original_keys = [key._original_key for key in versioned_keys]
data.update(self._get_many(client, original_keys, versioned_keys=versioned_keys))
return data
def set_many(self, data, timeout=None, version=None):
"""
Set a bunch of values in the cache at once from a dict of key/value
pairs. This is much more efficient than calling set() multiple times.
If timeout is given, that timeout will be used for the key; otherwise
the default cache timeout will be used.
"""
clients = self.shard(data.keys(), for_write=True, version=version)
if timeout is None:
for client, keys in clients.items():
subset = {}
for key in keys:
subset[key] = data[key._original_key]
self._set_many(client, subset)
return
for client, keys in clients.items():
pipeline = client.pipeline()
for key in keys:
self._set(key, data[key._original_key], timeout, client=pipeline)
pipeline.execute()
def incr(self, key, delta=1, version=None):
"""
Add delta to value in the cache. If the key does not exist, raise a
ValueError exception.
"""
client = self.get_client(key, for_write=True)
key = self.make_key(key, version=version)
return self._incr(client, key, delta=delta)
def incr_version(self, key, delta=1, version=None):
"""
Adds delta to the cache version for the supplied key. Returns the
new version.
"""
if version is None:
version = self.version
client = self.get_client(key, for_write=True)
old = self.make_key(key, version=version)
new = self.make_key(key, version=version + delta)
return self._incr_version(client, old, new, delta, version)
#####################
# Extra api methods #
#####################
def has_key(self, key, version=None):
client = self.get_client(key, for_write=False)
return self._has_key(client, key, version)
def ttl(self, key, version=None):
client = self.get_client(key, for_write=False)
key = self.make_key(key, version=version)
return self._ttl(client, key)
def delete_pattern(self, pattern, version=None):
pattern = self.make_key(pattern, version=version)
if self.master_client is None:
for client in self.clients.itervalues():
self._delete_pattern(client, pattern)
else:
self._delete_pattern(self.master_client, pattern)
def get_or_set(self, key, func, timeout=None, version=None):
client = self.get_client(key, for_write=True)
key = self.make_key(key, version=version)
return self._get_or_set(client, key, func, timeout)
def reinsert_keys(self):
"""
Reinsert cache entries using the current pickle protocol version.
"""
for client in self.clients.itervalues():
self._reinsert_keys(client)
print
