class RedisLock(Lock):
def __init__(self, namespace='lock', **redis_kwargs):
import redis
from redis.sentinel import Sentinel
redis_kwargs['decode_responses'] = True
if redis_kwargs.get('sentinel') and redis_kwargs.get('sentinel_service'):
sentinels = [tuple(l.split(':')) for l in redis_kwargs.pop('sentinel').split(',')]
sentinel_service = redis_kwargs.pop('sentinel_service')
kwargs = {k: v for k, v in redis_kwargs.items() if k in ["decode_responses", "password", "db", "socket_timeout"]}
self._redis = Sentinel(sentinels, **kwargs).master_for(sentinel_service)
else:
kwargs = {k: v for k, v in redis_kwargs.items() if "sentinel" not in k}
self._redis = redis.Redis(**kwargs)
self._redis.ping()
self._namespace = namespace
info("Created redis lock manager with socket_timeout of {}s".format(redis_kwargs['socket_timeout']))
def __key(self, name):
return "{}:{}".format(self._namespace, name)
def lock(self, name, owner, expiration=1, allow_owner_relock=False):
"""
Obtain the named lock.
:param allow_owner_relock: bool
:param name: str the name of the lock
:param owner: str a unique name for the locking node
:param expiration: int in seconds, 0 expiration means forever
"""
import redis
try:
if int(expiration) < 1:
expiration = 1
key = self.__key(name)
non_existing = not (allow_owner_relock and self._redis.get(key) == owner)
return self._redis.set(key, owner, ex=int(expiration), nx=non_existing)
except redis.exceptions.ResponseError as e:
exception("Unable to obtain lock, local state: name: %s, owner: %s, expiration: %s, allow_owner_relock: %s",
name, owner, expiration, allow_owner_relock)
def unlock(self, name, owner):
"""
Release the named lock.
:param name: str the name of the lock
:param owner: str a unique name for the locking node
"""
key = self.__key(name)
if self._redis.get(key) == owner:
self._redis.delete(key)
return True
return False
def check_lock(self, name):
return self._redis.get(self.__key(name)) is not None
def print_locks(self):
keys = self._redis.keys(self.__key('*'))
for key in keys:
print("{} locked by {}, expires in {} seconds".format(key, self._redis.get(key), self._redis.ttl(key)))
class RedisUniqueQueue(object):
def __init__(self, name, namespace="queue", **redis_kwargs):
import redis
from redis.sentinel import Sentinel
redis_kwargs["decode_responses"] = True
if redis_kwargs.get("sentinel") and redis_kwargs.get(
"sentinel_service"):
sentinels = [
tuple(l.split(":"))
for l in redis_kwargs.pop("sentinel").split(",")
]
sentinel_service = redis_kwargs.pop("sentinel_service")
kwargs = {
k: v
for k, v in redis_kwargs.items() if k in
["decode_responses", "password", "db", "socket_timeout"]
}
self._redis = Sentinel(sentinels,
**kwargs).master_for(sentinel_service)
else:
kwargs = {
k: v
for k, v in redis_kwargs.items() if "sentinel" not in k
}
self._redis = redis.Redis(**kwargs)
self._redis.ping()
self.key = "{}:{}".format(namespace, name)
info("Created redis queue with socket_timeout of {}s".format(
redis_kwargs["socket_timeout"]))
# clean up from previous implementations
if self._redis.type(self.key) != "zset":
self._redis.delete(self.key)
def qsize(self):
return self._redis.zcount(self.key, "-inf", "+inf")
def empty(self):
return self.qsize() == 0
def put(self, item):
self._redis.zadd(self.key, {item: time()}, nx=True)
def get(self, block=True, timeout=None):
if block:
item = self._redis.bzpopmin(self.key, timeout=timeout)
else:
item = self._redis.zpopmin(self.key)
if item:
item = item[1]
return item
def get_nowait(self):
return self.get(False)
class RedisUniqueQueue(object):
def __init__(self, name, namespace='queue', **redis_kwargs):
import redis
from redis.sentinel import Sentinel
redis_kwargs['decode_responses'] = True
if redis_kwargs.get('sentinel') and redis_kwargs.get(
'sentinel_service'):
sentinels = [
tuple(l.split(':'))
for l in redis_kwargs.pop('sentinel').split(',')
]
sentinel_service = redis_kwargs.pop('sentinel_service')
kwargs = {
k: v
for k, v in redis_kwargs.items()
if k in ["decode_responses", "password", "db"]
}
self._redis = Sentinel(sentinels,
**kwargs).master_for(sentinel_service)
else:
kwargs = {
k: v
for k, v in redis_kwargs.items() if "sentinel" not in k
}
self._redis = redis.Redis(**kwargs)
self._redis.ping()
self.key = "{}:{}".format(namespace, name)
# clean up from previous implementations
if self._redis.type(self.key) != 'zset':
self._redis.delete(self.key)
def qsize(self):
return self._redis.zcount(self.key, '-inf', '+inf')
def empty(self):
return self.qsize() == 0
def put(self, item):
self._redis.zadd(self.key, {item: time()}, nx=True)
def get(self, block=True, timeout=None):
if block:
item = self._redis.bzpopmin(self.key, timeout=timeout)
else:
item = self._redis.zpopmin(self.key)
if item:
item = item[1]
return item
def get_nowait(self):
return self.get(False)
class RedisUniqueQueue(object):
def __init__(self, name, namespace="queue", **redis_kwargs):
import redis
from redis.sentinel import Sentinel
redis_kwargs["decode_responses"] = True
if redis_kwargs.get("sentinel") and redis_kwargs.get(
"sentinel_service"):
sentinels = [
tuple(l.split(":"))
for l in redis_kwargs.pop("sentinel").split(",")
]
sentinel_service = redis_kwargs.pop("sentinel_service")
kwargs = {
k: v
for k, v in redis_kwargs.items() if k in
["decode_responses", "password", "db", "socket_timeout"]
}
self._redis = Sentinel(sentinels,
**kwargs).master_for(sentinel_service)
else:
kwargs = {
k: v
for k, v in redis_kwargs.items() if "sentinel" not in k
}
self._redis = redis.Redis(**kwargs)
self._redis.ping()
self.key = "{}:{}".format(namespace, name)
logger.info("Created redis queue with socket_timeout of {}s".format(
redis_kwargs["socket_timeout"]))
# clean up from previous implementations
if self._redis.type(self.key) != "zset":
self._redis.delete(self.key)
def qsize(self):
return self._redis.zcount(self.key, "-inf", "+inf")
def empty(self):
return self.qsize() == 0
def put(self, item):
self._redis.zadd(self.key, {item: time()}, nx=True)
def get(self, block=True, timeout=None):
try:
if block:
item = self._redis.bzpopmin(self.key, timeout=timeout)
else:
item = self._redis.zpopmin(self.key)
# Unfortunately we cannot use _redis.exceptions.ResponseError Exception here
# Since it would trigger another exception in queuemanager
except Exception as e:
logger.critical(
"BZPOPMIN/ZPOPMIN command failed: {}\nNote that redis >= 5.0 is required."
.format(e))
raise
if item:
item = item[1]
return item
def get_nowait(self):
return self.get(False)
