class RedisLock(Lock): def __init__(self, namespace='lock', **redis_kwargs): import redis from redis.sentinel import Sentinel redis_kwargs['decode_responses'] = True if redis_kwargs.get('sentinel') and redis_kwargs.get('sentinel_service'): sentinels = [tuple(l.split(':')) for l in redis_kwargs.pop('sentinel').split(',')] sentinel_service = redis_kwargs.pop('sentinel_service') kwargs = {k: v for k, v in redis_kwargs.items() if k in ["decode_responses", "password", "db", "socket_timeout"]} self._redis = Sentinel(sentinels, **kwargs).master_for(sentinel_service) else: kwargs = {k: v for k, v in redis_kwargs.items() if "sentinel" not in k} self._redis = redis.Redis(**kwargs) self._redis.ping() self._namespace = namespace info("Created redis lock manager with socket_timeout of {}s".format(redis_kwargs['socket_timeout'])) def __key(self, name): return "{}:{}".format(self._namespace, name) def lock(self, name, owner, expiration=1, allow_owner_relock=False): """ Obtain the named lock. :param allow_owner_relock: bool :param name: str the name of the lock :param owner: str a unique name for the locking node :param expiration: int in seconds, 0 expiration means forever """ import redis try: if int(expiration) < 1: expiration = 1 key = self.__key(name) non_existing = not (allow_owner_relock and self._redis.get(key) == owner) return self._redis.set(key, owner, ex=int(expiration), nx=non_existing) except redis.exceptions.ResponseError as e: exception("Unable to obtain lock, local state: name: %s, owner: %s, expiration: %s, allow_owner_relock: %s", name, owner, expiration, allow_owner_relock) def unlock(self, name, owner): """ Release the named lock. :param name: str the name of the lock :param owner: str a unique name for the locking node """ key = self.__key(name) if self._redis.get(key) == owner: self._redis.delete(key) return True return False def check_lock(self, name): return self._redis.get(self.__key(name)) is not None def print_locks(self): keys = self._redis.keys(self.__key('*')) for key in keys: print("{} locked by {}, expires in {} seconds".format(key, self._redis.get(key), self._redis.ttl(key)))
class RedisUniqueQueue(object): def __init__(self, name, namespace="queue", **redis_kwargs): import redis from redis.sentinel import Sentinel redis_kwargs["decode_responses"] = True if redis_kwargs.get("sentinel") and redis_kwargs.get( "sentinel_service"): sentinels = [ tuple(l.split(":")) for l in redis_kwargs.pop("sentinel").split(",") ] sentinel_service = redis_kwargs.pop("sentinel_service") kwargs = { k: v for k, v in redis_kwargs.items() if k in ["decode_responses", "password", "db", "socket_timeout"] } self._redis = Sentinel(sentinels, **kwargs).master_for(sentinel_service) else: kwargs = { k: v for k, v in redis_kwargs.items() if "sentinel" not in k } self._redis = redis.Redis(**kwargs) self._redis.ping() self.key = "{}:{}".format(namespace, name) info("Created redis queue with socket_timeout of {}s".format( redis_kwargs["socket_timeout"])) # clean up from previous implementations if self._redis.type(self.key) != "zset": self._redis.delete(self.key) def qsize(self): return self._redis.zcount(self.key, "-inf", "+inf") def empty(self): return self.qsize() == 0 def put(self, item): self._redis.zadd(self.key, {item: time()}, nx=True) def get(self, block=True, timeout=None): if block: item = self._redis.bzpopmin(self.key, timeout=timeout) else: item = self._redis.zpopmin(self.key) if item: item = item[1] return item def get_nowait(self): return self.get(False)
class RedisUniqueQueue(object): def __init__(self, name, namespace='queue', **redis_kwargs): import redis from redis.sentinel import Sentinel redis_kwargs['decode_responses'] = True if redis_kwargs.get('sentinel') and redis_kwargs.get( 'sentinel_service'): sentinels = [ tuple(l.split(':')) for l in redis_kwargs.pop('sentinel').split(',') ] sentinel_service = redis_kwargs.pop('sentinel_service') kwargs = { k: v for k, v in redis_kwargs.items() if k in ["decode_responses", "password", "db"] } self._redis = Sentinel(sentinels, **kwargs).master_for(sentinel_service) else: kwargs = { k: v for k, v in redis_kwargs.items() if "sentinel" not in k } self._redis = redis.Redis(**kwargs) self._redis.ping() self.key = "{}:{}".format(namespace, name) # clean up from previous implementations if self._redis.type(self.key) != 'zset': self._redis.delete(self.key) def qsize(self): return self._redis.zcount(self.key, '-inf', '+inf') def empty(self): return self.qsize() == 0 def put(self, item): self._redis.zadd(self.key, {item: time()}, nx=True) def get(self, block=True, timeout=None): if block: item = self._redis.bzpopmin(self.key, timeout=timeout) else: item = self._redis.zpopmin(self.key) if item: item = item[1] return item def get_nowait(self): return self.get(False)
class RedisUniqueQueue(object): def __init__(self, name, namespace="queue", **redis_kwargs): import redis from redis.sentinel import Sentinel redis_kwargs["decode_responses"] = True if redis_kwargs.get("sentinel") and redis_kwargs.get( "sentinel_service"): sentinels = [ tuple(l.split(":")) for l in redis_kwargs.pop("sentinel").split(",") ] sentinel_service = redis_kwargs.pop("sentinel_service") kwargs = { k: v for k, v in redis_kwargs.items() if k in ["decode_responses", "password", "db", "socket_timeout"] } self._redis = Sentinel(sentinels, **kwargs).master_for(sentinel_service) else: kwargs = { k: v for k, v in redis_kwargs.items() if "sentinel" not in k } self._redis = redis.Redis(**kwargs) self._redis.ping() self.key = "{}:{}".format(namespace, name) logger.info("Created redis queue with socket_timeout of {}s".format( redis_kwargs["socket_timeout"])) # clean up from previous implementations if self._redis.type(self.key) != "zset": self._redis.delete(self.key) def qsize(self): return self._redis.zcount(self.key, "-inf", "+inf") def empty(self): return self.qsize() == 0 def put(self, item): self._redis.zadd(self.key, {item: time()}, nx=True) def get(self, block=True, timeout=None): try: if block: item = self._redis.bzpopmin(self.key, timeout=timeout) else: item = self._redis.zpopmin(self.key) # Unfortunately we cannot use _redis.exceptions.ResponseError Exception here # Since it would trigger another exception in queuemanager except Exception as e: logger.critical( "BZPOPMIN/ZPOPMIN command failed: {}\nNote that redis >= 5.0 is required." .format(e)) raise if item: item = item[1] return item def get_nowait(self): return self.get(False)