Example #1
0
class RedisUniqueQueue(object):
    def __init__(self, name, namespace="queue", **redis_kwargs):
        import redis
        from redis.sentinel import Sentinel

        redis_kwargs["decode_responses"] = True
        if redis_kwargs.get("sentinel") and redis_kwargs.get(
                "sentinel_service"):
            sentinels = [
                tuple(l.split(":"))
                for l in redis_kwargs.pop("sentinel").split(",")
            ]
            sentinel_service = redis_kwargs.pop("sentinel_service")
            kwargs = {
                k: v
                for k, v in redis_kwargs.items() if k in
                ["decode_responses", "password", "db", "socket_timeout"]
            }
            self._redis = Sentinel(sentinels,
                                   **kwargs).master_for(sentinel_service)
        else:
            kwargs = {
                k: v
                for k, v in redis_kwargs.items() if "sentinel" not in k
            }
            self._redis = redis.Redis(**kwargs)
        self._redis.ping()
        self.key = "{}:{}".format(namespace, name)
        info("Created redis queue with socket_timeout of {}s".format(
            redis_kwargs["socket_timeout"]))

        # clean up from previous implementations
        if self._redis.type(self.key) != "zset":
            self._redis.delete(self.key)

    def qsize(self):
        return self._redis.zcount(self.key, "-inf", "+inf")

    def empty(self):
        return self.qsize() == 0

    def put(self, item):
        self._redis.zadd(self.key, {item: time()}, nx=True)

    def get(self, block=True, timeout=None):
        if block:
            item = self._redis.bzpopmin(self.key, timeout=timeout)
        else:
            item = self._redis.zpopmin(self.key)

        if item:
            item = item[1]
        return item

    def get_nowait(self):
        return self.get(False)
Example #2
0
class RedisUniqueQueue(object):
    def __init__(self, name, namespace='queue', **redis_kwargs):
        import redis
        from redis.sentinel import Sentinel
        redis_kwargs['decode_responses'] = True
        if redis_kwargs.get('sentinel') and redis_kwargs.get(
                'sentinel_service'):
            sentinels = [
                tuple(l.split(':'))
                for l in redis_kwargs.pop('sentinel').split(',')
            ]
            sentinel_service = redis_kwargs.pop('sentinel_service')
            kwargs = {
                k: v
                for k, v in redis_kwargs.items()
                if k in ["decode_responses", "password", "db"]
            }
            self._redis = Sentinel(sentinels,
                                   **kwargs).master_for(sentinel_service)
        else:
            kwargs = {
                k: v
                for k, v in redis_kwargs.items() if "sentinel" not in k
            }
            self._redis = redis.Redis(**kwargs)
        self._redis.ping()
        self.key = "{}:{}".format(namespace, name)

        # clean up from previous implementations
        if self._redis.type(self.key) != 'zset':
            self._redis.delete(self.key)

    def qsize(self):
        return self._redis.zcount(self.key, '-inf', '+inf')

    def empty(self):
        return self.qsize() == 0

    def put(self, item):
        self._redis.zadd(self.key, {item: time()}, nx=True)

    def get(self, block=True, timeout=None):
        if block:
            item = self._redis.bzpopmin(self.key, timeout=timeout)
        else:
            item = self._redis.zpopmin(self.key)

        if item:
            item = item[1]
        return item

    def get_nowait(self):
        return self.get(False)
Example #3
0
class RedisUniqueQueue(object):
    def __init__(self, name, namespace="queue", **redis_kwargs):
        import redis
        from redis.sentinel import Sentinel

        redis_kwargs["decode_responses"] = True
        if redis_kwargs.get("sentinel") and redis_kwargs.get(
                "sentinel_service"):
            sentinels = [
                tuple(l.split(":"))
                for l in redis_kwargs.pop("sentinel").split(",")
            ]
            sentinel_service = redis_kwargs.pop("sentinel_service")
            kwargs = {
                k: v
                for k, v in redis_kwargs.items() if k in
                ["decode_responses", "password", "db", "socket_timeout"]
            }
            self._redis = Sentinel(sentinels,
                                   **kwargs).master_for(sentinel_service)
        else:
            kwargs = {
                k: v
                for k, v in redis_kwargs.items() if "sentinel" not in k
            }
            self._redis = redis.Redis(**kwargs)
        self._redis.ping()
        self.key = "{}:{}".format(namespace, name)
        logger.info("Created redis queue with socket_timeout of {}s".format(
            redis_kwargs["socket_timeout"]))

        # clean up from previous implementations
        if self._redis.type(self.key) != "zset":
            self._redis.delete(self.key)

    def qsize(self):
        return self._redis.zcount(self.key, "-inf", "+inf")

    def empty(self):
        return self.qsize() == 0

    def put(self, item):
        self._redis.zadd(self.key, {item: time()}, nx=True)

    def get(self, block=True, timeout=None):
        try:
            if block:
                item = self._redis.bzpopmin(self.key, timeout=timeout)
            else:
                item = self._redis.zpopmin(self.key)
        # Unfortunately we cannot use _redis.exceptions.ResponseError Exception here
        # Since it would trigger another exception in queuemanager
        except Exception as e:
            logger.critical(
                "BZPOPMIN/ZPOPMIN command failed: {}\nNote that redis >= 5.0 is required."
                .format(e))
            raise

        if item:
            item = item[1]
        return item

    def get_nowait(self):
        return self.get(False)