Пример #1
0
 def run(self):
     limiter = TokenBucket(self.cwod.queue_updater_fill_rate)
     while True:
         if limiter.can_consume():
             for queue in self.cwod.queues.values():
                 queue.size = self.queue_size(queue)
         sleep(1)
Пример #2
0
 def run(self):
     limiter = TokenBucket(self.cwod.queue_updater_fill_rate)
     while True:
         if limiter.can_consume():
             for queue in self.cwod.queues.values():
                 queue.workers = self.queue_workers(queue)
         else:
             sleep_time = limiter.expected_time(1)
             logger.debug(f'QueueUpdater sleeping for {sleep_time} ' +
                          'seconds...')
             sleep(sleep_time)
Пример #3
0
    def _context(self,
                 sig,
                 rate_limits=True,
                 events=True,
                 utc=True,
                 limit=None):
        assert sig.type.Strategy
        assert sig.type.Request

        reserved = Mock()
        consumer = Mock()
        consumer.task_buckets = defaultdict(lambda: None)
        if limit:
            bucket = TokenBucket(rate(limit), capacity=1)
            consumer.task_buckets[sig.task] = bucket
        consumer.controller.state.revoked = set()
        consumer.disable_rate_limits = not rate_limits
        consumer.event_dispatcher.enabled = events
        s = sig.type.start_strategy(self.app, consumer, task_reserved=reserved)
        assert s

        message = self.task_message_from_sig(
            self.app,
            sig,
            utc=utc,
            TaskMessage=self.get_message_class(),
        )
        message = self.prepare_message(message)
        yield self.Context(sig, s, reserved, consumer, message)
Пример #4
0
 def add_task_type(self, name):
     task_type = self.task_registry[name]
     limit = getattr(task_type, 'rate_limit', None)
     limit = timeutils.rate(limit)
     bucket = self.buckets[name] = (TokenBucket(limit, capacity=1)
                                    if limit else None)
     return bucket
Пример #5
0
 def __init__(self, state, freq=1.0, maxrate=None,
              cleanup_freq=3600.0, timer=None, app=None):
     self.app = app_or_default(app)
     self.state = state
     self.freq = freq
     self.cleanup_freq = cleanup_freq
     self.timer = timer or self.timer
     self.logger = logger
     self.maxrate = maxrate and TokenBucket(rate(maxrate))
Пример #6
0
    def _context(self, sig,
                 rate_limits=True, events=True, utc=True, limit=None):
        self.assertTrue(sig.type.Strategy)

        reserved = Mock()
        consumer = Mock()
        consumer.task_buckets = defaultdict(lambda: None)
        if limit:
            bucket = TokenBucket(rate(limit), capacity=1)
            consumer.task_buckets[sig.task] = bucket
        consumer.disable_rate_limits = not rate_limits
        consumer.event_dispatcher.enabled = events
        s = sig.type.start_strategy(self.c, consumer, task_reserved=reserved)
        self.assertTrue(s)

        message = Mock()
        body = body_from_sig(self.c, sig, utc=utc)

        yield self.Context(sig, s, reserved, consumer, message, body)
Пример #7
0
 def bucket_for_task(self, type):
     limit = rate(getattr(type, 'rate_limit', None))
     return TokenBucket(limit, capacity=1) if limit else None
Пример #8
0
 def __init__(self, fill_rate, queue=None, capacity=1):
     self._bucket = TokenBucket(fill_rate, capacity)
     self.queue = queue
     if not self.queue:
         self.queue = Queue()
Пример #9
0
class TokenBucketQueue(object):
    """Queue with rate limited get operations.

    This uses the token bucket algorithm to rate limit the queue on get
    operations.

    :param fill_rate: The rate in tokens/second that the bucket will
                      be refilled.
    :keyword capacity: Maximum number of tokens in the bucket.
                       Default is 1.

    """
    RateLimitExceeded = RateLimitExceeded

    def __init__(self, fill_rate, queue=None, capacity=1):
        self._bucket = TokenBucket(fill_rate, capacity)
        self.queue = queue
        if not self.queue:
            self.queue = Queue()

    def put(self, item, block=True):
        """Put an item onto the queue."""
        self.queue.put(item, block=block)

    def put_nowait(self, item):
        """Put an item into the queue without blocking.

        :raises Queue.Full: If a free slot is not immediately available.

        """
        return self.put(item, block=False)

    def get(self, block=True):
        """Remove and return an item from the queue.

        :raises RateLimitExceeded: If a token could not be consumed from the
                                   token bucket (consuming from the queue
                                   too fast).
        :raises Queue.Empty: If an item is not immediately available.

        """
        get = block and self.queue.get or self.queue.get_nowait

        if not block and not self.items:
            raise Empty()

        if not self._bucket.can_consume(1):
            raise RateLimitExceeded()

        return get()

    def get_nowait(self):
        """Remove and return an item from the queue without blocking.

        :raises RateLimitExceeded: If a token could not be consumed from the
                                   token bucket (consuming from the queue
                                   too fast).
        :raises Queue.Empty: If an item is not immediately available.

        """
        return self.get(block=False)

    def qsize(self):
        """Returns the size of the queue."""
        return self.queue.qsize()

    def empty(self):
        """Returns :const:`True` if the queue is empty."""
        return self.queue.empty()

    def clear(self):
        """Delete all data in the queue."""
        return self.items.clear()

    def wait(self, block=False):
        """Wait until a token can be retrieved from the bucket and return
        the next item."""
        get = self.get
        expected_time = self.expected_time
        while 1:
            remaining = expected_time()
            if not remaining:
                return get(block=block)
            sleep(remaining)

    def expected_time(self, tokens=1):
        """Returns the expected time in seconds of when a new token should be
        available."""
        if not self.items:
            return 0
        return self._bucket.expected_time(tokens)

    @property
    def items(self):
        """Underlying data.  Do not modify."""
        return self.queue.queue
Пример #10
0
 def __init__(self, fill_rate, queue=None, capacity=1):
     self._bucket = TokenBucket(fill_rate, capacity)
     self.queue = queue
     if not self.queue:
         self.queue = Queue()
Пример #11
0
class TokenBucketQueue(object):
    """Queue with rate limited get operations.

    This uses the token bucket algorithm to rate limit the queue on get
    operations.

    :param fill_rate: The rate in tokens/second that the bucket will
                      be refilled.
    :keyword capacity: Maximum number of tokens in the bucket.
                       Default is 1.

    """
    RateLimitExceeded = RateLimitExceeded

    def __init__(self, fill_rate, queue=None, capacity=1):
        self._bucket = TokenBucket(fill_rate, capacity)
        self.queue = queue
        if not self.queue:
            self.queue = Queue()

    def put(self, item, block=True):
        """Put an item onto the queue."""
        self.queue.put(item, block=block)

    def put_nowait(self, item):
        """Put an item into the queue without blocking.

        :raises Queue.Full: If a free slot is not immediately available.

        """
        return self.put(item, block=False)

    def get(self, block=True):
        """Remove and return an item from the queue.

        :raises RateLimitExceeded: If a token could not be consumed from the
                                   token bucket (consuming from the queue
                                   too fast).
        :raises Queue.Empty: If an item is not immediately available.

        """
        get = block and self.queue.get or self.queue.get_nowait

        if not block and not self.items:
            raise Empty()

        if not self._bucket.can_consume(1):
            raise RateLimitExceeded()

        return get()

    def get_nowait(self):
        """Remove and return an item from the queue without blocking.

        :raises RateLimitExceeded: If a token could not be consumed from the
                                   token bucket (consuming from the queue
                                   too fast).
        :raises Queue.Empty: If an item is not immediately available.

        """
        return self.get(block=False)

    def qsize(self):
        """Returns the size of the queue."""
        return self.queue.qsize()

    def empty(self):
        """Returns :const:`True` if the queue is empty."""
        return self.queue.empty()

    def clear(self):
        """Delete all data in the queue."""
        return self.items.clear()

    def wait(self, block=False):
        """Wait until a token can be retrieved from the bucket and return
        the next item."""
        get = self.get
        expected_time = self.expected_time
        while 1:
            remaining = expected_time()
            if not remaining:
                return get(block=block)
            sleep(remaining)

    def expected_time(self, tokens=1):
        """Returns the expected time in seconds of when a new token should be
        available."""
        if not self.items:
            return 0
        return self._bucket.expected_time(tokens)

    @property
    def items(self):
        """Underlying data.  Do not modify."""
        return self.queue.queue
Пример #12
0
grp = try_import("grp")

try:
    from multiprocessing.process import current_process
except ImportError:
    current_process = None  # noqa

SYSTEM = _platform.system()
IS_OSX = SYSTEM == "Darwin"
IS_WINDOWS = SYSTEM == "Windows"

DAEMON_UMASK = 0
DAEMON_WORKDIR = "/"
DAEMON_REDIRECT_TO = getattr(os, "devnull", "/dev/null")

_setps_bucket = TokenBucket(0.5)  # 30/m, every 2 seconds


def pyimplementation():
    if hasattr(_platform, "python_implementation"):
        return _platform.python_implementation()
    elif sys.platform.startswith("java"):
        return "Jython %s" % (sys.platform, )
    elif hasattr(sys, "pypy_version_info"):
        v = ".".join(map(str, sys.pypy_version_info[:3]))
        if sys.pypy_version_info[3:]:
            v += "-" + "".join(map(str, sys.pypy_version_info[3:]))
        return "PyPy %s" % (v, )
    else:
        return "CPython"
Пример #13
0
 def restart_limit(self):
     # the AttributeError that can be catched from amqplib
     # poses problems for the too often restarts protection
     # in Connection.ensure_connection
     return TokenBucket(1)