def run(self):
limiter = TokenBucket(self.cwod.queue_updater_fill_rate)
while True:
if limiter.can_consume():
for queue in self.cwod.queues.values():
queue.size = self.queue_size(queue)
sleep(1)
def run(self):
limiter = TokenBucket(self.cwod.queue_updater_fill_rate)
while True:
if limiter.can_consume():
for queue in self.cwod.queues.values():
queue.workers = self.queue_workers(queue)
else:
sleep_time = limiter.expected_time(1)
logger.debug(f'QueueUpdater sleeping for {sleep_time} ' +
'seconds...')
sleep(sleep_time)
def _context(self,
sig,
rate_limits=True,
events=True,
utc=True,
limit=None):
assert sig.type.Strategy
assert sig.type.Request
reserved = Mock()
consumer = Mock()
consumer.task_buckets = defaultdict(lambda: None)
if limit:
bucket = TokenBucket(rate(limit), capacity=1)
consumer.task_buckets[sig.task] = bucket
consumer.controller.state.revoked = set()
consumer.disable_rate_limits = not rate_limits
consumer.event_dispatcher.enabled = events
s = sig.type.start_strategy(self.app, consumer, task_reserved=reserved)
assert s
message = self.task_message_from_sig(
self.app,
sig,
utc=utc,
TaskMessage=self.get_message_class(),
)
message = self.prepare_message(message)
yield self.Context(sig, s, reserved, consumer, message)
def add_task_type(self, name):
task_type = self.task_registry[name]
limit = getattr(task_type, 'rate_limit', None)
limit = timeutils.rate(limit)
bucket = self.buckets[name] = (TokenBucket(limit, capacity=1)
if limit else None)
return bucket
def __init__(self, state, freq=1.0, maxrate=None,
cleanup_freq=3600.0, timer=None, app=None):
self.app = app_or_default(app)
self.state = state
self.freq = freq
self.cleanup_freq = cleanup_freq
self.timer = timer or self.timer
self.logger = logger
self.maxrate = maxrate and TokenBucket(rate(maxrate))
def _context(self, sig,
rate_limits=True, events=True, utc=True, limit=None):
self.assertTrue(sig.type.Strategy)
reserved = Mock()
consumer = Mock()
consumer.task_buckets = defaultdict(lambda: None)
if limit:
bucket = TokenBucket(rate(limit), capacity=1)
consumer.task_buckets[sig.task] = bucket
consumer.disable_rate_limits = not rate_limits
consumer.event_dispatcher.enabled = events
s = sig.type.start_strategy(self.c, consumer, task_reserved=reserved)
self.assertTrue(s)
message = Mock()
body = body_from_sig(self.c, sig, utc=utc)
yield self.Context(sig, s, reserved, consumer, message, body)
def bucket_for_task(self, type):
limit = rate(getattr(type, 'rate_limit', None))
return TokenBucket(limit, capacity=1) if limit else None
def __init__(self, fill_rate, queue=None, capacity=1):
self._bucket = TokenBucket(fill_rate, capacity)
self.queue = queue
if not self.queue:
self.queue = Queue()
class TokenBucketQueue(object):
"""Queue with rate limited get operations.
This uses the token bucket algorithm to rate limit the queue on get
operations.
:param fill_rate: The rate in tokens/second that the bucket will
be refilled.
:keyword capacity: Maximum number of tokens in the bucket.
Default is 1.
"""
RateLimitExceeded = RateLimitExceeded
def __init__(self, fill_rate, queue=None, capacity=1):
self._bucket = TokenBucket(fill_rate, capacity)
self.queue = queue
if not self.queue:
self.queue = Queue()
def put(self, item, block=True):
"""Put an item onto the queue."""
self.queue.put(item, block=block)
def put_nowait(self, item):
"""Put an item into the queue without blocking.
:raises Queue.Full: If a free slot is not immediately available.
"""
return self.put(item, block=False)
def get(self, block=True):
"""Remove and return an item from the queue.
:raises RateLimitExceeded: If a token could not be consumed from the
token bucket (consuming from the queue
too fast).
:raises Queue.Empty: If an item is not immediately available.
"""
get = block and self.queue.get or self.queue.get_nowait
if not block and not self.items:
raise Empty()
if not self._bucket.can_consume(1):
raise RateLimitExceeded()
return get()
def get_nowait(self):
"""Remove and return an item from the queue without blocking.
:raises RateLimitExceeded: If a token could not be consumed from the
token bucket (consuming from the queue
too fast).
:raises Queue.Empty: If an item is not immediately available.
"""
return self.get(block=False)
def qsize(self):
"""Returns the size of the queue."""
return self.queue.qsize()
def empty(self):
"""Returns :const:`True` if the queue is empty."""
return self.queue.empty()
def clear(self):
"""Delete all data in the queue."""
return self.items.clear()
def wait(self, block=False):
"""Wait until a token can be retrieved from the bucket and return
the next item."""
get = self.get
expected_time = self.expected_time
while 1:
remaining = expected_time()
if not remaining:
return get(block=block)
sleep(remaining)
def expected_time(self, tokens=1):
"""Returns the expected time in seconds of when a new token should be
available."""
if not self.items:
return 0
return self._bucket.expected_time(tokens)
@property
def items(self):
"""Underlying data. Do not modify."""
return self.queue.queue
def __init__(self, fill_rate, queue=None, capacity=1):
self._bucket = TokenBucket(fill_rate, capacity)
self.queue = queue
if not self.queue:
self.queue = Queue()
class TokenBucketQueue(object):
"""Queue with rate limited get operations.
This uses the token bucket algorithm to rate limit the queue on get
operations.
:param fill_rate: The rate in tokens/second that the bucket will
be refilled.
:keyword capacity: Maximum number of tokens in the bucket.
Default is 1.
"""
RateLimitExceeded = RateLimitExceeded
def __init__(self, fill_rate, queue=None, capacity=1):
self._bucket = TokenBucket(fill_rate, capacity)
self.queue = queue
if not self.queue:
self.queue = Queue()
def put(self, item, block=True):
"""Put an item onto the queue."""
self.queue.put(item, block=block)
def put_nowait(self, item):
"""Put an item into the queue without blocking.
:raises Queue.Full: If a free slot is not immediately available.
"""
return self.put(item, block=False)
def get(self, block=True):
"""Remove and return an item from the queue.
:raises RateLimitExceeded: If a token could not be consumed from the
token bucket (consuming from the queue
too fast).
:raises Queue.Empty: If an item is not immediately available.
"""
get = block and self.queue.get or self.queue.get_nowait
if not block and not self.items:
raise Empty()
if not self._bucket.can_consume(1):
raise RateLimitExceeded()
return get()
def get_nowait(self):
"""Remove and return an item from the queue without blocking.
:raises RateLimitExceeded: If a token could not be consumed from the
token bucket (consuming from the queue
too fast).
:raises Queue.Empty: If an item is not immediately available.
"""
return self.get(block=False)
def qsize(self):
"""Returns the size of the queue."""
return self.queue.qsize()
def empty(self):
"""Returns :const:`True` if the queue is empty."""
return self.queue.empty()
def clear(self):
"""Delete all data in the queue."""
return self.items.clear()
def wait(self, block=False):
"""Wait until a token can be retrieved from the bucket and return
the next item."""
get = self.get
expected_time = self.expected_time
while 1:
remaining = expected_time()
if not remaining:
return get(block=block)
sleep(remaining)
def expected_time(self, tokens=1):
"""Returns the expected time in seconds of when a new token should be
available."""
if not self.items:
return 0
return self._bucket.expected_time(tokens)
@property
def items(self):
"""Underlying data. Do not modify."""
return self.queue.queue
grp = try_import("grp")
try:
from multiprocessing.process import current_process
except ImportError:
current_process = None # noqa
SYSTEM = _platform.system()
IS_OSX = SYSTEM == "Darwin"
IS_WINDOWS = SYSTEM == "Windows"
DAEMON_UMASK = 0
DAEMON_WORKDIR = "/"
DAEMON_REDIRECT_TO = getattr(os, "devnull", "/dev/null")
_setps_bucket = TokenBucket(0.5) # 30/m, every 2 seconds
def pyimplementation():
if hasattr(_platform, "python_implementation"):
return _platform.python_implementation()
elif sys.platform.startswith("java"):
return "Jython %s" % (sys.platform, )
elif hasattr(sys, "pypy_version_info"):
v = ".".join(map(str, sys.pypy_version_info[:3]))
if sys.pypy_version_info[3:]:
v += "-" + "".join(map(str, sys.pypy_version_info[3:]))
return "PyPy %s" % (v, )
else:
return "CPython"
def restart_limit(self):
# the AttributeError that can be catched from amqplib
# poses problems for the too often restarts protection
# in Connection.ensure_connection
return TokenBucket(1)
