def _get(self):
# If the first bucket is always returning items, we would never
# get to fetch items from the other buckets. So we always iterate over
# all the buckets and put any ready items into a queue called
# "immediate". This queue is always checked for cached items first.
try:
return 0, self._get_immediate()
except Empty:
pass
remaining_times = []
# NOTE(by seckcoder) : There are some problems here: if we
# put a huge number of tasks into a bucket, then the other
# tasks in other buckets will never be executed
for bucket in values(self.buckets):
remaining = bucket.expected_time()
if not remaining:
try:
# Just put any ready items into the immediate queue.
self.immediate.append(bucket.get_nowait())
except Empty:
pass
except RateLimitExceeded:
remaining_times.append(bucket.expected_time())
else:
remaining_times.append(remaining)
# Try the immediate queue again.
try:
return 0, self._get_immediate()
except Empty:
if not remaining_times:
# No items in any of the buckets.
raise
# There's items, but have to wait before we can retrieve them,
# return the shortest remaining time.
return min(remaining_times), None
def items(self):
"""Flattens the data in all of the buckets into a single list."""
# for queues with contents [(1, 2), (3, 4), (5, 6), (7, 8)]
# zips and flattens to [1, 3, 5, 7, 2, 4, 6, 8]
return [x for x in chain.from_iterable(zip_longest(*[bucket.items for bucket in values(self.buckets)])) if x]
def clear(self):
"""Delete the data in all of the buckets."""
for bucket in values(self.buckets):
bucket.clear()
def empty(self):
"""Returns :const:`True` if all of the buckets are empty."""
return all(bucket.empty() for bucket in values(self.buckets))
def qsize(self):
"""Get the total size of all the queues."""
return sum(bucket.qsize() for bucket in values(self.buckets))