def __init__(self,
callback=None,
max_workers_in_memory=5000,
max_tasks_in_memory=10000):
self.workers = LRUCache(limit=max_workers_in_memory)
self.tasks = LRUCache(limit=max_tasks_in_memory)
self.event_callback = callback
self._mutex = threading.Lock()
def __init__(self,
callback=None,
max_workers_in_memory=5000,
max_tasks_in_memory=10000):
self.workers = LRUCache(limit=max_workers_in_memory)
self.tasks = LRUCache(limit=max_tasks_in_memory)
self.event_callback = callback
self.group_handlers = {
'worker': self.worker_event,
'task': self.task_event
}
self._mutex = Lock()
def __init__(self,
callback=None,
max_workers_in_memory=5000,
max_tasks_in_memory=10000):
self.max_workers_in_memory = max_workers_in_memory
self.max_tasks_in_memory = 10000
self.workers = LRUCache(limit=self.max_workers_in_memory)
self.tasks = LRUCache(limit=self.max_tasks_in_memory)
self._taskheap = []
self.event_callback = callback
self.group_handlers = {
'worker': self.worker_event,
'task': self.task_event,
}
self._mutex = threading.Lock()
def assertSafeIter(self, method, interval=0.01, size=10000):
from threading import Thread, Event
from time import sleep
x = LRUCache(size)
x.update(zip(xrange(size), xrange(size)))
class Burglar(Thread):
def __init__(self, cache):
self.cache = cache
self._is_shutdown = Event()
self._is_stopped = Event()
Thread.__init__(self)
def run(self):
while not self._is_shutdown.isSet():
try:
self.cache.data.popitem(last=False)
except KeyError:
break
self._is_stopped.set()
def stop(self):
self._is_shutdown.set()
self._is_stopped.wait()
self.join(1e10)
burglar = Burglar(x)
burglar.start()
try:
for _ in getattr(x, method)():
sleep(0.0001)
finally:
burglar.stop()
def test_expires(self):
limit = 100
x = LRUCache(limit=limit)
slots = list(xrange(limit * 2))
for i in slots:
x[i] = i
self.assertListEqual(x.keys(), list(slots[limit:]))
def test_update_expires(self):
limit = 100
x = LRUCache(limit=limit)
slots = list(xrange(limit * 2))
for i in slots:
x.update({i: i})
self.assertListEqual(list(x.keys()), list(slots[limit:]))
def __init__(self,
callback=None,
workers=None,
tasks=None,
taskheap=None,
max_workers_in_memory=5000,
max_tasks_in_memory=10000):
self.event_callback = callback
self.workers = (LRUCache(max_workers_in_memory)
if workers is None else workers)
self.tasks = (LRUCache(max_tasks_in_memory)
if tasks is None else tasks)
self._taskheap = None # reserved for __reduce__ in 3.1
self.max_workers_in_memory = max_workers_in_memory
self.max_tasks_in_memory = max_tasks_in_memory
self._mutex = threading.Lock()
self.handlers = {'task': self.task_event, 'worker': self.worker_event}
self._get_handler = self.handlers.__getitem__
def __init__(self, app=None, serializer=None, max_cached_results=None,
**kwargs):
from celery.app import app_or_default
self.app = app_or_default(app)
self.serializer = serializer or self.app.conf.CELERY_RESULT_SERIALIZER
(self.content_type,
self.content_encoding,
self.encoder) = serialization.registry._encoders[self.serializer]
self._cache = LRUCache(limit=max_cached_results or
self.app.conf.CELERY_MAX_CACHED_RESULTS)
def test_least_recently_used(self):
x = LRUCache(3)
x[1], x[2], x[3] = 1, 2, 3
self.assertEqual(x.keys(), [1, 2, 3])
x[4], x[5] = 4, 5
self.assertEqual(x.keys(), [3, 4, 5])
# access 3, which makes it the last used key.
x[3]
x[6] = 6
self.assertEqual(x.keys(), [5, 3, 6])
x[7] = 7
self.assertEqual(x.keys(), [3, 6, 7])
def test_items(self):
c = LRUCache()
c.update(a=1, b=2, c=3)
self.assertTrue(c.items())
~~~~~~~~~~~~~~~~~~~~
THis is a simple program that dumps events to the console
as they happen. Think of it like a `tcpdump` for Celery events.
"""
from __future__ import absolute_import
import sys
from datetime import datetime
from celery.app import app_or_default
from celery.datastructures import LRUCache
TASK_NAMES = LRUCache(limit=0xFFF)
HUMAN_TYPES = {
'worker-offline': 'shutdown',
'worker-online': 'started',
'worker-heartbeat': 'heartbeat'
}
def humanize_type(type):
try:
return HUMAN_TYPES[type.lower()]
except KeyError:
return type.lower().replace('-', ' ')
def test_items(self):
c = LRUCache()
c.update(a=1, b=2, c=3)
self.assertTrue(list(items(c)))
def __init__(self, *args, **kwargs):
self.cache = LRUCache(limit=5000)
def __init__(self, *args, **kwargs):
super(BaseDictBackend, self).__init__(*args, **kwargs)
self._cache = LRUCache(limit=kwargs.get('max_cached_results')
or self.app.conf.CELERY_MAX_CACHED_RESULTS)
