def __init__(self, callback=None, max_workers_in_memory=5000, max_tasks_in_memory=10000): self.workers = LRUCache(limit=max_workers_in_memory) self.tasks = LRUCache(limit=max_tasks_in_memory) self.event_callback = callback self._mutex = threading.Lock()
def __init__(self, callback=None, max_workers_in_memory=5000, max_tasks_in_memory=10000): self.workers = LRUCache(limit=max_workers_in_memory) self.tasks = LRUCache(limit=max_tasks_in_memory) self.event_callback = callback self.group_handlers = { 'worker': self.worker_event, 'task': self.task_event } self._mutex = Lock()
def __init__(self, callback=None, max_workers_in_memory=5000, max_tasks_in_memory=10000): self.max_workers_in_memory = max_workers_in_memory self.max_tasks_in_memory = 10000 self.workers = LRUCache(limit=self.max_workers_in_memory) self.tasks = LRUCache(limit=self.max_tasks_in_memory) self._taskheap = [] self.event_callback = callback self.group_handlers = { 'worker': self.worker_event, 'task': self.task_event, } self._mutex = threading.Lock()
def assertSafeIter(self, method, interval=0.01, size=10000): from threading import Thread, Event from time import sleep x = LRUCache(size) x.update(zip(xrange(size), xrange(size))) class Burglar(Thread): def __init__(self, cache): self.cache = cache self._is_shutdown = Event() self._is_stopped = Event() Thread.__init__(self) def run(self): while not self._is_shutdown.isSet(): try: self.cache.data.popitem(last=False) except KeyError: break self._is_stopped.set() def stop(self): self._is_shutdown.set() self._is_stopped.wait() self.join(1e10) burglar = Burglar(x) burglar.start() try: for _ in getattr(x, method)(): sleep(0.0001) finally: burglar.stop()
def test_expires(self): limit = 100 x = LRUCache(limit=limit) slots = list(xrange(limit * 2)) for i in slots: x[i] = i self.assertListEqual(x.keys(), list(slots[limit:]))
def test_update_expires(self): limit = 100 x = LRUCache(limit=limit) slots = list(xrange(limit * 2)) for i in slots: x.update({i: i}) self.assertListEqual(list(x.keys()), list(slots[limit:]))
def __init__(self, callback=None, workers=None, tasks=None, taskheap=None, max_workers_in_memory=5000, max_tasks_in_memory=10000): self.event_callback = callback self.workers = (LRUCache(max_workers_in_memory) if workers is None else workers) self.tasks = (LRUCache(max_tasks_in_memory) if tasks is None else tasks) self._taskheap = None # reserved for __reduce__ in 3.1 self.max_workers_in_memory = max_workers_in_memory self.max_tasks_in_memory = max_tasks_in_memory self._mutex = threading.Lock() self.handlers = {'task': self.task_event, 'worker': self.worker_event} self._get_handler = self.handlers.__getitem__
def __init__(self, app=None, serializer=None, max_cached_results=None, **kwargs): from celery.app import app_or_default self.app = app_or_default(app) self.serializer = serializer or self.app.conf.CELERY_RESULT_SERIALIZER (self.content_type, self.content_encoding, self.encoder) = serialization.registry._encoders[self.serializer] self._cache = LRUCache(limit=max_cached_results or self.app.conf.CELERY_MAX_CACHED_RESULTS)
def test_least_recently_used(self): x = LRUCache(3) x[1], x[2], x[3] = 1, 2, 3 self.assertEqual(x.keys(), [1, 2, 3]) x[4], x[5] = 4, 5 self.assertEqual(x.keys(), [3, 4, 5]) # access 3, which makes it the last used key. x[3] x[6] = 6 self.assertEqual(x.keys(), [5, 3, 6]) x[7] = 7 self.assertEqual(x.keys(), [3, 6, 7])
def test_items(self): c = LRUCache() c.update(a=1, b=2, c=3) self.assertTrue(c.items())
~~~~~~~~~~~~~~~~~~~~ THis is a simple program that dumps events to the console as they happen. Think of it like a `tcpdump` for Celery events. """ from __future__ import absolute_import import sys from datetime import datetime from celery.app import app_or_default from celery.datastructures import LRUCache TASK_NAMES = LRUCache(limit=0xFFF) HUMAN_TYPES = { 'worker-offline': 'shutdown', 'worker-online': 'started', 'worker-heartbeat': 'heartbeat' } def humanize_type(type): try: return HUMAN_TYPES[type.lower()] except KeyError: return type.lower().replace('-', ' ')
def test_items(self): c = LRUCache() c.update(a=1, b=2, c=3) self.assertTrue(list(items(c)))
def __init__(self, *args, **kwargs): self.cache = LRUCache(limit=5000)
def __init__(self, *args, **kwargs): super(BaseDictBackend, self).__init__(*args, **kwargs) self._cache = LRUCache(limit=kwargs.get('max_cached_results') or self.app.conf.CELERY_MAX_CACHED_RESULTS)