def test_run(self): ready_queue = Queue() condition = [None] def mycallback(value): condition[0].set() m = Mediator(ready_queue, mycallback) condition[0] = m._is_shutdown ready_queue.put(MockTask('Elaine M. Benes')) m.run() self.assertTrue(m._is_shutdown.isSet()) self.assertTrue(m._is_stopped.isSet())
def test_mediator_body_exception(self): ready_queue = Queue() def mycallback(value): raise KeyError('foo') m = Mediator(ready_queue, mycallback) ready_queue.put(MockTask('Elaine M. Benes')) m.body()
def test_mediator_body(self): ready_queue = Queue() got = {} def mycallback(value): got['value'] = value.value m = Mediator(ready_queue, mycallback) ready_queue.put(MockTask('George Costanza')) m.body() self.assertEqual(got['value'], 'George Costanza') ready_queue.put(MockTask('Jerry Seinfeld')) m._does_debug = False m.body() self.assertEqual(got['value'], 'Jerry Seinfeld')
def test_mediator_crash(self, _exit): ms = [None] class _Mediator(Mediator): def body(self): try: raise KeyError('foo') finally: ms[0]._is_shutdown.set() ready_queue = Queue() ms[0] = m = _Mediator(ready_queue, None) ready_queue.put(MockTask('George Constanza')) stderr = Mock() p, sys.stderr = sys.stderr, stderr try: m.run() finally: sys.stderr = p self.assertTrue(_exit.call_count) self.assertTrue(stderr.write.call_count)
def test_poll_result(self): results = Queue() class Message(object): def __init__(self, **merge): self.payload = dict({'status': states.STARTED, 'result': None}, **merge) class MockBinding(object): def __init__(self, *args, **kwargs): pass def __call__(self, *args, **kwargs): return self def declare(self): pass def get(self, no_ack=False): try: return results.get(block=False) except Empty: pass class MockBackend(AMQPBackend): Queue = MockBinding backend = MockBackend() # FFWD's to the latest state. results.put(Message(status=states.RECEIVED, seq=1)) results.put(Message(status=states.STARTED, seq=2)) results.put(Message(status=states.FAILURE, seq=3)) r1 = backend.get_task_meta(uuid()) self.assertDictContainsSubset({'status': states.FAILURE, 'seq': 3}, r1, 'FFWDs to the last state') # Caches last known state. results.put(Message()) tid = uuid() backend.get_task_meta(tid) self.assertIn(tid, backend._cache, 'Caches last known state') # Returns cache if no new states. results.queue.clear() assert not results.qsize() backend._cache[tid] = 'hello' self.assertEqual(backend.get_task_meta(tid), 'hello', 'Returns cache if no new states')
class TokenBucketQueue(object): """Queue with rate limited get operations. This uses the token bucket algorithm to rate limit the queue on get operations. :param fill_rate: The rate in tokens/second that the bucket will be refilled. :keyword capacity: Maximum number of tokens in the bucket. Default is 1. """ RateLimitExceeded = RateLimitExceeded def __init__(self, fill_rate, queue=None, capacity=1): self._bucket = TokenBucket(fill_rate, capacity) self.queue = queue if not self.queue: self.queue = Queue() def put(self, item, block=True): """Put an item onto the queue.""" self.queue.put(item, block=block) def put_nowait(self, item): """Put an item into the queue without blocking. :raises Queue.Full: If a free slot is not immediately available. """ return self.put(item, block=False) def get(self, block=True): """Remove and return an item from the queue. :raises RateLimitExceeded: If a token could not be consumed from the token bucket (consuming from the queue too fast). :raises Queue.Empty: If an item is not immediately available. """ get = block and self.queue.get or self.queue.get_nowait if not block and not self.items: raise Empty() if not self._bucket.can_consume(1): raise RateLimitExceeded() return get() def get_nowait(self): """Remove and return an item from the queue without blocking. :raises RateLimitExceeded: If a token could not be consumed from the token bucket (consuming from the queue too fast). :raises Queue.Empty: If an item is not immediately available. """ return self.get(block=False) def qsize(self): """Returns the size of the queue.""" return self.queue.qsize() def empty(self): """Returns :const:`True` if the queue is empty.""" return self.queue.empty() def clear(self): """Delete all data in the queue.""" return self.items.clear() def wait(self, block=False): """Wait until a token can be retrieved from the bucket and return the next item.""" get = self.get expected_time = self.expected_time while 1: remaining = expected_time() if not remaining: return get(block=block) sleep(remaining) def expected_time(self, tokens=1): """Returns the expected time in seconds of when a new token should be available.""" if not self.items: return 0 return self._bucket.expected_time(tokens) @property def items(self): """Underlying data. Do not modify.""" return self.queue.queue
def test_poll_result(self): results = Queue() class Message(object): def __init__(self, **merge): self.payload = dict({'status': states.STARTED, 'result': None}, **merge) self.body = pickle.dumps(self.payload) self.content_type = 'application/x-python-serialize' self.content_encoding = 'binary' class MockBinding(object): def __init__(self, *args, **kwargs): self.channel = Mock() def __call__(self, *args, **kwargs): return self def declare(self): pass def get(self, no_ack=False): try: return results.get(block=False) except Empty: pass def is_bound(self): return True class MockBackend(AMQPBackend): Queue = MockBinding backend = MockBackend() backend._republish = Mock() # FFWD's to the latest state. results.put(Message(status=states.RECEIVED, seq=1)) results.put(Message(status=states.STARTED, seq=2)) results.put(Message(status=states.FAILURE, seq=3)) r1 = backend.get_task_meta(uuid()) self.assertDictContainsSubset({'status': states.FAILURE, 'seq': 3}, r1, 'FFWDs to the last state') # Caches last known state. results.put(Message()) tid = uuid() backend.get_task_meta(tid) self.assertIn(tid, backend._cache, 'Caches last known state') self.assertTrue(backend._republish.called) # Returns cache if no new states. results.queue.clear() assert not results.qsize() backend._cache[tid] = 'hello' self.assertEqual(backend.get_task_meta(tid), 'hello', 'Returns cache if no new states')
def test_poll_result(self): results = Queue() class Message(object): def __init__(self, **merge): self.payload = dict({ 'status': states.STARTED, 'result': None }, **merge) self.body = pickle.dumps(self.payload) self.content_type = 'application/x-python-serialize' self.content_encoding = 'binary' class MockBinding(object): def __init__(self, *args, **kwargs): self.channel = Mock() def __call__(self, *args, **kwargs): return self def declare(self): pass def get(self, no_ack=False): try: return results.get(block=False) except Empty: pass def is_bound(self): return True class MockBackend(AMQPBackend): Queue = MockBinding backend = MockBackend() backend._republish = Mock() # FFWD's to the latest state. results.put(Message(status=states.RECEIVED, seq=1)) results.put(Message(status=states.STARTED, seq=2)) results.put(Message(status=states.FAILURE, seq=3)) r1 = backend.get_task_meta(uuid()) self.assertDictContainsSubset({ 'status': states.FAILURE, 'seq': 3 }, r1, 'FFWDs to the last state') # Caches last known state. results.put(Message()) tid = uuid() backend.get_task_meta(tid) self.assertIn(tid, backend._cache, 'Caches last known state') self.assertTrue(backend._republish.called) # Returns cache if no new states. results.queue.clear() assert not results.qsize() backend._cache[tid] = 'hello' self.assertEqual(backend.get_task_meta(tid), 'hello', 'Returns cache if no new states')