class Events(threading.Thread): def __init__(self, capp, **kwargs): threading.Thread.__init__(self) self.daemon = True self.capp = capp self.state = EventsState(**kwargs) def run(self): try_interval = 1 while True: logger.info("Starting to capture events...") try: import celery try_interval *= 2 with self.capp.connection() as conn: recv = EventReceiver(conn, handlers={"*": self.on_event}, app=self.capp) try_interval = 1 recv.capture(limit=None, timeout=None, wakeup=True) except Exception as e: logger.error( "Failed to capture events: '%s', " "trying again in %s seconds.", e, try_interval) logger.debug(e, exc_info=True) def on_event(self, event): self.state.event(event)
def test_callback(self): scratch = {} def callback(state, event): scratch["recv"] = True s = State(callback=callback) s.event({"type": "worker-online"}) self.assertTrue(scratch.get("recv"))
def test_callback(self): scratch = {} def callback(state, event): scratch['recv'] = True s = State(callback=callback) s.event({'type': 'worker-online'}) assert scratch.get('recv')
def test_callback(self): scratch = {} def callback(state, event): scratch['recv'] = True s = State(callback=callback) s.event({'type': 'worker-online'}) assert scratch.get('recv')
def test_callback(self): scratch = {} def callback(state, event): scratch["recv"] = True s = State(callback=callback) s.event({"type": "worker-online"}) self.assertTrue(scratch.get("recv"))
def test_survives_unknown_worker_event(self): s = State() s.event({ 'type': 'worker-unknown-event-xxx', 'foo': 'bar', }) s.event({ 'type': 'worker-unknown-event-xxx', 'hostname': 'xxx', 'foo': 'bar', })
def test_survives_unknown_task_event(self): s = State() s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'x', 'hostname': 'y', 'timestamp': time(), 'local_received': time(), 'clock': 0, })
def test_survives_unknown_worker_event(self): s = State() s.event({ 'type': 'worker-unknown-event-xxx', 'foo': 'bar', }) s.event({ 'type': 'worker-unknown-event-xxx', 'hostname': 'xxx', 'foo': 'bar', })
def test_survives_unknown_task_event(self): s = State() s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'x', 'hostname': 'y', 'timestamp': time(), 'local_received': time(), 'clock': 0, })
def test_survives_unknown_task_event(self): s = State() s.event( { "type": "task-unknown-event-xxx", "foo": "bar", "uuid": "x", "hostname": "y", "timestamp": time(), "local_received": time(), "clock": 0, } )
def test_limits_maxtasks(self): s = State(max_tasks_in_memory=1) s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'x', 'hostname': 'y', 'clock': 3, 'timestamp': time(), 'local_received': time(), }) s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'y', 'hostname': 'y', 'clock': 4, 'timestamp': time(), 'local_received': time(), }) s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'z', 'hostname': 'y', 'clock': 5, 'timestamp': time(), 'local_received': time(), }) self.assertEqual(len(s._taskheap), 2) self.assertEqual(s._taskheap[0].clock, 4) self.assertEqual(s._taskheap[1].clock, 5) s._taskheap.append(s._taskheap[0]) self.assertTrue(list(s.tasks_by_time()))
def test_deepcopy(self): import copy s = State() s.event({ 'type': 'task-success', 'root_id': 'x', 'uuid': 'x', 'hostname': 'y', 'clock': 3, 'timestamp': time(), 'local_received': time(), }) s.event({ 'type': 'task-success', 'root_id': 'y', 'uuid': 'y', 'hostname': 'y', 'clock': 4, 'timestamp': time(), 'local_received': time(), }) copy.deepcopy(s)
def test_on_node_join_callback(self): s = State(on_node_join=Mock(name='on_node_join')) (worker, created), subject = s.event({ 'type': 'worker-online', 'hostname': '*****@*****.**', 'timestamp': time(), 'local_received': time(), 'clock': 34314, }) self.assertTrue(worker) self.assertTrue(created) self.assertEqual(subject, 'online') self.assertIn('*****@*****.**', s.workers) s.on_node_join.assert_called_with(worker)
def test_survives_unknown_worker_leaving(self): s = State(on_node_leave=Mock(name='on_node_leave')) (worker, created), subject = s.event({ 'type': 'worker-offline', 'hostname': '*****@*****.**', 'timestamp': time(), 'local_received': time(), 'clock': 301030134894833, }) self.assertEqual(worker, Worker('*****@*****.**')) self.assertFalse(created) self.assertEqual(subject, 'offline') self.assertNotIn('*****@*****.**', s.workers) s.on_node_leave.assert_called_with(worker)
def test_survives_unknown_worker_leaving(self): s = State(on_node_leave=Mock(name='on_node_leave')) (worker, created), subject = s.event({ 'type': 'worker-offline', 'hostname': '*****@*****.**', 'timestamp': time(), 'local_received': time(), 'clock': 301030134894833, }) assert worker == Worker('*****@*****.**') assert not created assert subject == 'offline' assert '*****@*****.**' not in s.workers s.on_node_leave.assert_called_with(worker)
def test_on_node_join_callback(self): s = State(on_node_join=Mock(name='on_node_join')) (worker, created), subject = s.event({ 'type': 'worker-online', 'hostname': '*****@*****.**', 'timestamp': time(), 'local_received': time(), 'clock': 34314, }) assert worker assert created assert subject == 'online' assert '*****@*****.**' in s.workers s.on_node_join.assert_called_with(worker)
def test_survives_unknown_worker_leaving(self): s = State(on_node_leave=Mock(name='on_node_leave')) (worker, created), subject = s.event({ 'type': 'worker-offline', 'hostname': '*****@*****.**', 'timestamp': time(), 'local_received': time(), 'clock': 301030134894833, }) self.assertEqual(worker, Worker('*****@*****.**')) self.assertFalse(created) self.assertEqual(subject, 'offline') self.assertNotIn('*****@*****.**', s.workers) s.on_node_leave.assert_called_with(worker)
def test_on_node_join_callback(self): s = State(on_node_join=Mock(name='on_node_join')) (worker, created), subject = s.event({ 'type': 'worker-online', 'hostname': '*****@*****.**', 'timestamp': time(), 'local_received': time(), 'clock': 34314, }) self.assertTrue(worker) self.assertTrue(created) self.assertEqual(subject, 'online') self.assertIn('*****@*****.**', s.workers) s.on_node_join.assert_called_with(worker)
def test_on_node_join_callback(self): s = State(on_node_join=Mock(name='on_node_join')) (worker, created), subject = s.event({ 'type': 'worker-online', 'hostname': '*****@*****.**', 'timestamp': time(), 'local_received': time(), 'clock': 34314, }) assert worker assert created assert subject == 'online' assert '*****@*****.**' in s.workers s.on_node_join.assert_called_with(worker)
def test_survives_unknown_worker_leaving(self): s = State(on_node_leave=Mock(name='on_node_leave')) (worker, created), subject = s.event({ 'type': 'worker-offline', 'hostname': '*****@*****.**', 'timestamp': time(), 'local_received': time(), 'clock': 301030134894833, }) assert worker == Worker('*****@*****.**') assert not created assert subject == 'offline' assert '*****@*****.**' not in s.workers s.on_node_leave.assert_called_with(worker)
def test_limits_maxtasks(self): s = State(max_tasks_in_memory=1) s.heap_multiplier = 2 s.event( { "type": "task-unknown-event-xxx", "foo": "bar", "uuid": "x", "hostname": "y", "clock": 3, "timestamp": time(), "local_received": time(), } ) s.event( { "type": "task-unknown-event-xxx", "foo": "bar", "uuid": "y", "hostname": "y", "clock": 4, "timestamp": time(), "local_received": time(), } ) s.event( { "type": "task-unknown-event-xxx", "foo": "bar", "uuid": "z", "hostname": "y", "clock": 5, "timestamp": time(), "local_received": time(), } ) self.assertEqual(len(s._taskheap), 2) self.assertEqual(s._taskheap[0].clock, 4) self.assertEqual(s._taskheap[1].clock, 5) s._taskheap.append(s._taskheap[0]) self.assertTrue(list(s.tasks_by_time()))
def test_limits_maxtasks(self): s = State(max_tasks_in_memory=1) s.heap_multiplier = 2 s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'x', 'hostname': 'y', 'clock': 3, 'timestamp': time(), 'local_received': time(), }) s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'y', 'hostname': 'y', 'clock': 4, 'timestamp': time(), 'local_received': time(), }) s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'z', 'hostname': 'y', 'clock': 5, 'timestamp': time(), 'local_received': time(), }) assert len(s._taskheap) == 2 assert s._taskheap[0].clock == 4 assert s._taskheap[1].clock == 5 s._taskheap.append(s._taskheap[0]) assert list(s.tasks_by_time())
def test_limits_maxtasks(self): s = State(max_tasks_in_memory=1) s.heap_multiplier = 2 s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'x', 'hostname': 'y', 'clock': 3, 'timestamp': time(), 'local_received': time(), }) s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'y', 'hostname': 'y', 'clock': 4, 'timestamp': time(), 'local_received': time(), }) s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'z', 'hostname': 'y', 'clock': 5, 'timestamp': time(), 'local_received': time(), }) assert len(s._taskheap) == 2 assert s._taskheap[0].clock == 4 assert s._taskheap[1].clock == 5 s._taskheap.append(s._taskheap[0]) assert list(s.tasks_by_time())
class EventListener(object): """Listens for celery events. Server object, to capture events and handle tasks and workers. Attributes: _app (Celery): a configured celery app instance _queue_output (Queue): to send to streaming dispatcher memory (State): LRU storage object to keep tasks and workers _use_result_backend (bool): if True, there's a result backend to fetch results from """ def __init__(self, broker, queue_output, backend=None, max_tasks_in_memory=None, max_workers_in_memory=None): """Constructs an event listener instance. Args: broker (str): the broker being used by the celery system. queue_output (Queue): to send to streaming dispatcher. backend (str): the result backend being used by the celery system. max_tasks_in_memory (int): max tasks stored max_workers_in_memory (int): max workers stored """ self._app = Celery(broker=broker, backend=backend) self._queue_output = queue_output from celery.backends.base import DisabledBackend self._use_result_backend = not isinstance(self._app.backend, DisabledBackend) logger.info('Creating %s: max_tasks=%d; max_workers=%d', EventListener.__name__, max_tasks_in_memory, max_workers_in_memory) logger.info('Celery broker=%s; backend=%s; using_result_backend=%s', broker, backend, self._use_result_backend) # events handling: storage and filling missing states. self.memory = State( max_tasks_in_memory=max_tasks_in_memory, max_workers_in_memory=max_workers_in_memory, ) # type: State # running engine (should be asyncio in the future) self._listener_thread = None # type:threading.Thread self._celery_receiver = None # type:EventReceiver # concurrency control self._wait_event = threading.Event() # detect shutdown. def sigterm_handler(_signo, _stack_frame): # pragma: no cover self.__stop() signal.signal(signal.SIGTERM, sigterm_handler) self.__start() def __start(self): # pragma: no cover """Starts the real-time engine that captures events.""" assert not self._listener_thread self._listener_thread = threading.Thread(target=self.__run_listener, name='clearly-listener') self._listener_thread.daemon = True self._listener_thread.start() self._wait_event.wait() self._wait_event.clear() def __stop(self): # pragma: no cover """Stops the background engine.""" if not self._listener_thread: return logger.info('Stopping listener') self._celery_receiver.should_stop = True self._listener_thread.join() self._listener_thread = self._celery_receiver = None def __run_listener(self): # pragma: no cover logger.info('Starting listener: %s', threading.current_thread()) with self._app.connection() as connection: self._celery_receiver = self._app.events.Receiver( connection, handlers={ '*': self._process_event, }) # type: EventReceiver self._wait_event.set() self._celery_receiver.capture(limit=None, timeout=None, wakeup=True) logger.info('Listener stopped: %s', threading.current_thread()) def _process_event(self, event): event_type = event['type'] if event_type.startswith('task'): data = self._process_task_event(event) elif event_type.startswith('worker'): data = self._process_worker_event(event) else: logger.warning('unknown event: %s', event) return self._queue_output.put(data) def _process_task_event(self, event): task = self.memory.tasks.get(event['uuid']) pre_state, created = (task.state, False) if task else (states.PENDING, True) (task, _), _ = self.memory.event(event) # the `created` is broken in celery. if task.state == states.SUCCESS: try: # verify if the celery task result is truncated. task.result = EventListener.compile_task_result(task) except SyntaxError: # use result backend as fallback if allowed and available. if self._use_result_backend: # pragma: no cover task.result = repr(self._app.AsyncResult(task.uuid).result) return immutable_task(task, task.state, pre_state, created) def _process_worker_event(self, event): worker = self.memory.workers.get(event['hostname']) pre_state, created = (worker.status_string, False) \ if worker else (worker_states.OFFLINE, True) (worker, _), _ = self.memory.event(event) return immutable_worker(worker, worker.status_string, pre_state, created) @staticmethod def compile_task_result(task): result = task.result # celery 4 sends results converted as strings, sometimes truncated (...) if task.worker.sw_ver < '4': # celery 3 tasks' results are converted twice. result = safe_compile_text(result, raises=True) # compile with `raises`, to detect truncated results. safe_compile_text(result, raises=True) # if compilable, returns the same, as the clients will be able to too. return result
def test_survives_unknown_worker_event(self): s = State() s.event({"type": "worker-unknown-event-xxx", "foo": "bar"}) s.event({"type": "worker-unknown-event-xxx", "hostname": "xxx", "foo": "bar"})