class Events(threading.Thread):
def __init__(self, capp, **kwargs):
threading.Thread.__init__(self)
self.daemon = True
self.capp = capp
self.state = EventsState(**kwargs)
def run(self):
try_interval = 1
while True:
logger.info("Starting to capture events...")
try:
import celery
try_interval *= 2
with self.capp.connection() as conn:
recv = EventReceiver(conn,
handlers={"*": self.on_event},
app=self.capp)
try_interval = 1
recv.capture(limit=None, timeout=None, wakeup=True)
except Exception as e:
logger.error(
"Failed to capture events: '%s', "
"trying again in %s seconds.", e, try_interval)
logger.debug(e, exc_info=True)
def on_event(self, event):
self.state.event(event)
def test_callback(self):
scratch = {}
def callback(state, event):
scratch["recv"] = True
s = State(callback=callback)
s.event({"type": "worker-online"})
self.assertTrue(scratch.get("recv"))
def test_callback(self):
scratch = {}
def callback(state, event):
scratch['recv'] = True
s = State(callback=callback)
s.event({'type': 'worker-online'})
assert scratch.get('recv')
def test_callback(self):
scratch = {}
def callback(state, event):
scratch['recv'] = True
s = State(callback=callback)
s.event({'type': 'worker-online'})
assert scratch.get('recv')
def test_callback(self):
scratch = {}
def callback(state, event):
scratch["recv"] = True
s = State(callback=callback)
s.event({"type": "worker-online"})
self.assertTrue(scratch.get("recv"))
def test_survives_unknown_worker_event(self):
s = State()
s.event({
'type': 'worker-unknown-event-xxx',
'foo': 'bar',
})
s.event({
'type': 'worker-unknown-event-xxx',
'hostname': 'xxx',
'foo': 'bar',
})
def test_survives_unknown_task_event(self):
s = State()
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'timestamp': time(),
'local_received': time(),
'clock': 0,
})
def test_survives_unknown_worker_event(self):
s = State()
s.event({
'type': 'worker-unknown-event-xxx',
'foo': 'bar',
})
s.event({
'type': 'worker-unknown-event-xxx',
'hostname': 'xxx',
'foo': 'bar',
})
def test_survives_unknown_task_event(self):
s = State()
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'timestamp': time(),
'local_received': time(),
'clock': 0,
})
def test_survives_unknown_task_event(self):
s = State()
s.event(
{
"type": "task-unknown-event-xxx",
"foo": "bar",
"uuid": "x",
"hostname": "y",
"timestamp": time(),
"local_received": time(),
"clock": 0,
}
)
def test_limits_maxtasks(self):
s = State(max_tasks_in_memory=1)
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'clock': 3,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'y',
'hostname': 'y',
'clock': 4,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'z',
'hostname': 'y',
'clock': 5,
'timestamp': time(),
'local_received': time(),
})
self.assertEqual(len(s._taskheap), 2)
self.assertEqual(s._taskheap[0].clock, 4)
self.assertEqual(s._taskheap[1].clock, 5)
s._taskheap.append(s._taskheap[0])
self.assertTrue(list(s.tasks_by_time()))
def test_deepcopy(self):
import copy
s = State()
s.event({
'type': 'task-success',
'root_id': 'x',
'uuid': 'x',
'hostname': 'y',
'clock': 3,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-success',
'root_id': 'y',
'uuid': 'y',
'hostname': 'y',
'clock': 4,
'timestamp': time(),
'local_received': time(),
})
copy.deepcopy(s)
def test_on_node_join_callback(self):
s = State(on_node_join=Mock(name='on_node_join'))
(worker, created), subject = s.event({
'type': 'worker-online',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 34314,
})
self.assertTrue(worker)
self.assertTrue(created)
self.assertEqual(subject, 'online')
self.assertIn('*****@*****.**', s.workers)
s.on_node_join.assert_called_with(worker)
def test_survives_unknown_worker_leaving(self):
s = State(on_node_leave=Mock(name='on_node_leave'))
(worker, created), subject = s.event({
'type': 'worker-offline',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 301030134894833,
})
self.assertEqual(worker, Worker('*****@*****.**'))
self.assertFalse(created)
self.assertEqual(subject, 'offline')
self.assertNotIn('*****@*****.**', s.workers)
s.on_node_leave.assert_called_with(worker)
def test_survives_unknown_worker_leaving(self):
s = State(on_node_leave=Mock(name='on_node_leave'))
(worker, created), subject = s.event({
'type': 'worker-offline',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 301030134894833,
})
assert worker == Worker('*****@*****.**')
assert not created
assert subject == 'offline'
assert '*****@*****.**' not in s.workers
s.on_node_leave.assert_called_with(worker)
def test_on_node_join_callback(self):
s = State(on_node_join=Mock(name='on_node_join'))
(worker, created), subject = s.event({
'type': 'worker-online',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 34314,
})
assert worker
assert created
assert subject == 'online'
assert '*****@*****.**' in s.workers
s.on_node_join.assert_called_with(worker)
def test_survives_unknown_worker_leaving(self):
s = State(on_node_leave=Mock(name='on_node_leave'))
(worker, created), subject = s.event({
'type': 'worker-offline',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 301030134894833,
})
self.assertEqual(worker, Worker('*****@*****.**'))
self.assertFalse(created)
self.assertEqual(subject, 'offline')
self.assertNotIn('*****@*****.**', s.workers)
s.on_node_leave.assert_called_with(worker)
def test_on_node_join_callback(self):
s = State(on_node_join=Mock(name='on_node_join'))
(worker, created), subject = s.event({
'type': 'worker-online',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 34314,
})
self.assertTrue(worker)
self.assertTrue(created)
self.assertEqual(subject, 'online')
self.assertIn('*****@*****.**', s.workers)
s.on_node_join.assert_called_with(worker)
def test_on_node_join_callback(self):
s = State(on_node_join=Mock(name='on_node_join'))
(worker, created), subject = s.event({
'type': 'worker-online',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 34314,
})
assert worker
assert created
assert subject == 'online'
assert '*****@*****.**' in s.workers
s.on_node_join.assert_called_with(worker)
def test_survives_unknown_worker_leaving(self):
s = State(on_node_leave=Mock(name='on_node_leave'))
(worker, created), subject = s.event({
'type': 'worker-offline',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 301030134894833,
})
assert worker == Worker('*****@*****.**')
assert not created
assert subject == 'offline'
assert '*****@*****.**' not in s.workers
s.on_node_leave.assert_called_with(worker)
def test_limits_maxtasks(self):
s = State(max_tasks_in_memory=1)
s.heap_multiplier = 2
s.event(
{
"type": "task-unknown-event-xxx",
"foo": "bar",
"uuid": "x",
"hostname": "y",
"clock": 3,
"timestamp": time(),
"local_received": time(),
}
)
s.event(
{
"type": "task-unknown-event-xxx",
"foo": "bar",
"uuid": "y",
"hostname": "y",
"clock": 4,
"timestamp": time(),
"local_received": time(),
}
)
s.event(
{
"type": "task-unknown-event-xxx",
"foo": "bar",
"uuid": "z",
"hostname": "y",
"clock": 5,
"timestamp": time(),
"local_received": time(),
}
)
self.assertEqual(len(s._taskheap), 2)
self.assertEqual(s._taskheap[0].clock, 4)
self.assertEqual(s._taskheap[1].clock, 5)
s._taskheap.append(s._taskheap[0])
self.assertTrue(list(s.tasks_by_time()))
def test_limits_maxtasks(self):
s = State(max_tasks_in_memory=1)
s.heap_multiplier = 2
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'clock': 3,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'y',
'hostname': 'y',
'clock': 4,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'z',
'hostname': 'y',
'clock': 5,
'timestamp': time(),
'local_received': time(),
})
assert len(s._taskheap) == 2
assert s._taskheap[0].clock == 4
assert s._taskheap[1].clock == 5
s._taskheap.append(s._taskheap[0])
assert list(s.tasks_by_time())
def test_limits_maxtasks(self):
s = State(max_tasks_in_memory=1)
s.heap_multiplier = 2
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'clock': 3,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'y',
'hostname': 'y',
'clock': 4,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'z',
'hostname': 'y',
'clock': 5,
'timestamp': time(),
'local_received': time(),
})
assert len(s._taskheap) == 2
assert s._taskheap[0].clock == 4
assert s._taskheap[1].clock == 5
s._taskheap.append(s._taskheap[0])
assert list(s.tasks_by_time())
class EventListener(object):
"""Listens for celery events.
Server object, to capture events and handle tasks and workers.
Attributes:
_app (Celery): a configured celery app instance
_queue_output (Queue): to send to streaming dispatcher
memory (State): LRU storage object to keep tasks and workers
_use_result_backend (bool): if True, there's a result backend to fetch results from
"""
def __init__(self,
broker,
queue_output,
backend=None,
max_tasks_in_memory=None,
max_workers_in_memory=None):
"""Constructs an event listener instance.
Args:
broker (str): the broker being used by the celery system.
queue_output (Queue): to send to streaming dispatcher.
backend (str): the result backend being used by the celery system.
max_tasks_in_memory (int): max tasks stored
max_workers_in_memory (int): max workers stored
"""
self._app = Celery(broker=broker, backend=backend)
self._queue_output = queue_output
from celery.backends.base import DisabledBackend
self._use_result_backend = not isinstance(self._app.backend,
DisabledBackend)
logger.info('Creating %s: max_tasks=%d; max_workers=%d',
EventListener.__name__, max_tasks_in_memory,
max_workers_in_memory)
logger.info('Celery broker=%s; backend=%s; using_result_backend=%s',
broker, backend, self._use_result_backend)
# events handling: storage and filling missing states.
self.memory = State(
max_tasks_in_memory=max_tasks_in_memory,
max_workers_in_memory=max_workers_in_memory,
) # type: State
# running engine (should be asyncio in the future)
self._listener_thread = None # type:threading.Thread
self._celery_receiver = None # type:EventReceiver
# concurrency control
self._wait_event = threading.Event()
# detect shutdown.
def sigterm_handler(_signo, _stack_frame): # pragma: no cover
self.__stop()
signal.signal(signal.SIGTERM, sigterm_handler)
self.__start()
def __start(self): # pragma: no cover
"""Starts the real-time engine that captures events."""
assert not self._listener_thread
self._listener_thread = threading.Thread(target=self.__run_listener,
name='clearly-listener')
self._listener_thread.daemon = True
self._listener_thread.start()
self._wait_event.wait()
self._wait_event.clear()
def __stop(self): # pragma: no cover
"""Stops the background engine."""
if not self._listener_thread:
return
logger.info('Stopping listener')
self._celery_receiver.should_stop = True
self._listener_thread.join()
self._listener_thread = self._celery_receiver = None
def __run_listener(self): # pragma: no cover
logger.info('Starting listener: %s', threading.current_thread())
with self._app.connection() as connection:
self._celery_receiver = self._app.events.Receiver(
connection, handlers={
'*': self._process_event,
}) # type: EventReceiver
self._wait_event.set()
self._celery_receiver.capture(limit=None,
timeout=None,
wakeup=True)
logger.info('Listener stopped: %s', threading.current_thread())
def _process_event(self, event):
event_type = event['type']
if event_type.startswith('task'):
data = self._process_task_event(event)
elif event_type.startswith('worker'):
data = self._process_worker_event(event)
else:
logger.warning('unknown event: %s', event)
return
self._queue_output.put(data)
def _process_task_event(self, event):
task = self.memory.tasks.get(event['uuid'])
pre_state, created = (task.state, False) if task else (states.PENDING,
True)
(task,
_), _ = self.memory.event(event) # the `created` is broken in celery.
if task.state == states.SUCCESS:
try:
# verify if the celery task result is truncated.
task.result = EventListener.compile_task_result(task)
except SyntaxError:
# use result backend as fallback if allowed and available.
if self._use_result_backend: # pragma: no cover
task.result = repr(self._app.AsyncResult(task.uuid).result)
return immutable_task(task, task.state, pre_state, created)
def _process_worker_event(self, event):
worker = self.memory.workers.get(event['hostname'])
pre_state, created = (worker.status_string, False) \
if worker else (worker_states.OFFLINE, True)
(worker, _), _ = self.memory.event(event)
return immutable_worker(worker, worker.status_string, pre_state,
created)
@staticmethod
def compile_task_result(task):
result = task.result
# celery 4 sends results converted as strings, sometimes truncated (...)
if task.worker.sw_ver < '4':
# celery 3 tasks' results are converted twice.
result = safe_compile_text(result, raises=True)
# compile with `raises`, to detect truncated results.
safe_compile_text(result, raises=True)
# if compilable, returns the same, as the clients will be able to too.
return result
def test_survives_unknown_worker_event(self):
s = State()
s.event({"type": "worker-unknown-event-xxx", "foo": "bar"})
s.event({"type": "worker-unknown-event-xxx", "hostname": "xxx", "foo": "bar"})
