def test_clear_tasks(self): s = State() r = ev_snapshot(s) r.play() self.assertTrue(s.tasks) s.clear_tasks(ready=False) self.assertFalse(s.tasks)
def test_survives_unknown_task_event(self): s = State() s.task_event("task-unknown-event-xxx", { "foo": "bar", "uuid": "x", "hostname": "y" })
def test_clear_tasks(self): s = State() r = ev_snapshot(s) r.play() assert s.tasks s.clear_tasks(ready=False) assert not s.tasks
def test_limits_maxtasks(self): s = State() s.max_tasks_in_memory = 1 s.task_event('task-unknown-event-xxx', { 'foo': 'bar', 'uuid': 'x', 'hostname': 'y', 'clock': 3 }) s.task_event('task-unknown-event-xxx', { 'foo': 'bar', 'uuid': 'y', 'hostname': 'y', 'clock': 4 }) s.task_event('task-unknown-event-xxx', { 'foo': 'bar', 'uuid': 'z', 'hostname': 'y', 'clock': 5 }) self.assertEqual(len(s._taskheap), 2) self.assertEqual(s._taskheap[0].clock, 4) self.assertEqual(s._taskheap[1].clock, 5) s._taskheap.append(s._taskheap[0]) self.assertTrue(list(s.tasks_by_time()))
def test_survives_unknown_task_event(self): s = State() s.task_event('task-unknown-event-xxx', { 'foo': 'bar', 'uuid': 'x', 'hostname': 'y' })
class Events(threading.Thread): def __init__(self, capp, **kwargs): threading.Thread.__init__(self) self.daemon = True self.capp = capp self.state = EventsState(**kwargs) def run(self): try_interval = 1 while True: logger.info("Starting to capture events...") try: import celery try_interval *= 2 with self.capp.connection() as conn: recv = EventReceiver(conn, handlers={"*": self.on_event}, app=self.capp) try_interval = 1 recv.capture(limit=None, timeout=None, wakeup=True) except Exception as e: logger.error( "Failed to capture events: '%s', " "trying again in %s seconds.", e, try_interval) logger.debug(e, exc_info=True) def on_event(self, event): self.state.event(event)
def test_get_or_create_worker(self): state = State() worker, created = state.get_or_create_worker('*****@*****.**') self.assertEqual(worker.hostname, '*****@*****.**') self.assertTrue(created) worker2, created2 = state.get_or_create_worker('*****@*****.**') self.assertIs(worker2, worker) self.assertFalse(created2)
def test_get_or_create_task(self): state = State() task, created = state.get_or_create_task('id1') self.assertEqual(task.uuid, 'id1') self.assertTrue(created) task2, created2 = state.get_or_create_task('id1') self.assertIs(task2, task) self.assertFalse(created2)
def test_get_or_create_task(self): state = State() task, created = state.get_or_create_task('id1') assert task.uuid == 'id1' assert created task2, created2 = state.get_or_create_task('id1') assert task2 is task assert not created2
def test_get_or_create_worker(self): state = State() worker, created = state.get_or_create_worker('*****@*****.**') assert worker.hostname == '*****@*****.**' assert created worker2, created2 = state.get_or_create_worker('*****@*****.**') assert worker2 is worker assert not created2
def test_thaw__no_replay(self): s = State() r = ev_snapshot(s) s.freeze(buffer=True) r.play() s.thaw(replay=False) self.assertFalse(s.buffer) self.assertStateEmpty(s)
def test_callback(self): scratch = {} def callback(state, event): scratch['recv'] = True s = State(callback=callback) s.event({'type': 'worker-online'}) assert scratch.get('recv')
def test_callback(self): scratch = {} def callback(state, event): scratch["recv"] = True s = State(callback=callback) s.event({"type": "worker-online"}) self.assertTrue(scratch.get("recv"))
def test_freeze_while(self): s = State() r = ev_snapshot(s) r.play() def work(): pass s.freeze_while(work, clear_after=True) self.assertFalse(s.event_count)
def test_freeze_while(self): s = State() r = ev_snapshot(s) def work(): r.play() self.assertStateEmpty(s) s.freeze_while(work) self.assertState(s)
def test_survives_unknown_worker_event(self): s = State() s.event({ 'type': 'worker-unknown-event-xxx', 'foo': 'bar', }) s.event({ 'type': 'worker-unknown-event-xxx', 'hostname': 'xxx', 'foo': 'bar', })
def test_survives_unknown_task_event(self): s = State() s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'x', 'hostname': 'y', 'timestamp': time(), 'local_received': time(), 'clock': 0, })
def test_freeze_thaw__buffering(self): s = State() r = ev_snapshot(s) s.freeze(buffer=True) self.assertTrue(s._buffering) r.play() self.assertStateEmpty(s) self.assertTrue(s.buffer) s.thaw() self.assertState(s) self.assertFalse(s.buffer)
def test_survives_unknown_task_event(self): s = State() s.event( { "type": "task-unknown-event-xxx", "foo": "bar", "uuid": "x", "hostname": "y", "timestamp": time(), "local_received": time(), "clock": 0, } )
def test_limits_maxtasks(self): s = State(max_tasks_in_memory=1) s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'x', 'hostname': 'y', 'clock': 3, 'timestamp': time(), 'local_received': time(), }) s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'y', 'hostname': 'y', 'clock': 4, 'timestamp': time(), 'local_received': time(), }) s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'z', 'hostname': 'y', 'clock': 5, 'timestamp': time(), 'local_received': time(), }) self.assertEqual(len(s._taskheap), 2) self.assertEqual(s._taskheap[0].clock, 4) self.assertEqual(s._taskheap[1].clock, 5) s._taskheap.append(s._taskheap[0]) self.assertTrue(list(s.tasks_by_time()))
def test_survives_unknown_worker_leaving(self): s = State(on_node_leave=Mock(name='on_node_leave')) (worker, created), subject = s.event({ 'type': 'worker-offline', 'hostname': '*****@*****.**', 'timestamp': time(), 'local_received': time(), 'clock': 301030134894833, }) self.assertEqual(worker, Worker('*****@*****.**')) self.assertFalse(created) self.assertEqual(subject, 'offline') self.assertNotIn('*****@*****.**', s.workers) s.on_node_leave.assert_called_with(worker)
def test_get_or_create_worker__with_defaults(self): state = State() worker, created = state.get_or_create_worker( '*****@*****.**', pid=30, ) self.assertEqual(worker.hostname, '*****@*****.**') self.assertEqual(worker.pid, 30) self.assertTrue(created) worker2, created2 = state.get_or_create_worker( '*****@*****.**', pid=40, ) self.assertIs(worker2, worker) self.assertEqual(worker2.pid, 40) self.assertFalse(created2)
def test_on_node_join_callback(self): s = State(on_node_join=Mock(name='on_node_join')) (worker, created), subject = s.event({ 'type': 'worker-online', 'hostname': '*****@*****.**', 'timestamp': time(), 'local_received': time(), 'clock': 34314, }) self.assertTrue(worker) self.assertTrue(created) self.assertEqual(subject, 'online') self.assertIn('*****@*****.**', s.workers) s.on_node_join.assert_called_with(worker)
def test_get_or_create_worker__with_defaults(self): state = State() worker, created = state.get_or_create_worker( '*****@*****.**', pid=30, ) assert worker.hostname == '*****@*****.**' assert worker.pid == 30 assert created worker2, created2 = state.get_or_create_worker( '*****@*****.**', pid=40, ) assert worker2 is worker assert worker2.pid == 40 assert not created2
def test_on_node_join_callback(self): s = State(on_node_join=Mock(name='on_node_join')) (worker, created), subject = s.event({ 'type': 'worker-online', 'hostname': '*****@*****.**', 'timestamp': time(), 'local_received': time(), 'clock': 34314, }) assert worker assert created assert subject == 'online' assert '*****@*****.**' in s.workers s.on_node_join.assert_called_with(worker)
def test_survives_unknown_worker_leaving(self): s = State(on_node_leave=Mock(name='on_node_leave')) (worker, created), subject = s.event({ 'type': 'worker-offline', 'hostname': '*****@*****.**', 'timestamp': time(), 'local_received': time(), 'clock': 301030134894833, }) assert worker == Worker('*****@*****.**') assert not created assert subject == 'offline' assert '*****@*****.**' not in s.workers s.on_node_leave.assert_called_with(worker)
def test_task_children_set_if_received_in_wrong_order(self): r = ev_task_states(State()) r.events.insert(0, r.events.pop()) r.play() self.assertIn(r.state.tasks[r.tid2], r.state.tasks[r.tid].children) self.assertIs(r.state.tasks[r.tid2].root, r.state.tasks[r.tid]) self.assertIs(r.state.tasks[r.tid2].parent, r.state.tasks[r.tid])
def __init__(self, broker: str, backend: Optional[str] = None, max_tasks: Optional[int] = None, max_workers: Optional[int] = None): # pragma: no cover """Construct a Clearly Server instance. Args: broker: the broker being used by the celery system backend: the result backend being used by the celery system max_tasks: max tasks stored max_workers: max workers stored """ max_tasks, max_workers = max_tasks or 10000, max_workers or 100 logger.info('Creating memory: max_tasks=%d; max_workers=%d', max_tasks, max_workers) self._memory = State(max_tasks_in_memory=max_tasks, max_workers_in_memory=max_workers) queue_tasks, queue_workers = Queue(), Queue( ) # hands new events to be distributed. try: self._listener = EventListener(broker, queue_tasks, queue_workers, self._memory, backend) except TimeoutError as e: logger.critical(e) sys.exit(1) self._dispatcher_tasks = StreamingDispatcher(queue_tasks, Role.TASKS) self._dispatcher_workers = StreamingDispatcher(queue_workers, Role.WORKERS) self._rpc = RPCService(self._memory, self._dispatcher_tasks, self._dispatcher_workers)
def eventtop(): sys.stderr.write("-> celeryev: starting capture...\n") state = State() display = CursesMonitor(state) display.init_screen() refresher = DisplayThread(display) refresher.start() conn = establish_connection() recv = EventReceiver(conn, handlers={"*": state.event}) try: consumer = recv.consumer() consumer.consume() while True: try: conn.connection.drain_events() except (socket.timeout, socket.error): pass except Exception: refresher.shutdown = True refresher.join() display.resetscreen() raise except (KeyboardInterrupt, SystemExit): conn and conn.close() refresher.shutdown = True refresher.join() display.resetscreen()
def test_task_children_set_if_received_in_wrong_order(self): r = ev_task_states(State()) r.events.insert(0, r.events.pop()) r.play() assert r.state.tasks[r.tid2] in r.state.tasks[r.tid].children assert r.state.tasks[r.tid2].root is r.state.tasks[r.tid] assert r.state.tasks[r.tid2].parent is r.state.tasks[r.tid]
def __init__(self, broker, queue_output, backend=None, max_tasks_in_memory=None, max_workers_in_memory=None): """Constructs an event listener instance. Args: broker (str): the broker being used by the celery system. queue_output (Queue): to send to streaming dispatcher. backend (str): the result backend being used by the celery system. max_tasks_in_memory (int): max tasks stored max_workers_in_memory (int): max workers stored """ self._app = Celery(broker=broker, backend=backend) self._queue_output = queue_output from celery.backends.base import DisabledBackend self._use_result_backend = not isinstance(self._app.backend, DisabledBackend) logger.info('Creating %s: max_tasks=%d; max_workers=%d', EventListener.__name__, max_tasks_in_memory, max_workers_in_memory) logger.info('Celery broker=%s; backend=%s; using_result_backend=%s', broker, backend, self._use_result_backend) # events handling: storage and filling missing states. self.memory = State( max_tasks_in_memory=max_tasks_in_memory, max_workers_in_memory=max_workers_in_memory, ) # type: State # running engine (should be asyncio in the future) self._listener_thread = None # type:threading.Thread self._celery_receiver = None # type:EventReceiver # concurrency control self._wait_event = threading.Event() # detect shutdown. def sigterm_handler(_signo, _stack_frame): # pragma: no cover self.__stop() signal.signal(signal.SIGTERM, sigterm_handler) self.__start()
def test_tasks_by_worker(self): r = ev_snapshot(State()) r.play() assert len(list(r.state.tasks_by_worker('utest1'))) == 10 assert len(list(r.state.tasks_by_worker('utest2'))) == 10 assert len(r.state.tasks_by_worker['utest1']) == 10 assert len(r.state.tasks_by_worker['utest2']) == 10
def test_tasks_by_type(self): r = ev_snapshot(State()) r.play() assert len(list(r.state.tasks_by_type('task1'))) == 10 assert len(list(r.state.tasks_by_type('task2'))) == 10 assert len(r.state.tasks_by_type['task1']) == 10 assert len(r.state.tasks_by_type['task2']) == 10
def test_worker_heartbeat_expire(self): r = ev_worker_heartbeats(State()) next(r) assert not list(r.state.alive_workers()) assert not r.state.workers['utest1'].alive r.play() assert list(r.state.alive_workers()) assert r.state.workers['utest1'].alive
def test_worker_online_offline(self): r = ev_worker_online_offline(State()) next(r) assert list(r.state.alive_workers()) assert r.state.workers['utest1'].alive r.play() assert not list(r.state.alive_workers()) assert not r.state.workers['utest1'].alive
def test_worker_heartbeat_expire(self): r = ev_worker_heartbeats(State()) next(r) self.assertFalse(r.state.alive_workers()) self.assertFalse(r.state.workers['utest1'].alive) r.play() self.assertTrue(r.state.alive_workers()) self.assertTrue(r.state.workers['utest1'].alive)
def test_task_descending_clock_ordering(self): state = State() r = ev_logical_clock_ordering(state) tA, tB, tC = r.uids r.play() now = list(state.tasks_by_time(reverse=False)) assert now[0][0] == tA assert now[1][0] == tB assert now[2][0] == tC for _ in range(1000): shuffle(r.uids) tA, tB, tC = r.uids r.rewind_with_offset(r.current_clock + 1, r.uids) r.play() now = list(state.tasks_by_time(reverse=False)) assert now[0][0] == tB assert now[1][0] == tC assert now[2][0] == tA
def test_task_logical_clock_ordering(self): state = State() r = ev_logical_clock_ordering(state) tA, tB, tC = r.uids r.play() now = list(state.tasks_by_time()) self.assertEqual(now[0][0], tA) self.assertEqual(now[1][0], tC) self.assertEqual(now[2][0], tB) for _ in range(1000): shuffle(r.uids) tA, tB, tC = r.uids r.rewind_with_offset(r.current_clock + 1, r.uids) r.play() now = list(state.tasks_by_time()) self.assertEqual(now[0][0], tA) self.assertEqual(now[1][0], tC) self.assertEqual(now[2][0], tB)
def test_task_descending_clock_ordering(self): raise SkipTest('not working') state = State() r = ev_logical_clock_ordering(state) tA, tB, tC = r.uids r.play() now = list(state.tasks_by_time(reverse=False)) self.assertEqual(now[0][0], tA) self.assertEqual(now[1][0], tB) self.assertEqual(now[2][0], tC) for _ in range(1000): shuffle(r.uids) tA, tB, tC = r.uids r.rewind_with_offset(r.current_clock + 1, r.uids) r.play() now = list(state.tasks_by_time(reverse=False)) self.assertEqual(now[0][0], tB) self.assertEqual(now[1][0], tC) self.assertEqual(now[2][0], tA)
def test_limits_maxtasks(self): s = State(max_tasks_in_memory=1) s.heap_multiplier = 2 s.event( { "type": "task-unknown-event-xxx", "foo": "bar", "uuid": "x", "hostname": "y", "clock": 3, "timestamp": time(), "local_received": time(), } ) s.event( { "type": "task-unknown-event-xxx", "foo": "bar", "uuid": "y", "hostname": "y", "clock": 4, "timestamp": time(), "local_received": time(), } ) s.event( { "type": "task-unknown-event-xxx", "foo": "bar", "uuid": "z", "hostname": "y", "clock": 5, "timestamp": time(), "local_received": time(), } ) self.assertEqual(len(s._taskheap), 2) self.assertEqual(s._taskheap[0].clock, 4) self.assertEqual(s._taskheap[1].clock, 5) s._taskheap.append(s._taskheap[0]) self.assertTrue(list(s.tasks_by_time()))
def test_limits_maxtasks(self): s = State() s.max_tasks_in_memory = 1 s.task_event("task-unknown-event-xxx", {"foo": "bar", "uuid": "x", "hostname": "y", "clock": 3}) s.task_event("task-unknown-event-xxx", {"foo": "bar", "uuid": "y", "hostname": "y", "clock": 4}) s.task_event("task-unknown-event-xxx", {"foo": "bar", "uuid": "z", "hostname": "y", "clock": 5}) self.assertEqual(len(s._taskheap), 2) self.assertEqual(s._taskheap[0].clock, 4) self.assertEqual(s._taskheap[1].clock, 5) s._taskheap.append(s._taskheap[0]) self.assertTrue(list(s.tasks_by_time()))
def test_limits_maxtasks(self): s = State() s.max_tasks_in_memory = 1 s.task_event('task-unknown-event-xxx', {'foo': 'bar', 'uuid': 'x', 'hostname': 'y', 'clock': 3}) s.task_event('task-unknown-event-xxx', {'foo': 'bar', 'uuid': 'y', 'hostname': 'y', 'clock': 4}) s.task_event('task-unknown-event-xxx', {'foo': 'bar', 'uuid': 'z', 'hostname': 'y', 'clock': 5}) self.assertEqual(len(s._taskheap), 2) self.assertEqual(s._taskheap[0].clock, 4) self.assertEqual(s._taskheap[1].clock, 5) s._taskheap.append(s._taskheap[0]) self.assertTrue(list(s.tasks_by_time()))
def test_limits_maxtasks(self): s = State(max_tasks_in_memory=1) s.heap_multiplier = 2 s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'x', 'hostname': 'y', 'clock': 3, 'timestamp': time(), 'local_received': time(), }) s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'y', 'hostname': 'y', 'clock': 4, 'timestamp': time(), 'local_received': time(), }) s.event({ 'type': 'task-unknown-event-xxx', 'foo': 'bar', 'uuid': 'z', 'hostname': 'y', 'clock': 5, 'timestamp': time(), 'local_received': time(), }) assert len(s._taskheap) == 2 assert s._taskheap[0].clock == 4 assert s._taskheap[1].clock == 5 s._taskheap.append(s._taskheap[0]) assert list(s.tasks_by_time())
def test_freeze_while(self): s = State() r = ev_snapshot(s) r.play() def work(): pass s.freeze_while(work, clear_after=True) assert not s.event_count s2 = State() r = ev_snapshot(s2) r.play() s2.freeze_while(work, clear_after=False) assert s2.event_count