def test_survives_unknown_task_event(self):
s = State()
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'timestamp': time(),
'local_received': time(),
'clock': 0,
})
def test_task_states(self):
r = ev_task_states(State())
# RECEIVED
r.next()
self.assertTrue(r.uuid in r.state.tasks)
task = r.state.tasks[r.uuid]
self.assertEqual(task.state, "RECEIVED")
self.assertTrue(task.received)
self.assertEqual(task.timestamp, task.received)
self.assertEqual(task.worker.hostname, "utest1")
# STARTED
r.next()
self.assertTrue(r.state.workers["utest1"].alive,
"any task event adds worker heartbeat")
self.assertEqual(task.state, states.STARTED)
self.assertTrue(task.started)
self.assertEqual(task.timestamp, task.started)
self.assertEqual(task.worker.hostname, "utest1")
# SUCCESS
r.next()
self.assertEqual(task.state, states.SUCCESS)
self.assertTrue(task.succeeded)
self.assertEqual(task.timestamp, task.succeeded)
self.assertEqual(task.worker.hostname, "utest1")
self.assertEqual(task.result, "4")
self.assertEqual(task.runtime, 0.1234)
# FAILURE
r.next()
self.assertEqual(task.state, states.FAILURE)
self.assertTrue(task.failed)
self.assertEqual(task.timestamp, task.failed)
self.assertEqual(task.worker.hostname, "utest1")
self.assertEqual(task.exception, "KeyError('foo')")
self.assertEqual(task.traceback, "line 1 at main")
# RETRY
r.next()
self.assertEqual(task.state, states.RETRY)
self.assertTrue(task.retried)
self.assertEqual(task.timestamp, task.retried)
self.assertEqual(task.worker.hostname, "utest1")
self.assertEqual(task.exception, "KeyError('bar')")
self.assertEqual(task.traceback, "line 2 at main")
# REVOKED
r.next()
self.assertEqual(task.state, states.REVOKED)
self.assertTrue(task.revoked)
self.assertEqual(task.timestamp, task.revoked)
self.assertEqual(task.worker.hostname, "utest1")
def test_survives_unknown_worker_event(self):
s = State()
s.event({
'type': 'worker-unknown-event-xxx',
'foo': 'bar',
})
s.event({
'type': 'worker-unknown-event-xxx',
'hostname': 'xxx',
'foo': 'bar',
})
def test_task_states(self):
r = ev_task_states(State())
# RECEIVED
next(r)
self.assertTrue(r.tid in r.state.tasks)
task = r.state.tasks[r.tid]
self.assertEqual(task.state, states.RECEIVED)
self.assertTrue(task.received)
self.assertEqual(task.timestamp, task.received)
self.assertEqual(task.worker.hostname, 'utest1')
# STARTED
next(r)
self.assertTrue(r.state.workers['utest1'].alive,
'any task event adds worker heartbeat')
self.assertEqual(task.state, states.STARTED)
self.assertTrue(task.started)
self.assertEqual(task.timestamp, task.started)
self.assertEqual(task.worker.hostname, 'utest1')
# REVOKED
next(r)
self.assertEqual(task.state, states.REVOKED)
self.assertTrue(task.revoked)
self.assertEqual(task.timestamp, task.revoked)
self.assertEqual(task.worker.hostname, 'utest1')
# RETRY
next(r)
self.assertEqual(task.state, states.RETRY)
self.assertTrue(task.retried)
self.assertEqual(task.timestamp, task.retried)
self.assertEqual(task.worker.hostname, 'utest1')
self.assertEqual(task.exception, "KeyError('bar')")
self.assertEqual(task.traceback, 'line 2 at main')
# FAILURE
next(r)
self.assertEqual(task.state, states.FAILURE)
self.assertTrue(task.failed)
self.assertEqual(task.timestamp, task.failed)
self.assertEqual(task.worker.hostname, 'utest1')
self.assertEqual(task.exception, "KeyError('foo')")
self.assertEqual(task.traceback, 'line 1 at main')
# SUCCESS
next(r)
self.assertEqual(task.state, states.SUCCESS)
self.assertTrue(task.succeeded)
self.assertEqual(task.timestamp, task.succeeded)
self.assertEqual(task.worker.hostname, 'utest1')
self.assertEqual(task.result, '4')
self.assertEqual(task.runtime, 0.1234)
def test_survives_unknown_worker_leaving(self):
s = State(on_node_leave=Mock(name='on_node_leave'))
(worker, created), subject = s.event({
'type': 'worker-offline',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 301030134894833,
})
self.assertEqual(worker, Worker('*****@*****.**'))
self.assertFalse(created)
self.assertEqual(subject, 'offline')
self.assertNotIn('*****@*****.**', s.workers)
s.on_node_leave.assert_called_with(worker)
def test_on_node_join_callback(self):
s = State(on_node_join=Mock(name='on_node_join'))
(worker, created), subject = s.event({
'type': 'worker-online',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 34314,
})
assert worker
assert created
assert subject == 'online'
assert '*****@*****.**' in s.workers
s.on_node_join.assert_called_with(worker)
def test_get_or_create_worker__with_defaults(self):
state = State()
worker, created = state.get_or_create_worker(
'*****@*****.**', pid=30,
)
assert worker.hostname == '*****@*****.**'
assert worker.pid == 30
assert created
worker2, created2 = state.get_or_create_worker(
'*****@*****.**', pid=40,
)
assert worker2 is worker
assert worker2.pid == 40
assert not created2
def test_survives_unknown_worker_leaving(self):
s = State(on_node_leave=Mock(name='on_node_leave'))
(worker, created), subject = s.event({
'type': 'worker-offline',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 301030134894833,
})
assert worker == Worker('*****@*****.**')
assert not created
assert subject == 'offline'
assert '*****@*****.**' not in s.workers
s.on_node_leave.assert_called_with(worker)
def kill_airodump_proc():
app.control.revoke(
[uuid for uuid, _ in State().tasks_by_type('start_airodump')])
import os
folder = AIRODUMP_CSV_ROOT
for the_file in os.listdir(folder):
file_path = os.path.join(folder, the_file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
# elif os.path.isdir(file_path): shutil.rmtree(file_path)
except Exception as e:
print(e)
def test_on_node_join_callback(self):
s = State(on_node_join=Mock(name='on_node_join'))
(worker, created), subject = s.event({
'type': 'worker-online',
'hostname': '*****@*****.**',
'timestamp': time(),
'local_received': time(),
'clock': 34314,
})
self.assertTrue(worker)
self.assertTrue(created)
self.assertEqual(subject, 'online')
self.assertIn('*****@*****.**', s.workers)
s.on_node_join.assert_called_with(worker)
def test_clear(self):
r = ev_snapshot(State())
r.play()
self.assertTrue(r.state.event_count)
self.assertTrue(r.state.workers)
self.assertTrue(r.state.tasks)
self.assertTrue(r.state.task_count)
r.state.clear()
self.assertFalse(r.state.event_count)
self.assertFalse(r.state.workers)
self.assertTrue(r.state.tasks)
self.assertFalse(r.state.task_count)
r.state.clear(False)
self.assertFalse(r.state.tasks)
def test_get_or_create_worker__with_defaults(self):
state = State()
worker, created = state.get_or_create_worker(
'*****@*****.**',
pid=30,
)
self.assertEqual(worker.hostname, '*****@*****.**')
self.assertEqual(worker.pid, 30)
self.assertTrue(created)
worker2, created2 = state.get_or_create_worker(
'*****@*****.**',
pid=40,
)
self.assertIs(worker2, worker)
self.assertEqual(worker2.pid, 40)
self.assertFalse(created2)
def test_clear(self):
r = ev_snapshot(State())
r.play()
assert r.state.event_count
assert r.state.workers
assert r.state.tasks
assert r.state.task_count
r.state.clear()
assert not r.state.event_count
assert not r.state.workers
assert r.state.tasks
assert not r.state.task_count
r.state.clear(False)
assert not r.state.tasks
def __init__(self,
broker,
queue_output,
backend=None,
max_tasks_in_memory=None,
max_workers_in_memory=None):
"""Constructs an event listener instance.
Args:
broker (str): the broker being used by the celery system.
queue_output (Queue): to send to streaming dispatcher.
backend (str): the result backend being used by the celery system.
max_tasks_in_memory (int): max tasks stored
max_workers_in_memory (int): max workers stored
"""
self._app = Celery(broker=broker, backend=backend)
self._queue_output = queue_output
from celery.backends.base import DisabledBackend
self._use_result_backend = not isinstance(self._app.backend,
DisabledBackend)
logger.info('Creating %s: max_tasks=%d; max_workers=%d',
EventListener.__name__, max_tasks_in_memory,
max_workers_in_memory)
logger.info('Celery broker=%s; backend=%s; using_result_backend=%s',
broker, backend, self._use_result_backend)
# events handling: storage and filling missing states.
self.memory = State(
max_tasks_in_memory=max_tasks_in_memory,
max_workers_in_memory=max_workers_in_memory,
) # type: State
# running engine (should be asyncio in the future)
self._listener_thread = None # type:threading.Thread
self._celery_receiver = None # type:EventReceiver
# concurrency control
self._wait_event = threading.Event()
# detect shutdown.
def sigterm_handler(_signo, _stack_frame): # pragma: no cover
self.__stop()
signal.signal(signal.SIGTERM, sigterm_handler)
self.__start()
def test_task_descending_clock_ordering(self):
state = State()
r = ev_logical_clock_ordering(state)
tA, tB, tC = r.uids
r.play()
now = list(state.tasks_by_time(reverse=False))
assert now[0][0] == tA
assert now[1][0] == tB
assert now[2][0] == tC
for _ in range(1000):
shuffle(r.uids)
tA, tB, tC = r.uids
r.rewind_with_offset(r.current_clock + 1, r.uids)
r.play()
now = list(state.tasks_by_time(reverse=False))
assert now[0][0] == tB
assert now[1][0] == tC
assert now[2][0] == tA
def test_task_logical_clock_ordering(self):
state = State()
r = ev_logical_clock_ordering(state)
tA, tB, tC = r.uids
r.play()
now = list(state.tasks_by_time())
self.assertEqual(now[0][0], tA)
self.assertEqual(now[1][0], tC)
self.assertEqual(now[2][0], tB)
for _ in range(1000):
shuffle(r.uids)
tA, tB, tC = r.uids
r.rewind_with_offset(r.current_clock + 1, r.uids)
r.play()
now = list(state.tasks_by_time())
self.assertEqual(now[0][0], tA)
self.assertEqual(now[1][0], tC)
self.assertEqual(now[2][0], tB)
def test_task_descending_clock_ordering(self):
raise SkipTest('not working')
state = State()
r = ev_logical_clock_ordering(state)
tA, tB, tC = r.uids
r.play()
now = list(state.tasks_by_time(reverse=False))
self.assertEqual(now[0][0], tA)
self.assertEqual(now[1][0], tB)
self.assertEqual(now[2][0], tC)
for _ in range(1000):
shuffle(r.uids)
tA, tB, tC = r.uids
r.rewind_with_offset(r.current_clock + 1, r.uids)
r.play()
now = list(state.tasks_by_time(reverse=False))
self.assertEqual(now[0][0], tB)
self.assertEqual(now[1][0], tC)
self.assertEqual(now[2][0], tA)
def evtop():
sys.stderr.write("-> evtop: starting capture...\n")
state = State()
display = CursesMonitor(state)
display.init_screen()
refresher = DisplayThread(display)
refresher.start()
conn = establish_connection()
recv = EventReceiver(conn, handlers={"*": state.event})
try:
recv.capture(limit=None)
except Exception:
refresher.shutdown = True
refresher.join()
display.resetscreen()
raise
except (KeyboardInterrupt, SystemExit):
conn and conn.close()
refresher.shutdown = True
refresher.join()
display.resetscreen()
def test_deepcopy(self):
import copy
s = State()
s.event({
'type': 'task-success',
'root_id': 'x',
'uuid': 'x',
'hostname': 'y',
'clock': 3,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-success',
'root_id': 'y',
'uuid': 'y',
'hostname': 'y',
'clock': 4,
'timestamp': time(),
'local_received': time(),
})
copy.deepcopy(s)
def evcam(camera, freq=1.0, maxrate=None, loglevel=0,
logfile=None):
if not isinstance(loglevel, int):
loglevel = conf.LOG_LEVELS[loglevel.upper()]
logger = log.setup_logger(loglevel=loglevel,
logfile=logfile,
name="celery.evcam")
logger.info(
"-> evcam: Taking snapshots with %s (every %s secs.)\n" % (
camera, freq))
state = State()
cam = instantiate(camera, state,
freq=freq, maxrate=maxrate, logger=logger)
cam.install()
conn = establish_connection()
recv = EventReceiver(conn, handlers={"*": state.event})
try:
try:
recv.capture(limit=None)
except KeyboardInterrupt:
raise SystemExit
finally:
cam.cancel()
conn.close()
def test_limits_maxtasks(self):
s = State(max_tasks_in_memory=1)
s.heap_multiplier = 2
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'x',
'hostname': 'y',
'clock': 3,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'y',
'hostname': 'y',
'clock': 4,
'timestamp': time(),
'local_received': time(),
})
s.event({
'type': 'task-unknown-event-xxx',
'foo': 'bar',
'uuid': 'z',
'hostname': 'y',
'clock': 5,
'timestamp': time(),
'local_received': time(),
})
assert len(s._taskheap) == 2
assert s._taskheap[0].clock == 4
assert s._taskheap[1].clock == 5
s._taskheap.append(s._taskheap[0])
assert list(s.tasks_by_time())
def test_constructor(self):
cam = self.Camera(State())
self.assertTrue(cam.state)
self.assertTrue(cam.freq)
self.assertTrue(cam.cleanup_freq)
self.assertTrue(cam.logger)
def setUp(self):
self.state = State()
self.cam = self.Camera(self.state)
def test_alive_workers(self):
r = ev_snapshot(State())
r.play()
assert len(list(r.state.alive_workers())) == 3
def test_repr(self):
assert repr(State())
def test_tasks_by_time(self):
r = ev_snapshot(State())
r.play()
assert len(list(r.state.tasks_by_time())) == 20
assert len(list(r.state.tasks_by_time(reverse=False))) == 20
def test_task_types(self):
r = ev_snapshot(State())
r.play()
assert sorted(r.state.task_types()) == ['task1', 'task2']
def test_pickleable(self):
state = State()
r = ev_logical_clock_ordering(state)
r.play()
assert pickle.loads(pickle.dumps(state))
def test_task_states(self):
r = ev_task_states(State())
# RECEIVED
next(r)
assert r.tid in r.state.tasks
task = r.state.tasks[r.tid]
assert task.state == states.RECEIVED
assert task.received
assert task.timestamp == task.received
assert task.worker.hostname == 'utest1'
# STARTED
next(r)
assert r.state.workers['utest1'].alive
assert task.state == states.STARTED
assert task.started
assert task.timestamp == task.started
assert task.worker.hostname == 'utest1'
# REVOKED
next(r)
assert task.state == states.REVOKED
assert task.revoked
assert task.timestamp == task.revoked
assert task.worker.hostname == 'utest1'
# RETRY
next(r)
assert task.state == states.RETRY
assert task.retried
assert task.timestamp == task.retried
assert task.worker.hostname, 'utest1'
assert task.exception == "KeyError('bar')"
assert task.traceback == 'line 2 at main'
# FAILURE
next(r)
assert task.state == states.FAILURE
assert task.failed
assert task.timestamp == task.failed
assert task.worker.hostname == 'utest1'
assert task.exception == "KeyError('foo')"
assert task.traceback == 'line 1 at main'
# SUCCESS
next(r)
assert task.state == states.SUCCESS
assert task.succeeded
assert task.timestamp == task.succeeded
assert task.worker.hostname == 'utest1'
assert task.result == '4'
assert task.runtime == 0.1234
# children, parent, root
r.play()
assert r.tid2 in r.state.tasks
task2 = r.state.tasks[r.tid2]
assert task2.parent is task
assert task2.root is task
assert task2 in task.children
def test_itertasks(self):
s = State()
s.tasks = {'a': 'a', 'b': 'b', 'c': 'c', 'd': 'd'}
assert len(list(s.itertasks(limit=2))) == 2
