def setup(self): self.events = [ Event("worker-heartbeat", hostname="utest1", timestamp=time() - HEARTBEAT_EXPIRE * 2), Event("worker-heartbeat", hostname="utest1"), ]
def test_metrics(self): state = EventsState() worker_name = 'worker1' task_name = 'task1' state.get_or_create_worker(worker_name) events = [ Event('worker-online', hostname=worker_name), Event('worker-heartbeat', hostname=worker_name, active=1) ] events += task_succeeded_events(worker=worker_name, name=task_name, id='123') for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state metrics = self.get('/metrics').body.decode('utf-8') events = dict( re.findall( 'flower_events_total{task="task1",type="(task-.*)",worker="worker1"} (.*)', metrics)) self.assertTrue('task-received' in events) self.assertTrue('task-started' in events) self.assertTrue('task-succeeded' in events) self.assertTrue( f'flower_worker_online{{worker="{worker_name}"}} 1.0' in metrics) self.assertTrue( f'flower_worker_number_of_currently_executing_tasks{{worker="{worker_name}"}} 1.0' in metrics)
def test_process_event(self, mocker, app): monitor = MonitorThread(app=app) task_uuid = uuid.uuid4() hostname = 'myhost' local_received = time() latency_before_started = 123.45 runtime = 234.5 event_kwargs = { 'uuid': task_uuid, 'name': self.TASK_NAME, 'hostname': hostname, } monitor._process_event(Event( 'task-received', **event_kwargs, args='()', kwargs='{}', retries=0, eta=None, clock=0, local_received=local_received) ) self.assert_only_states({celery.states.RECEIVED}) monitor._process_event(Event( 'task-started', **event_kwargs, clock=1, local_received=local_received + latency_before_started) ) self.assert_only_states({celery.states.STARTED}) monitor._process_event(Event( 'task-succeeded', **event_kwargs, runtime=runtime, clock=2, local_received=local_received + latency_before_started + runtime)) self.assert_only_states({celery.states.SUCCESS})
def test_tasks_events(self): task_uuid = uuid() hostname = 'myhost' local_received = time() latency_before_started = 123.45 runtime = 234.5 m = MonitorThread(app=self.app) self._assert_task_states(celery.states.ALL_STATES, 0) assert REGISTRY.get_sample_value('celery_task_latency_count') == 0 assert REGISTRY.get_sample_value('celery_task_latency_sum') == 0 m._process_event(Event( 'task-received', uuid=task_uuid, name=self.task, args='()', kwargs='{}', retries=0, eta=None, hostname=hostname, clock=0, local_received=local_received)) self._assert_all_states({celery.states.RECEIVED}) m._process_event(Event( 'task-started', uuid=task_uuid, hostname=hostname, clock=1, name=self.task, local_received=local_received + latency_before_started)) self._assert_all_states({celery.states.STARTED}) m._process_event(Event( 'task-succeeded', uuid=task_uuid, result='42', runtime=runtime, hostname=hostname, clock=2, local_received=local_received + latency_before_started + runtime)) self._assert_all_states({celery.states.SUCCESS}) assert REGISTRY.get_sample_value('celery_task_latency_count') == 1 self.assertAlmostEqual(REGISTRY.get_sample_value( 'celery_task_latency_sum'), latency_before_started)
def test_task_received(self): state = EventsState() state.get_or_create_worker('worker1') state.get_or_create_worker('worker2') events = [ Event('worker-online', hostname='worker1'), Event('worker-online', hostname='worker2'), Event('task-received', uuid=uuid(), name='task1', args='(2, 2)', kwargs="{'foo': 'bar'}", retries=0, eta=None, hostname='worker1') ] for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state r = self.get('/dashboard') table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(2, len(table.rows())) self.assertEqual(['worker1', 'True', '0', '1', '0', '0', '0', None], table.get_row('worker1')) self.assertEqual(['worker2', 'True', '0', '0', '0', '0', '0', None], table.get_row('worker2'))
def test_on_shutter(self): state = self.state cam = self.cam ws = ['worker1.ex.com', 'worker2.ex.com', 'worker3.ex.com'] uus = [gen_unique_id() for i in xrange(50)] events = [Event('worker-online', hostname=ws[0]), Event('worker-online', hostname=ws[1]), Event('worker-online', hostname=ws[2]), Event('task-received', uuid=uus[0], name='A', hostname=ws[0]), Event('task-started', uuid=uus[0], name='A', hostname=ws[0]), Event('task-received', uuid=uus[1], name='B', hostname=ws[1]), Event('task-revoked', uuid=uus[2], name='C', hostname=ws[2])] for event in events: event['local_received'] = time() state.event(event) cam.on_shutter(state) for host in ws: worker = models.WorkerState.objects.get(hostname=host) self.assertTrue(worker.is_alive()) t1 = models.TaskState.objects.get(task_id=uus[0]) self.assertEqual(t1.state, 'STARTED') self.assertEqual(t1.name, 'A') t2 = models.TaskState.objects.get(task_id=uus[1]) self.assertEqual(t2.state, 'RECEIVED') t3 = models.TaskState.objects.get(task_id=uus[2]) self.assertEqual(t3.state, 'REVOKED') events = [Event('task-succeeded', uuid=uus[0], hostname=ws[0], result=42), Event('task-failed', uuid=uus[1], exception="KeyError('foo')", hostname=ws[1]), Event('worker-offline', hostname=ws[0])] map(state.event, events) cam._last_worker_write.clear() cam.on_shutter(state) w1 = models.WorkerState.objects.get(hostname=ws[0]) self.assertFalse(w1.is_alive()) t1 = models.TaskState.objects.get(task_id=uus[0]) self.assertEqual(t1.state, 'SUCCESS') self.assertEqual(t1.result, '42') self.assertEqual(t1.worker, w1) t2 = models.TaskState.objects.get(task_id=uus[1]) self.assertEqual(t2.state, 'FAILURE') self.assertEqual(t2.result, "KeyError('foo')") self.assertEqual(t2.worker.hostname, ws[1]) cam.on_shutter(state)
class ev_worker_heartbeats(replay): events = [ Event("worker-heartbeat", hostname="utest1", timestamp=time() - HEARTBEAT_EXPIRE * 2), Event("worker-heartbeat", hostname="utest1"), ]
def setup(self): self.events = [ Event('worker-heartbeat', hostname='utest1', timestamp=time() - HEARTBEAT_EXPIRE_WINDOW * 2), Event('worker-heartbeat', hostname='utest1'), ]
def test_on_shutter(self): state = self.state cam = self.cam ws = ["worker1.ex.com", "worker2.ex.com", "worker3.ex.com"] uus = [gen_unique_id() for i in xrange(50)] events = [Event("worker-online", hostname=ws[0]), Event("worker-online", hostname=ws[1]), Event("worker-online", hostname=ws[2]), Event("task-received", uuid=uus[0], name="A", hostname=ws[0]), Event("task-started", uuid=uus[0], name="A", hostname=ws[0]), Event("task-received", uuid=uus[1], name="B", hostname=ws[1]), Event("task-revoked", uuid=uus[2], name="C", hostname=ws[2])] map(state.event, events) cam.on_shutter(state) for host in ws: worker = models.WorkerState.objects.get(hostname=host) self.assertTrue(worker.is_alive()) t1 = models.TaskState.objects.get(task_id=uus[0]) self.assertEqual(t1.state, "STARTED") self.assertEqual(t1.name, "A") t2 = models.TaskState.objects.get(task_id=uus[1]) self.assertEqual(t2.state, "RECEIVED") t3 = models.TaskState.objects.get(task_id=uus[2]) self.assertEqual(t3.state, "REVOKED") events = [Event("task-succeeded", uuid=uus[0], hostname=ws[0], result=42), Event("task-failed", uuid=uus[1], exception="KeyError('foo')", hostname=ws[1]), Event("worker-offline", hostname=ws[0])] map(state.event, events) cam._last_worker_write.clear() cam.on_shutter(state) w1 = models.WorkerState.objects.get(hostname=ws[0]) self.assertFalse(w1.is_alive()) t1 = models.TaskState.objects.get(task_id=uus[0]) self.assertEqual(t1.state, "SUCCESS") self.assertEqual(t1.result, u'42') self.assertEqual(t1.worker, w1) t2 = models.TaskState.objects.get(task_id=uus[1]) self.assertEqual(t2.state, "FAILURE") self.assertEqual(t2.result, u"KeyError('foo')") self.assertEqual(t2.worker.hostname, ws[1]) cam.on_shutter(state)
class ev_snapshot(replay): events = [ Event("worker-online", hostname="utest1"), Event("worker-online", hostname="utest2"), Event("worker-online", hostname="utest3"), ] for i in range(20): worker = not i % 2 and "utest2" or "utest1" type = not i % 2 and "task2" or "task1" events.append(Event("task-received", name=type, uuid=gen_unique_id(), hostname=worker))
def setup(self): self.events = [ Event('worker-online', hostname='utest1'), Event('worker-online', hostname='utest2'), Event('worker-online', hostname='utest3'), ] for i in range(20): worker = not i % 2 and 'utest2' or 'utest1' type = not i % 2 and 'task2' or 'task1' self.events.append(Event('task-received', name=type, uuid=uuid(), hostname=worker))
def setup(self): self.events = [ Event("worker-online", hostname="utest1"), Event("worker-online", hostname="utest2"), Event("worker-online", hostname="utest3"), ] for i in range(20): worker = not i % 2 and "utest2" or "utest1" type = not i % 2 and "task2" or "task1" self.events.append( Event("task-received", name=type, uuid=uuid(), hostname=worker))
def test_success_cycle(self): worker = 'headworker1' task_id = uuid() state = EventsState() state.get_or_create_worker(worker) events = [Event('worker-online', hostname=worker)] cycle = '20000101_0000z' cycle_events = cycle_task(worker, cycle, 'SUCCESS', id=task_id) events.extend(cycle_events) for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state r = self.get('/cycles') self.assertEqual(200, r.code) self.assertIn('/task/' + task_id, r.body.decode("utf-8")) params = dict(draw=1, start=0, length=10) params['search[value]'] = '' params['order[0][column]'] = 0 params['columns[0][data]'] = 'name' params['order[0][dir]'] = 'asc' r = self.get('/cycles/datatable?' + '&'.join(map(lambda x: '%s=%s' % x, params.items()))) table = json.loads(r.body.decode("utf-8")) self.assertEqual(200, r.code) self.assertEqual(0, table['recordsTotal'])
def __init__(self): """Initializes celery events and dispatchers """ self.app = app_or_default() self.event = Event(type='chatrooms') self.dispatcher = self.app.events.Dispatcher( connection=self.app.broker_connection(), enabled=True)
def QTEV(type, uuid, hostname, clock, timestamp=None): """Quick task event.""" return Event('task-{0}'.format(type), uuid=uuid, hostname=hostname, clock=clock, timestamp=timestamp or time())
def test_failed_task(self): state = EventsState() state.get_or_create_worker('worker1') events = [Event('worker-online', hostname='worker1')] events += task_failed_events(worker='worker1', name='task1', id='123') for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state params = dict(draw=1, start=0, length=10) params['search[value]'] = '' params['order[0][column]'] = 0 params['columns[0][data]'] = 'name' params['order[0][dir]'] = 'asc' r = self.get('/tasks/datatable?' + '&'.join(map(lambda x: '%s=%s' % x, params.items()))) table = json.loads(r.body.decode("utf-8")) self.assertEqual(200, r.code) self.assertEqual(1, table['recordsTotal']) self.assertEqual(1, table['recordsFiltered']) tasks = table['data'] self.assertEqual(1, len(tasks)) self.assertEqual('FAILURE', tasks[0]['state']) self.assertEqual('task1', tasks[0]['name']) self.assertEqual('123', tasks[0]['uuid']) self.assertEqual('worker1', tasks[0]['worker'])
def test_running_cycle_with_children_subprocess(self): worker = 'headworker1' state = EventsState() state.get_or_create_worker(worker) cycle_id = uuid() events = [Event('worker-online', hostname=worker)] cycle = '20000101_0000z' cycle_events = cycle_task(worker, cycle, 'RUNNING', id=cycle_id) child_id = uuid() child_task_events = task_succeeded_events(worker, child_id, 'wrappers.SubprocessTask', kwargs={ 'action_id': 'test_action', 'cycle_dt': cycle, 'parent': cycle_id }) events.extend(cycle_events) events.extend(child_task_events) for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state r = self.get('/cycles') self.assertEqual(200, r.code) self.assertIn('/task/' + cycle_id, r.body.decode("utf-8")) params = dict(draw=1, start=0, length=10, selected='previous') params['search[value]'] = '' params['order[0][column]'] = 0 params['columns[0][data]'] = 'name' params['order[0][dir]'] = 'asc' r = self.get('/cycles/datatable?' + '&'.join(map(lambda x: '%s=%s' % x, params.items()))) table = json.loads(r.body.decode("utf-8")) self.assertEqual(200, r.code) self.assertEqual(0, table['recordsTotal']) params = dict(draw=1, start=0, length=10, selected='active') params['search[value]'] = '' params['order[0][column]'] = 0 params['columns[0][data]'] = 'name' params['order[0][dir]'] = 'asc' r = self.get('/cycles/datatable?' + '&'.join(map(lambda x: '%s=%s' % x, params.items()))) table = json.loads(r.body.decode("utf-8")) self.assertEqual(200, r.code) self.assertEqual(1, table['recordsTotal']) tasks = table['data'] self.assertEqual('SUCCESS', tasks[0]['state']) self.assertEqual('wrappers.SubprocessTask', tasks[0]['name']) self.assertEqual(child_id, tasks[0]['uuid']) self.assertEqual(worker, tasks[0]['worker'])
def QTEV(type, uuid, hostname, clock, name=None, timestamp=None): """Quick task event.""" return Event(f'task-{type}', uuid=uuid, hostname=hostname, clock=clock, name=name, timestamp=timestamp or time())
def setup(self): tid = self.tid = uuid() self.events = [ Event('task-received', uuid=tid, name='task1', args='(2, 2)', kwargs="{'foo': 'bar'}", retries=0, eta=None, hostname='utest1'), Event('task-started', uuid=tid, hostname='utest1'), Event('task-revoked', uuid=tid, hostname='utest1'), Event('task-retried', uuid=tid, exception="KeyError('bar')", traceback='line 2 at main', hostname='utest1'), Event('task-failed', uuid=tid, exception="KeyError('foo')", traceback='line 1 at main', hostname='utest1'), Event('task-succeeded', uuid=tid, result='4', runtime=0.1234, hostname='utest1'), Event('foo-bar'), ]
def test_no_received_event(self, app): monitor = MonitorThread(app=app) monitor._process_event(Event( 'task-started', uuid=uuid.uuid4(), name=self.TASK_NAME, hostname='myhost', clock=1, local_received=time()) ) self.assert_only_states({celery.states.STARTED})
def task_succeeded_events(worker, id=None, name=None): id = id or uuid() name = name or 'sometask' return [ Event('task-received', uuid=id, name=name, args='(2, 2)', kwargs="{'foo': 'bar'}", retries=0, eta=None, hostname=worker), Event('task-started', uuid=id, hostname=worker), Event('task-succeeded', uuid=id, result='4', runtime=0.1234, hostname=worker) ]
def task_failed_events(worker, id=None, name=None): id = id or uuid() name = name or 'sometask' return [ Event('task-received', uuid=id, name=name, args='(2, 2)', kwargs="{'foo': 'bar'}", retries=0, eta=None, hostname=worker), Event('task-started', uuid=id, hostname=worker), Event('task-failed', uuid=id, exception="KeyError('foo')", traceback='line 1 at main', hostname=worker) ]
class ev_task_states(replay): uuid = gen_unique_id() events = [ Event("task-received", uuid=uuid, name="task1", args="(2, 2)", kwargs="{'foo': 'bar'}", retries=0, eta=None, hostname="utest1"), Event("task-started", uuid=uuid, hostname="utest1"), Event("task-revoked", uuid=uuid, hostname="utest1"), Event("task-retried", uuid=uuid, exception="KeyError('bar')", traceback="line 2 at main", hostname="utest1"), Event("task-failed", uuid=uuid, exception="KeyError('foo')", traceback="line 1 at main", hostname="utest1"), Event("task-succeeded", uuid=uuid, result="4", runtime=0.1234, hostname="utest1"), ]
def setup(self): tid = self.tid = uuid() self.events = [ Event("task-received", uuid=tid, name="task1", args="(2, 2)", kwargs="{'foo': 'bar'}", retries=0, eta=None, hostname="utest1"), Event("task-started", uuid=tid, hostname="utest1"), Event("task-revoked", uuid=tid, hostname="utest1"), Event("task-retried", uuid=tid, exception="KeyError('bar')", traceback="line 2 at main", hostname="utest1"), Event("task-failed", uuid=tid, exception="KeyError('foo')", traceback="line 1 at main", hostname="utest1"), Event("task-succeeded", uuid=tid, result="4", runtime=0.1234, hostname="utest1"), ]
def test_purge_offline_workers(self): state = EventsState() state.get_or_create_worker('worker1') state.event( Event('worker-online', hostname='worker1', local_received=time.time())) state.event( Event('worker-offline', hostname='worker1', local_received=time.time())) self.app.events.state = state with patch('flower.views.dashboard.options') as mock_options: mock_options.purge_offline_workers = 0 r = self.get('/dashboard') table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(0, len(table.rows()))
def test_exception(self, app): monitor = MonitorThread(app=app) monitor._process_event(Event( 'task-failed', uuid=uuid.uuid4(), name=self.TASK_NAME, hostname='myhost', clock=2, local_received=time(), exception='ZeroDivisionError()') ) exc_count = REGISTRY.get_sample_value( 'celery_task_exceptions', labels={'name': 'unknown', 'exception': 'ZeroDivisionError'} ) assert exc_count == 1
def test_single_workers_offline(self): state = EventsState() state.get_or_create_worker('worker1') state.event( Event('worker-online', hostname='worker1', local_received=time.time())) state.event( Event('worker-offline', hostname='worker1', local_received=time.time())) self.app.events.state = state r = self.get('/dashboard') table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(1, len(table.rows())) self.assertTrue(table.get_row('worker1')) self.assertEqual(['worker1', 'False', '0', '0', '0', '0', '0', None], table.get_row('worker1')) self.assertFalse(table.get_row('worker2'))
def test_worker_prefetched_tasks_metric(self): state = EventsState() worker_name = 'worker2' task_name = 'task1' task_id = uuid() state.get_or_create_worker(worker_name) events = [ Event('task-received', uuid=task_id, name=task_name, args='(2, 2)', kwargs="{'foo': 'bar'}", retries=1, eta=None, hostname=worker_name), Event('task-received', uuid=uuid(), name=task_name, args='(2, 2)', kwargs="{'foo': 'bar'}", retries=1, eta=None, hostname=worker_name), Event('task-started', uuid=task_id, hostname=worker_name), ] for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state metrics = self.get('/metrics').body.decode('utf-8') self.assertTrue( f'flower_worker_prefetched_tasks{{task="{task_name}",worker="{worker_name}"}} 1.0' in metrics)
def test_tasks(self): state = EventsState() state.get_or_create_worker('worker1') state.get_or_create_worker('worker2') state.get_or_create_worker('worker3') events = [ Event('worker-online', hostname='worker1'), Event('worker-online', hostname='worker2') ] for i in range(100): events += task_succeeded_events(worker='worker1') for i in range(10): events += task_succeeded_events(worker='worker3') for i in range(13): events += task_failed_events(worker='worker3') for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state r = self.get('/dashboard') table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(3, len(table.rows())) self.assertEqual( ['worker1', 'True', '0', '100', '0', '100', '0', None], table.get_row('worker1')) self.assertEqual(['worker2', 'True', '0', '0', '0', '0', '0', None], table.get_row('worker2')) self.assertEqual(['worker3', 'True', '0', '23', '13', '10', '0', None], table.get_row('worker3'))
def test_worker_online_metric_worker_is_offline(self): state = EventsState() worker_name = 'worker1' state.get_or_create_worker(worker_name) events = [Event('worker-offline', hostname=worker_name)] for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state metrics = self.get('/metrics').body.decode('utf-8') self.assertTrue( f'flower_worker_online{{worker="{worker_name}"}} 0.0' in metrics)