def setup(self):
self.events = [
Event("worker-heartbeat",
hostname="utest1",
timestamp=time() - HEARTBEAT_EXPIRE * 2),
Event("worker-heartbeat", hostname="utest1"),
]
def test_metrics(self):
state = EventsState()
worker_name = 'worker1'
task_name = 'task1'
state.get_or_create_worker(worker_name)
events = [
Event('worker-online', hostname=worker_name),
Event('worker-heartbeat', hostname=worker_name, active=1)
]
events += task_succeeded_events(worker=worker_name,
name=task_name,
id='123')
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
metrics = self.get('/metrics').body.decode('utf-8')
events = dict(
re.findall(
'flower_events_total{task="task1",type="(task-.*)",worker="worker1"} (.*)',
metrics))
self.assertTrue('task-received' in events)
self.assertTrue('task-started' in events)
self.assertTrue('task-succeeded' in events)
self.assertTrue(
f'flower_worker_online{{worker="{worker_name}"}} 1.0' in metrics)
self.assertTrue(
f'flower_worker_number_of_currently_executing_tasks{{worker="{worker_name}"}} 1.0'
in metrics)
def test_process_event(self, mocker, app):
monitor = MonitorThread(app=app)
task_uuid = uuid.uuid4()
hostname = 'myhost'
local_received = time()
latency_before_started = 123.45
runtime = 234.5
event_kwargs = {
'uuid': task_uuid,
'name': self.TASK_NAME,
'hostname': hostname,
}
monitor._process_event(Event(
'task-received', **event_kwargs,
args='()', kwargs='{}', retries=0, eta=None, clock=0, local_received=local_received)
)
self.assert_only_states({celery.states.RECEIVED})
monitor._process_event(Event(
'task-started', **event_kwargs,
clock=1, local_received=local_received + latency_before_started)
)
self.assert_only_states({celery.states.STARTED})
monitor._process_event(Event(
'task-succeeded', **event_kwargs,
runtime=runtime, clock=2,
local_received=local_received + latency_before_started + runtime))
self.assert_only_states({celery.states.SUCCESS})
def test_tasks_events(self):
task_uuid = uuid()
hostname = 'myhost'
local_received = time()
latency_before_started = 123.45
runtime = 234.5
m = MonitorThread(app=self.app)
self._assert_task_states(celery.states.ALL_STATES, 0)
assert REGISTRY.get_sample_value('celery_task_latency_count') == 0
assert REGISTRY.get_sample_value('celery_task_latency_sum') == 0
m._process_event(Event(
'task-received', uuid=task_uuid, name=self.task,
args='()', kwargs='{}', retries=0, eta=None, hostname=hostname,
clock=0,
local_received=local_received))
self._assert_all_states({celery.states.RECEIVED})
m._process_event(Event(
'task-started', uuid=task_uuid, hostname=hostname,
clock=1, name=self.task,
local_received=local_received + latency_before_started))
self._assert_all_states({celery.states.STARTED})
m._process_event(Event(
'task-succeeded', uuid=task_uuid, result='42',
runtime=runtime, hostname=hostname, clock=2,
local_received=local_received + latency_before_started + runtime))
self._assert_all_states({celery.states.SUCCESS})
assert REGISTRY.get_sample_value('celery_task_latency_count') == 1
self.assertAlmostEqual(REGISTRY.get_sample_value(
'celery_task_latency_sum'), latency_before_started)
def test_task_received(self):
state = EventsState()
state.get_or_create_worker('worker1')
state.get_or_create_worker('worker2')
events = [
Event('worker-online', hostname='worker1'),
Event('worker-online', hostname='worker2'),
Event('task-received',
uuid=uuid(),
name='task1',
args='(2, 2)',
kwargs="{'foo': 'bar'}",
retries=0,
eta=None,
hostname='worker1')
]
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
r = self.get('/dashboard')
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(2, len(table.rows()))
self.assertEqual(['worker1', 'True', '0', '1', '0', '0', '0', None],
table.get_row('worker1'))
self.assertEqual(['worker2', 'True', '0', '0', '0', '0', '0', None],
table.get_row('worker2'))
def test_on_shutter(self):
state = self.state
cam = self.cam
ws = ['worker1.ex.com', 'worker2.ex.com', 'worker3.ex.com']
uus = [gen_unique_id() for i in xrange(50)]
events = [Event('worker-online', hostname=ws[0]),
Event('worker-online', hostname=ws[1]),
Event('worker-online', hostname=ws[2]),
Event('task-received',
uuid=uus[0], name='A', hostname=ws[0]),
Event('task-started',
uuid=uus[0], name='A', hostname=ws[0]),
Event('task-received',
uuid=uus[1], name='B', hostname=ws[1]),
Event('task-revoked',
uuid=uus[2], name='C', hostname=ws[2])]
for event in events:
event['local_received'] = time()
state.event(event)
cam.on_shutter(state)
for host in ws:
worker = models.WorkerState.objects.get(hostname=host)
self.assertTrue(worker.is_alive())
t1 = models.TaskState.objects.get(task_id=uus[0])
self.assertEqual(t1.state, 'STARTED')
self.assertEqual(t1.name, 'A')
t2 = models.TaskState.objects.get(task_id=uus[1])
self.assertEqual(t2.state, 'RECEIVED')
t3 = models.TaskState.objects.get(task_id=uus[2])
self.assertEqual(t3.state, 'REVOKED')
events = [Event('task-succeeded',
uuid=uus[0], hostname=ws[0], result=42),
Event('task-failed',
uuid=uus[1], exception="KeyError('foo')",
hostname=ws[1]),
Event('worker-offline', hostname=ws[0])]
map(state.event, events)
cam._last_worker_write.clear()
cam.on_shutter(state)
w1 = models.WorkerState.objects.get(hostname=ws[0])
self.assertFalse(w1.is_alive())
t1 = models.TaskState.objects.get(task_id=uus[0])
self.assertEqual(t1.state, 'SUCCESS')
self.assertEqual(t1.result, '42')
self.assertEqual(t1.worker, w1)
t2 = models.TaskState.objects.get(task_id=uus[1])
self.assertEqual(t2.state, 'FAILURE')
self.assertEqual(t2.result, "KeyError('foo')")
self.assertEqual(t2.worker.hostname, ws[1])
cam.on_shutter(state)
class ev_worker_heartbeats(replay):
events = [
Event("worker-heartbeat",
hostname="utest1",
timestamp=time() - HEARTBEAT_EXPIRE * 2),
Event("worker-heartbeat", hostname="utest1"),
]
def setup(self):
self.events = [
Event('worker-heartbeat',
hostname='utest1',
timestamp=time() - HEARTBEAT_EXPIRE_WINDOW * 2),
Event('worker-heartbeat', hostname='utest1'),
]
def test_on_shutter(self):
state = self.state
cam = self.cam
ws = ["worker1.ex.com", "worker2.ex.com", "worker3.ex.com"]
uus = [gen_unique_id() for i in xrange(50)]
events = [Event("worker-online", hostname=ws[0]),
Event("worker-online", hostname=ws[1]),
Event("worker-online", hostname=ws[2]),
Event("task-received", uuid=uus[0], name="A",
hostname=ws[0]),
Event("task-started", uuid=uus[0], name="A",
hostname=ws[0]),
Event("task-received", uuid=uus[1], name="B",
hostname=ws[1]),
Event("task-revoked", uuid=uus[2], name="C",
hostname=ws[2])]
map(state.event, events)
cam.on_shutter(state)
for host in ws:
worker = models.WorkerState.objects.get(hostname=host)
self.assertTrue(worker.is_alive())
t1 = models.TaskState.objects.get(task_id=uus[0])
self.assertEqual(t1.state, "STARTED")
self.assertEqual(t1.name, "A")
t2 = models.TaskState.objects.get(task_id=uus[1])
self.assertEqual(t2.state, "RECEIVED")
t3 = models.TaskState.objects.get(task_id=uus[2])
self.assertEqual(t3.state, "REVOKED")
events = [Event("task-succeeded", uuid=uus[0],
hostname=ws[0],
result=42),
Event("task-failed", uuid=uus[1],
exception="KeyError('foo')",
hostname=ws[1]),
Event("worker-offline", hostname=ws[0])]
map(state.event, events)
cam._last_worker_write.clear()
cam.on_shutter(state)
w1 = models.WorkerState.objects.get(hostname=ws[0])
self.assertFalse(w1.is_alive())
t1 = models.TaskState.objects.get(task_id=uus[0])
self.assertEqual(t1.state, "SUCCESS")
self.assertEqual(t1.result, u'42')
self.assertEqual(t1.worker, w1)
t2 = models.TaskState.objects.get(task_id=uus[1])
self.assertEqual(t2.state, "FAILURE")
self.assertEqual(t2.result, u"KeyError('foo')")
self.assertEqual(t2.worker.hostname, ws[1])
cam.on_shutter(state)
class ev_snapshot(replay):
events = [
Event("worker-online", hostname="utest1"),
Event("worker-online", hostname="utest2"),
Event("worker-online", hostname="utest3"),
]
for i in range(20):
worker = not i % 2 and "utest2" or "utest1"
type = not i % 2 and "task2" or "task1"
events.append(Event("task-received", name=type,
uuid=gen_unique_id(), hostname=worker))
def setup(self):
self.events = [
Event('worker-online', hostname='utest1'),
Event('worker-online', hostname='utest2'),
Event('worker-online', hostname='utest3'),
]
for i in range(20):
worker = not i % 2 and 'utest2' or 'utest1'
type = not i % 2 and 'task2' or 'task1'
self.events.append(Event('task-received', name=type,
uuid=uuid(), hostname=worker))
def setup(self):
self.events = [
Event("worker-online", hostname="utest1"),
Event("worker-online", hostname="utest2"),
Event("worker-online", hostname="utest3"),
]
for i in range(20):
worker = not i % 2 and "utest2" or "utest1"
type = not i % 2 and "task2" or "task1"
self.events.append(
Event("task-received", name=type, uuid=uuid(),
hostname=worker))
def test_success_cycle(self):
worker = 'headworker1'
task_id = uuid()
state = EventsState()
state.get_or_create_worker(worker)
events = [Event('worker-online', hostname=worker)]
cycle = '20000101_0000z'
cycle_events = cycle_task(worker, cycle, 'SUCCESS', id=task_id)
events.extend(cycle_events)
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
r = self.get('/cycles')
self.assertEqual(200, r.code)
self.assertIn('/task/' + task_id, r.body.decode("utf-8"))
params = dict(draw=1, start=0, length=10)
params['search[value]'] = ''
params['order[0][column]'] = 0
params['columns[0][data]'] = 'name'
params['order[0][dir]'] = 'asc'
r = self.get('/cycles/datatable?' +
'&'.join(map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"))
self.assertEqual(200, r.code)
self.assertEqual(0, table['recordsTotal'])
def __init__(self):
"""Initializes celery events and dispatchers
"""
self.app = app_or_default()
self.event = Event(type='chatrooms')
self.dispatcher = self.app.events.Dispatcher(
connection=self.app.broker_connection(), enabled=True)
def QTEV(type, uuid, hostname, clock, timestamp=None):
"""Quick task event."""
return Event('task-{0}'.format(type),
uuid=uuid,
hostname=hostname,
clock=clock,
timestamp=timestamp or time())
def test_failed_task(self):
state = EventsState()
state.get_or_create_worker('worker1')
events = [Event('worker-online', hostname='worker1')]
events += task_failed_events(worker='worker1', name='task1', id='123')
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
params = dict(draw=1, start=0, length=10)
params['search[value]'] = ''
params['order[0][column]'] = 0
params['columns[0][data]'] = 'name'
params['order[0][dir]'] = 'asc'
r = self.get('/tasks/datatable?' +
'&'.join(map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"))
self.assertEqual(200, r.code)
self.assertEqual(1, table['recordsTotal'])
self.assertEqual(1, table['recordsFiltered'])
tasks = table['data']
self.assertEqual(1, len(tasks))
self.assertEqual('FAILURE', tasks[0]['state'])
self.assertEqual('task1', tasks[0]['name'])
self.assertEqual('123', tasks[0]['uuid'])
self.assertEqual('worker1', tasks[0]['worker'])
def test_running_cycle_with_children_subprocess(self):
worker = 'headworker1'
state = EventsState()
state.get_or_create_worker(worker)
cycle_id = uuid()
events = [Event('worker-online', hostname=worker)]
cycle = '20000101_0000z'
cycle_events = cycle_task(worker, cycle, 'RUNNING', id=cycle_id)
child_id = uuid()
child_task_events = task_succeeded_events(worker,
child_id,
'wrappers.SubprocessTask',
kwargs={
'action_id':
'test_action',
'cycle_dt': cycle,
'parent': cycle_id
})
events.extend(cycle_events)
events.extend(child_task_events)
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
r = self.get('/cycles')
self.assertEqual(200, r.code)
self.assertIn('/task/' + cycle_id, r.body.decode("utf-8"))
params = dict(draw=1, start=0, length=10, selected='previous')
params['search[value]'] = ''
params['order[0][column]'] = 0
params['columns[0][data]'] = 'name'
params['order[0][dir]'] = 'asc'
r = self.get('/cycles/datatable?' +
'&'.join(map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"))
self.assertEqual(200, r.code)
self.assertEqual(0, table['recordsTotal'])
params = dict(draw=1, start=0, length=10, selected='active')
params['search[value]'] = ''
params['order[0][column]'] = 0
params['columns[0][data]'] = 'name'
params['order[0][dir]'] = 'asc'
r = self.get('/cycles/datatable?' +
'&'.join(map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"))
self.assertEqual(200, r.code)
self.assertEqual(1, table['recordsTotal'])
tasks = table['data']
self.assertEqual('SUCCESS', tasks[0]['state'])
self.assertEqual('wrappers.SubprocessTask', tasks[0]['name'])
self.assertEqual(child_id, tasks[0]['uuid'])
self.assertEqual(worker, tasks[0]['worker'])
def QTEV(type, uuid, hostname, clock, name=None, timestamp=None):
"""Quick task event."""
return Event(f'task-{type}',
uuid=uuid,
hostname=hostname,
clock=clock,
name=name,
timestamp=timestamp or time())
def setup(self):
tid = self.tid = uuid()
self.events = [
Event('task-received',
uuid=tid,
name='task1',
args='(2, 2)',
kwargs="{'foo': 'bar'}",
retries=0,
eta=None,
hostname='utest1'),
Event('task-started', uuid=tid, hostname='utest1'),
Event('task-revoked', uuid=tid, hostname='utest1'),
Event('task-retried',
uuid=tid,
exception="KeyError('bar')",
traceback='line 2 at main',
hostname='utest1'),
Event('task-failed',
uuid=tid,
exception="KeyError('foo')",
traceback='line 1 at main',
hostname='utest1'),
Event('task-succeeded',
uuid=tid,
result='4',
runtime=0.1234,
hostname='utest1'),
Event('foo-bar'),
]
def test_no_received_event(self, app):
monitor = MonitorThread(app=app)
monitor._process_event(Event(
'task-started',
uuid=uuid.uuid4(), name=self.TASK_NAME, hostname='myhost',
clock=1, local_received=time())
)
self.assert_only_states({celery.states.STARTED})
def task_succeeded_events(worker, id=None, name=None):
id = id or uuid()
name = name or 'sometask'
return [
Event('task-received',
uuid=id,
name=name,
args='(2, 2)',
kwargs="{'foo': 'bar'}",
retries=0,
eta=None,
hostname=worker),
Event('task-started', uuid=id, hostname=worker),
Event('task-succeeded',
uuid=id,
result='4',
runtime=0.1234,
hostname=worker)
]
def task_failed_events(worker, id=None, name=None):
id = id or uuid()
name = name or 'sometask'
return [
Event('task-received',
uuid=id,
name=name,
args='(2, 2)',
kwargs="{'foo': 'bar'}",
retries=0,
eta=None,
hostname=worker),
Event('task-started', uuid=id, hostname=worker),
Event('task-failed',
uuid=id,
exception="KeyError('foo')",
traceback='line 1 at main',
hostname=worker)
]
class ev_task_states(replay):
uuid = gen_unique_id()
events = [
Event("task-received",
uuid=uuid,
name="task1",
args="(2, 2)",
kwargs="{'foo': 'bar'}",
retries=0,
eta=None,
hostname="utest1"),
Event("task-started", uuid=uuid, hostname="utest1"),
Event("task-revoked", uuid=uuid, hostname="utest1"),
Event("task-retried",
uuid=uuid,
exception="KeyError('bar')",
traceback="line 2 at main",
hostname="utest1"),
Event("task-failed",
uuid=uuid,
exception="KeyError('foo')",
traceback="line 1 at main",
hostname="utest1"),
Event("task-succeeded",
uuid=uuid,
result="4",
runtime=0.1234,
hostname="utest1"),
]
def setup(self):
tid = self.tid = uuid()
self.events = [
Event("task-received",
uuid=tid,
name="task1",
args="(2, 2)",
kwargs="{'foo': 'bar'}",
retries=0,
eta=None,
hostname="utest1"),
Event("task-started", uuid=tid, hostname="utest1"),
Event("task-revoked", uuid=tid, hostname="utest1"),
Event("task-retried",
uuid=tid,
exception="KeyError('bar')",
traceback="line 2 at main",
hostname="utest1"),
Event("task-failed",
uuid=tid,
exception="KeyError('foo')",
traceback="line 1 at main",
hostname="utest1"),
Event("task-succeeded",
uuid=tid,
result="4",
runtime=0.1234,
hostname="utest1"),
]
def test_purge_offline_workers(self):
state = EventsState()
state.get_or_create_worker('worker1')
state.event(
Event('worker-online',
hostname='worker1',
local_received=time.time()))
state.event(
Event('worker-offline',
hostname='worker1',
local_received=time.time()))
self.app.events.state = state
with patch('flower.views.dashboard.options') as mock_options:
mock_options.purge_offline_workers = 0
r = self.get('/dashboard')
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(0, len(table.rows()))
def test_exception(self, app):
monitor = MonitorThread(app=app)
monitor._process_event(Event(
'task-failed',
uuid=uuid.uuid4(), name=self.TASK_NAME, hostname='myhost',
clock=2, local_received=time(), exception='ZeroDivisionError()')
)
exc_count = REGISTRY.get_sample_value(
'celery_task_exceptions',
labels={'name': 'unknown', 'exception': 'ZeroDivisionError'}
)
assert exc_count == 1
def test_single_workers_offline(self):
state = EventsState()
state.get_or_create_worker('worker1')
state.event(
Event('worker-online',
hostname='worker1',
local_received=time.time()))
state.event(
Event('worker-offline',
hostname='worker1',
local_received=time.time()))
self.app.events.state = state
r = self.get('/dashboard')
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(1, len(table.rows()))
self.assertTrue(table.get_row('worker1'))
self.assertEqual(['worker1', 'False', '0', '0', '0', '0', '0', None],
table.get_row('worker1'))
self.assertFalse(table.get_row('worker2'))
def test_worker_prefetched_tasks_metric(self):
state = EventsState()
worker_name = 'worker2'
task_name = 'task1'
task_id = uuid()
state.get_or_create_worker(worker_name)
events = [
Event('task-received',
uuid=task_id,
name=task_name,
args='(2, 2)',
kwargs="{'foo': 'bar'}",
retries=1,
eta=None,
hostname=worker_name),
Event('task-received',
uuid=uuid(),
name=task_name,
args='(2, 2)',
kwargs="{'foo': 'bar'}",
retries=1,
eta=None,
hostname=worker_name),
Event('task-started', uuid=task_id, hostname=worker_name),
]
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
metrics = self.get('/metrics').body.decode('utf-8')
self.assertTrue(
f'flower_worker_prefetched_tasks{{task="{task_name}",worker="{worker_name}"}} 1.0'
in metrics)
def test_tasks(self):
state = EventsState()
state.get_or_create_worker('worker1')
state.get_or_create_worker('worker2')
state.get_or_create_worker('worker3')
events = [
Event('worker-online', hostname='worker1'),
Event('worker-online', hostname='worker2')
]
for i in range(100):
events += task_succeeded_events(worker='worker1')
for i in range(10):
events += task_succeeded_events(worker='worker3')
for i in range(13):
events += task_failed_events(worker='worker3')
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
r = self.get('/dashboard')
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(3, len(table.rows()))
self.assertEqual(
['worker1', 'True', '0', '100', '0', '100', '0', None],
table.get_row('worker1'))
self.assertEqual(['worker2', 'True', '0', '0', '0', '0', '0', None],
table.get_row('worker2'))
self.assertEqual(['worker3', 'True', '0', '23', '13', '10', '0', None],
table.get_row('worker3'))
def test_worker_online_metric_worker_is_offline(self):
state = EventsState()
worker_name = 'worker1'
state.get_or_create_worker(worker_name)
events = [Event('worker-offline', hostname=worker_name)]
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
metrics = self.get('/metrics').body.decode('utf-8')
self.assertTrue(
f'flower_worker_online{{worker="{worker_name}"}} 0.0' in metrics)
