def test_task_received(self): state = EventsState() state.get_or_create_worker('worker1') state.get_or_create_worker('worker2') events = [Event('worker-online', hostname='worker1'), Event('worker-online', hostname='worker2'), Event('task-received', uuid=uuid(), name='task1', args='(2, 2)', kwargs="{'foo': 'bar'}", retries=0, eta=None, hostname='worker1')] for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state r = self.get('/dashboard') table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(2, len(table.rows())) self.assertEqual(['worker1', 'True', '0', '1', '0', '0', '0', None], table.get_row('worker1')) self.assertEqual(['worker2', 'True', '0', '0', '0', '0', '0', None], table.get_row('worker2'))
def test_tasks(self): state = EventsState() state.get_or_create_worker("worker1") state.get_or_create_worker("worker2") state.get_or_create_worker("worker3") events = [Event("worker-online", hostname="worker1"), Event("worker-online", hostname="worker2")] for i in range(100): events += task_succeeded_events(worker="worker1") for i in range(10): events += task_succeeded_events(worker="worker3") for i in range(13): events += task_failed_events(worker="worker3") for i, e in enumerate(events): e["clock"] = i e["local_received"] = time.time() state.event(e) self.app.events.state = state r = self.get("/dashboard") table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(3, len(table.rows())) self.assertEqual(["worker1", "True", "0", "100", "0", "100", "0", None], table.get_row("worker1")) self.assertEqual(["worker2", "True", "0", "0", "0", "0", "0", None], table.get_row("worker2")) self.assertEqual(["worker3", "True", "0", "23", "13", "10", "0", None], table.get_row("worker3"))
def test_task_received(self): state = EventsState() state.get_or_create_worker("worker1") state.get_or_create_worker("worker2") events = [ Event("worker-online", hostname="worker1"), Event("worker-online", hostname="worker2"), Event( "task-received", uuid=uuid(), name="task1", args="(2, 2)", kwargs="{'foo': 'bar'}", retries=0, eta=None, hostname="worker1", ), ] for i, e in enumerate(events): e["clock"] = i e["local_received"] = time.time() state.event(e) self.app.events.state = state r = self.get("/dashboard") table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(2, len(table.rows())) self.assertEqual(["worker1", "True", "0", "1", "0", "0", "0", None], table.get_row("worker1")) self.assertEqual(["worker2", "True", "0", "0", "0", "0", "0", None], table.get_row("worker2"))
def test_tasks(self): state = EventsState() state.get_or_create_worker('worker1') state.get_or_create_worker('worker2') state.get_or_create_worker('worker3') events = [Event('worker-online', hostname='worker1'), Event('worker-online', hostname='worker2')] for i in range(100): events += task_succeeded_events(worker='worker1') for i in range(10): events += task_succeeded_events(worker='worker3') for i in range(13): events += task_failed_events(worker='worker3') for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state r = self.get('/dashboard') table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(3, len(table.rows())) self.assertEqual(['worker1', 'True', '0', '100', '0', '100', '0', None], table.get_row('worker1')) self.assertEqual(['worker2', 'True', '0', '0', '0', '0', '0', None], table.get_row('worker2')) self.assertEqual(['worker3', 'True', '0', '23', '13', '10', '0', None], table.get_row('worker3'))
def test_failed_task(self): state = EventsState() state.get_or_create_worker('worker1') events = [Event('worker-online', hostname='worker1')] events += task_failed_events(worker='worker1', name='task1', id='123') for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state params = dict(draw=1, start=0, length=10) params['search[value]'] = '' params['order[0][column]'] = 0 params['columns[0][data]'] = 'name' params['order[0][dir]'] = 'asc' r = self.get('/tasks/datatable?' + '&'.join( map(lambda x: '%s=%s' % x, params.items()))) table = json.loads(r.body.decode("utf-8")) self.assertEqual(200, r.code) self.assertEqual(1, table['recordsTotal']) self.assertEqual(1, table['recordsFiltered']) tasks = table['data'] self.assertEqual(1, len(tasks)) self.assertEqual('FAILURE', tasks[0]['state']) self.assertEqual('task1', tasks[0]['name']) self.assertEqual('123', tasks[0]['uuid']) self.assertEqual('worker1', tasks[0]['worker'])
def test_single_workers_online(self): state = EventsState() state.get_or_create_worker("worker1") state.event(Event("worker-online", hostname="worker1", local_received=time.time())) self.app.events.state = state r = self.get("/dashboard") table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(1, len(table.rows())) self.assertTrue(table.get_row("worker1")) self.assertEqual(["worker1", "True", "0", "0", "0", "0", "0", None], table.get_row("worker1")) self.assertFalse(table.get_row("worker2"))
def test_metrics(self): state = EventsState() state.get_or_create_worker('worker1') events = [Event('worker-online', hostname='worker1')] events += task_succeeded_events(worker='worker1', name='task1', id='123') for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state metrics = self.get('/metrics').body.decode('utf-8') events = dict(re.findall('flower_events_total{task="task1",type="(task-.*)",worker="worker1"} (.*)', metrics)) self.assertTrue('task-received' in events) self.assertTrue('task-started' in events) self.assertTrue('task-succeeded' in events)
def test_task_retried(self): state = EventsState() state.get_or_create_worker('worker1') state.get_or_create_worker('worker2') events = [ Event('worker-online', hostname='worker1'), Event('worker-online', hostname='worker2'), Event('task-received', uuid='123', name='task1', args='(2, 2)', kwargs="{'foo': 'bar'}", retries=0, eta=None, hostname='worker1'), Event('task-started', uuid='123', hostname='worker1'), Event('task-retried', uuid='123', exception="KeyError('bar')", traceback='line 2 at main', hostname='worker1'), Event('task-failed', uuid='123', exception="KeyError('foo')", traceback='line 1 at main', hostname='worker1') ] for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state r = self.get('/dashboard') table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(2, len(table.rows())) self.assertEqual(['worker1', 'True', '0', '1', '1', '0', '1', None], table.get_row('worker1')) self.assertEqual(['worker2', 'True', '0', '0', '0', '0', '0', None], table.get_row('worker2'))
def test_sort_runtime(self): state = EventsState() state.get_or_create_worker('worker1') events = [Event('worker-online', hostname='worker1')] events += task_succeeded_events(worker='worker1', name='task1', id='123', runtime=10.0) events += task_succeeded_events(worker='worker1', name='task1', id='456', runtime=2.0) for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state params = dict(draw=1, start=0, length=10) params['search[value]'] = '' params['order[0][column]'] = 0 params['columns[0][data]'] = 'runtime' params['order[0][dir]'] = 'asc' r = self.get('/tasks/datatable?' + '&'.join(map(lambda x: '%s=%s' % x, params.items()))) table = json.loads(r.body.decode("utf-8")) self.assertEqual(200, r.code) self.assertEqual(2, table['recordsTotal']) self.assertEqual(2, table['recordsFiltered']) tasks = table['data'] self.assertEqual(2, len(tasks)) self.assertEqual('SUCCESS', tasks[0]['state']) self.assertEqual('task1', tasks[0]['name']) self.assertEqual('456', tasks[0]['uuid']) self.assertEqual('worker1', tasks[0]['worker']) self.assertEqual(2.0, tasks[0]['runtime']) self.assertEqual('SUCCESS', tasks[1]['state']) self.assertEqual('task1', tasks[1]['name']) self.assertEqual('123', tasks[1]['uuid']) self.assertEqual('worker1', tasks[1]['worker']) self.assertEqual(10.0, tasks[1]['runtime'])
def test_single_workers_online(self): state = EventsState() state.get_or_create_worker('worker1') state.event(Event('worker-online', hostname='worker1', local_received=time.time())) self.app.events.state = state r = self.get('/dashboard') table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(1, len(table.rows())) self.assertTrue(table.get_row('worker1')) self.assertEqual(['worker1', 'True', '0', '0', '0', '0', '0', None], table.get_row('worker1')) self.assertFalse(table.get_row('worker2'))
def test_metrics(self): state = EventsState() worker_name = 'worker1' task_name = 'task1' state.get_or_create_worker(worker_name) events = [ Event('worker-online', hostname=worker_name), Event('worker-heartbeat', hostname=worker_name, active=1) ] events += task_succeeded_events(worker=worker_name, name=task_name, id='123') task_received = time.time() task_started = task_received + 3 for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() if e['type'] == 'task-received': e['timestamp'] = task_received if e['type'] == 'task-started': e['timestamp'] = task_started state.event(e) self.app.events.state = state metrics = self.get('/metrics').body.decode('utf-8') events = dict( re.findall( 'flower_events_total{task="task1",type="(task-.*)",worker="worker1"} (.*)', metrics)) self.assertTrue('task-received' in events) self.assertTrue('task-started' in events) self.assertTrue('task-succeeded' in events) self.assertTrue( f'flower_task_queuing_time_at_worker_seconds{{task="{task_name}",worker="{worker_name}"}} 3.0' in metrics) self.assertTrue( f'flower_worker_online{{worker="{worker_name}"}} 1.0' in metrics) self.assertTrue( f'flower_worker_number_of_currently_executing_tasks{{worker="{worker_name}"}} 1.0' in metrics)
def test_single_workers_online(self): state = EventsState() state.get_or_create_worker('worker1') state.event( Event('worker-online', hostname='worker1', local_received=time.time())) self.app.events.state = state r = self.get('/dashboard') table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(1, len(table.rows())) self.assertTrue(table.get_row('worker1')) self.assertEqual(['worker1', 'True', '0', '0', '0', '0', '0', None], table.get_row('worker1')) self.assertFalse(table.get_row('worker2'))
def test_dependencies_graph(self): worker = 'headworker1' state = EventsState() state.get_or_create_worker(worker) task_id = uuid() events = [Event('worker-online', hostname=worker)] cycle = '20000101_0000z' cycle_events = cycle_task(worker, cycle, 'RUNNING', id=task_id, workflow=['default']) events.extend(cycle_events) for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state resp = self.get('/pydot/%s' % task_id) self.assertEqual(200, resp.code) self.assertEqual(resp.headers['Content-Type'], 'image/png') self.assertTrue(os.path.exists('/tmp/%s.png' % task_id))
def test_does_not_compute_queuing_time_if_task_has_eta(self): state = EventsState() worker_name = 'worker2' task_name = 'task2' state.get_or_create_worker(worker_name) events = [Event('worker-online', hostname=worker_name)] events += task_succeeded_events(worker=worker_name, name=task_name, id='567', eta=datetime.now() + timedelta(hours=4)) for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state metrics = self.get('/metrics').body.decode('utf-8') self.assertFalse( f'flower_task_queuing_time_at_worker_seconds{{task="{task_name}",worker="{worker_name}"}} ' in metrics)
def test_purge_offline_workers(self): state = EventsState() state.get_or_create_worker('worker1') state.event( Event('worker-online', hostname='worker1', local_received=time.time())) state.event( Event('worker-offline', hostname='worker1', local_received=time.time())) self.app.events.state = state with patch('flower.views.dashboard.options') as mock_options: mock_options.purge_offline_workers = 0 r = self.get('/dashboard') table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(0, len(table.rows()))
def test_tasks(self): state = EventsState() state.get_or_create_worker('worker1') state.get_or_create_worker('worker2') state.get_or_create_worker('worker3') events = [ Event('worker-online', hostname='worker1'), Event('worker-online', hostname='worker2') ] for i in range(100): events += task_succeeded_events(worker='worker1') for i in range(10): events += task_succeeded_events(worker='worker3') for i in range(13): events += task_failed_events(worker='worker3') for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state r = self.get('/dashboard') table = HtmlTableParser() table.parse(str(r.body)) self.assertEqual(200, r.code) self.assertEqual(3, len(table.rows())) self.assertEqual( ['worker1', 'True', '0', '100', '0', '100', '0', None], table.get_row('worker1')) self.assertEqual(['worker2', 'True', '0', '0', '0', '0', '0', None], table.get_row('worker2')) self.assertEqual(['worker3', 'True', '0', '23', '13', '10', '0', None], table.get_row('worker3'))
def test_worker_prefetched_tasks_metric(self): state = EventsState() worker_name = 'worker2' task_name = 'task1' task_id = uuid() state.get_or_create_worker(worker_name) events = [ Event('task-received', uuid=task_id, name=task_name, args='(2, 2)', kwargs="{'foo': 'bar'}", retries=1, eta=None, hostname=worker_name), Event('task-received', uuid=uuid(), name=task_name, args='(2, 2)', kwargs="{'foo': 'bar'}", retries=1, eta=None, hostname=worker_name), Event('task-started', uuid=task_id, hostname=worker_name), ] for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state metrics = self.get('/metrics').body.decode('utf-8') self.assertTrue( f'flower_worker_prefetched_tasks{{task="{task_name}",worker="{worker_name}"}} 1.0' in metrics)
def test_tasks_pagination(self): state = EventsState() state.get_or_create_worker('worker1') events = [Event('worker-online', hostname='worker1')] events += task_succeeded_events(worker='worker1', name='task1', id='123') events += task_succeeded_events(worker='worker1', name='task2', id='456') events += task_succeeded_events(worker='worker1', name='task3', id='789') events += task_succeeded_events(worker='worker1', name='task4', id='666') # for i, e in enumerate(sorted(events, key=lambda event: event['uuid'])): for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state # Test limit 4 and offset 0 params = dict(limit=4, offset=0, sort_by='name') r = self.get('/api/tasks?' + '&'.join(map(lambda x: '%s=%s' % x, params.items()))) table = json.loads(r.body.decode("utf-8"), object_pairs_hook=OrderedDict) self.assertEqual(200, r.code) self.assertEqual(4, len(table)) firstFetchedTaskName = table[list(table)[0]]['name'] lastFetchedTaskName = table[list(table)[-1]]['name'] self.assertEqual("task1", firstFetchedTaskName) self.assertEqual("task4", lastFetchedTaskName) # Test limit 4 and offset 1 params = dict(limit=4, offset=1, sort_by='name') r = self.get('/api/tasks?' + '&'.join(map(lambda x: '%s=%s' % x, params.items()))) table = json.loads(r.body.decode("utf-8"), object_pairs_hook=OrderedDict) self.assertEqual(200, r.code) self.assertEqual(3, len(table)) firstFetchedTaskName = table[list(table)[0]]['name'] lastFetchedTaskName = table[list(table)[-1]]['name'] self.assertEqual("task2", firstFetchedTaskName) self.assertEqual("task4", lastFetchedTaskName) # Test limit 4 and offset -1 (-1 should act as 0) params = dict(limit=4, offset=-1, sort_by="name") r = self.get('/api/tasks?' + '&'.join(map(lambda x: '%s=%s' % x, params.items()))) table = json.loads(r.body.decode("utf-8"), object_pairs_hook=OrderedDict) self.assertEqual(200, r.code) self.assertEqual(4, len(table)) firstFetchedTaskName = table[list(table)[0]]['name'] lastFetchedTaskName = table[list(table)[-1]]['name'] self.assertEqual("task1", firstFetchedTaskName) self.assertEqual("task4", lastFetchedTaskName) # Test limit 2 and offset 1 params = dict(limit=2, offset=1, sort_by='name') r = self.get('/api/tasks?' + '&'.join(map(lambda x: '%s=%s' % x, params.items()))) table = json.loads(r.body.decode("utf-8"), object_pairs_hook=OrderedDict) self.assertEqual(200, r.code) self.assertEqual(2, len(table)) firstFetchedTaskName = table[list(table)[0]]['name'] lastFetchedTaskName = table[list(table)[-1]]['name'] self.assertEqual("task2", firstFetchedTaskName) self.assertEqual("task3", lastFetchedTaskName)
def test_running_cycle_with_children_allocate_chain_failed(self): worker = 'headworker1' state = EventsState() state.get_or_create_worker(worker) cycle_id = uuid() events = [Event('worker-online', hostname=worker)] cycle = '20000101_0000z' cycle_events = cycle_task(worker, cycle, 'RUNNING', id=cycle_id) child_id = uuid() child_task_events = task_failed_events(worker, child_id, 'chain.AllocateChainTask', kwargs={ 'action_id': 'test_action', 'cycle_dt': cycle, 'parent': cycle_id }) chain_child_id = uuid() child_task_events += task_failed_events(worker, chain_child_id, 'wrappers.WrapperTask', kwargs={ 'action_id': 'test_action', 'cycle_dt': cycle, 'parent': child_id }) events.extend(cycle_events) events.extend(child_task_events) for i, e in enumerate(events): e['clock'] = i e['local_received'] = time.time() state.event(e) self.app.events.state = state r = self.get('/cycles') self.assertEqual(200, r.code) self.assertIn('/task/' + cycle_id, r.body.decode("utf-8")) params = dict(draw=1, start=0, length=10, selected='previous') params['search[value]'] = '' params['order[0][column]'] = 0 params['columns[0][data]'] = 'name' params['order[0][dir]'] = 'asc' r = self.get('/cycles/datatable?' + '&'.join(map(lambda x: '%s=%s' % x, params.items()))) table = json.loads(r.body.decode("utf-8")) self.assertEqual(200, r.code) self.assertEqual(0, table['recordsTotal']) params = dict(draw=1, start=0, length=10, selected='active') params['search[value]'] = '' params['order[0][column]'] = 0 params['columns[0][data]'] = 'name' params['order[0][dir]'] = 'asc' r = self.get('/cycles/datatable?' + '&'.join(map(lambda x: '%s=%s' % x, params.items()))) table = json.loads(r.body.decode("utf-8")) self.assertEqual(200, r.code) self.assertEqual(1, table['recordsTotal']) tasks = table['data'] self.assertEqual('FAILURE', tasks[0]['state']) self.assertEqual('chain.AllocateChainTask', tasks[0]['name']) self.assertEqual(child_id, tasks[0]['uuid']) self.assertEqual(worker, tasks[0]['worker'])