def test_task_received(self):
state = EventsState()
state.get_or_create_worker('worker1')
state.get_or_create_worker('worker2')
events = [Event('worker-online', hostname='worker1'),
Event('worker-online', hostname='worker2'),
Event('task-received', uuid=uuid(), name='task1',
args='(2, 2)', kwargs="{'foo': 'bar'}",
retries=0, eta=None, hostname='worker1')]
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
r = self.get('/dashboard')
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(2, len(table.rows()))
self.assertEqual(['worker1', 'True', '0', '1', '0', '0', '0', None],
table.get_row('worker1'))
self.assertEqual(['worker2', 'True', '0', '0', '0', '0', '0', None],
table.get_row('worker2'))
def test_task_received(self):
state = EventsState()
state.get_or_create_worker("worker1")
state.get_or_create_worker("worker2")
events = [
Event("worker-online", hostname="worker1"),
Event("worker-online", hostname="worker2"),
Event(
"task-received",
uuid=uuid(),
name="task1",
args="(2, 2)",
kwargs="{'foo': 'bar'}",
retries=0,
eta=None,
hostname="worker1",
),
]
for i, e in enumerate(events):
e["clock"] = i
e["local_received"] = time.time()
state.event(e)
self.app.events.state = state
r = self.get("/dashboard")
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(2, len(table.rows()))
self.assertEqual(["worker1", "True", "0", "1", "0", "0", "0", None], table.get_row("worker1"))
self.assertEqual(["worker2", "True", "0", "0", "0", "0", "0", None], table.get_row("worker2"))
def test_failed_task(self):
state = EventsState()
state.get_or_create_worker('worker1')
events = [Event('worker-online', hostname='worker1')]
events += task_failed_events(worker='worker1', name='task1',
id='123')
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
params = dict(draw=1, start=0, length=10)
params['search[value]'] = ''
params['order[0][column]'] = 0
params['columns[0][data]'] = 'name'
params['order[0][dir]'] = 'asc'
r = self.get('/tasks/datatable?' + '&'.join(
map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"))
self.assertEqual(200, r.code)
self.assertEqual(1, table['recordsTotal'])
self.assertEqual(1, table['recordsFiltered'])
tasks = table['data']
self.assertEqual(1, len(tasks))
self.assertEqual('FAILURE', tasks[0]['state'])
self.assertEqual('task1', tasks[0]['name'])
self.assertEqual('123', tasks[0]['uuid'])
self.assertEqual('worker1', tasks[0]['worker'])
def test_single_workers_online(self):
state = EventsState()
state.get_or_create_worker("worker1")
state.event(Event("worker-online", hostname="worker1", local_received=time.time()))
self.app.events.state = state
r = self.get("/dashboard")
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(1, len(table.rows()))
self.assertTrue(table.get_row("worker1"))
self.assertEqual(["worker1", "True", "0", "0", "0", "0", "0", None], table.get_row("worker1"))
self.assertFalse(table.get_row("worker2"))
def test_tasks(self):
state = EventsState()
state.get_or_create_worker("worker1")
state.get_or_create_worker("worker2")
state.get_or_create_worker("worker3")
events = [Event("worker-online", hostname="worker1"), Event("worker-online", hostname="worker2")]
for i in range(100):
events += task_succeeded_events(worker="worker1")
for i in range(10):
events += task_succeeded_events(worker="worker3")
for i in range(13):
events += task_failed_events(worker="worker3")
for i, e in enumerate(events):
e["clock"] = i
e["local_received"] = time.time()
state.event(e)
self.app.events.state = state
r = self.get("/dashboard")
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(3, len(table.rows()))
self.assertEqual(["worker1", "True", "0", "100", "0", "100", "0", None], table.get_row("worker1"))
self.assertEqual(["worker2", "True", "0", "0", "0", "0", "0", None], table.get_row("worker2"))
self.assertEqual(["worker3", "True", "0", "23", "13", "10", "0", None], table.get_row("worker3"))
def test_tasks(self):
state = EventsState()
state.get_or_create_worker('worker1')
state.get_or_create_worker('worker2')
state.get_or_create_worker('worker3')
events = [Event('worker-online', hostname='worker1'),
Event('worker-online', hostname='worker2')]
for i in range(100):
events += task_succeeded_events(worker='worker1')
for i in range(10):
events += task_succeeded_events(worker='worker3')
for i in range(13):
events += task_failed_events(worker='worker3')
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
r = self.get('/dashboard')
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(3, len(table.rows()))
self.assertEqual(['worker1', 'True', '0', '100', '0', '100', '0', None],
table.get_row('worker1'))
self.assertEqual(['worker2', 'True', '0', '0', '0', '0', '0', None],
table.get_row('worker2'))
self.assertEqual(['worker3', 'True', '0', '23', '13', '10', '0', None],
table.get_row('worker3'))
def test_single_workers_online(self):
state = EventsState()
state.get_or_create_worker('worker1')
state.event(Event('worker-online', hostname='worker1',
local_received=time.time()))
self.app.events.state = state
r = self.get('/dashboard')
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(1, len(table.rows()))
self.assertTrue(table.get_row('worker1'))
self.assertEqual(['worker1', 'True', '0', '0', '0', '0', '0', None],
table.get_row('worker1'))
self.assertFalse(table.get_row('worker2'))
def test_running_cycle_with_children_subprocess(self):
worker = 'headworker1'
state = EventsState()
state.get_or_create_worker(worker)
cycle_id = uuid()
events = [Event('worker-online', hostname=worker)]
cycle = '20000101_0000z'
cycle_events = cycle_task(worker, cycle, 'RUNNING', id=cycle_id)
child_id = uuid()
child_task_events = task_succeeded_events(worker,
child_id,
'wrappers.SubprocessTask',
kwargs={
'action_id':
'test_action',
'cycle_dt': cycle,
'parent': cycle_id
})
events.extend(cycle_events)
events.extend(child_task_events)
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
r = self.get('/cycles')
self.assertEqual(200, r.code)
self.assertIn('/task/' + cycle_id, r.body.decode("utf-8"))
params = dict(draw=1, start=0, length=10, selected='previous')
params['search[value]'] = ''
params['order[0][column]'] = 0
params['columns[0][data]'] = 'name'
params['order[0][dir]'] = 'asc'
r = self.get('/cycles/datatable?' +
'&'.join(map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"))
self.assertEqual(200, r.code)
self.assertEqual(0, table['recordsTotal'])
params = dict(draw=1, start=0, length=10, selected='active')
params['search[value]'] = ''
params['order[0][column]'] = 0
params['columns[0][data]'] = 'name'
params['order[0][dir]'] = 'asc'
r = self.get('/cycles/datatable?' +
'&'.join(map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"))
self.assertEqual(200, r.code)
self.assertEqual(1, table['recordsTotal'])
tasks = table['data']
self.assertEqual('SUCCESS', tasks[0]['state'])
self.assertEqual('wrappers.SubprocessTask', tasks[0]['name'])
self.assertEqual(child_id, tasks[0]['uuid'])
self.assertEqual(worker, tasks[0]['worker'])
def test_task_prefetch_time_metric_successful_task_resets_metric_to_zero(
self):
state = EventsState()
worker_name = 'worker1'
task_name = 'task1'
state.get_or_create_worker(worker_name)
events = task_succeeded_events(worker=worker_name,
name=task_name,
id='123')
task_received = time.time()
task_started = task_received + 3
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
if e['type'] == 'task-received':
e['timestamp'] = task_received
if e['type'] == 'task-started':
e['timestamp'] = task_started
state.event(e)
self.app.events.state = state
metrics = self.get('/metrics').body.decode('utf-8')
self.assertTrue(
f'flower_task_prefetch_time_seconds{{task="{task_name}",worker="{worker_name}"}} 0.0'
in metrics)
def test_success_cycle(self):
worker = 'headworker1'
task_id = uuid()
state = EventsState()
state.get_or_create_worker(worker)
events = [Event('worker-online', hostname=worker)]
cycle = '20000101_0000z'
cycle_events = cycle_task(worker, cycle, 'SUCCESS', id=task_id)
events.extend(cycle_events)
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
r = self.get('/cycles')
self.assertEqual(200, r.code)
self.assertIn('/task/' + task_id, r.body.decode("utf-8"))
params = dict(draw=1, start=0, length=10)
params['search[value]'] = ''
params['order[0][column]'] = 0
params['columns[0][data]'] = 'name'
params['order[0][dir]'] = 'asc'
r = self.get('/cycles/datatable?' +
'&'.join(map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"))
self.assertEqual(200, r.code)
self.assertEqual(0, table['recordsTotal'])
def test_task_received(self):
state = EventsState()
state.get_or_create_worker('worker1')
state.get_or_create_worker('worker2')
events = [
Event('worker-online', hostname='worker1'),
Event('worker-online', hostname='worker2'),
Event('task-received',
uuid=uuid(),
name='task1',
args='(2, 2)',
kwargs="{'foo': 'bar'}",
retries=0,
eta=None,
hostname='worker1')
]
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
r = self.get('/dashboard')
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(2, len(table.rows()))
self.assertEqual(['worker1', 'True', '0', '1', '0', '0', '0', None],
table.get_row('worker1'))
self.assertEqual(['worker2', 'True', '0', '0', '0', '0', '0', None],
table.get_row('worker2'))
def test_failed_task(self):
state = EventsState()
state.get_or_create_worker('worker1')
events = [Event('worker-online', hostname='worker1')]
events += task_failed_events(worker='worker1', name='task1', id='123')
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
params = dict(draw=1, start=0, length=10)
params['search[value]'] = ''
params['order[0][column]'] = 0
params['columns[0][data]'] = 'name'
params['order[0][dir]'] = 'asc'
r = self.get('/tasks/datatable?' +
'&'.join(map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"))
self.assertEqual(200, r.code)
self.assertEqual(1, table['recordsTotal'])
self.assertEqual(1, table['recordsFiltered'])
tasks = table['data']
self.assertEqual(1, len(tasks))
self.assertEqual('FAILURE', tasks[0]['state'])
self.assertEqual('task1', tasks[0]['name'])
self.assertEqual('123', tasks[0]['uuid'])
self.assertEqual('worker1', tasks[0]['worker'])
def test_metrics(self):
state = EventsState()
worker_name = 'worker1'
task_name = 'task1'
state.get_or_create_worker(worker_name)
events = [Event('worker-online', hostname=worker_name)]
events += task_succeeded_events(worker=worker_name,
name=task_name,
id='123')
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
metrics = self.get('/metrics').body.decode('utf-8')
events = dict(
re.findall(
'flower_events_total{task="task1",type="(task-.*)",worker="worker1"} (.*)',
metrics))
self.assertTrue('task-received' in events)
self.assertTrue('task-started' in events)
self.assertTrue('task-succeeded' in events)
self.assertTrue(
f'flower_worker_online{{worker="{worker_name}"}} 1.0' in metrics)
self.assertTrue(
f'flower_task_queuing_time_at_worker_seconds{{task="{task_name}",worker="{worker_name}"}} '
in metrics)
def test_purge_offline_workers(self):
state = EventsState()
state.get_or_create_worker('worker1')
state.event(
Event('worker-online',
hostname='worker1',
local_received=time.time()))
state.event(
Event('worker-offline',
hostname='worker1',
local_received=time.time()))
self.app.events.state = state
with patch('flower.views.dashboard.options') as mock_options:
mock_options.purge_offline_workers = 0
r = self.get('/dashboard')
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(0, len(table.rows()))
def test_worker_online_metric_worker_is_offline(self):
state = EventsState()
worker_name = 'worker1'
state.get_or_create_worker(worker_name)
events = [Event('worker-offline', hostname=worker_name)]
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
metrics = self.get('/metrics').body.decode('utf-8')
self.assertTrue(
f'flower_worker_online{{worker="{worker_name}"}} 0.0' in metrics)
def test_single_workers_offline(self):
state = EventsState()
state.get_or_create_worker('worker1')
state.event(
Event('worker-online',
hostname='worker1',
local_received=time.time()))
state.event(
Event('worker-offline',
hostname='worker1',
local_received=time.time()))
self.app.events.state = state
r = self.get('/dashboard')
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(1, len(table.rows()))
self.assertTrue(table.get_row('worker1'))
self.assertEqual(['worker1', 'False', '0', '0', '0', '0', '0', None],
table.get_row('worker1'))
self.assertFalse(table.get_row('worker2'))
def test_tasks(self):
state = EventsState()
state.get_or_create_worker('worker1')
state.get_or_create_worker('worker2')
state.get_or_create_worker('worker3')
events = [
Event('worker-online', hostname='worker1'),
Event('worker-online', hostname='worker2')
]
for i in range(100):
events += task_succeeded_events(worker='worker1')
for i in range(10):
events += task_succeeded_events(worker='worker3')
for i in range(13):
events += task_failed_events(worker='worker3')
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
r = self.get('/dashboard')
table = HtmlTableParser()
table.parse(str(r.body))
self.assertEqual(200, r.code)
self.assertEqual(3, len(table.rows()))
self.assertEqual(
['worker1', 'True', '0', '100', '0', '100', '0', None],
table.get_row('worker1'))
self.assertEqual(['worker2', 'True', '0', '0', '0', '0', '0', None],
table.get_row('worker2'))
self.assertEqual(['worker3', 'True', '0', '23', '13', '10', '0', None],
table.get_row('worker3'))
def test_does_not_compute_queuing_time_if_task_has_eta(self):
state = EventsState()
worker_name = 'worker2'
task_name = 'task2'
state.get_or_create_worker(worker_name)
events = [Event('worker-online', hostname=worker_name)]
events += task_succeeded_events(worker=worker_name,
name=task_name,
id='567',
eta=datetime.now() +
timedelta(hours=4))
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
metrics = self.get('/metrics').body.decode('utf-8')
self.assertFalse(
f'flower_task_queuing_time_at_worker_seconds{{task="{task_name}",worker="{worker_name}"}} '
in metrics)
def test_dependencies_graph(self):
worker = 'headworker1'
state = EventsState()
state.get_or_create_worker(worker)
task_id = uuid()
events = [Event('worker-online', hostname=worker)]
cycle = '20000101_0000z'
cycle_events = cycle_task(worker,
cycle,
'RUNNING',
id=task_id,
workflow=['default'])
events.extend(cycle_events)
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
resp = self.get('/pydot/%s' % task_id)
self.assertEqual(200, resp.code)
self.assertEqual(resp.headers['Content-Type'], 'image/png')
self.assertTrue(os.path.exists('/tmp/%s.png' % task_id))
def test_worker_prefetched_tasks_metric(self):
state = EventsState()
worker_name = 'worker2'
task_name = 'task1'
task_id = uuid()
state.get_or_create_worker(worker_name)
events = [
Event('task-received',
uuid=task_id,
name=task_name,
args='(2, 2)',
kwargs="{'foo': 'bar'}",
retries=1,
eta=None,
hostname=worker_name),
Event('task-received',
uuid=uuid(),
name=task_name,
args='(2, 2)',
kwargs="{'foo': 'bar'}",
retries=1,
eta=None,
hostname=worker_name),
Event('task-started', uuid=task_id, hostname=worker_name),
]
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
metrics = self.get('/metrics').body.decode('utf-8')
self.assertTrue(
f'flower_worker_prefetched_tasks{{task="{task_name}",worker="{worker_name}"}} 1.0'
in metrics)
def test_tasks_pagination(self):
state = EventsState()
state.get_or_create_worker('worker1')
events = [Event('worker-online', hostname='worker1')]
events += task_succeeded_events(worker='worker1',
name='task1',
id='123')
events += task_succeeded_events(worker='worker1',
name='task2',
id='456')
events += task_succeeded_events(worker='worker1',
name='task3',
id='789')
events += task_succeeded_events(worker='worker1',
name='task4',
id='666')
# for i, e in enumerate(sorted(events, key=lambda event: event['uuid'])):
for i, e in enumerate(events):
e['clock'] = i
e['local_received'] = time.time()
state.event(e)
self.app.events.state = state
# Test limit 4 and offset 0
params = dict(limit=4, offset=0, sort_by='name')
r = self.get('/api/tasks?' +
'&'.join(map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"),
object_pairs_hook=OrderedDict)
self.assertEqual(200, r.code)
self.assertEqual(4, len(table))
firstFetchedTaskName = table[list(table)[0]]['name']
lastFetchedTaskName = table[list(table)[-1]]['name']
self.assertEqual("task1", firstFetchedTaskName)
self.assertEqual("task4", lastFetchedTaskName)
# Test limit 4 and offset 1
params = dict(limit=4, offset=1, sort_by='name')
r = self.get('/api/tasks?' +
'&'.join(map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"),
object_pairs_hook=OrderedDict)
self.assertEqual(200, r.code)
self.assertEqual(3, len(table))
firstFetchedTaskName = table[list(table)[0]]['name']
lastFetchedTaskName = table[list(table)[-1]]['name']
self.assertEqual("task2", firstFetchedTaskName)
self.assertEqual("task4", lastFetchedTaskName)
# Test limit 4 and offset -1 (-1 should act as 0)
params = dict(limit=4, offset=-1, sort_by="name")
r = self.get('/api/tasks?' +
'&'.join(map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"),
object_pairs_hook=OrderedDict)
self.assertEqual(200, r.code)
self.assertEqual(4, len(table))
firstFetchedTaskName = table[list(table)[0]]['name']
lastFetchedTaskName = table[list(table)[-1]]['name']
self.assertEqual("task1", firstFetchedTaskName)
self.assertEqual("task4", lastFetchedTaskName)
# Test limit 2 and offset 1
params = dict(limit=2, offset=1, sort_by='name')
r = self.get('/api/tasks?' +
'&'.join(map(lambda x: '%s=%s' % x, params.items())))
table = json.loads(r.body.decode("utf-8"),
object_pairs_hook=OrderedDict)
self.assertEqual(200, r.code)
self.assertEqual(2, len(table))
firstFetchedTaskName = table[list(table)[0]]['name']
lastFetchedTaskName = table[list(table)[-1]]['name']
self.assertEqual("task2", firstFetchedTaskName)
self.assertEqual("task3", lastFetchedTaskName)