def test_fetch_current_task():
worker = Worker('testworker', queues=[QueueFactory()])
queue = worker.queues[0]
queue.enqueue_call()
assert worker.fetch_current_task() is None
task = queue.dequeue(worker)
assert worker.fetch_current_task().id == task.id
def list_workers():
return {
'workers': [{
'name': worker.description,
'queues': [q.name for q in worker.queues],
'state': str(worker.state),
} for worker in Worker.all()]
}
def WorkerFactory(*, queues=None):
global worker_sequence
worker_sequence += 1
id = f'worker_{worker_sequence}'
description = f"Worker {worker_sequence}"
if queues is None:
queues = [QueueFactory()]
return Worker(id, description=description, queues=queues)
def list_jobs(queue_name, page):
if queue_name != '[running]':
if queue_name == '[failed]':
queue = failed_task_registry
reverse_order = True
elif queue_name == '[finished]':
queue = finished_task_registry
reverse_order = True
else:
queue = Queue(queue_name)
reverse_order = False
current_page = int(page)
per_page = 20
total_items = queue.count()
pages_numbers_in_window = pagination_window(total_items, current_page,
per_page)
pages_in_window = [
dict(number=p,
url=app.url_for('overview', queue_name=queue_name, page=p))
for p in pages_numbers_in_window
]
last_page = int(ceil(total_items / float(per_page)))
prev_page = None
if current_page > 1:
prev_page = dict(
url=app.url_for('overview', queue_name=queue_name, page=1))
next_page = None
if current_page < last_page:
next_page = dict(url=app.url_for(
'overview', queue_name=queue_name, page=last_page))
pagination = remove_none_values(
dict(pages_in_window=pages_in_window,
next_page=next_page,
prev_page=prev_page))
if reverse_order:
start = -1 - (current_page - 1) * per_page
end = start - per_page
jobs = reversed(queue.get_tasks(end, start))
else:
offset = (current_page - 1) * per_page
jobs = queue.get_tasks(offset, per_page)
jobs = [serialize_job(job) for job in jobs]
else:
jobs = sorted(
({
**serialize_job(Task.fetch(tid)), 'worker':
Worker.fetch(wid).description
} for wid, tid in worker_registry.get_running_tasks().items()),
key=itemgetter('worker'),
)
pagination = {}
return dict(name=queue_name, jobs=jobs, pagination=pagination)
def overview(queue_name, page):
queue = Queue(queue_name)
return app.render_template(
'rt_dashboard/dashboard.html',
workers=Worker.all(),
queue=queue,
page=page,
queues=Queue.all(),
rt_url_prefix=app.url_for('overview'),
poll_interval=2500,
)
def show_workers(queues, by_queue):
def state_symbol(state):
return {
WorkerState.BUSY: red('busy'),
WorkerState.IDLE: green('idle')
}.get(state, state)
workers = Worker.all()
all_queues = set(Queue.all()) | {q for w in workers for q in w.queues}
if queues:
workers = [w for w in workers if any(set(w.queues) & set(queues))]
else:
queues = all_queues
if not by_queue:
click.secho("Workers", bold=True)
else:
click.secho("Workers per Queue", bold=True)
print_separator()
if not by_queue:
for worker in sorted(workers, key=attrgetter('description')):
worker_queues = ', '.join(q.name for q in worker.queues)
click.echo(
f'{worker.description} {state_symbol(worker.state)}: {worker_queues}'
)
else:
queue_map = {q: [] for q in all_queues}
for w in workers:
for q in w.queues:
queue_map[q].append(w)
max_qname = max(len(q.name) for q in queues)
for queue in sorted(queues, key=attrgetter('name')):
q_workers = queue_map[queue]
workers_str = ", ".join(
sorted(f'{w.description} {state_symbol(w.state)}'
for w in q_workers))
if not workers_str:
workers_str = '–'
click.echo(f'{queue.name:>{max_qname}}: {workers_str}')
click.echo(f'{len(workers)} worker(s)')
def handle_died_workers(self): # TODO: Test
from redis_tasks.worker import Worker
died_worker_ids = self.get_dead_ids()
for worker_id in died_worker_ids:
worker = Worker.fetch(worker_id)
worker.died()
def test_died(time_mocker, connection, assert_atomic):
time = time_mocker('redis_tasks.worker.utcnow')
# Die while idle
worker = Worker('idleworker', queues=[QueueFactory()])
time.step()
worker.startup()
time.step()
assert connection.ttl(worker.key) == -1
assert worker.id in worker_registry.get_worker_ids()
with assert_atomic():
worker.died()
assert worker.id not in worker_registry.get_worker_ids()
for w in [worker, Worker.fetch(worker.id)]:
assert w.state == WorkerState.DEAD
assert w.shutdown_at == time.now
assert connection.ttl(worker.key) > 0
# die whith task in limbo
worker = Worker('limboworker', queues=[QueueFactory(), QueueFactory()])
queue = worker.queues[1]
time.step()
worker.startup()
time.step()
queue.enqueue_call()
task = queue.dequeue(worker)
with assert_atomic(exceptions=['hgetall']):
worker.died()
assert queue.get_task_ids() == [task.id]
assert worker.id not in worker_registry.get_worker_ids()
for w in [worker, Worker.fetch(worker.id)]:
assert w.state == WorkerState.DEAD
assert w.current_task_id is None
assert w.shutdown_at == time.now
assert connection.ttl(worker.key) > 0
# die while busy
worker = Worker('busyworker', queues=[QueueFactory(), QueueFactory()])
queue = worker.queues[1]
time.step()
worker.startup()
time.step()
queue.enqueue_call()
task = queue.dequeue(worker)
worker.start_task(task)
with assert_atomic(exceptions=['hgetall']):
worker.died()
assert queue.get_task_ids() == []
assert failed_task_registry.get_task_ids() == [task.id]
assert worker.id not in worker_registry.get_worker_ids()
for w in [worker, Worker.fetch(worker.id)]:
assert w.state == WorkerState.DEAD
assert w.current_task_id is None
assert w.shutdown_at == time.now
assert connection.ttl(worker.key) > 0
def test_persistence(assert_atomic, connection, time_mocker):
time = time_mocker('redis_tasks.worker.utcnow')
time.step()
fields = {
'description', 'state', 'queues', 'started_at', 'shutdown_at',
'current_task_id'
}
def randomize_data(worker):
string_fields = ['description', 'state', 'current_task_id']
date_fields = ['started_at', 'shutdown_at']
for f in string_fields:
setattr(worker, f, str(uuid.uuid4()))
for f in date_fields:
setattr(
worker, f,
datetime.datetime(random.randint(1000, 9999),
1,
1,
tzinfo=datetime.timezone.utc))
worker.args = tuple(str(uuid.uuid4()) for i in range(4))
worker.kwargs = {str(uuid.uuid4()): ["d"]}
worker.meta = {"x": [str(uuid.uuid4())]}
worker.aborted_runs = ["foo", "bar", str(uuid.uuid4())]
def as_dict(worker):
return {
f: getattr(worker, f)
if f != 'queues' else [q.name for q in worker.queues]
for f in fields
}
worker = Worker("testworker", queues=[QueueFactory()])
with assert_atomic():
worker.startup()
assert as_dict(worker) == as_dict(Worker.fetch(worker.id))
worker2 = Worker("worker2", queues=[QueueFactory() for i in range(5)])
worker2.startup()
assert as_dict(worker2) == as_dict(Worker.fetch(worker2.id))
assert as_dict(worker) != as_dict(worker2)
worker = Worker("testworker", queues=[QueueFactory()])
worker.startup()
with assert_atomic():
worker._save()
assert set(decode_list(connection.hkeys(worker.key))) <= fields
assert as_dict(Worker.fetch(worker.id)) == as_dict(worker)
randomize_data(worker)
worker._save()
assert as_dict(Worker.fetch(worker.id)) == as_dict(worker)
# only deletes
worker.started_at = None
worker._save(['started_at'])
assert as_dict(Worker.fetch(worker.id)) == as_dict(worker)
for i in range(5):
store = random.sample(fields, 3)
copy = Worker.fetch(worker.id)
randomize_data(copy)
copy._save(store)
for f in store:
setattr(worker, f, getattr(copy, f))
assert as_dict(Worker.fetch(worker.id)) == as_dict(worker)
worker = Worker("nonexist", queues=[QueueFactory()])
with pytest.raises(WorkerDoesNotExist):
worker.refresh()
with pytest.raises(WorkerDoesNotExist):
Worker.fetch("nonexist")
def test_state_transitions(time_mocker, connection, assert_atomic):
worker = Worker('myworker', queues=[QueueFactory(), QueueFactory()])
time = time_mocker('redis_tasks.worker.utcnow')
queue = worker.queues[0]
time.step()
assert not connection.exists(worker.key, worker.task_key)
assert worker.id not in worker_registry.get_worker_ids()
with assert_atomic():
worker.startup()
assert worker.id in worker_registry.get_worker_ids()
for w in [worker, Worker.fetch(worker.id)]:
assert w.started_at == time.now
assert w.state == WorkerState.IDLE
queue.enqueue_call()
with assert_atomic(exceptions=['hgetall']):
task = queue.dequeue(worker)
assert connection.exists(worker.key, worker.task_key) == 2
for w in [worker, Worker.fetch(worker.id)]:
assert w.current_task_id == task.id
assert w.state == WorkerState.IDLE
with assert_atomic():
worker.start_task(task)
for w in [worker, Worker.fetch(worker.id)]:
assert w.current_task_id == task.id
assert w.state == WorkerState.BUSY
with assert_atomic():
worker.end_task(task, TaskOutcome("success"))
for w in [worker, Worker.fetch(worker.id)]:
assert w.current_task_id is None
assert w.state == WorkerState.IDLE
time.step()
assert connection.ttl(worker.key) == -1
assert worker.id in worker_registry.get_worker_ids()
with assert_atomic():
worker.shutdown()
assert worker.id not in worker_registry.get_worker_ids()
for w in [worker, Worker.fetch(worker.id)]:
assert w.state == WorkerState.DEAD
assert w.shutdown_at == time.now
assert connection.ttl(worker.key) > 0
def test_heartbeat(mocker):
heartbeat = mocker.patch.object(worker_registry, 'heartbeat')
worker = Worker('testworker', queues=[QueueFactory()])
worker.startup()
worker.heartbeat()
assert heartbeat.called_once_with(worker)
def test_all():
w1 = Worker('w1', queues=[QueueFactory()])
w2 = Worker('w2', queues=[QueueFactory()])
assert Worker.all() == []
w1.startup()
w2.startup()
assert id_list(Worker.all()) == [w1.id, w2.id]
w1.shutdown()
assert id_list(Worker.all()) == [w2.id]
def get_history():
max_finished_tasks = 5000
max_failed_tasks = 1000
finished_tasks = finished_task_registry.get_tasks(-max_finished_tasks, -1)
failed_tasks = failed_task_registry.get_tasks(-max_failed_tasks, -1)
if len(finished_tasks) == max_finished_tasks:
failed_tasks = [
t for t in failed_tasks if t.ended_at >= finished_tasks[0].ended_at
]
if len(failed_tasks) == max_failed_tasks:
finished_tasks = [
t for t in finished_tasks if t.ended_at >= failed_tasks[0].ended_at
]
now = utcnow()
running_tasks = []
for wid, tid in worker_registry.get_running_tasks().items():
task = Task.fetch(tid)
task.ended_at = now
task.running_on = Worker.fetch(wid).description
running_tasks.append(task)
tasks = failed_tasks + finished_tasks + running_tasks
tasks.sort(key=lambda t: t.started_at)
by_func = defaultdict(list)
for t in tasks:
by_func[t.func_name].append(t)
# reconstruct worker-mapping
for group in by_func.values():
workers = []
for task in sorted(group, key=lambda t: t.started_at):
workers = [
None if not t or t.ended_at <= task.started_at else t
for t in workers
]
try:
task.worker = workers.index(None)
workers[task.worker] = task
except ValueError:
task.worker = len(workers)
workers.append(task)
groups = sorted(by_func.values(),
key=lambda group_tasks: (
min(t.started_at.timetuple()[3:] for t in group_tasks),
max(t.ended_at - t.started_at for t in group_tasks),
))
collapsed_groups = {
k
for k, v in by_func.items() if len(v) / len(tasks) < 0.02
}
tz = pytz.timezone(settings.TIMEZONE)
rows = []
for t in tasks:
t.started_at = t.started_at.astimezone(tz)
t.ended_at = t.ended_at.astimezone(tz)
keys = {
'group': t.func_name,
'subgroup': t.worker,
'start': t.started_at,
'title': task_tooltip(t),
}
if hasattr(t, 'running_on'):
keys.update({
'end': t.ended_at,
'type': 'range',
'content': t.running_on,
})
elif (t.func_name not in collapsed_groups
or (t.ended_at - t.started_at) > datetime.timedelta(minutes=1)):
keys.update({
'end': t.ended_at,
'type': 'range',
'content': f'[{t.ended_at - t.started_at}]',
})
else:
keys.update({
'type': 'point',
'content': t.started_at.strftime('%H:%M:%S'),
})
if t.status == 'failed':
keys['style'] = 'border-color: {0}; background-color: {0}'.format(
'#E69089')
elif t.status == 'running':
keys['style'] = 'border-color: {0}; background-color: {0}'.format(
'#D5F6D7')
keys = {
k: v.timestamp() if isinstance(v, datetime.datetime) else v
for k, v in keys.items()
}
rows.append(keys)
return {
"rows":
rows,
"groups": [{
'id': group[0].func_name,
'content': group[0].func_name,
'order': i
} for i, group in enumerate(groups)],
}