def test_delete_many(connection, assert_atomic):
tasks = [Task() for i in range(5)]
for t in tasks:
t._save()
with assert_atomic():
Task.delete_many([tasks[0].id, tasks[1].id])
for t in tasks[0:2]:
assert not connection.exists(t.key)
for t in tasks[2:5]:
assert connection.exists(t.key)
def test_process_shutdown(suprocess_socket):
# sanity checks against leftovers from previous tests
assert not PostponeShutdown._active
assert not PostponeShutdown._shutdown_delayed
task = Task(horsetaskwait)
horse_conn, conn2 = multiprocessing.Pipe()
horse = WorkHorse(task, conn2)
horse.start()
assert horse_conn.poll(1)
assert horse_conn.recv() is True
with suprocess_socket.accept() as taskconn:
assert taskconn.poll(1)
assert taskconn.recv() == "A"
# horse ignores SIGTERM
horse.send_signal(signal.SIGTERM)
taskconn.send("B")
assert taskconn.poll(1)
assert taskconn.recv() == "C"
# Trigger shutdown
horse.send_signal(signal.SIGUSR1)
horse.join(1)
assert not horse.is_alive()
assert horse_conn.poll()
outcome = horse_conn.recv()
assert not horse_conn.poll()
assert outcome.outcome == 'failure'
assert 'Worker shutdown' in outcome.message.splitlines()[-1]
def list_jobs(queue_name, page):
if queue_name != '[running]':
if queue_name == '[failed]':
queue = failed_task_registry
reverse_order = True
elif queue_name == '[finished]':
queue = finished_task_registry
reverse_order = True
else:
queue = Queue(queue_name)
reverse_order = False
current_page = int(page)
per_page = 20
total_items = queue.count()
pages_numbers_in_window = pagination_window(total_items, current_page,
per_page)
pages_in_window = [
dict(number=p,
url=app.url_for('overview', queue_name=queue_name, page=p))
for p in pages_numbers_in_window
]
last_page = int(ceil(total_items / float(per_page)))
prev_page = None
if current_page > 1:
prev_page = dict(
url=app.url_for('overview', queue_name=queue_name, page=1))
next_page = None
if current_page < last_page:
next_page = dict(url=app.url_for(
'overview', queue_name=queue_name, page=last_page))
pagination = remove_none_values(
dict(pages_in_window=pages_in_window,
next_page=next_page,
prev_page=prev_page))
if reverse_order:
start = -1 - (current_page - 1) * per_page
end = start - per_page
jobs = reversed(queue.get_tasks(end, start))
else:
offset = (current_page - 1) * per_page
jobs = queue.get_tasks(offset, per_page)
jobs = [serialize_job(job) for job in jobs]
else:
jobs = sorted(
({
**serialize_job(Task.fetch(tid)), 'worker':
Worker.fetch(wid).description
} for wid, tid in worker_registry.get_running_tasks().items()),
key=itemgetter('worker'),
)
pagination = {}
return dict(name=queue_name, jobs=jobs, pagination=pagination)
def test_init_save_fetch_delete(connection, assert_atomic, stub):
t = Task(stub, ["foo"])
assert not connection.exists(t.key)
with assert_atomic():
t._save()
assert connection.exists(t.key)
fetched = Task.fetch(t.id)
assert fetched.id == t.id
assert fetched.args == ["foo"]
Task.delete_many([t.id])
assert not connection.exists(t.key)
with pytest.raises(TaskDoesNotExist):
Task.fetch(t.id)
def test_fetch_many(stub):
tasks_data = [{
"args": []
}, {
"args": ["foo"]
}, {
"args": ["bar"]
}, {
"args": ["foo", "bar"]
}, {
"args": []
}]
tasks = [Task(stub, **d) for d in tasks_data]
for t in tasks:
t._save()
fetched = Task.fetch_many([t.id for t in tasks[1:4]])
assert len(fetched) == 3
for i, task in enumerate(fetched):
assert task.id == tasks[i + 1].id
assert task.args == tasks_data[i + 1]["args"]
def test_process():
task = Task(horsetask)
horse_conn, conn2 = multiprocessing.Pipe()
horse = WorkHorse(task, conn2)
horse.start()
horse.join()
assert horse_conn.poll()
assert horse_conn.recv() is True
assert horse_conn.poll()
outcome = horse_conn.recv()
assert horse_conn.poll() is False
assert outcome.outcome == 'failure'
assert outcome.message.splitlines()[-1] == 'ArithmeticError: work me'
def test_init(stub):
def closure():
pass
with pytest.raises(ValueError):
Task(closure)
with pytest.raises(ValueError):
Task(SomeClass.some_func)
with pytest.raises(ValueError):
Task(SomeClass().some_func)
with pytest.raises(ValueError):
Task(SomeClass().misleading_func)
with pytest.raises(TypeError):
Task(stub, args="foo")
with pytest.raises(TypeError):
Task(stub, kwargs=["foo"])
t = Task(name_func, ["foo"], {"bar": "a"})
assert t.description == "tests.test_task.name_func('foo', bar='a')"
def cancel_job_view(job_id):
Task.fetch(job_id).cancel()
return dict(status='OK')
def get_history():
max_finished_tasks = 5000
max_failed_tasks = 1000
finished_tasks = finished_task_registry.get_tasks(-max_finished_tasks, -1)
failed_tasks = failed_task_registry.get_tasks(-max_failed_tasks, -1)
if len(finished_tasks) == max_finished_tasks:
failed_tasks = [
t for t in failed_tasks if t.ended_at >= finished_tasks[0].ended_at
]
if len(failed_tasks) == max_failed_tasks:
finished_tasks = [
t for t in finished_tasks if t.ended_at >= failed_tasks[0].ended_at
]
now = utcnow()
running_tasks = []
for wid, tid in worker_registry.get_running_tasks().items():
task = Task.fetch(tid)
task.ended_at = now
task.running_on = Worker.fetch(wid).description
running_tasks.append(task)
tasks = failed_tasks + finished_tasks + running_tasks
tasks.sort(key=lambda t: t.started_at)
by_func = defaultdict(list)
for t in tasks:
by_func[t.func_name].append(t)
# reconstruct worker-mapping
for group in by_func.values():
workers = []
for task in sorted(group, key=lambda t: t.started_at):
workers = [
None if not t or t.ended_at <= task.started_at else t
for t in workers
]
try:
task.worker = workers.index(None)
workers[task.worker] = task
except ValueError:
task.worker = len(workers)
workers.append(task)
groups = sorted(by_func.values(),
key=lambda group_tasks: (
min(t.started_at.timetuple()[3:] for t in group_tasks),
max(t.ended_at - t.started_at for t in group_tasks),
))
collapsed_groups = {
k
for k, v in by_func.items() if len(v) / len(tasks) < 0.02
}
tz = pytz.timezone(settings.TIMEZONE)
rows = []
for t in tasks:
t.started_at = t.started_at.astimezone(tz)
t.ended_at = t.ended_at.astimezone(tz)
keys = {
'group': t.func_name,
'subgroup': t.worker,
'start': t.started_at,
'title': task_tooltip(t),
}
if hasattr(t, 'running_on'):
keys.update({
'end': t.ended_at,
'type': 'range',
'content': t.running_on,
})
elif (t.func_name not in collapsed_groups
or (t.ended_at - t.started_at) > datetime.timedelta(minutes=1)):
keys.update({
'end': t.ended_at,
'type': 'range',
'content': f'[{t.ended_at - t.started_at}]',
})
else:
keys.update({
'type': 'point',
'content': t.started_at.strftime('%H:%M:%S'),
})
if t.status == 'failed':
keys['style'] = 'border-color: {0}; background-color: {0}'.format(
'#E69089')
elif t.status == 'running':
keys['style'] = 'border-color: {0}; background-color: {0}'.format(
'#D5F6D7')
keys = {
k: v.timestamp() if isinstance(v, datetime.datetime) else v
for k, v in keys.items()
}
rows.append(keys)
return {
"rows":
rows,
"groups": [{
'id': group[0].func_name,
'content': group[0].func_name,
'order': i
} for i, group in enumerate(groups)],
}
def TaskFactory():
global task_sequence
task_sequence += 1
task = Task(func=_stub)
task.id = f'task_{task_sequence}'
return task
def test_state_transistions(assert_atomic, connection, time_mocker, stub):
time = time_mocker('redis_tasks.task.utcnow')
task = Task(reentrant_stub)
q = QueueFactory()
w = WorkerFactory()
w.startup()
# enqueue
time.step()
assert not connection.exists(task.key)
with assert_atomic():
task.enqueue(q)
assert q.get_task_ids() == [task.id]
assert connection.exists(task.key)
for t in [task, Task.fetch(task.id)]:
assert t.enqueued_at == time.now
assert t.status == TaskStatus.QUEUED
assert t.origin == q.name
# dequeue
task = q.dequeue(w)
assert q.get_task_ids() == []
assert worker_registry.get_running_tasks() == {w.id: task.id}
# set_running
time.step()
with assert_atomic():
w.start_task(task)
assert worker_registry.get_running_tasks() == {w.id: task.id}
for t in [task, Task.fetch(task.id)]:
assert t.status == TaskStatus.RUNNING
assert t.started_at == time.now
# requeue
time.step()
with assert_atomic():
w.end_task(task, TaskOutcome("requeue"))
assert worker_registry.get_running_tasks() == dict()
assert q.get_task_ids() == [task.id]
for t in [task, Task.fetch(task.id)]:
assert t.status == TaskStatus.QUEUED
assert t.started_at is None
# set_finished
task = q.dequeue(w)
w.start_task(task)
time.step()
with assert_atomic():
w.end_task(task, TaskOutcome("success"))
assert q.get_task_ids() == []
assert finished_task_registry.get_task_ids() == [task.id]
for t in [task, Task.fetch(task.id)]:
assert t.status == TaskStatus.FINISHED
assert t.ended_at == time.now
# set_failed
task = q.enqueue_call(stub)
task = q.dequeue(w)
w.start_task(task)
assert worker_registry.get_running_tasks() == {w.id: task.id}
time.step()
with assert_atomic():
w.end_task(task, TaskOutcome("failure", message="my error"))
assert worker_registry.get_running_tasks() == dict()
assert failed_task_registry.get_task_ids() == [task.id]
for t in [task, Task.fetch(task.id)]:
assert t.status == TaskStatus.FAILED
assert t.error_message == "my error"
assert t.ended_at == time.now
def test_persistence(assert_atomic, connection, stub):
fields = {
'func_name', 'args', 'kwargs', 'status', 'origin', 'description',
'error_message', 'enqueued_at', 'started_at', 'ended_at', 'meta',
'aborted_runs'
}
def randomize_data(task):
string_fields = [
'func_name', 'status', 'description', 'origin', 'error_message'
]
date_fields = ['enqueued_at', 'started_at', 'ended_at']
for f in string_fields:
setattr(task, f, str(uuid.uuid4()))
for f in date_fields:
setattr(
task, f,
datetime.datetime(random.randint(1000, 9999),
1,
1,
tzinfo=datetime.timezone.utc))
task.args = tuple(str(uuid.uuid4()) for i in range(4))
task.kwargs = {str(uuid.uuid4()): ["d"]}
task.meta = {"x": [str(uuid.uuid4())]}
task.aborted_runs = ["foo", "bar", str(uuid.uuid4())]
def as_dict(task):
return {f: getattr(task, f) for f in fields}
task = Task(stub)
with assert_atomic():
task._save()
assert set(decode_list(connection.hkeys(task.key))) <= fields
assert as_dict(Task.fetch(task.id)) == as_dict(task)
randomize_data(task)
task._save()
assert as_dict(Task.fetch(task.id)) == as_dict(task)
# only deletes
task.enqueued_at = None
task.error_message = None
task._save(['enqueued_at', 'error_message'])
assert task.enqueued_at is None
assert task.error_message is None
for i in range(5):
store = random.sample(fields, 7)
copy = Task.fetch(task.id)
randomize_data(copy)
copy._save(store)
for f in store:
setattr(task, f, getattr(copy, f))
assert as_dict(Task.fetch(task.id)) == as_dict(task)
copy = Task.fetch(task.id)
randomize_data(copy)
copy.meta = task.meta = {"new_meta": "here"}
copy.save_meta()
copy.refresh()
assert as_dict(copy) == as_dict(task)
assert as_dict(Task.fetch(task.id)) == as_dict(task)
# save() and save_meta() in same pipeline
with assert_atomic():
with connection.pipeline() as pipe:
task = Task(stub)
task._save(pipeline=pipe)
task.meta["a"] = "b"
task.save_meta(pipeline=pipe)
pipe.execute()
assert Task.fetch(task.id).meta == {"a": "b"}
task = Task(stub)
with pytest.raises(TaskDoesNotExist):
task.refresh()
with pytest.raises(TaskDoesNotExist):
Task.fetch('nonexist')
def test_get_properties(settings, stub):
assert not Task(stub).is_reentrant
assert Task(reentrant_stub).is_reentrant
assert Task(stub).timeout == settings.DEFAULT_TASK_TIMEOUT
assert Task(my_timeout_func).timeout == 42
def test_get_func(stub):
assert Task(stub)._get_func() == stub