def test_worker_death(assert_atomic, connection, stub):
def setup(func):
w = WorkerFactory()
w.startup()
q.enqueue_call(func)
task = q.dequeue(w)
return task, w
q = QueueFactory()
# Worker died before starting work on the task
task, w = setup(stub)
assert worker_registry.get_running_tasks() == {w.id: task.id}
with assert_atomic(exceptions=['hgetall']):
w.died()
assert q.get_task_ids() == [task.id]
assert worker_registry.get_running_tasks() == dict()
assert failed_task_registry.get_task_ids() == []
# Worker died after starting non-reentrant task
q.empty()
task, w = setup(stub)
assert worker_registry.get_running_tasks() == {w.id: task.id}
w.start_task(task)
with assert_atomic(exceptions=['hgetall']):
w.died()
assert q.get_task_ids() == []
assert worker_registry.get_running_tasks() == dict()
assert failed_task_registry.get_task_ids() == [task.id]
# Worker died after starting reentrant task
connection.delete(failed_task_registry.key)
task, w = setup(reentrant_stub)
assert worker_registry.get_running_tasks() == {w.id: task.id}
with assert_atomic():
w.start_task(task)
with assert_atomic(exceptions=['hgetall']):
w.died()
assert q.get_task_ids() == [task.id]
assert worker_registry.get_running_tasks() == dict()
assert failed_task_registry.get_task_ids() == []
def test_cancel(assert_atomic, connection):
q = QueueFactory()
task = q.enqueue_call()
with assert_atomic():
task.cancel()
assert q.get_task_ids() == []
assert task.status == TaskStatus.CANCELED
assert not connection.exists(task.key)
w = WorkerFactory()
w.startup()
task = q.enqueue_call()
q.dequeue(w)
with pytest.raises(InvalidOperation):
with assert_atomic():
task.cancel()
assert worker_registry.get_running_tasks() == {w.id: task.id}
assert connection.exists(task.key)
def test_state_transistions(assert_atomic, connection, time_mocker, stub):
time = time_mocker('redis_tasks.task.utcnow')
task = Task(reentrant_stub)
q = QueueFactory()
w = WorkerFactory()
w.startup()
# enqueue
time.step()
assert not connection.exists(task.key)
with assert_atomic():
task.enqueue(q)
assert q.get_task_ids() == [task.id]
assert connection.exists(task.key)
for t in [task, Task.fetch(task.id)]:
assert t.enqueued_at == time.now
assert t.status == TaskStatus.QUEUED
assert t.origin == q.name
# dequeue
task = q.dequeue(w)
assert q.get_task_ids() == []
assert worker_registry.get_running_tasks() == {w.id: task.id}
# set_running
time.step()
with assert_atomic():
w.start_task(task)
assert worker_registry.get_running_tasks() == {w.id: task.id}
for t in [task, Task.fetch(task.id)]:
assert t.status == TaskStatus.RUNNING
assert t.started_at == time.now
# requeue
time.step()
with assert_atomic():
w.end_task(task, TaskOutcome("requeue"))
assert worker_registry.get_running_tasks() == dict()
assert q.get_task_ids() == [task.id]
for t in [task, Task.fetch(task.id)]:
assert t.status == TaskStatus.QUEUED
assert t.started_at is None
# set_finished
task = q.dequeue(w)
w.start_task(task)
time.step()
with assert_atomic():
w.end_task(task, TaskOutcome("success"))
assert q.get_task_ids() == []
assert finished_task_registry.get_task_ids() == [task.id]
for t in [task, Task.fetch(task.id)]:
assert t.status == TaskStatus.FINISHED
assert t.ended_at == time.now
# set_failed
task = q.enqueue_call(stub)
task = q.dequeue(w)
w.start_task(task)
assert worker_registry.get_running_tasks() == {w.id: task.id}
time.step()
with assert_atomic():
w.end_task(task, TaskOutcome("failure", message="my error"))
assert worker_registry.get_running_tasks() == dict()
assert failed_task_registry.get_task_ids() == [task.id]
for t in [task, Task.fetch(task.id)]:
assert t.status == TaskStatus.FAILED
assert t.error_message == "my error"
assert t.ended_at == time.now