def test_dead_e2e():
app = App(
name="testapp",
retry_backoff=lambda retries: 0.01,
schedule_interval=0.1,
heartbeat_interval=0.1,
maintenance_interval=0.1,
processes=1,
concurrency=4,
prefetch_count=1,
)
@app.task(retries=0)
def example():
raise Chaos(f"Task failure")
x = example.delay()
# Process the queue, move the failure to the DLQ.
with worker(app):
state = wait_for_results(app, length=1, sleep=0.02, maxwait=1)
assert len(state.dead.messages) == 1
assert len(state.queue.messages) == 0
assert get_job(app, x.job.uuid).max_retries == 0
assert count_results(app) == 1
# Process the DLQ, move the tasks back to the main queue.
replay_dead(app)
state = get_state(app)
assert len(state.dead.messages) == 0
assert len(state.queue.messages) == 1
assert get_job(app, x.job.uuid).max_retries == 0
assert count_results(app) == 1
def test_purge(app, broker, failing_job, dead_job):
results = purge_dead(app)
assert len(results) == 1
state = get_state(app)
assert len(state.queue.messages) == 0
assert len(state.dead.messages) == 0
with pytest.raises(JobNotFound):
get_job(app, failing_job.uuid)
def task(application: str, uuid: Optional[str]) -> None:
"""
Print a JSON-encoded summary of job information.
"""
app = get_object(application)
try:
click.echo(render(get_job(app, uuid)))
except JobNotFound:
click.echo(render({"error": f"Task with uuid={uuid} not found"}))
def test_module(app, job, xid):
actions.send(app, job, xid=xid)
assert state.get_status(app, job.uuid) == SENT
assert state.get_job(app, job.uuid).status == SENT
assert state.count_jobs(app) == 1
assert state.get_messages(
app, app.keys.queue) == [Message(id=xid, uuid=job.uuid)]
with executor(app):
wait_for_results(app, length=1)
assert actions.result(app, job, timeout=1)["return_value"] == job.args[0]
assert state.get_info(app, app.keys.queue).groups == 1
assert state.get_groups(app, app.keys.queue)[0].name == app.keys.group
assert state.get_stream(app, app.keys.queue).key == app.keys.queue
assert state.count_results(app) == 1
with pytest.raises(JobNotFound):
state.get_job(app, "nope")
def test_replay(app, broker, failing_job, dead_job):
results = replay_dead(app)
assert len(results) == 1
state = get_state(app)
assert len(state.queue.messages) == 1
assert len(state.dead.messages) == 0
job = get_job(app, failing_job.uuid)
assert job.tries == 1
assert job.max_retries == 0
assert job.status == SENT
def dead_job(app, failing_job):
failing_job.tries = 1
failing_job.max_retries = 0
failing_job.status = DEAD
app.client.hmset(app.keys.status(failing_job), failing_job.serialise())
app.client.xadd(app.keys.dead, {"uuid": failing_job.uuid})
state = get_state(app)
assert len(state.queue.messages) == 0
assert len(state.dead.messages) == 1
job = get_job(app, failing_job.uuid)
assert job.tries == 1
assert job.max_retries == 0
assert job.status == DEAD