def test_worker_load_task_new_missing(app):
task_worker = worker.Worker(app=app, queues=["yay"])
with pytest.raises(exceptions.TaskNotFound):
task_worker.load_task("foobarbaz")
assert task_worker.known_missing_tasks == {"foobarbaz"}
async def test_run_job_pass_context(app):
result = []
@app.task(queue="yay", name="job", pass_context=True)
def task_func(test_context, a):
result.extend([test_context, a])
job = jobs.Job(
id=16,
task_kwargs={"a": 1},
lock="sherlock",
queueing_lock="houba",
task_name="job",
queue="yay",
)
test_worker = worker.Worker(app, queues=["yay"], name="my_worker")
context = job_context.JobContext(
worker_name="my_worker",
worker_id=3,
worker_queues=["yay"],
job=job,
task=task_func,
)
test_worker.current_contexts[3] = context
await test_worker.run_job(job=job, worker_id=3)
assert result == [
context,
1,
]
async def test_run_job_log_name(caplog, app, worker_name, logger_name,
record_worker_name):
caplog.set_level("INFO")
test_worker = worker.Worker(app, name=worker_name, wait=False)
@app.task
def task():
pass
await task.defer_async()
await test_worker.run()
# We're not interested in defer logs
records = [r for r in caplog.records if "worker" in r.name]
assert len(records)
record_names = [record.name for record in records]
assert all([name.endswith(logger_name) for name in record_names])
worker_names = [
getattr(record, "worker", {}).get("name") for record in records
]
assert all([name == record_worker_name for name in worker_names])
def test_context_for_worker_kwargs(app):
test_worker = worker.Worker(app=app, name="foo")
expected = job_context.JobContext(app=app, worker_id=3, worker_name="bar")
context = test_worker.context_for_worker(worker_id=3, worker_name="bar")
assert context == expected
async def running_worker(app):
running_worker = worker.Worker(app=app, queues=["some_queue"])
task = asyncio.ensure_future(running_worker.run())
running_worker.task = task
yield running_worker
running_worker.stop()
await asyncio.wait_for(task, timeout=0.5)
async def test_run_job_pass_context(app):
result = []
@app.task(queue="yay", name="job", pass_context=True)
def task_func(test_context, a):
result.extend([test_context, a])
job = jobs.Job(
id=16,
task_kwargs={"a": 1},
lock="sherlock",
queueing_lock="houba",
task_name="job",
queue="yay",
)
test_worker = worker.Worker(app,
queues=["yay"],
name="my_worker",
additional_context={"foo": "bar"})
context = test_worker.context_for_worker(worker_id=3)
await test_worker.run_job(job=job, worker_id=3)
context = context.evolve(task=task_func)
assert result == [
context,
1,
]
async def test_run_job_log_result(caplog, app, job_store):
caplog.set_level("INFO")
result = []
def task_func(a, b): # pylint: disable=unused-argument
s = a + b
result.append(s)
return s
task = tasks.Task(task_func, app=app, queue="yay", name="job")
app.tasks = {"task_func": task}
job = jobs.Job(
id=16,
task_kwargs={
"a": 9,
"b": 3
},
lock="sherlock",
task_name="task_func",
queue="yay",
)
test_worker = worker.Worker(app, queues=["yay"])
await test_worker.run_job(job)
assert result == [12]
records = [
record for record in caplog.records if record.action == "job_success"
]
assert len(records) == 1
record = records[0]
assert record.result == 12
def test_worker_find_task(app):
test_worker = worker.Worker(app=app, queues=["yay"])
@app.task(name="foo")
def task_func():
pass
assert test_worker.find_task("foo") == task_func
def test_worker_call_import_all(app, mocker):
import_all = mocker.patch("procrastinate.utils.import_all")
app.import_paths = ["hohoho"]
worker.Worker(app=app, queues=["yay"])
import_all.assert_called_with(import_paths=["hohoho"])
def test_worker_load_task_known_task(app):
task_worker = worker.Worker(app=app, queues=["yay"])
@app.task
def task_func():
pass
assert task_worker.load_task(
"tests.unit.test_worker_sync.task_func") == task_func
async def test_process_next_job_raise_stop_requested(app):
test_worker = worker.Worker(app)
@app.task
def empty():
test_worker.stop_requested = True
await empty.defer_async()
with pytest.raises(exceptions.StopRequested):
await test_worker.process_next_job()
def test_worker_copy_additional_context(app):
additional_context = {"foo": "bar"}
test_worker = worker.Worker(
app=app,
name="worker",
additional_context=additional_context,
)
# mutate the additional_context object and test that we have the original
# value in the worker
additional_context["foo"] = "baz"
assert test_worker.base_context.additional_context == {"foo": "bar"}
async def test_process_job_with_deletion(mocker, app, job_factory, connector,
side_effect, delete_jobs):
async def coro(*args, **kwargs):
pass
test_worker = worker.Worker(app, delete_jobs=delete_jobs)
test_worker.run_job = mocker.Mock(side_effect=side_effect or coro)
job = job_factory(id=1)
await test_worker.job_manager.defer_job_async(job)
await test_worker.process_job(job=job)
assert 1 not in connector.jobs
async def test_run_no_listen_notify(app):
running_worker = worker.Worker(app=app,
queues=["some_queue"],
listen_notify=False)
task = asyncio.ensure_future(running_worker.run())
try:
await asyncio.sleep(0.01)
assert app.connector.notify_event is None
finally:
running_worker.stop()
await asyncio.wait_for(task, timeout=0.5)
async def test_run_job_not_found(app, job_store):
job = jobs.Job(
id=16,
task_kwargs={
"a": 9,
"b": 3
},
lock="sherlock",
task_name="job",
queue="yay",
)
test_worker = worker.Worker(app, queues=["yay"])
with pytest.raises(exceptions.TaskNotFound):
await test_worker.run_job(job)
async def test_wait_for_activity_stop(pg_connector):
"""
Testing than calling job_store.stop() interrupts the wait
"""
pg_app = app.App(connector=pg_connector)
worker = worker_module.Worker(app=pg_app, timeout=2)
task = asyncio.ensure_future(worker.run())
await asyncio.sleep(0.2) # should be enough so that we're waiting
worker.stop()
try:
await asyncio.wait_for(task, timeout=0.2)
except asyncio.TimeoutError:
pytest.fail("Failed to stop worker within .2s")
async def test_process_next_job(mocker, app, job_factory, side_effect, status):
job = job_factory(id=1)
await app.job_store.defer_job(job)
test_worker = worker.Worker(app, queues=["queue"])
async def coro(*args, **kwargs):
pass
run_job = mocker.patch("procrastinate.worker.Worker.run_job",
side_effect=side_effect or coro)
await test_worker.process_next_job()
run_job.assert_called_with(job=job)
assert app.job_store.jobs[1]["status"] == status
async def test_wait_for_activity_stop_from_signal(aiopg_connector,
kill_own_pid):
"""
Testing than ctrl+c interrupts the wait
"""
pg_app = app.App(connector=aiopg_connector)
worker = worker_module.Worker(app=pg_app, timeout=2)
task = asyncio.ensure_future(worker.run())
await asyncio.sleep(0.2) # should be enough so that we're waiting
kill_own_pid()
try:
await asyncio.wait_for(task, timeout=0.2)
except asyncio.TimeoutError:
pytest.fail("Failed to stop worker within .2s")
def test_worker_load_task_unknown_task(app, caplog):
global unknown_task
test_worker = worker.Worker(app=app, queues=["yay"])
@app.task
def task_func():
pass
unknown_task = task_func
assert (
test_worker.load_task("tests.unit.test_worker_sync.unknown_task", worker_id=2)
== task_func
)
assert [record for record in caplog.records if record.action == "load_dynamic_task"]
def test_context_for_worker_with_additional_context(app):
additional_context = {"foo": "bar"}
test_worker = worker.Worker(
app=app,
name="worker",
additional_context=additional_context,
)
context1 = test_worker.context_for_worker(worker_id=3)
# mutate the additional_context object for one worker and test that it
# hasn't changed for other workers
context1.additional_context["foo"] = "baz"
context2 = test_worker.context_for_worker(worker_id=4)
assert context2.additional_context == {"foo": "bar"}
async def test_wait_for_activity(pg_connector):
"""
Testing that a new event interrupts the wait
"""
pg_app = app.App(connector=pg_connector)
worker = worker_module.Worker(app=pg_app, timeout=2)
worker.notify_event = asyncio.Event()
task = asyncio.ensure_future(worker.single_worker(worker_id=0))
await asyncio.sleep(0.2) # should be enough so that we're waiting
worker.stop_requested = True
worker.notify_event.set()
try:
await asyncio.wait_for(task, timeout=0.2)
except asyncio.TimeoutError:
pytest.fail("Failed to stop worker within .2s")
async def test_wait_for_activity_timeout(pg_connector):
"""
Testing that we timeout if nothing happens
"""
pg_app = app.App(connector=pg_connector)
worker = worker_module.Worker(app=pg_app, timeout=2)
worker.notify_event = asyncio.Event()
task = asyncio.ensure_future(worker.single_worker(worker_id=0))
try:
await asyncio.sleep(0.2) # should be enough so that we're waiting
worker.stop_requested = True
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(task, timeout=0.2)
finally:
worker.notify_event.set()
async def test_process_next_job_retry_failed_job(mocker, app, job_factory):
job = job_factory(id=1)
await app.job_store.defer_job(job)
mocker.patch(
"procrastinate.worker.Worker.run_job",
side_effect=exceptions.JobRetry(
scheduled_at=pendulum.datetime(2000, 1, 1, tz="UTC")),
)
test_worker = worker.Worker(app, queues=["queue"])
await test_worker.process_next_job()
new_job = app.job_store.jobs[1]
assert len(app.job_store.jobs) == 1
assert new_job["status"] == "todo"
assert new_job["id"] == 1
assert new_job["scheduled_at"] == pendulum.datetime(2000, 1, 1, tz="UTC")
async def test_wait_for_job_without_job(app, mocker):
test_worker = worker.Worker(app)
# notify_event is set to None initially, and we skip run()
test_worker.notify_event = mocker.Mock()
wait_for = mocker.Mock(side_effect=asyncio.TimeoutError)
async def mock(coro, timeout):
wait_for(coro, timeout=timeout)
mocker.patch("asyncio.wait_for", mock)
await test_worker.wait_for_job(timeout=42)
wait_for.assert_called_with(test_worker.notify_event.wait.return_value,
timeout=42)
assert test_worker.notify_event.mock_calls == [
mocker.call.clear(),
mocker.call.wait(),
]
async def test_run_job(app, job_store):
result = []
@app.task(queue="yay", name="task_func")
def task_func(a, b):
result.append(a + b)
job = jobs.Job(
id=16,
task_kwargs={
"a": 9,
"b": 3
},
lock="sherlock",
task_name="task_func",
queue="yay",
)
test_worker = worker.Worker(app, queues=["yay"])
await test_worker.run_job(job)
assert result == [12]
async def test_run_job_retry(app, job_store):
def job(a, b): # pylint: disable=unused-argument
raise ValueError("nope")
task = tasks.Task(job, app=app, queue="yay", name="job", retry=True)
task.func = job
app.tasks = {"job": task}
job = jobs.Job(
id=16,
task_kwargs={
"a": 9,
"b": 3
},
lock="sherlock",
task_name="job",
queue="yay",
)
test_worker = worker.Worker(app, queues=["yay"])
with pytest.raises(exceptions.JobRetry):
await test_worker.run_job(job)
async def test_run_job_concurrency_warning(app, caplog):
# Running a sync task with concurrency > 1 should raise a warning
result = []
caplog.set_level(logging.WARNING)
@app.task(queue="yay", name="job")
def task_func(a):
result.append(a)
job = jobs.Job(
id=16,
task_kwargs={"a": 1},
lock="sherlock",
queueing_lock="houba",
task_name="job",
queue="yay",
)
test_worker = worker.Worker(app, concurrency=2)
await test_worker.run_job(job=job, worker_id=0)
assert result == [1]
assert [(r.action, r.levelname) for r in caplog.records
] == [("concurrent_sync_task", "WARNING")], caplog.records
async def test_run_job_error(app):
def job(a, b): # pylint: disable=unused-argument
raise ValueError("nope")
task = tasks.Task(job, blueprint=app, queue="yay", name="job")
task.func = job
app.tasks = {"job": task}
job = jobs.Job(
id=16,
task_kwargs={
"a": 9,
"b": 3
},
lock="sherlock",
queueing_lock="houba",
task_name="job",
queue="yay",
)
test_worker = worker.Worker(app, queues=["yay"])
with pytest.raises(exceptions.JobError):
await test_worker.run_job(job=job, worker_id=3)
def _worker(self, **kwargs) -> "worker.Worker":
from procrastinate import worker
final_kwargs = {**self.worker_defaults, **kwargs}
return worker.Worker(app=self, **final_kwargs)
def test_worker(app):
return worker.Worker(app=app, queues=["yay"])
