def test_worker_load_task_new_missing(app): task_worker = worker.Worker(app=app, queues=["yay"]) with pytest.raises(exceptions.TaskNotFound): task_worker.load_task("foobarbaz") assert task_worker.known_missing_tasks == {"foobarbaz"}
async def test_run_job_pass_context(app): result = [] @app.task(queue="yay", name="job", pass_context=True) def task_func(test_context, a): result.extend([test_context, a]) job = jobs.Job( id=16, task_kwargs={"a": 1}, lock="sherlock", queueing_lock="houba", task_name="job", queue="yay", ) test_worker = worker.Worker(app, queues=["yay"], name="my_worker") context = job_context.JobContext( worker_name="my_worker", worker_id=3, worker_queues=["yay"], job=job, task=task_func, ) test_worker.current_contexts[3] = context await test_worker.run_job(job=job, worker_id=3) assert result == [ context, 1, ]
async def test_run_job_log_name(caplog, app, worker_name, logger_name, record_worker_name): caplog.set_level("INFO") test_worker = worker.Worker(app, name=worker_name, wait=False) @app.task def task(): pass await task.defer_async() await test_worker.run() # We're not interested in defer logs records = [r for r in caplog.records if "worker" in r.name] assert len(records) record_names = [record.name for record in records] assert all([name.endswith(logger_name) for name in record_names]) worker_names = [ getattr(record, "worker", {}).get("name") for record in records ] assert all([name == record_worker_name for name in worker_names])
def test_context_for_worker_kwargs(app): test_worker = worker.Worker(app=app, name="foo") expected = job_context.JobContext(app=app, worker_id=3, worker_name="bar") context = test_worker.context_for_worker(worker_id=3, worker_name="bar") assert context == expected
async def running_worker(app): running_worker = worker.Worker(app=app, queues=["some_queue"]) task = asyncio.ensure_future(running_worker.run()) running_worker.task = task yield running_worker running_worker.stop() await asyncio.wait_for(task, timeout=0.5)
async def test_run_job_pass_context(app): result = [] @app.task(queue="yay", name="job", pass_context=True) def task_func(test_context, a): result.extend([test_context, a]) job = jobs.Job( id=16, task_kwargs={"a": 1}, lock="sherlock", queueing_lock="houba", task_name="job", queue="yay", ) test_worker = worker.Worker(app, queues=["yay"], name="my_worker", additional_context={"foo": "bar"}) context = test_worker.context_for_worker(worker_id=3) await test_worker.run_job(job=job, worker_id=3) context = context.evolve(task=task_func) assert result == [ context, 1, ]
async def test_run_job_log_result(caplog, app, job_store): caplog.set_level("INFO") result = [] def task_func(a, b): # pylint: disable=unused-argument s = a + b result.append(s) return s task = tasks.Task(task_func, app=app, queue="yay", name="job") app.tasks = {"task_func": task} job = jobs.Job( id=16, task_kwargs={ "a": 9, "b": 3 }, lock="sherlock", task_name="task_func", queue="yay", ) test_worker = worker.Worker(app, queues=["yay"]) await test_worker.run_job(job) assert result == [12] records = [ record for record in caplog.records if record.action == "job_success" ] assert len(records) == 1 record = records[0] assert record.result == 12
def test_worker_find_task(app): test_worker = worker.Worker(app=app, queues=["yay"]) @app.task(name="foo") def task_func(): pass assert test_worker.find_task("foo") == task_func
def test_worker_call_import_all(app, mocker): import_all = mocker.patch("procrastinate.utils.import_all") app.import_paths = ["hohoho"] worker.Worker(app=app, queues=["yay"]) import_all.assert_called_with(import_paths=["hohoho"])
def test_worker_load_task_known_task(app): task_worker = worker.Worker(app=app, queues=["yay"]) @app.task def task_func(): pass assert task_worker.load_task( "tests.unit.test_worker_sync.task_func") == task_func
async def test_process_next_job_raise_stop_requested(app): test_worker = worker.Worker(app) @app.task def empty(): test_worker.stop_requested = True await empty.defer_async() with pytest.raises(exceptions.StopRequested): await test_worker.process_next_job()
def test_worker_copy_additional_context(app): additional_context = {"foo": "bar"} test_worker = worker.Worker( app=app, name="worker", additional_context=additional_context, ) # mutate the additional_context object and test that we have the original # value in the worker additional_context["foo"] = "baz" assert test_worker.base_context.additional_context == {"foo": "bar"}
async def test_process_job_with_deletion(mocker, app, job_factory, connector, side_effect, delete_jobs): async def coro(*args, **kwargs): pass test_worker = worker.Worker(app, delete_jobs=delete_jobs) test_worker.run_job = mocker.Mock(side_effect=side_effect or coro) job = job_factory(id=1) await test_worker.job_manager.defer_job_async(job) await test_worker.process_job(job=job) assert 1 not in connector.jobs
async def test_run_no_listen_notify(app): running_worker = worker.Worker(app=app, queues=["some_queue"], listen_notify=False) task = asyncio.ensure_future(running_worker.run()) try: await asyncio.sleep(0.01) assert app.connector.notify_event is None finally: running_worker.stop() await asyncio.wait_for(task, timeout=0.5)
async def test_run_job_not_found(app, job_store): job = jobs.Job( id=16, task_kwargs={ "a": 9, "b": 3 }, lock="sherlock", task_name="job", queue="yay", ) test_worker = worker.Worker(app, queues=["yay"]) with pytest.raises(exceptions.TaskNotFound): await test_worker.run_job(job)
async def test_wait_for_activity_stop(pg_connector): """ Testing than calling job_store.stop() interrupts the wait """ pg_app = app.App(connector=pg_connector) worker = worker_module.Worker(app=pg_app, timeout=2) task = asyncio.ensure_future(worker.run()) await asyncio.sleep(0.2) # should be enough so that we're waiting worker.stop() try: await asyncio.wait_for(task, timeout=0.2) except asyncio.TimeoutError: pytest.fail("Failed to stop worker within .2s")
async def test_process_next_job(mocker, app, job_factory, side_effect, status): job = job_factory(id=1) await app.job_store.defer_job(job) test_worker = worker.Worker(app, queues=["queue"]) async def coro(*args, **kwargs): pass run_job = mocker.patch("procrastinate.worker.Worker.run_job", side_effect=side_effect or coro) await test_worker.process_next_job() run_job.assert_called_with(job=job) assert app.job_store.jobs[1]["status"] == status
async def test_wait_for_activity_stop_from_signal(aiopg_connector, kill_own_pid): """ Testing than ctrl+c interrupts the wait """ pg_app = app.App(connector=aiopg_connector) worker = worker_module.Worker(app=pg_app, timeout=2) task = asyncio.ensure_future(worker.run()) await asyncio.sleep(0.2) # should be enough so that we're waiting kill_own_pid() try: await asyncio.wait_for(task, timeout=0.2) except asyncio.TimeoutError: pytest.fail("Failed to stop worker within .2s")
def test_worker_load_task_unknown_task(app, caplog): global unknown_task test_worker = worker.Worker(app=app, queues=["yay"]) @app.task def task_func(): pass unknown_task = task_func assert ( test_worker.load_task("tests.unit.test_worker_sync.unknown_task", worker_id=2) == task_func ) assert [record for record in caplog.records if record.action == "load_dynamic_task"]
def test_context_for_worker_with_additional_context(app): additional_context = {"foo": "bar"} test_worker = worker.Worker( app=app, name="worker", additional_context=additional_context, ) context1 = test_worker.context_for_worker(worker_id=3) # mutate the additional_context object for one worker and test that it # hasn't changed for other workers context1.additional_context["foo"] = "baz" context2 = test_worker.context_for_worker(worker_id=4) assert context2.additional_context == {"foo": "bar"}
async def test_wait_for_activity(pg_connector): """ Testing that a new event interrupts the wait """ pg_app = app.App(connector=pg_connector) worker = worker_module.Worker(app=pg_app, timeout=2) worker.notify_event = asyncio.Event() task = asyncio.ensure_future(worker.single_worker(worker_id=0)) await asyncio.sleep(0.2) # should be enough so that we're waiting worker.stop_requested = True worker.notify_event.set() try: await asyncio.wait_for(task, timeout=0.2) except asyncio.TimeoutError: pytest.fail("Failed to stop worker within .2s")
async def test_wait_for_activity_timeout(pg_connector): """ Testing that we timeout if nothing happens """ pg_app = app.App(connector=pg_connector) worker = worker_module.Worker(app=pg_app, timeout=2) worker.notify_event = asyncio.Event() task = asyncio.ensure_future(worker.single_worker(worker_id=0)) try: await asyncio.sleep(0.2) # should be enough so that we're waiting worker.stop_requested = True with pytest.raises(asyncio.TimeoutError): await asyncio.wait_for(task, timeout=0.2) finally: worker.notify_event.set()
async def test_process_next_job_retry_failed_job(mocker, app, job_factory): job = job_factory(id=1) await app.job_store.defer_job(job) mocker.patch( "procrastinate.worker.Worker.run_job", side_effect=exceptions.JobRetry( scheduled_at=pendulum.datetime(2000, 1, 1, tz="UTC")), ) test_worker = worker.Worker(app, queues=["queue"]) await test_worker.process_next_job() new_job = app.job_store.jobs[1] assert len(app.job_store.jobs) == 1 assert new_job["status"] == "todo" assert new_job["id"] == 1 assert new_job["scheduled_at"] == pendulum.datetime(2000, 1, 1, tz="UTC")
async def test_wait_for_job_without_job(app, mocker): test_worker = worker.Worker(app) # notify_event is set to None initially, and we skip run() test_worker.notify_event = mocker.Mock() wait_for = mocker.Mock(side_effect=asyncio.TimeoutError) async def mock(coro, timeout): wait_for(coro, timeout=timeout) mocker.patch("asyncio.wait_for", mock) await test_worker.wait_for_job(timeout=42) wait_for.assert_called_with(test_worker.notify_event.wait.return_value, timeout=42) assert test_worker.notify_event.mock_calls == [ mocker.call.clear(), mocker.call.wait(), ]
async def test_run_job(app, job_store): result = [] @app.task(queue="yay", name="task_func") def task_func(a, b): result.append(a + b) job = jobs.Job( id=16, task_kwargs={ "a": 9, "b": 3 }, lock="sherlock", task_name="task_func", queue="yay", ) test_worker = worker.Worker(app, queues=["yay"]) await test_worker.run_job(job) assert result == [12]
async def test_run_job_retry(app, job_store): def job(a, b): # pylint: disable=unused-argument raise ValueError("nope") task = tasks.Task(job, app=app, queue="yay", name="job", retry=True) task.func = job app.tasks = {"job": task} job = jobs.Job( id=16, task_kwargs={ "a": 9, "b": 3 }, lock="sherlock", task_name="job", queue="yay", ) test_worker = worker.Worker(app, queues=["yay"]) with pytest.raises(exceptions.JobRetry): await test_worker.run_job(job)
async def test_run_job_concurrency_warning(app, caplog): # Running a sync task with concurrency > 1 should raise a warning result = [] caplog.set_level(logging.WARNING) @app.task(queue="yay", name="job") def task_func(a): result.append(a) job = jobs.Job( id=16, task_kwargs={"a": 1}, lock="sherlock", queueing_lock="houba", task_name="job", queue="yay", ) test_worker = worker.Worker(app, concurrency=2) await test_worker.run_job(job=job, worker_id=0) assert result == [1] assert [(r.action, r.levelname) for r in caplog.records ] == [("concurrent_sync_task", "WARNING")], caplog.records
async def test_run_job_error(app): def job(a, b): # pylint: disable=unused-argument raise ValueError("nope") task = tasks.Task(job, blueprint=app, queue="yay", name="job") task.func = job app.tasks = {"job": task} job = jobs.Job( id=16, task_kwargs={ "a": 9, "b": 3 }, lock="sherlock", queueing_lock="houba", task_name="job", queue="yay", ) test_worker = worker.Worker(app, queues=["yay"]) with pytest.raises(exceptions.JobError): await test_worker.run_job(job=job, worker_id=3)
def _worker(self, **kwargs) -> "worker.Worker": from procrastinate import worker final_kwargs = {**self.worker_defaults, **kwargs} return worker.Worker(app=self, **final_kwargs)
def test_worker(app): return worker.Worker(app=app, queues=["yay"])