async def test_dynamic_coordination(timemachine, backend): if not backend.is_persistent: pytest.skip("This test requires persistence.") args = backend.get_args(timemachine) app1 = Pyncette(**args) app2 = Pyncette(**args) counter = MagicMock() @app1.dynamic_task() @app2.dynamic_task() async def hello(context: Context) -> None: counter.execute() async with app1.create() as ctx1, app2.create() as ctx2: task1 = asyncio.create_task(ctx1.run()) task2 = asyncio.create_task(ctx2.run()) await ctx1.schedule_task(hello, "1", interval=datetime.timedelta(seconds=2)) await timemachine.step(datetime.timedelta(seconds=10)) await ctx2.unschedule_task(hello, "1") await timemachine.step(datetime.timedelta(seconds=10)) ctx1.shutdown() ctx2.shutdown() await asyncio.gather(task1, task2) await timemachine.unwind() assert counter.execute.call_count == 5
async def test_persistence(timemachine, backend): if not backend.is_persistent: pytest.skip("This test requires persistence.") app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.dynamic_task() async def hello(context: Context) -> None: counter.execute() async with app.create() as ctx: await ctx.schedule_task(hello, "1", schedule="* * * * *") await timemachine.jump_to(datetime.timedelta(seconds=30)) async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step() ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 0 await timemachine.jump_to(datetime.timedelta(seconds=60)) async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step() ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 1
async def test_cancelling_run_should_cancel_executing_tasks( timemachine, backend, ): app = Pyncette(**backend.get_args(timemachine), concurrency_limit=1) counter = MagicMock() release = asyncio.Event() @app.task( interval=datetime.timedelta(seconds=1), execution_mode=ExecutionMode.AT_MOST_ONCE, ) async def long_running_task(context: Context) -> None: counter.execute() await release.wait() with pytest.raises(asyncio.CancelledError): async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=1.5)) task.cancel() await task assert counter.execute.call_count == 1
async def test_dynamic_successful_task_interval_small_batch_size(timemachine, backend): app = Pyncette(**backend.get_args(timemachine), batch_size=1) counter = MagicMock() @app.dynamic_task() async def hello(context: Context) -> None: counter.execute() async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await asyncio.gather( ctx.schedule_task(hello, "1", interval=datetime.timedelta(seconds=2)), ctx.schedule_task(hello, "2", interval=datetime.timedelta(seconds=2)), ctx.schedule_task(hello, "3", interval=datetime.timedelta(seconds=2)), ) await timemachine.step(datetime.timedelta(seconds=10)) await asyncio.gather( ctx.unschedule_task(hello, "1"), ctx.unschedule_task(hello, "2"), ctx.unschedule_task(hello, "3"), ) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 15
async def test_context_scheduled_at(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.task(interval=datetime.timedelta(seconds=2)) async def once_failing_task(context: Context) -> None: counter.offset(context.scheduled_at) async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.offset.call_count == 5 counter.offset.assert_any_call( datetime.datetime(2019, 1, 1, 0, 0, 2, tzinfo=dateutil.tz.UTC) ) counter.offset.assert_any_call( datetime.datetime(2019, 1, 1, 0, 0, 4, tzinfo=dateutil.tz.UTC) ) counter.offset.assert_any_call( datetime.datetime(2019, 1, 1, 0, 0, 6, tzinfo=dateutil.tz.UTC) ) counter.offset.assert_any_call( datetime.datetime(2019, 1, 1, 0, 0, 8, tzinfo=dateutil.tz.UTC) ) counter.offset.assert_any_call( datetime.datetime(2019, 1, 1, 0, 0, 10, tzinfo=dateutil.tz.UTC) )
async def test_dynamic_successful_task_interval_invalid_batch_size( timemachine, backend ): app = Pyncette(**backend.get_args(timemachine), batch_size=0) with pytest.raises(ValueError): async with app.create(): pass # pragma: no cover
async def test_dynamic_poll_after_unregister(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.dynamic_task() async def hello(context: Context) -> None: counter.execute() # pragma: no cover async with app.create() as ctx: task_instance = await ctx.schedule_task( hello, "1", interval=datetime.timedelta(seconds=1) ) await ctx.unschedule_task(hello, "1") with pytest.raises(PyncetteException, match="not found"): await ctx._repository.poll_task(timemachine.utcnow(), task_instance) task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=60)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 0
async def test_partitioned_successful_task_interval_selective(timemachine, backend): timemachine.spin_iterations = 20 app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.partitioned_task(partition_count=5, enabled_partitions=[0, 1]) async def hello(context: Context) -> None: counter.execute() async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await asyncio.gather( *[ ctx.schedule_task(hello, name, interval=datetime.timedelta(seconds=2)) for name in PARTITION_TASK_NAMES ] ) await timemachine.step(datetime.timedelta(seconds=10)) await asyncio.gather( *[ctx.unschedule_task(hello, name) for name in PARTITION_TASK_NAMES] ) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 10
async def test_disabled_partitioned_task(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.partitioned_task(partition_count=1, enabled=True) async def hello_1(context: Context) -> None: counter.execute() # pragma: no cover @app.partitioned_task(partition_count=1, enabled=False) async def hello_2(context: Context) -> None: counter.execute() # pragma: no cover async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await ctx.schedule_task(hello_1, "1", interval=datetime.timedelta(seconds=2)) await ctx.schedule_task(hello_2, "1", interval=datetime.timedelta(seconds=2)) await timemachine.step(datetime.timedelta(seconds=10)) await ctx.unschedule_task(hello_1, "1") await ctx.unschedule_task(hello_2, "1") await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 5
async def test_heartbeat_fails_if_lease_lost(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.task( interval=datetime.timedelta(seconds=2), lease_duration=datetime.timedelta(seconds=1), ) async def successful_task(context: Context) -> None: await asyncio.sleep(5) try: await context.heartbeat() counter.successes() except LeaseLostException: counter.failures() raise async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.failures.call_count == 8 assert counter.successes.call_count == 1
async def test_automatic_heartbeating_cancel_on_lease_expired(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() leases = [] @app.task( interval=datetime.timedelta(seconds=1), lease_duration=datetime.timedelta(seconds=1), ) @with_heartbeat(cancel_on_lease_lost=True) async def successful_task(context: Context) -> None: counter.started() # Fake getting an old lease for the 2nd execution leases.append(context._lease) context._lease = leases[0] await asyncio.sleep(6) counter.finished() async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.started.call_count == 4 assert counter.finished.call_count == 1
async def test_heartbeating_noop_on_best_effort_tasks(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.task( interval=datetime.timedelta(seconds=6), execution_mode=ExecutionMode.AT_MOST_ONCE, ) async def successful_task(context: Context) -> None: counter.lease(context._lease) await context.heartbeat() counter.lease(context._lease) async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() lease1 = counter.lease.call_args_list[0][0][0] lease2 = counter.lease.call_args_list[1][0][0] assert lease1 is lease2
async def test_execute_at_retry(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.dynamic_task(failure_mode=FailureMode.UNLOCK) async def hello(context: Context) -> None: getattr(counter, context.task.name).execute() raise RuntimeError("Oops") now = timemachine.utcnow() async with app.create() as ctx: await asyncio.gather( ctx.schedule_task( hello, "task1", execute_at=now + datetime.timedelta(seconds=1) ), ctx.schedule_task( hello, "task2", execute_at=now + datetime.timedelta(seconds=2) ), ctx.schedule_task( hello, "task3", execute_at=now + datetime.timedelta(seconds=3) ), ) task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=5)) ctx.shutdown() await task await timemachine.unwind() assert counter.task1.execute.call_count == 5 assert counter.task2.execute.call_count == 4 assert counter.task3.execute.call_count == 3
async def test_execute_at(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.dynamic_task() async def hello(context: Context) -> None: counter.execute() now = timemachine.utcnow() async with app.create() as ctx: await asyncio.gather( ctx.schedule_task( hello, "1", execute_at=now + datetime.timedelta(seconds=1) ), ctx.schedule_task( hello, "2", execute_at=now + datetime.timedelta(seconds=2) ), ctx.schedule_task( hello, "3", execute_at=now + datetime.timedelta(seconds=3) ), ) task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 3
async def test_fixture(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.fixture() async def hello(app_context: PyncetteContext): counter.entered() yield counter.executed counter.exited() @app.task(interval=datetime.timedelta(seconds=2)) async def successful_task(context: Context) -> None: context.hello() async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.entered.call_count == 1 assert counter.executed.call_count == 5 assert counter.exited.call_count == 1
async def test_does_not_catch_up_with_stale_executions_if_fast_forward_used_cronspec( timemachine, backend ): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.task( schedule="* * * * * */3", fast_forward=True, failure_mode=FailureMode.UNLOCK, ) async def once_failing_task(context: Context) -> None: counter.execute() if counter.execute.call_count == 1: await asyncio.sleep(10) raise RuntimeError("Oops") async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=20)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 4
async def test_dynamic_successful_task_interval(): app = Pyncette() @app.dynamic_task() async def hello(context: Context) -> None: pass # pragma: no cover with pytest.raises(ValueError, match="instance name must be provided"): async with app.create() as ctx: await ctx.unschedule_task(hello)
async def test_successful_task_interval(timemachine): app = Pyncette( repository_factory=wrap_factory(sqlite_repository, timemachine)) use_prometheus(app) counter = MagicMock() @app.dynamic_task() async def dynamic_task_1(context: Context) -> None: counter.execute() @app.task(interval=datetime.timedelta(seconds=2)) async def task_1(context: Context) -> None: counter.execute() async with app.create() as ctx: await ctx.schedule_task(dynamic_task_1, "1", interval=datetime.timedelta(seconds=2)) task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) await ctx.unschedule_task(dynamic_task_1, "1") ctx.shutdown() await task await timemachine.unwind() metrics = generate_latest().decode("ascii").splitlines() assert ( 'pyncette_repository_ops_total{operation="commit_task",task_name="dynamic_task_1"} 5.0' in metrics) assert ( 'pyncette_repository_ops_total{operation="commit_task",task_name="task_1"} 5.0' in metrics) assert ( 'pyncette_repository_ops_total{operation="poll_dynamic_task",task_name="dynamic_task_1"} 11.0' in metrics) assert ( 'pyncette_repository_ops_total{operation="poll_task",task_name="dynamic_task_1"} 5.0' in metrics) assert ( 'pyncette_repository_ops_total{operation="poll_task",task_name="task_1"} 11.0' in metrics) assert ( 'pyncette_repository_ops_total{operation="register_task",task_name="dynamic_task_1"} 1.0' in metrics) assert ( 'pyncette_repository_ops_total{operation="unregister_task",task_name="dynamic_task_1"} 1.0' in metrics) assert 'pyncette_tasks_total{task_name="dynamic_task_1"} 5.0' in metrics assert 'pyncette_tasks_total{task_name="task_1"} 5.0' in metrics
async def test_default_healthcheck_handler_healthy(timemachine): app = Pyncette( repository_factory=wrap_factory(sqlite_repository, timemachine)) async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=1.5)) is_healthy = await default_healthcheck(ctx) ctx.shutdown() await task await timemachine.unwind() assert is_healthy
async def test_middlewares(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.task(interval=datetime.timedelta(seconds=2)) async def task1(context: Context) -> None: pass @app.task(interval=datetime.timedelta(seconds=2)) async def task2(context: Context) -> None: raise Exception() @app.middleware async def switch1(context: Context, next: Callable[[], Awaitable[None]]): c = getattr(counter, context.task.name) try: c.enter(1) await next() c.success(1) except Exception: # pragma: no cover c.caught(1) @app.middleware async def switch2(context: Context, next: Callable[[], Awaitable[None]]): c = getattr(counter, context.task.name) try: c.enter(2) await next() c.success(2) except Exception: c.caught(2) async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=2)) ctx.shutdown() await task await timemachine.unwind() assert counter.task1.mock_calls == [ call.enter(1), call.enter(2), call.success(2), call.success(1), ] assert counter.task2.mock_calls == [ call.enter(1), call.enter(2), call.caught(2), call.success(1), ]
async def test_default_healthcheck_handler_unhealthy(timemachine): app = Pyncette( repository_factory=wrap_factory(sqlite_repository, timemachine)) async with app.create() as ctx: task = asyncio.create_task(ctx.run()) # Advance time without executing calbacks timemachine._update_offset(timemachine.offset + datetime.timedelta(hours=1)) is_healthy = await default_healthcheck(ctx) ctx.shutdown() await task await timemachine.unwind() assert not is_healthy
async def test_successful_task_interval(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.task(interval=datetime.timedelta(seconds=2)) async def successful_task(context: Context) -> None: counter.execute() async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 5
async def test_failed_task_retried_on_every_tick_if_unlock(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.task(interval=datetime.timedelta(seconds=2), failure_mode=FailureMode.UNLOCK) async def failing_task(context: Context) -> None: counter.execute() raise RuntimeError("Oops") async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 9
async def test_add_to_context(timemachine): app = Pyncette(**DefaultBackend().get_args(timemachine)) counter = MagicMock() @app.task(interval=datetime.timedelta(seconds=2)) async def successful_task(context: Context) -> None: context.hello() async with app.create() as ctx: ctx.add_to_context("hello", counter) task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.call_count == 5
async def test_dynamic_register_again(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.dynamic_task() async def hello(context: Context) -> None: counter.execute() async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await ctx.schedule_task(hello, "1", interval=datetime.timedelta(seconds=1)) await ctx.schedule_task(hello, "1", interval=datetime.timedelta(seconds=2)) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 5
async def test_fast_forward_cronspec(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.task(schedule="* * * * * */3", fast_forward=True) async def once_long_task(context: Context) -> None: counter.execute() if counter.execute.call_count == 1: await asyncio.sleep(10) async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=20)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 3
async def test_extra_args(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.task(interval=datetime.timedelta(seconds=2), foo="bar", quux=123) async def successful_task(context: Context) -> None: counter.foo = context.args["foo"] counter.quux = context.args["quux"] async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.foo == "bar" assert counter.quux == 123
async def test_catches_up_with_stale_executions(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.task(interval=datetime.timedelta(seconds=2), failure_mode=FailureMode.UNLOCK) async def once_failing_task(context: Context) -> None: counter.execute() if counter.execute.call_count == 1: await asyncio.sleep(10) raise RuntimeError("Oops") async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=20)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 10
async def test_failed_task_not_retried_if_best_effort(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.task( interval=datetime.timedelta(seconds=2), execution_mode=ExecutionMode.AT_MOST_ONCE, ) async def failing_task(context: Context) -> None: counter.execute() raise RuntimeError("Oops") async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 5
async def test_not_locked_while_executing_if_best_effort_is_used(timemachine, backend): app = Pyncette(**backend.get_args(timemachine)) counter = MagicMock() @app.task( interval=datetime.timedelta(seconds=2), execution_mode=ExecutionMode.AT_MOST_ONCE, ) async def successful_task(context: Context) -> None: counter.execute() await asyncio.sleep(5) async with app.create() as ctx: task = asyncio.create_task(ctx.run()) await timemachine.step(datetime.timedelta(seconds=10)) ctx.shutdown() await task await timemachine.unwind() assert counter.execute.call_count == 5