async def test_handle_outdated(
task_queue: WorkerTaskQueue,
worker: Tuple[WorkerTaskDescription, WorkerTaskDescription, WorkerTaskDescription],
performed_by: Dict[str, List[str]],
) -> None:
_, _, outdated_task = worker
all_tasks = [create_task(str(n), outdated_task.name) for n in range(0, 20)]
for t in all_tasks:
await task_queue.add_task(t)
await asyncio.sleep(0)
count_outstanding = 0
while task_queue.outstanding_tasks:
await task_queue.check_outdated_unassigned_tasks()
count_outstanding += 1
# every message is retried 3 times ==> 4 times to get rid of all messages
assert count_outstanding == 4
results = await asyncio.gather(*[a.callback for a in all_tasks], return_exceptions=True)
# make sure all results are failures
for r in results:
assert isinstance(r, Exception)
# 20 work items by 4 workers: 5 work items each + retried 3 times (15) => 20
by_worker = group_by(lambda x: x, (item for sublist in performed_by.values() for item in sublist))
assert len(by_worker) == 4
for work_done in by_worker.values():
assert len(work_done) == 20
def __init__(
self,
dependencies: CLIDependencies,
parts: List[CLICommand],
alias_names: Dict[str, str],
alias_templates: List[AliasTemplate],
):
super().__init__(dependencies, True)
self.all_parts = {p.name: p for p in parts + [self]}
self.parts = {
p.name: p
for p in parts + [self] if not isinstance(p, InternalPart)
}
self.alias_names = {
a: n
for a, n in alias_names.items()
if n in self.parts and a not in self.parts
}
self.reverse_alias_names: Dict[str, List[str]] = {
k: [e[0] for e in v]
for k, v in group_by(lambda a: a[1],
self.alias_names.items()).items()
}
self.alias_templates = {
a.name: a
for a in sorted(alias_templates, key=attrgetter("name"))
}
async def test_handle_work_successfully(
task_queue: WorkerTaskQueue,
worker: Tuple[WorkerTaskDescription, WorkerTaskDescription, WorkerTaskDescription],
performed_by: Dict[str, List[str]],
) -> None:
success_task, _, _ = worker
all_tasks = [create_task(str(n), success_task.name) for n in range(0, 20)]
for t in all_tasks:
await task_queue.add_task(t)
results = await asyncio.gather(*[a.callback for a in all_tasks])
assert results == [{"result": "done!"} for _ in range(0, 20)]
# make sure the work is split equally between all workers: 20 work items by 4 workers: 5 work items each
by_worker = group_by(lambda x: x, (item for sublist in performed_by.values() for item in sublist))
assert len(by_worker) == 4
for work_done in by_worker.values():
assert len(work_done) == 5
async def update_trigger(self, desc: TaskDescription, register: bool = True) -> None:
# safeguard: unregister all event trigger of this task
for existing in (tup for tup in self.registered_event_trigger if desc.id == tup[1].id):
self.registered_event_trigger.remove(existing)
# safeguard: unregister all schedule trigger of this task
for job in self.scheduler.list_jobs():
if str(job.id).startswith(desc.id):
job.remove()
# add all triggers
if register:
for trigger in desc.triggers:
if isinstance(trigger, EventTrigger):
self.registered_event_trigger.append((trigger, desc))
if isinstance(trigger, TimeTrigger):
uid = f"{desc.id}_{trigger.cron_expression}"
name = f"Trigger for task {desc.id} on cron expression {trigger.cron_expression}"
self.scheduler.cron(uid, name, trigger.cron_expression, self.time_triggered, desc, trigger)
# recompute the lookup table
self.registered_event_trigger_by_message_type = group_by(
lambda t: t[0].message_type, self.registered_event_trigger
)