async def __retry_tasks(self, tasks: List[WorkerTaskInProgress]) -> None:
for task in tasks:
if task.retry_counter > 0:
# todo: maybe it still in the queue of the worker?
# reschedule
await self.__add_task(task.task, task.retry_counter - 1)
else:
log.warning(f"Too many retried executing task {task.task.id}. Give up.")
self.outstanding_tasks.pop(task.task.id, None)
self.work_count[task.worker.worker_id] = self.work_count[task.worker.worker_id] - 1
set_future_result(task.task.callback, TimeoutError("Could not finish the task."))
async def __acknowledge_task(self, worker_id: WorkerId, task_id: TaskId, result: Optional[Json]) -> None:
# remove task from internal list
in_progress = self.outstanding_tasks.get(task_id, None)
if in_progress:
if in_progress.worker.worker_id == worker_id:
self.outstanding_tasks.pop(task_id, None)
self.work_count[worker_id] = self.work_count[worker_id] - 1
set_future_result(in_progress.task.callback, result)
# todo: remove task from database
else:
log.info(f"Got result for task {task_id} from wrong worker {worker_id}. outdated?")
async def __error_task(self, worker_id: WorkerId, task_id: TaskId, message: str) -> None:
log.warning(f"Task {task_id} yielded an error: {message}")
in_progress = self.outstanding_tasks.get(task_id, None)
if in_progress:
if in_progress.worker.worker_id == worker_id:
self.outstanding_tasks.pop(task_id, None)
self.work_count[worker_id] = self.work_count[worker_id] - 1
set_future_result(in_progress.task.callback, AttributeError(f"Error executing task: {message}"))
# todo: remove task from database
else:
log.info(f"Got error for task {task_id} from wrong worker {worker_id}. outdated?")
async def check_outdated_unassigned_tasks(self) -> None:
now = utc()
outstanding = [ip for ip in self.outstanding_tasks.values() if ip.deadline < now]
not_started_outdated = [ns for ns in self.unassigned_tasks.values() if ns.deadline < now]
async with self.lock:
await self.__retry_tasks(outstanding)
for ns in not_started_outdated:
log.info(f"No worker for task: {ns.task.id}. Give up.")
set_future_result(ns.task.callback, Exception(f"No worker for task: {ns.task.name}"))
self.unassigned_tasks.pop(ns.task.id, None)
# unassigned_task now only holds valid tasks
for ns in list(self.unassigned_tasks.values()):
if await self.__add_task(ns.task, ns.retry_counter):
self.unassigned_tasks.pop(ns.task.id, None)