async def wait(self, finished: Queue): """Wait for process to complete""" self.waiting.set() await self.finish_event.wait() if self.finish: logger.info(f"Completed, pid: {self.pid}") await finished.put(self.pid)
async def add(self, process: ProcessMock) -> (uuid.uuid1, None): async with self.processes_lock: if len(self.processes) < self.capacity: pid = await process.run(self.finished) self.processes[pid] = process logger.info(f"Created process with pid: {pid}, priority: {process.priority}") return pid logger.warning(f"Cannot create a new process. The maximum capacity {self.capacity} has been reached") return None
async def kill_group(self, priority: str): pids = list() async with self.processes_lock: for pid, process in self.processes.items(): if process.priority == priority: process.kill() pids.append(pid) for pid in pids: del self.processes[pid] logger.info(f"All ({len(pids)}) processes with priority {priority} killed")
async def kill(self, pid: uuid.uuid1): async with self.processes_lock: if pid not in self.processes.keys(): logger.warning(f"Cannot kill process. No such pid: {pid}") return process = self.processes[pid] priority = process.priority process.kill() del self.processes[pid] logger.info(f"Killed process with pid: {pid}, priority: {priority}")
async def monitor_finished_tasks(self): logger.info("Starting monitoring finished tasks...") async with self.running_lock: self.running.set() while True: async with self.running_lock: if not self.running.is_set(): return try: pid = await asyncio.wait_for(self.finished.get(), timeout=1) await self.clean_up_(pid) except: pass
async def add(self, process: ProcessMock) -> (uuid.uuid1, None): pid_to_remove = None async with self.processes_lock: if len(self.pids) >= self.capacity: pid_to_remove = self.pids.pop( ) # remove pid of the oldest process if pid_to_remove: logger.warning( f"The maximum capacity {self.capacity} has been reached. Removing process with pid {pid_to_remove}" ) await self.kill(pid_to_remove) # kill the oldest process pid = await process.run(self.finished) async with self.processes_lock: self.pids.append(pid) self.processes[pid] = process logger.info( f"Created process with pid: {pid}, priority: {process.priority}") return pid
async def list_all(t): listed = await t.list('priority') logger.info(f"Priority-based list with length {len(listed)}: {listed}") listed = await t.list('id') logger.info(f"Id-based list with length {len(listed)}: {listed}") listed = await t.list('time') logger.info(f"Time-based list with length {len(listed)}: {listed}")
async def add(self, process: ProcessMock) -> (uuid.uuid1, None): pid_to_remove = None logger.info(f"Add process with priority: {process.priority}...") async with self.processes_lock: if len(self.processes) >= self.capacity: pid_to_remove = self.get_lower_priority_pid_(process.priority) if pid_to_remove is None: logger.warning( "Cannot create a new process. The maximum capacity has been reached and nothing to remove") return priority_to_remove = self.processes[pid_to_remove].priority if pid_to_remove is not None: logger.warning( f"The maximum capacity {self.capacity} has been reached. " f"Removing process with pid {pid_to_remove}, priority: {priority_to_remove}") await self.kill(pid_to_remove) pid = await process.run(self.finished) async with self.processes_lock: self.priority_to_pid[process.priority].append(pid) self.processes[pid] = process logger.info(f"Created process with pid: {pid}, priority: {process.priority}") return pid
async def clean_up_(self, pid: uuid.uuid1()): async with self.processes_lock: logger.info(f"Clean up finished process with pid: {pid}") priority = self.processes[pid].priority del self.processes[pid] self.priority_to_pid[priority].remove(pid)
async def clean_up_(self, pid: uuid.uuid1()): async with self.processes_lock: logger.info(f"Clean up finished process with pid: {pid}") del self.processes[pid]
async def kill_all(self): async with self.processes_lock: for pid, process in self.processes.items(): process.kill() self.processes = dict() logger.info(f"All processes killed")
async def __aexit__(self, exc_type, exc, tb): logger.info("Closing Task Manager. Killing all...") await self.kill_all() async with self.running_lock: self.running.clear()
def main(behaviour: str, capacity: int, processes: dict): logger.setLevel(logging.INFO) logger.info(f"Task Manager service starting...") loop = asyncio.get_event_loop() loop.run_until_complete(scenario(behaviour, capacity, processes))