async def wait(self, finished: Queue):
"""Wait for process to complete"""
self.waiting.set()
await self.finish_event.wait()
if self.finish:
logger.info(f"Completed, pid: {self.pid}")
await finished.put(self.pid)
async def add(self, process: ProcessMock) -> (uuid.uuid1, None):
async with self.processes_lock:
if len(self.processes) < self.capacity:
pid = await process.run(self.finished)
self.processes[pid] = process
logger.info(f"Created process with pid: {pid}, priority: {process.priority}")
return pid
logger.warning(f"Cannot create a new process. The maximum capacity {self.capacity} has been reached")
return None
async def kill_group(self, priority: str):
pids = list()
async with self.processes_lock:
for pid, process in self.processes.items():
if process.priority == priority:
process.kill()
pids.append(pid)
for pid in pids:
del self.processes[pid]
logger.info(f"All ({len(pids)}) processes with priority {priority} killed")
async def kill(self, pid: uuid.uuid1):
async with self.processes_lock:
if pid not in self.processes.keys():
logger.warning(f"Cannot kill process. No such pid: {pid}")
return
process = self.processes[pid]
priority = process.priority
process.kill()
del self.processes[pid]
logger.info(f"Killed process with pid: {pid}, priority: {priority}")
async def monitor_finished_tasks(self):
logger.info("Starting monitoring finished tasks...")
async with self.running_lock:
self.running.set()
while True:
async with self.running_lock:
if not self.running.is_set():
return
try:
pid = await asyncio.wait_for(self.finished.get(), timeout=1)
await self.clean_up_(pid)
except:
pass
async def add(self, process: ProcessMock) -> (uuid.uuid1, None):
pid_to_remove = None
async with self.processes_lock:
if len(self.pids) >= self.capacity:
pid_to_remove = self.pids.pop(
) # remove pid of the oldest process
if pid_to_remove:
logger.warning(
f"The maximum capacity {self.capacity} has been reached. Removing process with pid {pid_to_remove}"
)
await self.kill(pid_to_remove) # kill the oldest process
pid = await process.run(self.finished)
async with self.processes_lock:
self.pids.append(pid)
self.processes[pid] = process
logger.info(
f"Created process with pid: {pid}, priority: {process.priority}")
return pid
async def list_all(t):
listed = await t.list('priority')
logger.info(f"Priority-based list with length {len(listed)}: {listed}")
listed = await t.list('id')
logger.info(f"Id-based list with length {len(listed)}: {listed}")
listed = await t.list('time')
logger.info(f"Time-based list with length {len(listed)}: {listed}")
async def add(self, process: ProcessMock) -> (uuid.uuid1, None):
pid_to_remove = None
logger.info(f"Add process with priority: {process.priority}...")
async with self.processes_lock:
if len(self.processes) >= self.capacity:
pid_to_remove = self.get_lower_priority_pid_(process.priority)
if pid_to_remove is None:
logger.warning(
"Cannot create a new process. The maximum capacity has been reached and nothing to remove")
return
priority_to_remove = self.processes[pid_to_remove].priority
if pid_to_remove is not None:
logger.warning(
f"The maximum capacity {self.capacity} has been reached. "
f"Removing process with pid {pid_to_remove}, priority: {priority_to_remove}")
await self.kill(pid_to_remove)
pid = await process.run(self.finished)
async with self.processes_lock:
self.priority_to_pid[process.priority].append(pid)
self.processes[pid] = process
logger.info(f"Created process with pid: {pid}, priority: {process.priority}")
return pid
async def clean_up_(self, pid: uuid.uuid1()):
async with self.processes_lock:
logger.info(f"Clean up finished process with pid: {pid}")
priority = self.processes[pid].priority
del self.processes[pid]
self.priority_to_pid[priority].remove(pid)
async def clean_up_(self, pid: uuid.uuid1()):
async with self.processes_lock:
logger.info(f"Clean up finished process with pid: {pid}")
del self.processes[pid]
async def kill_all(self):
async with self.processes_lock:
for pid, process in self.processes.items():
process.kill()
self.processes = dict()
logger.info(f"All processes killed")
async def __aexit__(self, exc_type, exc, tb):
logger.info("Closing Task Manager. Killing all...")
await self.kill_all()
async with self.running_lock:
self.running.clear()
def main(behaviour: str, capacity: int, processes: dict):
logger.setLevel(logging.INFO)
logger.info(f"Task Manager service starting...")
loop = asyncio.get_event_loop()
loop.run_until_complete(scenario(behaviour, capacity, processes))
