def handle_line(line): _label, value, unit = line.split() if unit == 'kB': return int(value) * 1024 logger.warning(f"unexpected unit {unit} in meminfo") return 0
async def _co_run(self, idle, lingering): """ both timeouts in seconds """ now = time.time() self.reload() # inspection remains None on InternalServerError if self.inspection is None: logger.info(f"BLIP weirdo (0) {self.name} - cannot inspect - ignored") return state = self.inspection['State']['Status'] if state in ('stopped', 'configured'): logger.info(f"BLIP weirdo (1) {self.name} - removing") logger.info(f"BLIP weirdo (1) detailed state was {self.inspection['State']}") self.remove_container() return # ignore non running containers if state != 'running': logger.info(f"BLIP weirdo (2) {self.name} - ignoring") logger.info(f"BLIP weirdo (2) detailed state was {self.inspection['State']}") return # count number of kernels and last activity await self.count_running_kernels() # last_activity may be 0 if no kernel is running inside that container # or None if we could not determine it properly if self.last_activity is None: # an unreachable container may be one that is just taking off # as unlikely as that sounds, it actually tends to happen much more # often than I at least had foreseen at first logger.info(f"unreachable (1) {self} - will try again in {GRACE}s") await asyncio.sleep(GRACE) await self.count_running_kernels() if self.last_activity is None: logger.info(f"Killing unreachable (2) {self}") self.kill_container() return # check there has been activity in the last grace_idle_in_minutes idle_minutes = (int)((now - self.last_activity) // 60) if (now - self.last_activity) < idle: logger.debug( f"Sparing running {self} that had activity {idle_minutes} mn ago") self.figures.count_container(True, self.nb_kernels) elif self.last_activity == 0: logger.info(f"running and empty (1) {self} - will try again in {GRACE}s") await asyncio.sleep(GRACE) await self.count_running_kernels() if self.last_activity == 0: logger.info( f"Killing (running and empty) (2) {self} " f"that has no kernel attached") self.kill_container() return else: logger.info( f"Killing (running & idle) {self} " f"that has been idle for {idle_minutes} mn") self.kill_container() return # if students accidentally leave stuff running in the background # last_activity may be misleading # so we kill all caontainers older than <lingering> # the unit here is seconds but the front CLI has it in hours created_time = self.creation_time() ellapsed = int(now - created_time) if ellapsed > lingering: created_days = (int)(ellapsed // (24 * 3600)) created_hours = (int)((ellapsed // 3600) % 24) logger.warning( f"Removing lingering {self} " f"that was created {created_days} days " f"{created_hours} hours ago (idle_minutes={idle_minutes})") self.kill_container() return