def _wait_and_ping(self): while True: try: yield from asyncio_process_wait_timeout( self.process, self.ping_timer) except asyncio.TimeoutError: logger.debug("pinging controller %s", self.name) ok = yield from self._ping() if not ok: logger.warning("Controller %s ping failed", self.name) yield from self._terminate() return else: break
def close(self, term_timeout=1.0): """Interrupts any I/O with the worker process and terminates the worker process. This method should always be called by the user to clean up, even if build() or examine() raises an exception.""" self.closed.set() yield from self.io_lock.acquire() try: if self.process is None: # Note the %s - self.rid can be None logger.debug("worker was not created (RID %s)", self.rid) return if self.process.returncode is not None: logger.debug("worker already terminated (RID %s)", self.rid) if self.process.returncode != 0: logger.warning( "worker finished with status code %d" " (RID %s)", self.process.returncode, self.rid) return obj = {"action": "terminate"} try: yield from self._send(obj, cancellable=False) except: logger.warning( "failed to send terminate command to worker" " (RID %s), killing", self.rid, exc_info=True) self.process.kill() yield from asyncio_process_wait(self.process) return try: yield from asyncio_process_wait_timeout( self.process, term_timeout) except asyncio.TimeoutError: logger.warning("worker did not exit (RID %s), killing", self.rid) self.process.kill() yield from asyncio_process_wait(self.process) else: logger.debug("worker exited gracefully (RID %s)", self.rid) finally: self.io_lock.release()
def _terminate(self): logger.info("Terminating controller %s", self.name) if self.process is not None and self.process.returncode is None: try: yield from asyncio.wait_for(self._call_controller("terminate"), self.term_timeout) except: logger.warning( "Controller %s did not respond to terminate " "command, killing", self.name) self.process.kill() try: yield from asyncio_process_wait_timeout( self.process, self.term_timeout) except: logger.warning("Controller %s failed to exit, killing", self.name) self.process.kill() yield from self.process.wait() logger.debug("Controller %s terminated", self.name)
def close(self, term_timeout=1.0): """Interrupts any I/O with the worker process and terminates the worker process. This method should always be called by the user to clean up, even if build() or examine() raises an exception.""" self.closed.set() yield from self.io_lock.acquire() try: if self.process is None: # Note the %s - self.rid can be None logger.debug("worker was not created (RID %s)", self.rid) return if self.process.returncode is not None: logger.debug("worker already terminated (RID %s)", self.rid) if self.process.returncode != 0: logger.warning("worker finished with status code %d" " (RID %s)", self.process.returncode, self.rid) return obj = {"action": "terminate"} try: yield from self._send(obj, cancellable=False) except: logger.warning("failed to send terminate command to worker" " (RID %s), killing", self.rid, exc_info=True) self.process.kill() yield from asyncio_process_wait(self.process) return try: yield from asyncio_process_wait_timeout(self.process, term_timeout) except asyncio.TimeoutError: logger.warning("worker did not exit (RID %s), killing", self.rid) self.process.kill() yield from asyncio_process_wait(self.process) else: logger.debug("worker exited gracefully (RID %s)", self.rid) finally: self.io_lock.release()