def _wait_and_ping(self):
while True:
try:
yield from asyncio_process_wait_timeout(
self.process, self.ping_timer)
except asyncio.TimeoutError:
logger.debug("pinging controller %s", self.name)
ok = yield from self._ping()
if not ok:
logger.warning("Controller %s ping failed", self.name)
yield from self._terminate()
return
else:
break
def close(self, term_timeout=1.0):
"""Interrupts any I/O with the worker process and terminates the
worker process.
This method should always be called by the user to clean up, even if
build() or examine() raises an exception."""
self.closed.set()
yield from self.io_lock.acquire()
try:
if self.process is None:
# Note the %s - self.rid can be None
logger.debug("worker was not created (RID %s)", self.rid)
return
if self.process.returncode is not None:
logger.debug("worker already terminated (RID %s)", self.rid)
if self.process.returncode != 0:
logger.warning(
"worker finished with status code %d"
" (RID %s)", self.process.returncode, self.rid)
return
obj = {"action": "terminate"}
try:
yield from self._send(obj, cancellable=False)
except:
logger.warning(
"failed to send terminate command to worker"
" (RID %s), killing",
self.rid,
exc_info=True)
self.process.kill()
yield from asyncio_process_wait(self.process)
return
try:
yield from asyncio_process_wait_timeout(
self.process, term_timeout)
except asyncio.TimeoutError:
logger.warning("worker did not exit (RID %s), killing",
self.rid)
self.process.kill()
yield from asyncio_process_wait(self.process)
else:
logger.debug("worker exited gracefully (RID %s)", self.rid)
finally:
self.io_lock.release()
def _terminate(self):
logger.info("Terminating controller %s", self.name)
if self.process is not None and self.process.returncode is None:
try:
yield from asyncio.wait_for(self._call_controller("terminate"),
self.term_timeout)
except:
logger.warning(
"Controller %s did not respond to terminate "
"command, killing", self.name)
self.process.kill()
try:
yield from asyncio_process_wait_timeout(
self.process, self.term_timeout)
except:
logger.warning("Controller %s failed to exit, killing",
self.name)
self.process.kill()
yield from self.process.wait()
logger.debug("Controller %s terminated", self.name)
def close(self, term_timeout=1.0):
"""Interrupts any I/O with the worker process and terminates the
worker process.
This method should always be called by the user to clean up, even if
build() or examine() raises an exception."""
self.closed.set()
yield from self.io_lock.acquire()
try:
if self.process is None:
# Note the %s - self.rid can be None
logger.debug("worker was not created (RID %s)", self.rid)
return
if self.process.returncode is not None:
logger.debug("worker already terminated (RID %s)", self.rid)
if self.process.returncode != 0:
logger.warning("worker finished with status code %d"
" (RID %s)", self.process.returncode,
self.rid)
return
obj = {"action": "terminate"}
try:
yield from self._send(obj, cancellable=False)
except:
logger.warning("failed to send terminate command to worker"
" (RID %s), killing", self.rid, exc_info=True)
self.process.kill()
yield from asyncio_process_wait(self.process)
return
try:
yield from asyncio_process_wait_timeout(self.process,
term_timeout)
except asyncio.TimeoutError:
logger.warning("worker did not exit (RID %s), killing", self.rid)
self.process.kill()
yield from asyncio_process_wait(self.process)
else:
logger.debug("worker exited gracefully (RID %s)", self.rid)
finally:
self.io_lock.release()
