def test_timeout():
sec = 2
with pytest.raises(timeout.TimeoutError):
with timeout.timeout(seconds=sec):
print(
"Waiting and testing timeout, it should happen in {} seconds.".
format(sec))
time.sleep(sec + 1)
def _wait_for_workers(self): # type: () -> None
logger.info("Waiting for MPI workers to establish their SSH connections")
workers = [host for host in self._hosts if host != self._master_hostname]
with timeout.timeout(seconds=self.timeout_in_seconds):
for host in workers:
while not _can_connect(host):
time.sleep(self._interval)
logger.info("Worker %s available for communication", host)
def _wait_for_workers(self): # type: () -> None
logger.info(
"Waiting for MPI workers to establish their SSH connections")
workers = [
host for host in self._hosts if host != self._master_hostname
]
try:
with timeout.timeout(seconds=self.timeout_in_seconds):
for host in workers:
while not _can_connect(host):
time.sleep(self._interval)
logger.info("Worker %s available for communication", host)
except timeout.TimeoutError:
logger.exception(
"Connection between the hosts couldn't established. Aborting the training."
)
raise