def wait(self, timeout_sec=600):
"""Wait no more than timeout_sec for all worker threads to finish.
raise TimeoutException if all worker threads do not finish within timeout_sec
"""
super(BackgroundThreadService, self).wait()
start = time.time()
end = start + timeout_sec
for idx, worker in enumerate(self.worker_threads, 1):
if end > time.time():
self.logger.debug("Waiting for worker thread %s finish", worker.name)
current_timeout = end - time.time()
worker.join(current_timeout)
alive_workers = [worker.name for worker in self.worker_threads if worker.is_alive()]
if len(alive_workers) > 0:
raise TimeoutError("Timed out waiting %s seconds for background threads to finish. " % str(timeout_sec) +
"These threads are still alive: " + str(alive_workers))
# Propagate exceptions thrown in background threads
with self.lock:
if len(self.worker_errors) > 0:
raise Exception(str(self.worker_errors))
def recv(self, timeout=1800000):
if timeout is None:
# use default value of 1800000 or 30 minutes
timeout = 1800000
self.socket.RCVTIMEO = timeout
try:
message = self.socket.recv()
except zmq.Again:
raise TimeoutError("runner client unresponsive")
return self.serde.deserialize(message)
def wait_until(condition, timeout_sec, backoff_sec=.1, err_msg=""):
"""Block until condition evaluates as true or timeout expires, whichever comes first.
return silently if condition becomes true within the timeout window, otherwise raise Exception with the given
error message.
"""
start = time.time()
stop = start + timeout_sec
while time.time() < stop:
if condition():
return
else:
time.sleep(backoff_sec)
raise TimeoutError(err_msg)
def wait_until(condition,
timeout_sec,
backoff_sec=.1,
err_msg="",
retry_on_exc=False):
"""Block until condition evaluates as true or timeout expires, whichever comes first.
:param condition: callable that returns True if the condition is met, False otherwise
:param timeout_sec: number of seconds to check the condition for before failing
:param backoff_sec: number of seconds to back off between each failure to meet the condition before checking again
:param err_msg: a string or callable returning a string that will be included as the exception message if the
condition is not met
:param retry_on_exc: if True, will retry if condition raises an exception. If condition raised exception on last
iteration, that exception will be raised as a cause of TimeoutError.
If False and condition raises an exception, that exception will be forwarded to the caller
immediately.
Defaults to False (original ducktape behavior).
# TODO: [1.0.0] flip this to True
:return: silently if condition becomes true within the timeout window, otherwise raise Exception with the given
error message.
"""
start = time.time()
stop = start + timeout_sec
last_exception = None
while time.time() < stop:
try:
if condition():
return
else:
# reset last_exception if last iteration didn't raise any exception, but simply returned False
last_exception = None
except BaseException as e:
# save last raised exception for logging it later
last_exception = e
if not retry_on_exc:
raise e
finally:
time.sleep(backoff_sec)
# it is safe to call Exception from None - will be just treated as a normal exception
raise TimeoutError(
err_msg() if callable(err_msg) else err_msg) from last_exception
def wait_until(condition, timeout_sec, backoff_sec=.1, err_msg=""):
"""Block until condition evaluates as true or timeout expires, whichever comes first.
:param condition: callable that returns True if the condition is met, False otherwise
:param timeout_sec: number of seconds to check the condition for before failing
:param backoff_sec: number of seconds to back off between each failure to meet the condition before checking again
:param err_msg: a string or callable returning a string that will be included as the exception message if the
condition is not met
:return: silently if condition becomes true within the timeout window, otherwise raise Exception with the given
error message.
"""
start = time.time()
stop = start + timeout_sec
while time.time() < stop:
if condition():
return
else:
time.sleep(backoff_sec)
raise TimeoutError(err_msg() if callable(err_msg) else err_msg)
def wait(self, timeout_sec=600):
"""Wait for the service to finish.
This only makes sense for tasks with a fixed amount of work to do. For services that generate
output, it is only guaranteed to be available after this call returns.
"""
unfinished_nodes = []
start = time.time()
end = start + timeout_sec
for node in self.nodes:
now = time.time()
node_name = self.who_am_i(node)
if end > now:
self.logger.debug("%s: waiting for node", node_name)
if not self.wait_node(node, end - now):
unfinished_nodes.append(node_name)
else:
unfinished_nodes.append(node_name)
if unfinished_nodes:
raise TimeoutError("Timed out waiting %s seconds for service nodes to finish. " % str(timeout_sec)
+ "These nodes are still alive: " + str(unfinished_nodes))