def wait_for_cluster_to_resume_processing(runners, timeout=30):
# Wait until all workers have resumed processing
waiting = set()
for r in runners:
if not r.is_alive():
continue
obs = ObservabilityNotifier(cluster_status_query,
r.external,
tests=is_processing,
timeout=timeout)
waiting.add(obs)
obs.start()
# Cycle through waiting until its empty or error
t0 = time.time()
while True:
for obs in list(waiting):
# short join
obs.join(0.05)
if obs.error:
raise obs.error
if obs.is_alive():
continue
else:
waiting.remove(obs)
# check completion
if waiting:
# check timeout!
if time.time() - t0 > timeout:
raise TimeoutError(
"Timed out after {} seconds while waiting "
"for cluster to resume processing.".format(timeout))
else:
return # done!
def query_observability(self, query, args, tests, timeout=30, period=2):
logging.debug("query_observability(query={}, args={}, tests={}, "
"timeout={}, period={})".format(
query, args, tests, timeout, period))
obs = ObservabilityNotifier(query, args, tests, timeout, period)
obs.start()
obs.join()
if obs.error:
raise obs.error
def query_observability(self, query, args, tests, timeout=30, period=2):
logging.log(1, "query_observability(query={}, args={}, tests={}, "
"timeout={}, period={})".format(
query, args, tests, timeout, period))
obs = ObservabilityNotifier(query, args, tests, timeout, period)
self._stoppables.add(obs)
obs.start()
obs.join()
self._stoppables.discard(obs)
if obs.error:
raise obs.error
def run(self):
waiting = set()
for r in self.runners:
if not r.is_alive():
continue
obs = ObservabilityNotifier(cluster_status_query,
r.external,
tests=is_processing,
timeout=self.timeout)
waiting.add(obs)
obs.start()
# Cycle through waiting until its empty or error
t0 = time.time()
while not self.stopped():
for obs in list(waiting):
# short join
obs.join(self.interval)
if obs.error:
self.stop(obs.error)
break
if obs.is_alive():
continue
else:
logging.log(
1, "ObservabilityNotifier completed: {}".format(obs))
waiting.remove(obs)
# check completion
if waiting:
# check timeout!
if time.time() - t0 > self.timeout:
self.stop(
TimeoutError(
"Timed out after {} seconds while waiting "
"for cluster to resume processing.".format(
self.timeout)))
break
else:
break # done!