def result(self, timeout=None):
"""
:param timeout: int,
:return:
"""
now = time.time()
sleep = 0.5
count = 0
if timeout:
future = now + timeout
else:
future = float("inf")
while not self._is_done() and time.time() < future:
self._update_status()
count += 1
time.sleep(exp_backoff(sleep, count))
if time.time() > future and not self._is_done():
raise TimeoutError()
# result should be ready:
results = []
while True:
result = self.ag.actors.getOneExecutionResult(
actorId=self.actor_id, executionId=self.execution_id).content
if not result:
break
results.append(cloudpickle.loads(result))
return results
def wait_for_completion(self, timeout=None):
"""Wait for the upload to complete.
Parameters
----------
timeout : int, optional
If specified, will wait up to specified number of seconds and will raise
a `concurrent.futures.TimeoutError` if the upload has not completed.
Raises
------
concurrent.futures.TimeoutError
If the specified timeout elapses and the upload has not completed.
"""
if self.status in self._TERMINAL_STATES:
return
if timeout:
timeout = time.time() + timeout
intervals = itertools.chain(
self._POLLING_INTERVALS,
itertools.repeat(self._POLLING_INTERVALS[-1]))
while True:
self.reload()
if self.status in self._TERMINAL_STATES:
return
interval = next(intervals)
if timeout:
t = timeout - time.time()
if t <= 0:
raise TimeoutError()
t = min(t, interval)
else:
t = interval
time.sleep(t)
def result(self, timeout=None):
"""Return the result of the call that the future represents.
Args:
timeout: The number of seconds to wait for the result if the future
isn't done. If None, then there is no limit on the wait time.
Returns:
The result of the call that the future represents.
Raises:
CancelledError: If the future was cancelled.
TimeoutError: If the future didn't finish executing before the
given timeout.
Exception: If the call raised then that exception will be raised.
"""
with self._condition:
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
raise CancelledError()
elif self._state == FINISHED:
return self.__get_result()
self._condition.wait(timeout)
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
raise CancelledError()
elif self._state == FINISHED:
return self.__get_result()
else:
raise TimeoutError()
def wait_for_completion(self, timeout=None):
"""Wait for the upload to complete.
Parameters
----------
timeout : int, optional
If specified, will wait up to specified number of seconds and will raise
a concurrent.futures.TimeoutError if the upload has not completed.
Raises
------
:py:exc:`~concurrent.futures.TimeoutError`
If the specified timeout elapses and the upload has not completed
"""
if self.status in self._TERMINAL_STATES:
return
if timeout:
timeout = time.time() + timeout
while True:
self.reload()
if self.status in self._TERMINAL_STATES:
return
if timeout:
t = timeout - time.time()
if t <= 0:
raise TimeoutError()
t = min(t, self._POLLING_INTERVAL)
else:
t = self._POLLING_INTERVAL
time.sleep(t)
def exception(self, timeout=None):
"""Return the exception raised by the call that the future represents.
Args:
timeout: The number of seconds to wait for the exception if the
future isn't done. If None, then there is no limit on the wait
time.
Returns:
The exception raised by the call that the future represents or None
if the call completed without raising.
Raises:
CancelledError: If the future was cancelled.
TimeoutError: If the future didn't finish executing before the
given timeout.
"""
with self._condition:
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
raise CancelledError()
elif self._state == FINISHED:
return self._exception
self._condition.wait(timeout)
if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
raise CancelledError()
elif self._state == FINISHED:
return self._exception
else:
raise TimeoutError()
def as_completed_with_timeout_reset(futures, timeout):
"""
Yields any finished future within time alloted by timeout, raises TimeoutError otherwise.
Futures are allowed to continue over timeout, if any processing in caller is slow.
Timeout is not applied to futures themselves but rather to wait() call, which means
that this function allows to yield all quicker futures and tries to wait for slow ones.
Parameters:
futures: List[Future]
list of futures
timeout: int or float
Timeout used for wait() calls
Returns:
Future
finished Future object
Throws:
TimeoutError, if waiting for some Future to finish in wait() call takes more
time than given timeout.
"""
total_futures = len(futures)
pending = futures
while pending:
finished, pending = wait(pending,
timeout=timeout,
return_when=FIRST_COMPLETED)
# no future finished in timeout given, raise TimeoutError
if not finished:
raise TimeoutError("%d (of %d) futures unfinished" %
(len(pending), total_futures))
for fs in finished:
yield fs
def as_completed(fs, timeout=None):
"""An iterator over the given futures that yields each as it completes.
Args:
fs: The sequence of Futures (possibly created by different
Executors) to iterate over.
timeout: The maximum number of seconds to wait. If None, then there
is no limit on the wait time.
Returns:
An iterator that yields the given Futures as they complete
(finished or cancelled). If any given Futures are duplicated, they
will be returned once.
Raises:
TimeoutError: If the entire result iterator could not be generated
before the given timeout.
"""
if timeout is not None:
end_time = timeout + time.time()
fs = set(fs)
with _AcquireFutures(fs):
finished = set(
f for f in fs
if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
pending = fs - finished
waiter = _create_and_install_waiters(fs, _AS_COMPLETED)
try:
for future in finished:
yield future
while pending:
if timeout is None:
wait_timeout = None
else:
wait_timeout = end_time - time.time()
if wait_timeout < 0:
raise TimeoutError('%d (of %d) futures unfinished' % (
len(pending), len(fs)))
waiter.event.wait(wait_timeout)
with waiter.lock:
finished = waiter.finished_futures
waiter.finished_futures = []
waiter.event.clear()
for future in finished:
yield future
pending.remove(future)
finally:
for f in fs:
with f._condition:
f._waiters.remove(waiter)
def run_and_await_k(jobs: List[callable],
k: int,
timeout_after_k: Optional[float] = 0,
timeout_total: Optional[float] = None):
"""
Runs all :jobs: asynchronously, awaits for at least k of them to finish
:param jobs: functions to call asynchronously
:param k: how many functions should finish for call to be successful
:param timeout_after_k: after reaching k finished jobs, wait for this long before cancelling
:param timeout_total: if specified, terminate cancel jobs after this many seconds
:returns: a list of either results or exceptions for each job
"""
jobs = list(jobs)
assert k <= len(jobs), f"Can't await {k} out of {len(jobs)} jobs."
start_time = time.time()
future_to_ix = {run_in_background(job): i for i, job in enumerate(jobs)}
outputs = [None] * len(jobs)
success_count = 0
try:
# await first k futures for as long as it takes
for future in as_completed(list(future_to_ix.keys()),
timeout=timeout_total):
success_count += int(not future.exception())
outputs[future_to_ix.pop(future)] = future.result(
) if not future.exception() else future.exception()
if success_count >= k:
break # we have enough futures to succeed
if len(outputs) + len(future_to_ix) < k:
failed = len(jobs) - len(outputs) - len(future_to_ix)
raise ValueError(
f"Couldn't get enough results: too many jobs failed ({failed} / {len(outputs)})"
)
# await stragglers for at most self.timeout_after_k_min or whatever time is left
if timeout_after_k is not None and timeout_total is not None:
time_left = min(timeout_after_k,
timeout_total - time.time() + start_time)
else:
time_left = timeout_after_k if timeout_after_k is not None else timeout_total
for future in as_completed(list(future_to_ix.keys()),
timeout=time_left):
success_count += int(not future.exception())
outputs[future_to_ix.pop(future)] = future.result(
) if not future.exception() else future.exception()
except TimeoutError:
if len(outputs) < k:
raise TimeoutError(
f"Couldn't get enough results: time limit exceeded (got {len(outputs)} of {k})"
)
finally:
for future, index in future_to_ix.items():
future.cancel()
outputs[index] = future.result(
) if not future.exception() else future.exception()
return outputs
def update_tasks(self):
"""Handles timing out Tasks."""
for task in self.task_manager.timeout_tasks():
self.task_manager.task_done(
task.id, TimeoutError("Task timeout", task.timeout))
self.worker_manager.stop_worker(task.worker_id)
for task in self.task_manager.cancelled_tasks():
self.task_manager.task_done(task.id, CancelledError())
self.worker_manager.stop_worker(task.worker_id)
def _wait_queue_depletion(self, timeout):
tick = time.time()
while self.active:
if timeout is not None and time.time() - tick > timeout:
raise TimeoutError("Tasks are still being executed")
elif self._context.task_queue.unfinished_tasks:
time.sleep(SLEEP_UNIT)
else:
return
async def read_lock(self) -> AsyncIterator[None]:
if self._check_lock():
yield
return
for delay in self._read_retry_delay:
await asyncio.sleep(delay)
if not os.path.exists(self._path):
yield
break
else:
raise TimeoutError()
def test_request_to_notify_with_pubsub_timeout_error(self):
"""Tests if the future.result() raises a TimeoutError then the function raises a RasNotifyError"""
future = MagicMock()
future.result.side_effect = TimeoutError("bad")
publisher = MagicMock()
publisher.publish.return_value = future
# Given a mocked notify gateway
notify = NotifyGateway(current_app.config)
notify.publisher = publisher
with self.assertRaises(RasNotifyError):
notify.request_to_notify('*****@*****.**', 'notify_account_locked')
def __init__(self,
seconds=1,
exception=TimeoutError('Timeout.'),
thread=True):
"""
seconds :
Note `signal.alarm`
thread :
If True, `concurrent.futures.ThreadPoolExecutor` is used,
otherwise `signal.signal` is used.
Only takes effect in decorator mode.
"""
self.seconds = seconds
self.thread = thread
if isinstance(exception, type) and issubclass(exception, Exception):
self.exception = exception('Timeout.')
elif isinstance(exception, Exception):
self.exception = exception
else:
self.exception = TimeoutError(exception)
def _poll_generator(self, timeout=None):
sleeptime = self._executor.poll_interval
logger = self._executor.logger
if timeout is not None:
sleeptime = min(sleeptime, timeout)
endtime = time.time() + timeout
while True:
yield
if timeout is not None and time.time() >= endtime:
raise TimeoutError()
logger.debug('Waiting for job (sleeptime=%s): %s', sleeptime,
self.job_name)
time.sleep(sleeptime)
def wait_for_completion(self, timeout=None, warn_transient_errors=True):
"""Wait for the upload to complete.
Parameters
----------
timeout : int, optional
If specified, will wait up to specified number of seconds and will raise
a `concurrent.futures.TimeoutError` if the upload has not completed.
warn_transient_errors : bool, optional, default True
Any transient errors while periodically checking upload status are suppressed.
If True, those errors will be printed as warnings.
Raises
------
concurrent.futures.TimeoutError
If the specified timeout elapses and the upload has not completed.
"""
if self.status in self._TERMINAL_STATES:
return
if timeout:
timeout = time.time() + timeout
intervals = itertools.chain(
self._POLLING_INTERVALS, itertools.repeat(self._POLLING_INTERVALS[-1])
)
while True:
try:
self.reload()
except (
ServerError,
urllib3.exceptions.MaxRetryError,
requests.exceptions.RetryError,
urllib3.exceptions.TimeoutError,
) as e:
# If a reload fails, just try again on the next interval
if warn_transient_errors:
warnings.warn(
"In wait_for_completion: error fetching status for ImageUpload {!r}; "
"will retry: {}".format(self.id, e)
)
if self.status in self._TERMINAL_STATES:
return
interval = next(intervals)
if timeout:
t = timeout - time.time()
if t <= 0:
raise TimeoutError()
t = min(t, interval)
else:
t = interval
time.sleep(t)
def _doMoveAbs(self, future, pos):
"""
Blocking and cancellable absolute move
future (Future): the future it handles
_pos (dict str -> float): axis name -> absolute target position
raise:
SmarPodError: if the controller reported an error
CancelledError: if cancelled before the end of the move
"""
last_upd = time.time()
dur = 30 # TODO: Calculate an estimated move duration
end = time.time() + dur
max_dur = dur * 2 + 1
logging.debug("Expecting a move of %g s, will wait up to %g s", dur,
max_dur)
timeout = last_upd + max_dur
with future._moving_lock:
self.Move(pos)
while not future._must_stop.is_set():
status = self.GetMoveStatus()
# check if move is done
if status.value == SmarPodDLL.SMARPOD_STOPPED.value:
break
now = time.time()
if now > timeout:
logging.warning("Stopping move due to timeout after %g s.",
max_dur)
self.stop()
raise TimeoutError("Move is not over after %g s, while "
"expected it takes only %g s" %
(max_dur, dur))
# Update the position from time to time (10 Hz)
if now - last_upd > 0.1:
self._updatePosition()
last_upd = time.time()
# Wait half of the time left (maximum 0.1 s)
left = end - time.time()
sleept = max(0.001, min(left / 2, 0.1))
future._must_stop.wait(sleept)
else:
self.stop()
future._was_stopped = True
raise CancelledError()
self._updatePosition()
logging.debug("move successfully completed")
def await_first(*events: Event, k=1, timeout=None):
"""
wait until first k (default=1) events are set, return True if event was set fast
# Note: after k successes we manually *set* all events to avoid memory leak.
"""
events_done = CountdownEvent(count_to=k)
for event in events:
add_event_callback(event, callback=events_done.increment, timeout=timeout)
if events_done.wait(timeout=timeout):
[event.set() for event in events]
return True
else:
raise TimeoutError()
def run_and_await_k(jobs: callable, k, timeout_after_k=0, timeout_total=None):
"""
Runs all :jobs: asynchronously, awaits for at least k of them to finish
:param jobs: functions to call
:param k: how many functions should finish
:param timeout_after_k: after reaching k finished jobs, wait for this long before cancelling
:param timeout_total: if specified, terminate cancel jobs after this many seconds
:returns: a list of either results or exceptions for each job
"""
assert k <= len(jobs)
start_time = time.time()
min_successful_jobs = CountdownEvent(count_to=k)
max_failed_jobs = CountdownEvent(count_to=len(jobs) - k + 1)
def _run_and_increment(run_job: callable):
try:
result = run_job()
min_successful_jobs.increment()
return result
except Exception as e:
max_failed_jobs.increment()
return e
def _run_and_await(run_job: callable):
# call function asynchronously. Increment counter after finished
future = run_in_background(_run_and_increment, run_job)
try: # await for success counter to reach k OR for fail counter to reach n - k + 1
await_first(min_successful_jobs, max_failed_jobs,
timeout=None if timeout_total is None else timeout_total - time.time() + start_time)
except TimeoutError as e: # counter didn't reach k jobs in timeout_total
return future.result() if future.done() else e
try: # await for subsequent jobs if asked to
return future.result(timeout=timeout_after_k)
except TimeoutError as e:
future.cancel()
return e
except Exception as e: # job failed with exception. Ignore it.
return e
results = [run_in_background(_run_and_await, f) for f in jobs]
results = [result.result() for result in results]
if min_successful_jobs.is_set():
return results
elif max_failed_jobs.is_set():
raise ValueError("Could not get enough results: too many jobs failed.")
else:
raise TimeoutError("Could not get enough results: reached timeout_total.")
def wait_route53(change_id, domain_zone_name):
try:
# wait 5 minutes (300 seconds)
for i in range(31):
if i == 30:
raise TimeoutError()
response = route53_client.get_change(Id=change_id)
status = response["ChangeInfo"]["Status"]
print('DNS propagation for %s is %s' % (domain_zone_name, status))
if status == "INSYNC":
return True
time.sleep(10)
except TimeoutError:
return False
def stop(self, timeout=5.0):
"""
Stops all the service processes. Waits for full stop.
Args:
timeout (float): How long (in seconds) to wait before raising an error.
Raises:
TimeoutError: If all of the services didn't stop in time.
"""
stop_futures = [
self._executor.submit(service.stop) for service in self._services
]
results = concurrent.futures.wait(stop_futures, timeout=timeout)
if results.not_done:
raise TimeoutError('Not all processes stopped in time.')
def start(self, timeout=5.0):
"""Starts all the service processes and waits them to start accepting connections.
Args:
timeout (float): How long (in seconds) to wait before raising an error.
Raises:
TimeoutError: If all of the services didn't start in time.
"""
start_futures = [
self._executor.submit(service.start) for service in self._services
]
results = concurrent.futures.wait(start_futures, timeout=timeout)
if results.not_done:
raise TimeoutError('Not all processes started in time.')
def _get_result(future, pipe, timeout):
"""Waits for result and handles communication errors."""
counter = count(step=SLEEP_UNIT)
try:
while not pipe.poll(SLEEP_UNIT):
if timeout is not None and next(counter) >= timeout:
return TimeoutError('Task Timeout', timeout)
elif future.cancelled():
return CancelledError()
return pipe.recv()
except (EOFError, OSError):
return ProcessExpired('Abnormal termination')
except Exception as error:
return error
def exception(self, timeout: int = None):
start = time.time()
state, error = self._get_state_and_error()
while state.is_executing():
if timeout is not None and (time.time() - start) > timeout:
raise TimeoutError(
f"{self.job_id} did not finish running before timeout of {timeout}s"
)
time.sleep(10)
state, error = self._get_state_and_error()
logging.info(f"Future for {self.job_id} has state {state}")
if state.is_cancelled():
raise CancelledError(f"{self.job_id}: " + error)
if state is State.FAILED:
return AipError(f"{self.job_id}: " + error)
async def write_lock(self) -> AsyncIterator[None]:
if self._check_lock() and self._try_lock():
try:
yield
finally:
self._unlock()
return
for delay in self._write_retry_delay:
await asyncio.sleep(delay)
if self._try_lock():
try:
yield
finally:
self._unlock()
break
else:
raise TimeoutError()
def wait_for_requests(self, count=1, timeout=5.0):
"""Wait until a number of requests arrive to the imposter.
Args:
count (int): How many requests to wait for.
timeout (float): How long to wait for requests.
Returns:
list[`ImposterRequest`]: The requests made on the impostor.
"""
start_time = time.perf_counter()
while True:
received_requests = self.requests()
if len(received_requests) >= count:
return received_requests
else:
time.sleep(0.01)
if time.perf_counter() - start_time >= timeout:
raise TimeoutError('Waited too long for requests on stub.')
def within(self, duration):
"""
Return a Promise whose state is guaranteed to be resolved within `duration`
seconds. If this Promise completes before `duration` seconds expire, it will
contain this Promise's contents. If this Promise is not resolved by then, the
resulting Promise will fail with a `TimeoutError`.
Parameters
----------
duration : number
Number of seconds to wait before resolving a `TimeoutError`
Returns
-------
result : Promise
Promise guaranteed to resolve in `duration` seconds.
"""
e = TimeoutError(
u"Promise did not finish in {} seconds".format(duration))
return self.or_(
Promise.wait(duration).flatmap(lambda v: Promise.exception(e)))
def wait_for_port(port, host='localhost', timeout=5.0):
"""Wait until a port starts accepting TCP connections.
Args:
port (int): Port number.
host (str): Host address on which the port should exist.
timeout (float): In seconds. How long to wait before raising errors.
Raises:
TimeoutError: The port isn't accepting connection after time specified in `timeout`.
"""
start_time = time.time()
while True:
try:
sock = socket.create_connection((host, port))
sock.close()
break
except socket.error:
time.sleep(0.01)
if time.time() - start_time >= timeout:
raise TimeoutError(
'Waited too long for the port {} on host {} to start accepting '
'connections.'.format(port, host))
def wait_id(self, timeout=None):
"""Blocking id getter.
Return the submitted problem ID, but unlike :meth:`.id`, block until the
ID becomes known, or until `timeout` expires.
Args:
timeout (float, default=None):
Timeout in seconds. By default, wait indefinitely for problem
id to become known/available.
Returns:
str:
Problem ID, as returned by SAPI.
Raises:
:exc:`concurrent.futures.TimeoutError`:
When `timeout` exceeded, and problem id not ready.
"""
if not self._id_ready_event.wait(timeout=timeout):
raise TimeoutError("problem id not available yet")
return self._id
def _waitEndMove(self, future, axes, end=0):
"""
Wait until all the given axes are finished moving, or a request to
stop has been received.
future (Future): the future it handles
axes (set of int): the axes IDs to check
end (float): expected end time
raise:
TimeoutError: if took too long to finish the move
CancelledError: if cancelled before the end of the move
"""
moving_axes = set(axes)
last_upd = time.time()
dur = max(0.01, min(end - last_upd, 60))
max_dur = dur * 2 + 1
logging.debug("Expecting a move of %g s, will wait up to %g s", dur,
max_dur)
timeout = last_upd + max_dur
last_axes = moving_axes.copy()
try:
while not future._must_stop.is_set():
for aid in moving_axes.copy(
): # need copy to remove during iteration
if self.GetMotionDone(aid):
moving_axes.discard(aid)
if not moving_axes:
# no more axes to wait for
break
now = time.time()
if now > timeout:
logging.warning("Stopping move due to timeout after %g s.",
max_dur)
for i in moving_axes:
self.StopMotion(i)
raise TimeoutError("Move is not over after %g s, while "
"expected it takes only %g s" %
(max_dur, dur))
# Update the position from time to time (10 Hz)
if now - last_upd > 0.1 or last_axes != moving_axes:
last_names = set(n for n, i in self._axis_map.items()
if i in last_axes)
self._updatePosition(last_names)
last_upd = time.time()
last_axes = moving_axes.copy()
# Wait half of the time left (maximum 0.1 s)
left = end - time.time()
sleept = max(0.001, min(left / 2, 0.1))
future._must_stop.wait(sleept)
else:
logging.debug("Move of axes %s cancelled before the end", axes)
# stop all axes still moving them
for i in moving_axes:
self.StopMotion(i)
future._was_stopped = True
raise CancelledError()
finally:
# TODO: check if the move succeded ? (= Not failed due to stallguard/limit switch)
self._updatePosition() # update (all axes) with final position
def _read(self, subscriber_i):
"""Iterate over incoming messages in order.
Your thread will sleep until the next message is available, or timeout
expires (in which case MailboxReadTimeout is raised)
"""
self.log.debug("Start reading")
next_number = 0
last_message = False
while not last_message:
with self._lock:
# Wait for new messages
def next_ready():
return self._has_msg(next_number) or self.killed
if not next_ready():
self.log.debug(f"Checking/waiting for {next_number}")
if not self._read_condition.wait_for(next_ready, self.timeout):
raise MailboxReadTimeout(
f"{self.name} did not get {next_number} in time")
if self.killed:
self.log.debug(f"Reader finds {self.name} killed")
raise MailboxKilled(self.killed_because)
# Grab all messages we can yield
to_yield = []
while self._has_msg(next_number):
msg = self._get_msg(next_number)
if msg is StopIteration:
self.log.debug(f"{next_number} is StopIteration")
last_message = True
to_yield.append((next_number, msg))
next_number += 1
if len(to_yield) > 1:
self.log.debug(f"Read {to_yield[0][0]}-{to_yield[-1][0]}")
else:
self.log.debug(f"Read {to_yield[0][0]}")
self._subscribers_have_read[subscriber_i] = next_number - 1
# Clean up the mailbox
while (len(self._mailbox) and (min(self._subscribers_have_read)
>= self._lowest_msg_number)):
heapq.heappop(self._mailbox)
self._write_condition.notify_all()
for msg_number, msg in to_yield:
if msg is StopIteration:
return
elif isinstance(msg, Future):
if not msg.done():
self.log.debug(f"Waiting for future {msg_number}")
try:
res = msg.result(timeout=self.timeout)
except TimeoutError:
raise TimeoutError(
f"Future {msg_number} timed out!")
self.log.debug(f"Future {msg_number} completed")
else:
res = msg.result()
self.log.debug(f"Future {msg_number} was already done")
yield res
else:
yield msg
self.log.debug("Done reading")