def result(self, timeout=None): """ :param timeout: int, :return: """ now = time.time() sleep = 0.5 count = 0 if timeout: future = now + timeout else: future = float("inf") while not self._is_done() and time.time() < future: self._update_status() count += 1 time.sleep(exp_backoff(sleep, count)) if time.time() > future and not self._is_done(): raise TimeoutError() # result should be ready: results = [] while True: result = self.ag.actors.getOneExecutionResult( actorId=self.actor_id, executionId=self.execution_id).content if not result: break results.append(cloudpickle.loads(result)) return results
def wait_for_completion(self, timeout=None): """Wait for the upload to complete. Parameters ---------- timeout : int, optional If specified, will wait up to specified number of seconds and will raise a `concurrent.futures.TimeoutError` if the upload has not completed. Raises ------ concurrent.futures.TimeoutError If the specified timeout elapses and the upload has not completed. """ if self.status in self._TERMINAL_STATES: return if timeout: timeout = time.time() + timeout intervals = itertools.chain( self._POLLING_INTERVALS, itertools.repeat(self._POLLING_INTERVALS[-1])) while True: self.reload() if self.status in self._TERMINAL_STATES: return interval = next(intervals) if timeout: t = timeout - time.time() if t <= 0: raise TimeoutError() t = min(t, interval) else: t = interval time.sleep(t)
def result(self, timeout=None): """Return the result of the call that the future represents. Args: timeout: The number of seconds to wait for the result if the future isn't done. If None, then there is no limit on the wait time. Returns: The result of the call that the future represents. Raises: CancelledError: If the future was cancelled. TimeoutError: If the future didn't finish executing before the given timeout. Exception: If the call raised then that exception will be raised. """ with self._condition: if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: raise CancelledError() elif self._state == FINISHED: return self.__get_result() self._condition.wait(timeout) if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: raise CancelledError() elif self._state == FINISHED: return self.__get_result() else: raise TimeoutError()
def wait_for_completion(self, timeout=None): """Wait for the upload to complete. Parameters ---------- timeout : int, optional If specified, will wait up to specified number of seconds and will raise a concurrent.futures.TimeoutError if the upload has not completed. Raises ------ :py:exc:`~concurrent.futures.TimeoutError` If the specified timeout elapses and the upload has not completed """ if self.status in self._TERMINAL_STATES: return if timeout: timeout = time.time() + timeout while True: self.reload() if self.status in self._TERMINAL_STATES: return if timeout: t = timeout - time.time() if t <= 0: raise TimeoutError() t = min(t, self._POLLING_INTERVAL) else: t = self._POLLING_INTERVAL time.sleep(t)
def exception(self, timeout=None): """Return the exception raised by the call that the future represents. Args: timeout: The number of seconds to wait for the exception if the future isn't done. If None, then there is no limit on the wait time. Returns: The exception raised by the call that the future represents or None if the call completed without raising. Raises: CancelledError: If the future was cancelled. TimeoutError: If the future didn't finish executing before the given timeout. """ with self._condition: if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: raise CancelledError() elif self._state == FINISHED: return self._exception self._condition.wait(timeout) if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: raise CancelledError() elif self._state == FINISHED: return self._exception else: raise TimeoutError()
def as_completed_with_timeout_reset(futures, timeout): """ Yields any finished future within time alloted by timeout, raises TimeoutError otherwise. Futures are allowed to continue over timeout, if any processing in caller is slow. Timeout is not applied to futures themselves but rather to wait() call, which means that this function allows to yield all quicker futures and tries to wait for slow ones. Parameters: futures: List[Future] list of futures timeout: int or float Timeout used for wait() calls Returns: Future finished Future object Throws: TimeoutError, if waiting for some Future to finish in wait() call takes more time than given timeout. """ total_futures = len(futures) pending = futures while pending: finished, pending = wait(pending, timeout=timeout, return_when=FIRST_COMPLETED) # no future finished in timeout given, raise TimeoutError if not finished: raise TimeoutError("%d (of %d) futures unfinished" % (len(pending), total_futures)) for fs in finished: yield fs
def as_completed(fs, timeout=None): """An iterator over the given futures that yields each as it completes. Args: fs: The sequence of Futures (possibly created by different Executors) to iterate over. timeout: The maximum number of seconds to wait. If None, then there is no limit on the wait time. Returns: An iterator that yields the given Futures as they complete (finished or cancelled). If any given Futures are duplicated, they will be returned once. Raises: TimeoutError: If the entire result iterator could not be generated before the given timeout. """ if timeout is not None: end_time = timeout + time.time() fs = set(fs) with _AcquireFutures(fs): finished = set( f for f in fs if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]) pending = fs - finished waiter = _create_and_install_waiters(fs, _AS_COMPLETED) try: for future in finished: yield future while pending: if timeout is None: wait_timeout = None else: wait_timeout = end_time - time.time() if wait_timeout < 0: raise TimeoutError('%d (of %d) futures unfinished' % ( len(pending), len(fs))) waiter.event.wait(wait_timeout) with waiter.lock: finished = waiter.finished_futures waiter.finished_futures = [] waiter.event.clear() for future in finished: yield future pending.remove(future) finally: for f in fs: with f._condition: f._waiters.remove(waiter)
def run_and_await_k(jobs: List[callable], k: int, timeout_after_k: Optional[float] = 0, timeout_total: Optional[float] = None): """ Runs all :jobs: asynchronously, awaits for at least k of them to finish :param jobs: functions to call asynchronously :param k: how many functions should finish for call to be successful :param timeout_after_k: after reaching k finished jobs, wait for this long before cancelling :param timeout_total: if specified, terminate cancel jobs after this many seconds :returns: a list of either results or exceptions for each job """ jobs = list(jobs) assert k <= len(jobs), f"Can't await {k} out of {len(jobs)} jobs." start_time = time.time() future_to_ix = {run_in_background(job): i for i, job in enumerate(jobs)} outputs = [None] * len(jobs) success_count = 0 try: # await first k futures for as long as it takes for future in as_completed(list(future_to_ix.keys()), timeout=timeout_total): success_count += int(not future.exception()) outputs[future_to_ix.pop(future)] = future.result( ) if not future.exception() else future.exception() if success_count >= k: break # we have enough futures to succeed if len(outputs) + len(future_to_ix) < k: failed = len(jobs) - len(outputs) - len(future_to_ix) raise ValueError( f"Couldn't get enough results: too many jobs failed ({failed} / {len(outputs)})" ) # await stragglers for at most self.timeout_after_k_min or whatever time is left if timeout_after_k is not None and timeout_total is not None: time_left = min(timeout_after_k, timeout_total - time.time() + start_time) else: time_left = timeout_after_k if timeout_after_k is not None else timeout_total for future in as_completed(list(future_to_ix.keys()), timeout=time_left): success_count += int(not future.exception()) outputs[future_to_ix.pop(future)] = future.result( ) if not future.exception() else future.exception() except TimeoutError: if len(outputs) < k: raise TimeoutError( f"Couldn't get enough results: time limit exceeded (got {len(outputs)} of {k})" ) finally: for future, index in future_to_ix.items(): future.cancel() outputs[index] = future.result( ) if not future.exception() else future.exception() return outputs
def update_tasks(self): """Handles timing out Tasks.""" for task in self.task_manager.timeout_tasks(): self.task_manager.task_done( task.id, TimeoutError("Task timeout", task.timeout)) self.worker_manager.stop_worker(task.worker_id) for task in self.task_manager.cancelled_tasks(): self.task_manager.task_done(task.id, CancelledError()) self.worker_manager.stop_worker(task.worker_id)
def _wait_queue_depletion(self, timeout): tick = time.time() while self.active: if timeout is not None and time.time() - tick > timeout: raise TimeoutError("Tasks are still being executed") elif self._context.task_queue.unfinished_tasks: time.sleep(SLEEP_UNIT) else: return
async def read_lock(self) -> AsyncIterator[None]: if self._check_lock(): yield return for delay in self._read_retry_delay: await asyncio.sleep(delay) if not os.path.exists(self._path): yield break else: raise TimeoutError()
def test_request_to_notify_with_pubsub_timeout_error(self): """Tests if the future.result() raises a TimeoutError then the function raises a RasNotifyError""" future = MagicMock() future.result.side_effect = TimeoutError("bad") publisher = MagicMock() publisher.publish.return_value = future # Given a mocked notify gateway notify = NotifyGateway(current_app.config) notify.publisher = publisher with self.assertRaises(RasNotifyError): notify.request_to_notify('*****@*****.**', 'notify_account_locked')
def __init__(self, seconds=1, exception=TimeoutError('Timeout.'), thread=True): """ seconds : Note `signal.alarm` thread : If True, `concurrent.futures.ThreadPoolExecutor` is used, otherwise `signal.signal` is used. Only takes effect in decorator mode. """ self.seconds = seconds self.thread = thread if isinstance(exception, type) and issubclass(exception, Exception): self.exception = exception('Timeout.') elif isinstance(exception, Exception): self.exception = exception else: self.exception = TimeoutError(exception)
def _poll_generator(self, timeout=None): sleeptime = self._executor.poll_interval logger = self._executor.logger if timeout is not None: sleeptime = min(sleeptime, timeout) endtime = time.time() + timeout while True: yield if timeout is not None and time.time() >= endtime: raise TimeoutError() logger.debug('Waiting for job (sleeptime=%s): %s', sleeptime, self.job_name) time.sleep(sleeptime)
def wait_for_completion(self, timeout=None, warn_transient_errors=True): """Wait for the upload to complete. Parameters ---------- timeout : int, optional If specified, will wait up to specified number of seconds and will raise a `concurrent.futures.TimeoutError` if the upload has not completed. warn_transient_errors : bool, optional, default True Any transient errors while periodically checking upload status are suppressed. If True, those errors will be printed as warnings. Raises ------ concurrent.futures.TimeoutError If the specified timeout elapses and the upload has not completed. """ if self.status in self._TERMINAL_STATES: return if timeout: timeout = time.time() + timeout intervals = itertools.chain( self._POLLING_INTERVALS, itertools.repeat(self._POLLING_INTERVALS[-1]) ) while True: try: self.reload() except ( ServerError, urllib3.exceptions.MaxRetryError, requests.exceptions.RetryError, urllib3.exceptions.TimeoutError, ) as e: # If a reload fails, just try again on the next interval if warn_transient_errors: warnings.warn( "In wait_for_completion: error fetching status for ImageUpload {!r}; " "will retry: {}".format(self.id, e) ) if self.status in self._TERMINAL_STATES: return interval = next(intervals) if timeout: t = timeout - time.time() if t <= 0: raise TimeoutError() t = min(t, interval) else: t = interval time.sleep(t)
def _doMoveAbs(self, future, pos): """ Blocking and cancellable absolute move future (Future): the future it handles _pos (dict str -> float): axis name -> absolute target position raise: SmarPodError: if the controller reported an error CancelledError: if cancelled before the end of the move """ last_upd = time.time() dur = 30 # TODO: Calculate an estimated move duration end = time.time() + dur max_dur = dur * 2 + 1 logging.debug("Expecting a move of %g s, will wait up to %g s", dur, max_dur) timeout = last_upd + max_dur with future._moving_lock: self.Move(pos) while not future._must_stop.is_set(): status = self.GetMoveStatus() # check if move is done if status.value == SmarPodDLL.SMARPOD_STOPPED.value: break now = time.time() if now > timeout: logging.warning("Stopping move due to timeout after %g s.", max_dur) self.stop() raise TimeoutError("Move is not over after %g s, while " "expected it takes only %g s" % (max_dur, dur)) # Update the position from time to time (10 Hz) if now - last_upd > 0.1: self._updatePosition() last_upd = time.time() # Wait half of the time left (maximum 0.1 s) left = end - time.time() sleept = max(0.001, min(left / 2, 0.1)) future._must_stop.wait(sleept) else: self.stop() future._was_stopped = True raise CancelledError() self._updatePosition() logging.debug("move successfully completed")
def await_first(*events: Event, k=1, timeout=None): """ wait until first k (default=1) events are set, return True if event was set fast # Note: after k successes we manually *set* all events to avoid memory leak. """ events_done = CountdownEvent(count_to=k) for event in events: add_event_callback(event, callback=events_done.increment, timeout=timeout) if events_done.wait(timeout=timeout): [event.set() for event in events] return True else: raise TimeoutError()
def run_and_await_k(jobs: callable, k, timeout_after_k=0, timeout_total=None): """ Runs all :jobs: asynchronously, awaits for at least k of them to finish :param jobs: functions to call :param k: how many functions should finish :param timeout_after_k: after reaching k finished jobs, wait for this long before cancelling :param timeout_total: if specified, terminate cancel jobs after this many seconds :returns: a list of either results or exceptions for each job """ assert k <= len(jobs) start_time = time.time() min_successful_jobs = CountdownEvent(count_to=k) max_failed_jobs = CountdownEvent(count_to=len(jobs) - k + 1) def _run_and_increment(run_job: callable): try: result = run_job() min_successful_jobs.increment() return result except Exception as e: max_failed_jobs.increment() return e def _run_and_await(run_job: callable): # call function asynchronously. Increment counter after finished future = run_in_background(_run_and_increment, run_job) try: # await for success counter to reach k OR for fail counter to reach n - k + 1 await_first(min_successful_jobs, max_failed_jobs, timeout=None if timeout_total is None else timeout_total - time.time() + start_time) except TimeoutError as e: # counter didn't reach k jobs in timeout_total return future.result() if future.done() else e try: # await for subsequent jobs if asked to return future.result(timeout=timeout_after_k) except TimeoutError as e: future.cancel() return e except Exception as e: # job failed with exception. Ignore it. return e results = [run_in_background(_run_and_await, f) for f in jobs] results = [result.result() for result in results] if min_successful_jobs.is_set(): return results elif max_failed_jobs.is_set(): raise ValueError("Could not get enough results: too many jobs failed.") else: raise TimeoutError("Could not get enough results: reached timeout_total.")
def wait_route53(change_id, domain_zone_name): try: # wait 5 minutes (300 seconds) for i in range(31): if i == 30: raise TimeoutError() response = route53_client.get_change(Id=change_id) status = response["ChangeInfo"]["Status"] print('DNS propagation for %s is %s' % (domain_zone_name, status)) if status == "INSYNC": return True time.sleep(10) except TimeoutError: return False
def stop(self, timeout=5.0): """ Stops all the service processes. Waits for full stop. Args: timeout (float): How long (in seconds) to wait before raising an error. Raises: TimeoutError: If all of the services didn't stop in time. """ stop_futures = [ self._executor.submit(service.stop) for service in self._services ] results = concurrent.futures.wait(stop_futures, timeout=timeout) if results.not_done: raise TimeoutError('Not all processes stopped in time.')
def start(self, timeout=5.0): """Starts all the service processes and waits them to start accepting connections. Args: timeout (float): How long (in seconds) to wait before raising an error. Raises: TimeoutError: If all of the services didn't start in time. """ start_futures = [ self._executor.submit(service.start) for service in self._services ] results = concurrent.futures.wait(start_futures, timeout=timeout) if results.not_done: raise TimeoutError('Not all processes started in time.')
def _get_result(future, pipe, timeout): """Waits for result and handles communication errors.""" counter = count(step=SLEEP_UNIT) try: while not pipe.poll(SLEEP_UNIT): if timeout is not None and next(counter) >= timeout: return TimeoutError('Task Timeout', timeout) elif future.cancelled(): return CancelledError() return pipe.recv() except (EOFError, OSError): return ProcessExpired('Abnormal termination') except Exception as error: return error
def exception(self, timeout: int = None): start = time.time() state, error = self._get_state_and_error() while state.is_executing(): if timeout is not None and (time.time() - start) > timeout: raise TimeoutError( f"{self.job_id} did not finish running before timeout of {timeout}s" ) time.sleep(10) state, error = self._get_state_and_error() logging.info(f"Future for {self.job_id} has state {state}") if state.is_cancelled(): raise CancelledError(f"{self.job_id}: " + error) if state is State.FAILED: return AipError(f"{self.job_id}: " + error)
async def write_lock(self) -> AsyncIterator[None]: if self._check_lock() and self._try_lock(): try: yield finally: self._unlock() return for delay in self._write_retry_delay: await asyncio.sleep(delay) if self._try_lock(): try: yield finally: self._unlock() break else: raise TimeoutError()
def wait_for_requests(self, count=1, timeout=5.0): """Wait until a number of requests arrive to the imposter. Args: count (int): How many requests to wait for. timeout (float): How long to wait for requests. Returns: list[`ImposterRequest`]: The requests made on the impostor. """ start_time = time.perf_counter() while True: received_requests = self.requests() if len(received_requests) >= count: return received_requests else: time.sleep(0.01) if time.perf_counter() - start_time >= timeout: raise TimeoutError('Waited too long for requests on stub.')
def within(self, duration): """ Return a Promise whose state is guaranteed to be resolved within `duration` seconds. If this Promise completes before `duration` seconds expire, it will contain this Promise's contents. If this Promise is not resolved by then, the resulting Promise will fail with a `TimeoutError`. Parameters ---------- duration : number Number of seconds to wait before resolving a `TimeoutError` Returns ------- result : Promise Promise guaranteed to resolve in `duration` seconds. """ e = TimeoutError( u"Promise did not finish in {} seconds".format(duration)) return self.or_( Promise.wait(duration).flatmap(lambda v: Promise.exception(e)))
def wait_for_port(port, host='localhost', timeout=5.0): """Wait until a port starts accepting TCP connections. Args: port (int): Port number. host (str): Host address on which the port should exist. timeout (float): In seconds. How long to wait before raising errors. Raises: TimeoutError: The port isn't accepting connection after time specified in `timeout`. """ start_time = time.time() while True: try: sock = socket.create_connection((host, port)) sock.close() break except socket.error: time.sleep(0.01) if time.time() - start_time >= timeout: raise TimeoutError( 'Waited too long for the port {} on host {} to start accepting ' 'connections.'.format(port, host))
def wait_id(self, timeout=None): """Blocking id getter. Return the submitted problem ID, but unlike :meth:`.id`, block until the ID becomes known, or until `timeout` expires. Args: timeout (float, default=None): Timeout in seconds. By default, wait indefinitely for problem id to become known/available. Returns: str: Problem ID, as returned by SAPI. Raises: :exc:`concurrent.futures.TimeoutError`: When `timeout` exceeded, and problem id not ready. """ if not self._id_ready_event.wait(timeout=timeout): raise TimeoutError("problem id not available yet") return self._id
def _waitEndMove(self, future, axes, end=0): """ Wait until all the given axes are finished moving, or a request to stop has been received. future (Future): the future it handles axes (set of int): the axes IDs to check end (float): expected end time raise: TimeoutError: if took too long to finish the move CancelledError: if cancelled before the end of the move """ moving_axes = set(axes) last_upd = time.time() dur = max(0.01, min(end - last_upd, 60)) max_dur = dur * 2 + 1 logging.debug("Expecting a move of %g s, will wait up to %g s", dur, max_dur) timeout = last_upd + max_dur last_axes = moving_axes.copy() try: while not future._must_stop.is_set(): for aid in moving_axes.copy( ): # need copy to remove during iteration if self.GetMotionDone(aid): moving_axes.discard(aid) if not moving_axes: # no more axes to wait for break now = time.time() if now > timeout: logging.warning("Stopping move due to timeout after %g s.", max_dur) for i in moving_axes: self.StopMotion(i) raise TimeoutError("Move is not over after %g s, while " "expected it takes only %g s" % (max_dur, dur)) # Update the position from time to time (10 Hz) if now - last_upd > 0.1 or last_axes != moving_axes: last_names = set(n for n, i in self._axis_map.items() if i in last_axes) self._updatePosition(last_names) last_upd = time.time() last_axes = moving_axes.copy() # Wait half of the time left (maximum 0.1 s) left = end - time.time() sleept = max(0.001, min(left / 2, 0.1)) future._must_stop.wait(sleept) else: logging.debug("Move of axes %s cancelled before the end", axes) # stop all axes still moving them for i in moving_axes: self.StopMotion(i) future._was_stopped = True raise CancelledError() finally: # TODO: check if the move succeded ? (= Not failed due to stallguard/limit switch) self._updatePosition() # update (all axes) with final position
def _read(self, subscriber_i): """Iterate over incoming messages in order. Your thread will sleep until the next message is available, or timeout expires (in which case MailboxReadTimeout is raised) """ self.log.debug("Start reading") next_number = 0 last_message = False while not last_message: with self._lock: # Wait for new messages def next_ready(): return self._has_msg(next_number) or self.killed if not next_ready(): self.log.debug(f"Checking/waiting for {next_number}") if not self._read_condition.wait_for(next_ready, self.timeout): raise MailboxReadTimeout( f"{self.name} did not get {next_number} in time") if self.killed: self.log.debug(f"Reader finds {self.name} killed") raise MailboxKilled(self.killed_because) # Grab all messages we can yield to_yield = [] while self._has_msg(next_number): msg = self._get_msg(next_number) if msg is StopIteration: self.log.debug(f"{next_number} is StopIteration") last_message = True to_yield.append((next_number, msg)) next_number += 1 if len(to_yield) > 1: self.log.debug(f"Read {to_yield[0][0]}-{to_yield[-1][0]}") else: self.log.debug(f"Read {to_yield[0][0]}") self._subscribers_have_read[subscriber_i] = next_number - 1 # Clean up the mailbox while (len(self._mailbox) and (min(self._subscribers_have_read) >= self._lowest_msg_number)): heapq.heappop(self._mailbox) self._write_condition.notify_all() for msg_number, msg in to_yield: if msg is StopIteration: return elif isinstance(msg, Future): if not msg.done(): self.log.debug(f"Waiting for future {msg_number}") try: res = msg.result(timeout=self.timeout) except TimeoutError: raise TimeoutError( f"Future {msg_number} timed out!") self.log.debug(f"Future {msg_number} completed") else: res = msg.result() self.log.debug(f"Future {msg_number} was already done") yield res else: yield msg self.log.debug("Done reading")