def _serviceActions(self) -> int: """ Run all pending actions in the action queue. :return: number of actions executed. """ if self.aqStash: tm = time.perf_counter() if tm > self.aqNextCheck: earliest = float('inf') for d in list(self.aqStash): nxt, action = d if tm > nxt: self.actionQueue.appendleft(action) self.aqStash.remove(d) if nxt < earliest: earliest = nxt self.aqNextCheck = earliest count = len(self.actionQueue) while self.actionQueue: action, aid = self.actionQueue.popleft() assert action in self.scheduled if aid in self.scheduled[action]: self.scheduled[action].remove(aid) logger.trace("{} running action {} with id {}".format( self, get_func_name(action), aid)) action() else: logger.trace( "{} not running cancelled action {} with id {}".format( self, get_func_name(action), aid)) return count
def _serviceActions(self) -> int: """ Run all pending actions in the action queue. :return: number of actions executed. """ if self.aqStash: tm = time.perf_counter() if tm > self.aqNextCheck: earliest = float('inf') for d in list(self.aqStash): nxt, action = d if tm > nxt: self.actionQueue.appendleft(action) self.aqStash.remove(d) if nxt < earliest: earliest = nxt self.aqNextCheck = earliest count = len(self.actionQueue) while self.actionQueue: action, aid = self.actionQueue.popleft() assert action in self.scheduled if aid in self.scheduled[action]: self.scheduled[action].remove(aid) logger.trace("{} running action {} with id {}". format(self, get_func_name(action), aid)) action() else: logger.trace("{} not running cancelled action {} with id {}". format(self, get_func_name(action), aid)) return count
def _schedule(self, action: Callable, seconds: int = 0) -> int: """ Schedule an action to be executed after `seconds` seconds. :param action: a callable to be scheduled :param seconds: the time in seconds after which the action must be executed """ self.aid += 1 if seconds > 0: nxt = time.perf_counter() + seconds if nxt < self.aqNextCheck: self.aqNextCheck = nxt logger.trace("{} scheduling action {} with id {} to run in {} " "seconds".format(self, get_func_name(action), self.aid, seconds)) self.aqStash.append((nxt, (action, self.aid))) else: logger.trace( "{} scheduling action {} with id {} to run now".format( self, get_func_name(action), self.aid)) self.actionQueue.append((action, self.aid)) if action not in self.scheduled: self.scheduled[action] = [] self.scheduled[action].append(self.aid) return self.aid
def _schedule(self, action: Callable, seconds: int=0) -> int: """ Schedule an action to be executed after `seconds` seconds. :param action: a callable to be scheduled :param seconds: the time in seconds after which the action must be executed """ self.aid += 1 if seconds > 0: nxt = time.perf_counter() + seconds if nxt < self.aqNextCheck: self.aqNextCheck = nxt logger.trace("{} scheduling action {} with id {} to run in {} " "seconds".format(self, get_func_name(action), self.aid, seconds)) self.aqStash.append((nxt, (action, self.aid))) else: logger.trace("{} scheduling action {} with id {} to run now". format(self, get_func_name(action), self.aid)) self.actionQueue.append((action, self.aid)) if action not in self.scheduled: self.scheduled[action] = [] self.scheduled[action].append(self.aid) return self.aid
def stopRepeating(self, action: Callable, strict=True): try: self.repeatingActions.remove(action) logger.debug('{} will not be repeating'.format( get_func_name(action))) except KeyError: msg = '{} not found in repeating actions'.format( get_func_name(action)) if strict: raise KeyError(msg) else: logger.debug(msg)
def startRepeating(self, action: Callable, seconds: int): @wraps(action) def wrapper(): if action in self.repeatingActions: action() self._schedule(wrapper, seconds) if action not in self.repeatingActions: logger.debug('{} will be repeating every {} seconds'.format( get_func_name(action), seconds)) self.repeatingActions.add(action) self._schedule(wrapper, seconds) else: logger.debug('{} is already repeating'.format( get_func_name(action)))
def startRepeating(self, action: Callable, seconds: int): @wraps(action) def wrapper(): if action in self.repeatingActions: action() self._schedule(wrapper, seconds) if action not in self.repeatingActions: logger.debug('{} will be repeating every {} seconds'. format(get_func_name(action), seconds)) self.repeatingActions.add(action) self._schedule(wrapper, seconds) else: logger.trace('{} is already repeating'.format( get_func_name(action)))
async def eventuallyAll( *coroFuncs: FlexFunc, # (use functools.partials if needed) totalTimeout: float, retryWait: float = 0.1, acceptableExceptions=None, acceptableFails: int = 0, override_timeout_limit=False): # TODO: Bug when `acceptableFails` > 0 if the first check fails, it will # exhaust the entire timeout. """ :param coroFuncs: iterable of no-arg functions :param totalTimeout: :param retryWait: :param acceptableExceptions: :param acceptableFails: how many of the passed in coroutines can ultimately fail and still be ok :return: """ start = time.perf_counter() def remaining(): return totalTimeout + start - time.perf_counter() funcNames = [] others = 0 fails = 0 rem = None for cf in coroFuncs: if len(funcNames) < 2: funcNames.append(get_func_name(cf)) else: others += 1 # noinspection PyBroadException try: rem = remaining() if rem <= 0: break await eventually(cf, retryWait=retryWait, timeout=rem, acceptableExceptions=acceptableExceptions, verbose=True, override_timeout_limit=override_timeout_limit) except Exception as ex: if acceptableExceptions and type(ex) not in acceptableExceptions: raise fails += 1 logger.debug( "a coro {} with args {} timed out without succeeding; fail count: " "{}, acceptable: {}".format(get_func_name(cf), get_func_args(cf), fails, acceptableFails)) if fails > acceptableFails: raise if rem is not None and rem <= 0: fails += 1 if fails > acceptableFails: err = 'All checks could not complete successfully since total timeout ' \ 'expired {} sec ago'.format(-1 * rem if rem < 0 else 0) raise EventuallyTimeoutException(err) if others: funcNames.append("and {} others".format(others)) desc = ", ".join(funcNames) logger.debug("{} succeeded with {:.2f} seconds to spare".format( desc, remaining()))
async def eventually(coroFunc: FlexFunc, *args, retryWait: float = 0.1, timeout: float = 5, ratchetSteps: Optional[int] = None, acceptableExceptions=None, verbose=True, override_timeout_limit=False) -> T: assert timeout > 0, 'Need a timeout value of greater than 0 but got {} instead'.format( timeout) if not override_timeout_limit: assert timeout < 240, '`eventually` timeout ({:.2f} sec) is huge. ' \ 'Is it expected?'.format(timeout) else: logger.debug('Overriding timeout limit to {} for evaluating {}'.format( timeout, coroFunc)) if acceptableExceptions and not isinstance(acceptableExceptions, Iterable): acceptableExceptions = [acceptableExceptions] start = time.perf_counter() ratchet = Ratchet.fromGoalDuration(retryWait * slowFactor, ratchetSteps, timeout * slowFactor).gen() \ if ratchetSteps else None fname = get_func_name(coroFunc) while True: remain = 0 try: remain = start + timeout * slowFactor - time.perf_counter() if remain < 0: # this provides a convenient breakpoint for a debugger logger.debug("{} last try...".format(fname), extra={"cli": False}) # noinspection PyCallingNonCallable res = coroFunc(*args) if isawaitable(res): result = await res else: result = res if verbose: recordSuccess(fname, timeout, timeout * slowFactor, remain) logger.debug( "{} succeeded with {:.2f} seconds to spare".format( fname, remain)) return result except Exception as ex: if acceptableExceptions and type(ex) not in acceptableExceptions: raise if remain >= 0: sleep_dur = next(ratchet) if ratchet else retryWait if verbose: logger.trace("{} not succeeded yet, {:.2f} seconds " "remaining..., will sleep for {}".format( fname, remain, sleep_dur)) await asyncio.sleep(sleep_dur) else: recordFail(fname, timeout) logger.error("{} failed; not trying any more because {} " "seconds have passed; args were {}".format( fname, timeout, args)) raise ex
async def eventuallyAll(*coroFuncs: FlexFunc, # (use functools.partials if needed) totalTimeout: float, retryWait: float=0.1, acceptableExceptions=None, acceptableFails: int=0, override_timeout_limit=False): # TODO: Bug when `acceptableFails` > 0 if the first check fails, it will # exhaust the entire timeout. """ :param coroFuncs: iterable of no-arg functions :param totalTimeout: :param retryWait: :param acceptableExceptions: :param acceptableFails: how many of the passed in coroutines can ultimately fail and still be ok :return: """ start = time.perf_counter() def remaining(): return totalTimeout + start - time.perf_counter() funcNames = [] others = 0 fails = 0 rem = None for cf in coroFuncs: if len(funcNames) < 2: funcNames.append(get_func_name(cf)) else: others += 1 # noinspection PyBroadException try: rem = remaining() if rem <= 0: break await eventually(cf, retryWait=retryWait, timeout=rem, acceptableExceptions=acceptableExceptions, verbose=True, override_timeout_limit=override_timeout_limit) except Exception as ex: if acceptableExceptions and type(ex) not in acceptableExceptions: raise fails += 1 logger.debug("a coro {} with args {} timed out without succeeding; fail count: " "{}, acceptable: {}". format(get_func_name(cf), get_func_args(cf), fails, acceptableFails)) if fails > acceptableFails: raise if rem is not None and rem <= 0: fails += 1 if fails > acceptableFails: err = 'All checks could not complete successfully since total timeout ' \ 'expired {} sec ago'.format(-1 * rem if rem < 0 else 0) raise EventuallyTimeoutException(err) if others: funcNames.append("and {} others".format(others)) desc = ", ".join(funcNames) logger.debug("{} succeeded with {:.2f} seconds to spare". format(desc, remaining()))
async def eventually(coroFunc: FlexFunc, *args, retryWait: float=0.1, timeout: float=5, ratchetSteps: Optional[int]=None, acceptableExceptions=None, verbose=True, override_timeout_limit=False) -> T: if not timeout > 0: raise PlenumValueError('timeout', timeout, '> 0') if not override_timeout_limit: if timeout > 240: raise PlenumValueError( 'timeout', timeout, "< 240 or override_timeout_limit=True" ) else: logger.debug('Overriding timeout limit to {} for evaluating {}' .format(timeout, coroFunc)) if acceptableExceptions and not isinstance(acceptableExceptions, Iterable): acceptableExceptions = [acceptableExceptions] start = time.perf_counter() ratchet = Ratchet.fromGoalDuration(retryWait * slowFactor, ratchetSteps, timeout * slowFactor).gen() \ if ratchetSteps else None fname = get_func_name(coroFunc) while True: remain = 0 try: remain = start + timeout * slowFactor - time.perf_counter() if remain < 0: # this provides a convenient breakpoint for a debugger logger.debug("{} last try...".format(fname), extra={"cli": False}) # noinspection PyCallingNonCallable res = coroFunc(*args) if isawaitable(res): result = await res else: result = res if verbose: recordSuccess(fname, timeout, timeout * slowFactor, remain) logger.debug("{} succeeded with {:.2f} seconds to spare". format(fname, remain)) return result except Exception as ex: if acceptableExceptions and type(ex) not in acceptableExceptions: raise if remain >= 0: sleep_dur = next(ratchet) if ratchet else retryWait if verbose: logger.trace("{} not succeeded yet, {:.2f} seconds " "remaining..., will sleep for {}".format(fname, remain, sleep_dur)) await asyncio.sleep(sleep_dur) else: recordFail(fname, timeout) logger.error("{} failed; not trying any more because {} " "seconds have passed; args were {}". format(fname, timeout, args)) raise ex