def _serviceActions(self) -> int:
"""
Run all pending actions in the action queue.
:return: number of actions executed.
"""
if self.aqStash:
tm = time.perf_counter()
if tm > self.aqNextCheck:
earliest = float('inf')
for d in list(self.aqStash):
nxt, action = d
if tm > nxt:
self.actionQueue.appendleft(action)
self.aqStash.remove(d)
if nxt < earliest:
earliest = nxt
self.aqNextCheck = earliest
count = len(self.actionQueue)
while self.actionQueue:
action, aid = self.actionQueue.popleft()
assert action in self.scheduled
if aid in self.scheduled[action]:
self.scheduled[action].remove(aid)
logger.trace("{} running action {} with id {}".format(
self, get_func_name(action), aid))
action()
else:
logger.trace(
"{} not running cancelled action {} with id {}".format(
self, get_func_name(action), aid))
return count
def _serviceActions(self) -> int:
"""
Run all pending actions in the action queue.
:return: number of actions executed.
"""
if self.aqStash:
tm = time.perf_counter()
if tm > self.aqNextCheck:
earliest = float('inf')
for d in list(self.aqStash):
nxt, action = d
if tm > nxt:
self.actionQueue.appendleft(action)
self.aqStash.remove(d)
if nxt < earliest:
earliest = nxt
self.aqNextCheck = earliest
count = len(self.actionQueue)
while self.actionQueue:
action, aid = self.actionQueue.popleft()
assert action in self.scheduled
if aid in self.scheduled[action]:
self.scheduled[action].remove(aid)
logger.trace("{} running action {} with id {}".
format(self, get_func_name(action), aid))
action()
else:
logger.trace("{} not running cancelled action {} with id {}".
format(self, get_func_name(action), aid))
return count
def _schedule(self, action: Callable, seconds: int = 0) -> int:
"""
Schedule an action to be executed after `seconds` seconds.
:param action: a callable to be scheduled
:param seconds: the time in seconds after which the action must be executed
"""
self.aid += 1
if seconds > 0:
nxt = time.perf_counter() + seconds
if nxt < self.aqNextCheck:
self.aqNextCheck = nxt
logger.trace("{} scheduling action {} with id {} to run in {} "
"seconds".format(self, get_func_name(action),
self.aid, seconds))
self.aqStash.append((nxt, (action, self.aid)))
else:
logger.trace(
"{} scheduling action {} with id {} to run now".format(
self, get_func_name(action), self.aid))
self.actionQueue.append((action, self.aid))
if action not in self.scheduled:
self.scheduled[action] = []
self.scheduled[action].append(self.aid)
return self.aid
def _schedule(self, action: Callable, seconds: int=0) -> int:
"""
Schedule an action to be executed after `seconds` seconds.
:param action: a callable to be scheduled
:param seconds: the time in seconds after which the action must be executed
"""
self.aid += 1
if seconds > 0:
nxt = time.perf_counter() + seconds
if nxt < self.aqNextCheck:
self.aqNextCheck = nxt
logger.trace("{} scheduling action {} with id {} to run in {} "
"seconds".format(self, get_func_name(action),
self.aid, seconds))
self.aqStash.append((nxt, (action, self.aid)))
else:
logger.trace("{} scheduling action {} with id {} to run now".
format(self, get_func_name(action), self.aid))
self.actionQueue.append((action, self.aid))
if action not in self.scheduled:
self.scheduled[action] = []
self.scheduled[action].append(self.aid)
return self.aid
def stopRepeating(self, action: Callable, strict=True):
try:
self.repeatingActions.remove(action)
logger.debug('{} will not be repeating'.format(
get_func_name(action)))
except KeyError:
msg = '{} not found in repeating actions'.format(
get_func_name(action))
if strict:
raise KeyError(msg)
else:
logger.debug(msg)
def stopRepeating(self, action: Callable, strict=True):
try:
self.repeatingActions.remove(action)
logger.debug('{} will not be repeating'.format(
get_func_name(action)))
except KeyError:
msg = '{} not found in repeating actions'.format(
get_func_name(action))
if strict:
raise KeyError(msg)
else:
logger.debug(msg)
def startRepeating(self, action: Callable, seconds: int):
@wraps(action)
def wrapper():
if action in self.repeatingActions:
action()
self._schedule(wrapper, seconds)
if action not in self.repeatingActions:
logger.debug('{} will be repeating every {} seconds'.format(
get_func_name(action), seconds))
self.repeatingActions.add(action)
self._schedule(wrapper, seconds)
else:
logger.debug('{} is already repeating'.format(
get_func_name(action)))
def startRepeating(self, action: Callable, seconds: int):
@wraps(action)
def wrapper():
if action in self.repeatingActions:
action()
self._schedule(wrapper, seconds)
if action not in self.repeatingActions:
logger.debug('{} will be repeating every {} seconds'.
format(get_func_name(action), seconds))
self.repeatingActions.add(action)
self._schedule(wrapper, seconds)
else:
logger.trace('{} is already repeating'.format(
get_func_name(action)))
async def eventuallyAll(
*coroFuncs: FlexFunc, # (use functools.partials if needed)
totalTimeout: float,
retryWait: float = 0.1,
acceptableExceptions=None,
acceptableFails: int = 0,
override_timeout_limit=False):
# TODO: Bug when `acceptableFails` > 0 if the first check fails, it will
# exhaust the entire timeout.
"""
:param coroFuncs: iterable of no-arg functions
:param totalTimeout:
:param retryWait:
:param acceptableExceptions:
:param acceptableFails: how many of the passed in coroutines can
ultimately fail and still be ok
:return:
"""
start = time.perf_counter()
def remaining():
return totalTimeout + start - time.perf_counter()
funcNames = []
others = 0
fails = 0
rem = None
for cf in coroFuncs:
if len(funcNames) < 2:
funcNames.append(get_func_name(cf))
else:
others += 1
# noinspection PyBroadException
try:
rem = remaining()
if rem <= 0:
break
await eventually(cf,
retryWait=retryWait,
timeout=rem,
acceptableExceptions=acceptableExceptions,
verbose=True,
override_timeout_limit=override_timeout_limit)
except Exception as ex:
if acceptableExceptions and type(ex) not in acceptableExceptions:
raise
fails += 1
logger.debug(
"a coro {} with args {} timed out without succeeding; fail count: "
"{}, acceptable: {}".format(get_func_name(cf),
get_func_args(cf), fails,
acceptableFails))
if fails > acceptableFails:
raise
if rem is not None and rem <= 0:
fails += 1
if fails > acceptableFails:
err = 'All checks could not complete successfully since total timeout ' \
'expired {} sec ago'.format(-1 * rem if rem < 0 else 0)
raise EventuallyTimeoutException(err)
if others:
funcNames.append("and {} others".format(others))
desc = ", ".join(funcNames)
logger.debug("{} succeeded with {:.2f} seconds to spare".format(
desc, remaining()))
async def eventually(coroFunc: FlexFunc,
*args,
retryWait: float = 0.1,
timeout: float = 5,
ratchetSteps: Optional[int] = None,
acceptableExceptions=None,
verbose=True,
override_timeout_limit=False) -> T:
assert timeout > 0, 'Need a timeout value of greater than 0 but got {} instead'.format(
timeout)
if not override_timeout_limit:
assert timeout < 240, '`eventually` timeout ({:.2f} sec) is huge. ' \
'Is it expected?'.format(timeout)
else:
logger.debug('Overriding timeout limit to {} for evaluating {}'.format(
timeout, coroFunc))
if acceptableExceptions and not isinstance(acceptableExceptions, Iterable):
acceptableExceptions = [acceptableExceptions]
start = time.perf_counter()
ratchet = Ratchet.fromGoalDuration(retryWait * slowFactor,
ratchetSteps,
timeout * slowFactor).gen() \
if ratchetSteps else None
fname = get_func_name(coroFunc)
while True:
remain = 0
try:
remain = start + timeout * slowFactor - time.perf_counter()
if remain < 0:
# this provides a convenient breakpoint for a debugger
logger.debug("{} last try...".format(fname),
extra={"cli": False})
# noinspection PyCallingNonCallable
res = coroFunc(*args)
if isawaitable(res):
result = await res
else:
result = res
if verbose:
recordSuccess(fname, timeout, timeout * slowFactor, remain)
logger.debug(
"{} succeeded with {:.2f} seconds to spare".format(
fname, remain))
return result
except Exception as ex:
if acceptableExceptions and type(ex) not in acceptableExceptions:
raise
if remain >= 0:
sleep_dur = next(ratchet) if ratchet else retryWait
if verbose:
logger.trace("{} not succeeded yet, {:.2f} seconds "
"remaining..., will sleep for {}".format(
fname, remain, sleep_dur))
await asyncio.sleep(sleep_dur)
else:
recordFail(fname, timeout)
logger.error("{} failed; not trying any more because {} "
"seconds have passed; args were {}".format(
fname, timeout, args))
raise ex
async def eventuallyAll(*coroFuncs: FlexFunc, # (use functools.partials if needed)
totalTimeout: float,
retryWait: float=0.1,
acceptableExceptions=None,
acceptableFails: int=0,
override_timeout_limit=False):
# TODO: Bug when `acceptableFails` > 0 if the first check fails, it will
# exhaust the entire timeout.
"""
:param coroFuncs: iterable of no-arg functions
:param totalTimeout:
:param retryWait:
:param acceptableExceptions:
:param acceptableFails: how many of the passed in coroutines can
ultimately fail and still be ok
:return:
"""
start = time.perf_counter()
def remaining():
return totalTimeout + start - time.perf_counter()
funcNames = []
others = 0
fails = 0
rem = None
for cf in coroFuncs:
if len(funcNames) < 2:
funcNames.append(get_func_name(cf))
else:
others += 1
# noinspection PyBroadException
try:
rem = remaining()
if rem <= 0:
break
await eventually(cf,
retryWait=retryWait,
timeout=rem,
acceptableExceptions=acceptableExceptions,
verbose=True,
override_timeout_limit=override_timeout_limit)
except Exception as ex:
if acceptableExceptions and type(ex) not in acceptableExceptions:
raise
fails += 1
logger.debug("a coro {} with args {} timed out without succeeding; fail count: "
"{}, acceptable: {}".
format(get_func_name(cf), get_func_args(cf), fails, acceptableFails))
if fails > acceptableFails:
raise
if rem is not None and rem <= 0:
fails += 1
if fails > acceptableFails:
err = 'All checks could not complete successfully since total timeout ' \
'expired {} sec ago'.format(-1 * rem if rem < 0 else 0)
raise EventuallyTimeoutException(err)
if others:
funcNames.append("and {} others".format(others))
desc = ", ".join(funcNames)
logger.debug("{} succeeded with {:.2f} seconds to spare".
format(desc, remaining()))
async def eventually(coroFunc: FlexFunc,
*args,
retryWait: float=0.1,
timeout: float=5,
ratchetSteps: Optional[int]=None,
acceptableExceptions=None,
verbose=True,
override_timeout_limit=False) -> T:
if not timeout > 0:
raise PlenumValueError('timeout', timeout, '> 0')
if not override_timeout_limit:
if timeout > 240:
raise PlenumValueError(
'timeout', timeout,
"< 240 or override_timeout_limit=True"
)
else:
logger.debug('Overriding timeout limit to {} for evaluating {}'
.format(timeout, coroFunc))
if acceptableExceptions and not isinstance(acceptableExceptions, Iterable):
acceptableExceptions = [acceptableExceptions]
start = time.perf_counter()
ratchet = Ratchet.fromGoalDuration(retryWait * slowFactor,
ratchetSteps,
timeout * slowFactor).gen() \
if ratchetSteps else None
fname = get_func_name(coroFunc)
while True:
remain = 0
try:
remain = start + timeout * slowFactor - time.perf_counter()
if remain < 0:
# this provides a convenient breakpoint for a debugger
logger.debug("{} last try...".format(fname),
extra={"cli": False})
# noinspection PyCallingNonCallable
res = coroFunc(*args)
if isawaitable(res):
result = await res
else:
result = res
if verbose:
recordSuccess(fname, timeout, timeout * slowFactor, remain)
logger.debug("{} succeeded with {:.2f} seconds to spare".
format(fname, remain))
return result
except Exception as ex:
if acceptableExceptions and type(ex) not in acceptableExceptions:
raise
if remain >= 0:
sleep_dur = next(ratchet) if ratchet else retryWait
if verbose:
logger.trace("{} not succeeded yet, {:.2f} seconds "
"remaining..., will sleep for {}".format(fname, remain, sleep_dur))
await asyncio.sleep(sleep_dur)
else:
recordFail(fname, timeout)
logger.error("{} failed; not trying any more because {} "
"seconds have passed; args were {}".
format(fname, timeout, args))
raise ex
