def __init__(self, registry: Dict[str, HA]):
self.registry = registry
self.lastcheck = {} # type: Dict[int, Tuple[int, float]]
self.ratchet = Ratchet(a=8, b=0.198, c=-4, base=8, peak=3600)
# holds the last time we checked remotes
self.nextCheck = 0
async def eventually(coroFunc: FlexFunc,
*args,
retryWait: float = 0.1,
timeout: float = 5,
ratchetSteps: Optional[int] = None,
acceptableExceptions=None,
verbose=True) -> T:
if acceptableExceptions and not isinstance(acceptableExceptions, Iterable):
acceptableExceptions = [acceptableExceptions]
start = time.perf_counter()
ratchet = Ratchet.fromGoalDuration(retryWait*slowFactor,
ratchetSteps,
timeout*slowFactor).gen() \
if ratchetSteps else None
fname = getFuncName(coroFunc)
while True:
remain = 0
try:
remain = start + timeout * slowFactor - time.perf_counter()
if remain < 0:
# this provides a convenient breakpoint for a debugger
logger.warning("{} last try...".format(fname),
extra={"cli": False})
# noinspection PyCallingNonCallable
res = coroFunc(*args)
if isawaitable(res):
result = await res
else:
result = res
if verbose:
recordSuccess(fname, timeout, timeout * slowFactor, remain)
logger.debug(
"{} succeeded with {:.2f} seconds to spare".format(
fname, remain))
return result
except Exception as ex:
if acceptableExceptions and type(ex) not in acceptableExceptions:
raise
if remain >= 0:
if verbose:
logger.trace("{} not succeeded yet, {:.2f} seconds "
"remaining...".format(fname, remain))
await asyncio.sleep(next(ratchet) if ratchet else retryWait)
else:
recordFail(fname, timeout)
logger.error("{} failed; not trying any more because {} "
"seconds have passed; args were {}".format(
fname, timeout, args))
raise ex
def test_throttler_case2():
"""
Tests throttler with custom delay function
"""
windowSize = 10
testIterations = windowSize - 2
ratchet = Ratchet(a=2, b=0.05, c=1, base=2, peak=windowSize)
throttler = Throttler(windowSize, ratchet.get)
cooldowns = [
time.sleep(1) or throttler.acquire()[1] for i in range(testIterations)
]
middle = len(cooldowns) // 2
firstIteration, secondIteration = cooldowns[:middle], cooldowns[middle:]
for a, b in zip(firstIteration, secondIteration):
if not a == b == 0:
assert b > a
def initInsChngThrottling(self):
windowSize = self.node.config.ViewChangeWindowSize
ratchet = Ratchet(a=2, b=0.05, c=1, base=2, peak=windowSize)
self.insChngThrottler = Throttler(windowSize, ratchet.get)
async def eventually(coroFunc: FlexFunc,
*args,
retryWait: float=0.1,
timeout: float=5,
ratchetSteps: Optional[int]=None,
acceptableExceptions=None,
verbose=True,
override_timeout_limit=False) -> T:
assert timeout > 0, 'Need a timeout value of greater than 0 but got {} instead'.format(timeout)
if not override_timeout_limit:
assert timeout < 240, '`eventually` timeout ({:.2f} sec) is huge. ' \
'Is it expected?'.format(timeout)
else:
logger.debug('Overriding timeout limit to {} for evaluating {}'
.format(timeout, coroFunc))
if acceptableExceptions and not isinstance(acceptableExceptions, Iterable):
acceptableExceptions = [acceptableExceptions]
start = time.perf_counter()
ratchet = Ratchet.fromGoalDuration(retryWait*slowFactor,
ratchetSteps,
timeout*slowFactor).gen() \
if ratchetSteps else None
fname = getFuncName(coroFunc)
while True:
remain = 0
try:
remain = start + timeout*slowFactor - time.perf_counter()
if remain < 0:
# this provides a convenient breakpoint for a debugger
logger.warning("{} last try...".format(fname),
extra={"cli": False})
# noinspection PyCallingNonCallable
res = coroFunc(*args)
if isawaitable(res):
result = await res
else:
result = res
if verbose:
recordSuccess(fname, timeout, timeout*slowFactor, remain)
logger.debug("{} succeeded with {:.2f} seconds to spare".
format(fname, remain))
return result
except Exception as ex:
if acceptableExceptions and type(ex) not in acceptableExceptions:
raise
if remain >= 0:
sleep_dur = next(ratchet) if ratchet else retryWait
if verbose:
logger.trace("{} not succeeded yet, {:.2f} seconds "
"remaining..., will sleep for {}".format(fname, remain, sleep_dur))
await asyncio.sleep(sleep_dur)
else:
recordFail(fname, timeout)
logger.error("{} failed; not trying any more because {} "
"seconds have passed; args were {}".
format(fname, timeout, args))
raise ex
async def eventually(coroFunc: FlexFunc,
*args,
retryWait: float=0.1,
timeout: float=5,
ratchetSteps: Optional[int]=None,
acceptableExceptions=None,
verbose=True,
override_timeout_limit=False) -> T:
if not timeout > 0:
raise PlenumValueError('timeout', timeout, '> 0')
if not override_timeout_limit:
if timeout > 240:
raise PlenumValueError(
'timeout', timeout,
"< 240 or override_timeout_limit=True"
)
else:
logger.debug('Overriding timeout limit to {} for evaluating {}'
.format(timeout, coroFunc))
if acceptableExceptions and not isinstance(acceptableExceptions, Iterable):
acceptableExceptions = [acceptableExceptions]
start = time.perf_counter()
ratchet = Ratchet.fromGoalDuration(retryWait * slowFactor,
ratchetSteps,
timeout * slowFactor).gen() \
if ratchetSteps else None
fname = get_func_name(coroFunc)
while True:
remain = 0
try:
remain = start + timeout * slowFactor - time.perf_counter()
if remain < 0:
# this provides a convenient breakpoint for a debugger
logger.debug("{} last try...".format(fname),
extra={"cli": False})
# noinspection PyCallingNonCallable
res = coroFunc(*args)
if isawaitable(res):
result = await res
else:
result = res
if verbose:
recordSuccess(fname, timeout, timeout * slowFactor, remain)
logger.debug("{} succeeded with {:.2f} seconds to spare".
format(fname, remain))
return result
except Exception as ex:
if acceptableExceptions and type(ex) not in acceptableExceptions:
raise
if remain >= 0:
sleep_dur = next(ratchet) if ratchet else retryWait
if verbose:
logger.trace("{} not succeeded yet, {:.2f} seconds "
"remaining..., will sleep for {}".format(fname, remain, sleep_dur))
await asyncio.sleep(sleep_dur)
else:
recordFail(fname, timeout)
logger.error("{} failed; not trying any more because {} "
"seconds have passed; args were {}".
format(fname, timeout, args))
raise ex