def wrapper(*args, **kwargs):
nonlocal backoff
start = dt.now()
LOGGER.debug("Starting function call retries.")
n = 1
while True:
LOGGER.info(f"Running try number {n}.")
try:
res = func(*args, **kwargs)
LOGGER.info("Call succeeded.")
return res
except on_exception:
LOGGER.warning(f"Call failed (raised {on_exception}).")
n += 1
now = dt.now()
delta = now - start
if delta > timeout:
LOGGER.error("Timeout reached, exiting.")
raise
LOGGER.info(f"Sleeping for {backoff}.")
time.sleep(backoff.total_seconds())
backoff *= 2 # exponential
def _run(cmd, subcmd, flags=None, **kwargs) -> CompletedProcess:
if flags is None:
flags = []
else:
if isinstance(flags, str):
# Allow flags to be a string, so that we don't have to create a list for a
# single flag argument (`["--all"]` vs. `"--all"`).
flags = split(flags)
try:
return text_run(cmd + split(subcmd) + flags, **kwargs)
except CalledTextProcessError as e:
# Examine the standard error output, trying to find matches. If a match is found,
# convert raised exception to a more fitting one.
for regex, exception in _STDERR_CONVERSIONS.items():
match = regex.search(e.stderr)
if match:
LOGGER.warning(
f"Process' stderr matched against {regex}, raising {exception}"
)
raise exception(**vars(e)) from e
raise