def test_increment_linear(self, iteration, backoff):
eb = rate_limits.ExponentialBackOff(2, 64, 0)
for _ in range(iteration):
next(eb)
assert next(eb) == backoff
def test_increment_jitter(self, iteration, backoff):
abs_tol = 1
eb = rate_limits.ExponentialBackOff(2, 64, abs_tol)
for _ in range(iteration):
next(eb)
assert math.isclose(next(eb), backoff, abs_tol=abs_tol)
def test_increment_maximum(self):
max_bound = 64
eb = rate_limits.ExponentialBackOff(2, max_bound, 0)
iterations = math.ceil(math.log2(max_bound))
for _ in range(iterations):
next(eb)
assert next(eb) == max_bound
def test_increment_raises_on_numerical_limitation(self):
power = math.log(sys.float_info.max, 5) + 0.5
eb = rate_limits.ExponentialBackOff(base=5,
maximum=sys.float_info.max,
jitter_multiplier=0.0,
initial_increment=power)
assert next(eb) == sys.float_info.max
def test_increment_does_not_increment_when_on_maximum(self):
eb = rate_limits.ExponentialBackOff(2,
32,
initial_increment=5,
jitter_multiplier=0)
assert eb.increment == 5
assert next(eb) == 32
assert eb.increment == 5
def test_iter_returns_self(self):
eb = rate_limits.ExponentialBackOff(2, 64, 123)
assert iter(eb) is eb
def test___init___raises_on_not_finite_jitter_multiplier(self):
with pytest.raises(ValueError,
match="jitter_multiplier must be a finite number"):
rate_limits.ExponentialBackOff(jitter_multiplier=float("inf"))
def test_reset(self):
eb = rate_limits.ExponentialBackOff()
eb.increment = 10
eb.reset()
assert eb.increment == 0
def test___init___raises_on_not_finite_maximum(self):
with pytest.raises(ValueError,
match="maximum must be a finite number"):
rate_limits.ExponentialBackOff(maximum=float("nan"))
def test___init___raises_on_not_finite_base(self):
with pytest.raises(ValueError, match="base must be a finite number"):
rate_limits.ExponentialBackOff(base=float("inf"))
def test___init___raises_on_too_large_int_jitter_multiplier(self):
jitter_multiplier = int(sys.float_info.max) + int(
sys.float_info.max * 1 / 300)
with pytest.raises(ValueError,
match="int too large to be represented as a float"):
rate_limits.ExponentialBackOff(jitter_multiplier=jitter_multiplier)
def test___init___raises_on_too_large_int_maximum(self):
maximum = int(sys.float_info.max) + int(sys.float_info.max * 1 / 200)
with pytest.raises(ValueError,
match="int too large to be represented as a float"):
rate_limits.ExponentialBackOff(maximum=maximum)
async def _run(self) -> None:
self._closed.clear()
self._closing.clear()
last_started_at = -float("inf")
backoff = rate_limits.ExponentialBackOff(
base=_BACKOFF_BASE,
maximum=_BACKOFF_CAP,
initial_increment=_BACKOFF_INCREMENT_START,
)
try:
while not self._closing.is_set() and not self._closed.is_set():
if time.monotonic() - last_started_at < _BACKOFF_WINDOW:
backoff_time = next(backoff)
self._logger.info("backing off reconnecting for %.2fs",
backoff_time)
try:
await asyncio.wait_for(self._closing.wait(),
timeout=backoff_time)
# We were told to close.
return
except asyncio.TimeoutError:
# We are going to run once.
pass
try:
last_started_at = time.monotonic()
should_restart = await self._run_once()
if not should_restart:
self._logger.info(
"shard has disconnected and shut down normally")
return
except errors.GatewayConnectionError as ex:
self._logger.error(
"failed to communicate with server, reason was: %s. Will retry shortly",
ex.__cause__,
)
except errors.GatewayServerClosedConnectionError as ex:
if not ex.can_reconnect:
raise
self._logger.info(
"server has closed connection, will reconnect if possible [code:%s, reason:%s]",
ex.code,
ex.reason,
)
# We don't want to back off from this. If Discord keep closing the connection, it is their issue.
# If we back off here, we'll find a mass outage will prevent shards from becoming healthy on
# reconnect in large sharded bots for a very long period of time.
backoff.reset()
except errors.GatewayError as ex:
self._logger.error("encountered generic gateway error",
exc_info=ex)
raise
except Exception as ex:
self._logger.error("encountered some unhandled error",
exc_info=ex)
raise
finally:
self._closing.set()
self._closed.set()
