async def _loop(self, *args, **kwargs):
backoff = ExponentialBackoff()
await self._call_loop_function('before_loop')
try:
while True:
try:
await self.coro(*args, **kwargs)
except self._valid_exception as exc:
if not self.reconnect:
raise
await asyncio.sleep(backoff.delay())
else:
if self._stop_next_iteration:
return
self._current_loop += 1
if self._current_loop == self.count:
break
await asyncio.sleep(self._sleep)
except asyncio.CancelledError:
self._is_being_cancelled = True
raise
except Exception:
self._has_failed = True
log.exception('Internal background task failed.')
raise
finally:
await self._call_loop_function('after_loop')
self._is_being_cancelled = False
self._current_loop = 0
self._stop_next_iteration = False
self._has_failed = False
async def _multi_try_connect(self, uri):
backoff = ExponentialBackoff()
attempt = 1
if self._ws is not None:
await self._ws.close(code=4006, message=b"Reconnecting")
while self._is_shutdown is False and (self._ws is None
or self._ws.closed):
self._retries += 1
try:
ws = await self.session.ws_connect(url=uri,
headers=self.headers,
heartbeat=60)
except (OSError, aiohttp.ClientConnectionError):
delay = backoff.delay()
ws_ll_log.error("Failed connect attempt %s, retrying in %s",
attempt, delay)
await asyncio.sleep(delay)
attempt += 1
if attempt > 5:
raise asyncio.TimeoutError
except aiohttp.WSServerHandshakeError:
ws_ll_log.error("Failed connect WSServerHandshakeError")
raise asyncio.TimeoutError
else:
self.session_resumed = ws._response.headers.get(
"Session-Resumed", False)
if self._ws is not None and self.session_resumed:
ws_ll_log.info("WEBSOCKET Resumed Session with key: %s",
self._resume_key)
self._ws = ws
break
async def _connect(self):
backoff = ExponentialBackoff(5, integral=True)
while not self.connected:
try:
logger.info(
f'Attempting to establish websocket connection to {self.name}'
)
self.ws = await self.session.ws_connect(self.uri,
headers=self.headers)
except aiohttp.ClientConnectorError:
logger.warning(
f'[{self.name}] Invalid response received; this may indicate that '
'Lavalink is not running, or is running on a port different '
'to the one you passed to `add_node`.')
except aiohttp.WSServerHandshakeError as ce:
if ce.status in (401, 403):
logger.error(
f'Authentication failed when establishing a connection to {self.name}'
)
return
logger.warning(
f'{self.name} returned a code {ce.status} which was unexpected'
)
else:
logger.info(f'Connection established to {self.name}')
self.listen_task = asyncio.create_task(self.listen())
return
delay = backoff.delay()
logger.error(f"Connection refused, trying again in {delay}s")
await asyncio.sleep(delay)
async def _reconnect(self):
self._ready_event.clear()
if self._is_shutdown is True:
ws_ll_log.info("[NODE] | Shutting down Lavalink WS.")
return
if self.state != NodeState.CONNECTING:
self.update_state(NodeState.RECONNECTING)
if self.state != NodeState.RECONNECTING:
return
backoff = ExponentialBackoff(base=1)
attempt = 1
while self.state == NodeState.RECONNECTING:
attempt += 1
try:
await self.connect()
except asyncio.TimeoutError:
delay = backoff.delay()
ws_ll_log.info(
"[NODE] | Failed to reconnect to the Lavalink node.")
ws_ll_log.info(
"[NODE] | Lavalink WS reconnect connect attempt %s, retrying in %s",
attempt,
delay,
)
else:
ws_ll_log.info("[NODE] | Reconnect successful.")
self.dispatch_reconnect()
self._retries = 0
async def wrapper(*args, **kwargs):
if isinstance(args[0], commands.Bot):
bot = args[0]
elif not isinstance(args[0].bot, commands.Bot):
raise utils.MissingInstance(
f'Missing an instance of Bot in task <{func.__name__}>.')
else:
bot = args[0].bot
backoff = ExponentialBackoff()
retrys = 0
if until_ready:
await bot.wait_until_ready()
while not bot.is_closed():
await func(*args, *kwargs)
while bot.is_reconnecting():
retrys += 1
retry = backoff.delay()
await asyncio.sleep(retry)
if retrys:
print(
f'Resumed Task: {func.__name__} after {retrys} trys.\n'
)
retrys = 0
async def connect(self, *, reconnect=True):
"""Override connect and add a reconnecting state."""
backoff = ExponentialBackoff()
while not self.is_closed():
try:
await self._connect()
except (OSError,
discord.HTTPException,
discord.GatewayNotFound,
discord.ConnectionClosed,
aiohttp.ClientError,
asyncio.TimeoutError,
websockets.InvalidHandshake,
websockets.WebSocketProtocolError) as e:
if not reconnect:
await self.close()
if isinstance(e, discord.ConnectionClosed) and e.code == 1000:
# clean close, don't re-raise this
return
raise
if self.is_closed():
return
if isinstance(e, discord.ConnectionClosed):
if e.code != 1000:
await self.close()
raise
retry = backoff.delay()
self._reconnecting.set()
await asyncio.sleep(retry, loop=self.loop)
async def _multi_try_connect(self, uri):
backoff = ExponentialBackoff()
attempt = 1
if self._listener_task is not None:
self._listener_task.cancel()
if self._ws is not None:
await self._ws.close(code=4006, message=b"Reconnecting")
while self._is_shutdown is False and (self._ws is None
or self._ws.closed):
self._retries += 1
if self._is_shutdown is True:
ws_ll_log.error(
"Lavalink node was shutdown during a connect attempt.")
raise asyncio.CancelledError
try:
ws = await self.session.ws_connect(url=uri,
headers=self.headers,
heartbeat=60)
except (OSError, aiohttp.ClientConnectionError):
if attempt > 5:
raise asyncio.TimeoutError
delay = backoff.delay()
ws_ll_log.warning("Failed connect attempt %s, retrying in %s",
attempt, delay)
await asyncio.sleep(delay)
attempt += 1
except aiohttp.WSServerHandshakeError:
ws_ll_log.error("Failed connect WSServerHandshakeError")
raise asyncio.TimeoutError
else:
if self._is_shutdown is True:
ws_ll_log.error(
"Lavalink node was shutdown during a connect attempt.")
raise asyncio.CancelledError
self.session_resumed = ws._response.headers.get(
"Session-Resumed", False)
if self._ws is not None and self.session_resumed:
ws_ll_log.info("WEBSOCKET Resumed Session with key: %s",
self._resume_key)
self._ws = ws
break
if self._is_shutdown is True:
raise asyncio.CancelledError
ws_ll_log.info("Lavalink WS connected to %s", uri)
ws_ll_log.debug("Creating Lavalink WS listener.")
if self._is_shutdown is False:
self._listener_task = asyncio.create_task(self.listener())
asyncio.create_task(self._configure_resume())
if self._queue:
temp = self._queue.copy()
self._queue.clear()
for data in temp:
await self.send(data)
self._ready_event.set()
self.update_state(NodeState.READY)
async def _loop(self, *args: Any, **kwargs: Any) -> None:
backoff = ExponentialBackoff()
await self._call_loop_function('before_loop')
self._last_iteration_failed = False
if self._time is not MISSING:
# the time index should be prepared every time the internal loop is started
self._prepare_time_index()
self._next_iteration = self._get_next_sleep_time()
else:
self._next_iteration = datetime.datetime.now(datetime.timezone.utc)
await asyncio.sleep(0) # allows canceling in before_loop
try:
if self._stop_next_iteration: # allow calling stop() before first iteration
return
while True:
# sleep before the body of the task for explicit time intervals
if self._time is not MISSING:
await self._try_sleep_until(self._next_iteration)
if not self._last_iteration_failed:
self._last_iteration = self._next_iteration
self._next_iteration = self._get_next_sleep_time()
try:
await self.coro(*args, **kwargs)
self._last_iteration_failed = False
except self._valid_exception:
self._last_iteration_failed = True
if not self.reconnect:
raise
await asyncio.sleep(backoff.delay())
else:
if self._stop_next_iteration:
return
# sleep after the body of the task for relative time intervals
if self._time is MISSING:
await self._try_sleep_until(self._next_iteration)
self._current_loop += 1
if self._current_loop == self.count:
break
except asyncio.CancelledError:
self._is_being_cancelled = True
raise
except Exception as exc:
self._has_failed = True
await self._call_loop_function('error', exc)
raise exc
finally:
await self._call_loop_function('after_loop')
self._handle.cancel()
self._is_being_cancelled = False
self._current_loop = 0
self._stop_next_iteration = False
self._has_failed = False
async def _loop(self, *args, **kwargs):
backoff = ExponentialBackoff()
await self._call_loop_function('before_loop')
sleep_until = discord.utils.sleep_until
self._last_iteration_failed = False
if self._time is not None:
# the time index should be prepared every time the internal loop is started
self._prepare_time_index()
self._next_iteration = self._get_next_sleep_time()
else:
self._next_iteration = datetime.datetime.now(datetime.timezone.utc)
try:
await self._try_sleep_until(self._next_iteration)
while True:
if not self._last_iteration_failed:
self._last_iteration = self._next_iteration
self._next_iteration = self._get_next_sleep_time()
try:
await self.coro(*args, **kwargs)
self._last_iteration_failed = False
except self._valid_exception:
self._last_iteration_failed = True
if not self.reconnect:
raise
await asyncio.sleep(backoff.delay())
else:
await self._try_sleep_until(self._next_iteration)
if self._stop_next_iteration:
return
now = datetime.datetime.now(datetime.timezone.utc)
if now > self._next_iteration:
self._next_iteration = now
if self._time is not None:
self._prepare_time_index(now)
self._current_loop += 1
if self._current_loop == self.count:
break
except asyncio.CancelledError:
self._is_being_cancelled = True
raise
except Exception as exc:
self._has_failed = True
await self._call_loop_function('error', exc)
raise exc
finally:
await self._call_loop_function('after_loop')
self._handle.cancel()
self._is_being_cancelled = False
self._current_loop = 0
self._stop_next_iteration = False
self._has_failed = False
async def _multi_try_connect(self, uri):
backoff = ExponentialBackoff()
attempt = 1
while not SHUTDOWN.is_set() and (self._ws is None
or not self._ws.open):
try:
self._ws = await websockets.connect(uri,
extra_headers=self.headers)
except OSError:
delay = backoff.delay()
log.debug("Failed connect attempt {}, retrying in {}".format(
attempt, delay))
await asyncio.sleep(delay)
attempt += 1
async def reaction_backoff(partial): b = None for _ in range(5): try: await partial() except discord.HTTPException: if b is None: b = ExponentialBackoff(base=2.0) await asyncio.sleep(b.delay()) continue except (discord.Forbidden, discord.NotFound): break else: return True
async def _connect(self):
backoff = ExponentialBackoff(2)
while not (self.ws and self.ws.open):
try:
self.ws = await websockets.connect(self.uri, extra_headers=self.headers)
task = asyncio.create_task(self.listen())
# asyncio.create_task(hello())
self.keep_alive = KeepAlive(self, 3)
self.keep_alive.start()
# await task
except OSError:
delay = backoff.delay()
logger.error(f"Connection refused, trying again in {delay:.2f}s")
await asyncio.sleep(delay)
async def handle_event(self, event: "node.LavalinkEvents", extra):
"""
Handles various Lavalink Events.
If the event is TRACK END, extra will be TrackEndReason.
If the event is TRACK EXCEPTION, extra will be the string reason.
If the event is TRACK STUCK, extra will be the threshold ms.
Parameters
----------
event : node.LavalinkEvents
extra
"""
log.debug("Received player event for player: %r - %r - %r.", self,
event, extra)
if event == LavalinkEvents.TRACK_END:
self._is_playing = False
if extra == TrackEndReason.FINISHED:
await self.play()
elif event == LavalinkEvents.WEBSOCKET_CLOSED:
code = extra.get("code")
if code in (4015, 4014, 4009, 4006, 4000, 1006):
if not self._con_delay:
self._con_delay = ExponentialBackoff(base=1)
async def _loop(self, *args, **kwargs):
backoff = ExponentialBackoff()
await self._call_loop_function("before_loop")
sleep_until = discord.utils.sleep_until
self._last_iteration_failed = False
self._next_iteration = datetime.datetime.now(datetime.timezone.utc)
if self._realignment:
self._next_iteration += datetime.timedelta(seconds=self._realignment)
await sleep_until(self._next_iteration)
try:
await asyncio.sleep(0) # allows canceling in before_loop
while True:
if not self._last_iteration_failed:
self._last_iteration = self._next_iteration
self._next_iteration = self._get_next_sleep_time()
try:
await self.coro(*args, **kwargs)
self._last_iteration_failed = False
now = datetime.datetime.now(datetime.timezone.utc)
if now > self._next_iteration:
self._next_iteration = now
except self._valid_exception as exc:
self._last_iteration_failed = True
if not self.reconnect:
raise
await asyncio.sleep(backoff.delay())
else:
if self._stop_next_iteration:
return
self._current_loop += 1
if self._current_loop == self.count:
break
await sleep_until(self._next_iteration)
except asyncio.CancelledError:
self._is_being_cancelled = True
raise
except Exception as exc:
self._has_failed = True
await self._call_loop_function("error", exc)
raise exc
finally:
await self._call_loop_function("after_loop")
self._is_being_cancelled = False
self._current_loop = 0
self._stop_next_iteration = False
self._has_failed = False
async def _loop(self, *args, **kwargs):
backoff = ExponentialBackoff()
await self._call_loop_function('before_loop')
sleep_until = discord.utils.sleep_until
self._last_iteration_failed = False
self._next_iteration = datetime.datetime.now(datetime.timezone.utc)
try:
await asyncio.sleep(0) # allows canceling in before_loop
while True:
if not self._last_iteration_failed:
self._last_iteration = self._next_iteration
self._next_iteration = self._get_next_sleep_time()
try:
if self._current_loop >= self.delay:
await self.coro(*args, **kwargs)
self._last_iteration_failed = False
now = datetime.datetime.now(datetime.timezone.utc)
if now > self._next_iteration:
self._next_iteration = now
except self._valid_exception as exc:
log.warning(
f"Task: '{self.coro.__name__}' raised exception: {exc}"
)
self._last_iteration_failed = True
if not self.reconnect:
raise
await asyncio.sleep(backoff.delay())
else:
if self._stop_next_iteration:
return
self._current_loop += 1
if self._current_loop == self.count:
break
await sleep_until(self._next_iteration)
except asyncio.CancelledError:
self._is_being_cancelled = True
except Exception as exc:
self._has_failed = True
await self._call_loop_function('error', exc)
finally:
await self._call_loop_function('after_loop')
self._is_being_cancelled = False
self._current_loop = 0
self._stop_next_iteration = False
self._has_failed = False
async def _do_send(self, command, kwargs):
backoff = ExponentialBackoff()
for i in range(5):
try:
if i != 0:
await asyncio.sleep(backoff.delay())
msg = await getattr(self.original_ctx, command)(**kwargs)
if self.interaction_payload:
self.interaction_payload.message_callback(msg, kwargs)
return msg
except ClientError as e:
log.warning(
f"ContextWrapper: Network error when sending message on try {i}!"
f"\n {e}")
log.error(
f"ContextWrapper: Network error when sending message after 5 retries! Giving up..."
)
raise UnexpectedError(
"ContextWrapper could not send message after 5 retries!")
async def _connect(self):
backoff = ExponentialBackoff(2)
while not (self.ws and self.ws.open):
try:
self.ws = await websockets.client.connect(
self.uri,
ping_interval=5,
max_size=None,
max_queue=None,
extra_headers=self.headers)
asyncio.create_task(self.poll())
asyncio.create_task(self._send_task())
# self.listener_thread = ListenerThread(self)
# self.listener_thread.start()
except OSError:
delay = backoff.delay()
logger.error(
f"Connection refused, trying again in {delay:.2f}s")
await asyncio.sleep(delay)
async def _multi_try_connect(self, uri):
backoff = ExponentialBackoff()
attempt = 1
while self._is_shutdown is False and (self._ws is None or self._ws.closed):
try:
ws = await self.session.ws_connect(url=uri, headers=self.headers, heartbeat=60)
except OSError:
delay = backoff.delay()
log.debug("Failed connect attempt %s, retrying in %s", attempt, delay)
await asyncio.sleep(delay)
attempt += 1
except aiohttp.WSServerHandshakeError:
return None
else:
self.session_resumed = ws._response.headers.get("Session-Resumed", False)
if self._ws is not None and self.session_resumed:
log.info(f"WEBSOCKET Resumed Session with key: {self._resume_key}")
self._ws = ws
return self._ws
async def get_tracks(self, query, tries=5, retry_on_failure=True):
# Fetch tracks from the Lavalink node using its REST API
params = {"identifier": query}
backoff = ExponentialBackoff(base=1)
for attempt in range(tries):
async with self.session.get(self.rest_uri + "/loadtracks",
params=params) as resp:
if resp.status != 200 and retry_on_failure:
retry = backoff.delay()
logger.error(
f"Received status code ({resp.status}) while retrieving tracks, retrying in {retry} seconds. Attempt {attempt+1}/{tries}"
)
continue
elif resp.status != 200 and not retry_on_failure:
logger.error(
f"Received status code ({resp.status}) while retrieving tracks, not retrying."
)
return {}
res = await resp.json()
return res
async def _loop(self, *args, **kwargs):
backoff = ExponentialBackoff()
await self._call_loop_function('before_loop')
try:
while True:
try:
await self.coro(*args, **kwargs)
except asyncio.CancelledError:
break
except self._valid_exception as exc:
if not self.reconnect:
raise
await asyncio.sleep(backoff.delay())
else:
self._current_loop += 1
if self._current_loop == self.count:
break
await asyncio.sleep(self._sleep)
finally:
await self._call_loop_function('after_loop')
async def _loop(self, *args, **kwargs):
backoff = ExponentialBackoff()
await self._call_loop_function('before_loop')
sleep_until = discord.utils.sleep_until
self._next_iteration = datetime.datetime.now(datetime.timezone.utc)
try:
while True:
self._last_iteration = self._next_iteration
self._next_iteration = self._get_next_sleep_time()
try:
await self.coro(*args, **kwargs)
now = datetime.datetime.now(datetime.timezone.utc)
if now > self._next_iteration:
self._next_iteration = now
except self._valid_exception as exc:
if not self.reconnect:
raise
await asyncio.sleep(backoff.delay())
else:
if self._stop_next_iteration:
return
self._current_loop += 1
if self._current_loop == self.count:
break
await sleep_until(self._next_iteration)
except asyncio.CancelledError:
self._is_being_cancelled = True
raise
except Exception:
self._has_failed = True
log.exception('Internal background task failed.')
raise
finally:
await self._call_loop_function('after_loop')
self._is_being_cancelled = False
self._current_loop = 0
self._stop_next_iteration = False
self._has_failed = False
async def reconnect_handler(self, reconnect, timeout):
backoff = ExponentialBackoff()
loop = asyncio.get_running_loop()
while True:
try:
await self._connection.run(loop)
except _native_voice.ConnectionClosed as e:
log.info('Voice connection got a clean close %s', e)
await self.disconnect()
return
except _native_voice.ConnectionError as e:
log.exception('Internal voice error: %s', e)
await self.disconnect()
return
except (_native_voice.ReconnectError) as e:
if not reconnect:
await self.disconnect()
raise
retry = backoff.delay()
log.exception(
'Disconnected from voice... Reconnecting in %.2fs.', retry)
await asyncio.sleep(retry)
await self.voice_disconnect()
try:
await self.connect(reconnect=True, timeout=timeout)
except asyncio.TimeoutError:
# at this point we've retried 5 times... let's continue the loop.
log.warning('Could not connect to voice... Retrying...')
continue
else:
# The function above is actually a loop already
# So if we're here then it exited normally
await self.disconnect()
return
async def _reconnect(self):
if SHUTDOWN.is_set():
log.debug('Shutting down Lavalink WS.')
return
log.debug("Attempting Lavalink WS reconnect.")
backoff = ExponentialBackoff()
for x in range(5):
delay = backoff.delay()
log.debug("Reconnecting in {} seconds".format(delay))
await asyncio.sleep(delay)
try:
await self.connect()
except OSError:
log.debug("Could not reconnect.")
else:
log.debug("Reconnect successful.")
break
else:
log.debug(
"Failed to reconnect, please reinitialize lavalink when ready."
)
async def _reconnect(self, *, shutdown: bool = False):
self._ready_event.clear()
if self._is_shutdown is True or shutdown:
ws_ll_log.info("[NODE] | Shutting down Lavalink WS.")
return
if self.state != NodeState.CONNECTING:
self.update_state(NodeState.RECONNECTING)
if self.state != NodeState.RECONNECTING:
return
backoff = ExponentialBackoff(base=1)
attempt = 1
while self.state == NodeState.RECONNECTING:
attempt += 1
if attempt > 10:
ws_ll_log.info(
"[NODE] | Failed reconnection attempt too many times, aborting ..."
)
asyncio.create_task(self.disconnect())
return
try:
await self.connect(shutdown=shutdown)
except AbortingNodeConnection:
return
except asyncio.TimeoutError:
delay = backoff.delay()
ws_ll_log.warning(
"[NODE] | Lavalink WS reconnect attempt %s, retrying in %s",
attempt,
delay,
)
await asyncio.sleep(delay)
else:
ws_ll_log.info("[NODE] | Reconnect successful.")
self.dispatch_reconnect()
self._retries = 0
async def _start(self):
self.failed_exc = None
try:
if self.on_start:
await self.on_start()
await sleep_until(self.timedelta.to_datetime_now())
while True:
try:
await self.on_end()
return
except _VALID_EXCEPTIONS as exc:
if not self._reconnect:
raise
await asyncio.sleep(ExponentialBackoff().delay())
except asyncio.CancelledError:
raise
except Exception as exc:
self.failed_exc = exc
raise exc
async def _run(self):
from discord.backoff import ExponentialBackoff
backoff = ExponentialBackoff()
try:
self.session = aiohttp.ClientSession()
dbot = None
streamer = None
bot = None
allow_starts = True
fails = CooldownMapping.from_cooldown(5, 300)
if not self.config.get("tokens", "twitch_bot_token", fallback=None) \
or not self.config.get("tokens", "twitch_streamer_token", fallback=None) \
or not self.config.get("tokens", "discord_bot", fallback=None):
logger.error(
"Refusing to start clients: must have tokens to start!")
allow_starts = False
else:
if not await self.api.do_hello():
logger.warning("Did not receive an OK from the xlydn api")
allow_starts = False
try:
while self.alive:
if not allow_starts and self.config.get("tokens", "twitch_bot_token", fallback=None) \
and self.config.get("tokens", "twitch_streamer_token", fallback=None) \
and self.config.get("tokens", "discord_bot", fallback=None):
allow_starts = True
await self.api.do_hello()
elif not allow_starts:
await asyncio.sleep(1)
continue
if self.bot_run_event.is_set() and (bot is None
or bot.done()):
if fails.update_rate_limit("bot.twitch"):
self.bot_run_event.clear()
self.interface.token_disconnect_bot()
logger.warning("Failed to start Twitch Bot Client")
continue
bot = self.loop.create_task(
self.twitch_bot.try_start(
self.config.get("tokens", "twitch_bot_token")))
if not self.bot_run_event.is_set(
) and bot is not None and not bot.done():
bot.cancel()
if self.streamer_run_event.is_set() and (
streamer is None or streamer.done()):
if fails.update_rate_limit("streamer.twitch"):
self.streamer_run_event.clear()
self.interface.token_disconnect_streamer()
logger.warning(
"Failed to start Twitch Streamer Client")
continue
streamer = self.loop.create_task(
self.twitch_streamer.try_start(
self.config.get("tokens",
"twitch_streamer_token")))
if not self.streamer_run_event.is_set(
) and streamer is not None and not streamer.done():
streamer.cancel()
if self.discord_run_event.is_set() and (dbot is None
or dbot.done()):
if fails.update_rate_limit("bot.discord"):
self.discord_run_event.clear()
self.interface.token_disconnect_discord()
logger.error(
"Failed to start Discord Bot",
exc_info=dbot.exception() if dbot else None)
continue
# for whatever reason, attempting to reuse the bot object causes a variety of asyncio issues
self.discord_bot = discord_bot(
self,
command_prefix=self.get_dpy_prefix,
activity=discord.Game(
name=self.config.get("general",
"discord_presence",
fallback="Xlydn bot")),
intents=discord.Intents.all())
import gc
gc.collect()
dbot = self.loop.create_task(
self.discord_bot.try_start(
self.config.get("tokens", "discord_bot")))
if not self.discord_run_event.is_set(
) and dbot is not None and not dbot.done():
await self.discord_bot.close()
logger.debug("Closed discord client")
await asyncio.sleep(0)
except KeyboardInterrupt:
pass
except Exception as e:
self.end_event.set_exception(e)
async def _websocket_closed_handler(
self,
guild: discord.Guild,
player: lavalink.Player,
extra: Dict,
deafen: bool,
disconnect: bool,
) -> None:
guild_id = guild.id
node = player.node
voice_ws: DiscordWebSocket = node.get_voice_ws(guild_id)
code = extra.get("code")
by_remote = extra.get("byRemote", "")
reason = extra.get("reason", "").strip()
if self._ws_resume[guild_id].is_set():
ws_audio_log.debug(
f"WS EVENT | Discarding WS Closed event for guild {guild_id} -> "
f"Socket Closed {voice_ws.socket._closing or voice_ws.socket.closed}. "
f"Code: {code} -- Remote: {by_remote} -- {reason}")
return
self._ws_resume[guild_id].set()
if player.channel:
current_perms = player.channel.permissions_for(
player.channel.guild.me)
has_perm = current_perms.speak and current_perms.connect
else:
has_perm = False
channel_id = player.channel.id
if voice_ws.socket._closing or voice_ws.socket.closed or not voice_ws.open:
if player._con_delay:
delay = player._con_delay.delay()
else:
player._con_delay = ExponentialBackoff(base=1)
delay = player._con_delay.delay()
ws_audio_log.warning(
"YOU CAN IGNORE THIS UNLESS IT'S CONSISTENTLY REPEATING FOR THE SAME GUILD - "
f"Voice websocket closed for guild {guild_id} -> "
f"Socket Closed {voice_ws.socket._closing or voice_ws.socket.closed}. "
f"Code: {code} -- Remote: {by_remote} -- {reason}")
ws_audio_log.debug(
f"Reconnecting to channel {channel_id} in guild: {guild_id} | {delay:.2f}s"
)
await asyncio.sleep(delay)
while voice_ws.socket._closing or voice_ws.socket.closed or not voice_ws.open:
voice_ws = node.get_voice_ws(guild_id)
await asyncio.sleep(0.1)
if has_perm and player.current and player.is_playing:
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.connect(deafen=deafen)
await player.resume(player.current, start=player.position)
ws_audio_log.info(
"Voice websocket reconnected "
f"to channel {channel_id} in guild: {guild_id} | "
f"Reason: Error code {code} & Currently playing.")
elif has_perm and player.paused and player.current:
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.connect(deafen=deafen)
await player.pause(pause=True)
ws_audio_log.info(
"Voice websocket reconnected "
f"to channel {channel_id} in guild: {guild_id} | "
f"Reason: Error code {code} & Currently Paused.")
elif has_perm and (not disconnect) and (not player.is_playing):
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.connect(deafen=deafen)
ws_audio_log.info(
"Voice websocket reconnected "
f"to channel {channel_id} in guild: {guild_id} | "
f"Reason: Error code {code} & Not playing, but auto disconnect disabled."
)
self._ll_guild_updates.discard(guild_id)
elif not has_perm:
self.bot.dispatch("red_audio_audio_disconnect", guild)
ws_audio_log.info(
"Voice websocket disconnected "
f"from channel {channel_id} in guild: {guild_id} | "
f"Reason: Error code {code} & Missing permissions.")
self._ll_guild_updates.discard(guild_id)
player.store("autoplay_notified", False)
await player.stop()
await player.disconnect()
await self.config.guild_from_id(
guild_id=guild_id).currently_auto_playing_in.set([])
else:
self.bot.dispatch("red_audio_audio_disconnect", guild)
ws_audio_log.info(
"Voice websocket disconnected "
f"from channel {channel_id} in guild: {guild_id} | "
f"Reason: Error code {code} & Unknown.")
self._ll_guild_updates.discard(guild_id)
player.store("autoplay_notified", False)
await player.stop()
await player.disconnect()
await self.config.guild_from_id(
guild_id=guild_id).currently_auto_playing_in.set([])
elif code in (42069, ):
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.connect(deafen=deafen)
await player.resume(player.current, start=player.position)
ws_audio_log.info(
f"Player resumed in channel {channel_id} in guild: {guild_id} | "
f"Reason: Error code {code} & {reason}.")
elif code in (4015, 4014, 4009, 4006, 1006):
if (code == 4006 and has_perm and player._last_resume
and player._last_resume + datetime.timedelta(seconds=5) >
datetime.datetime.now(tz=datetime.timezone.utc)):
attempts = player.fetch("resume_attempts", 0)
player.store("resume_attempts", attempts + 1)
channel = self.bot.get_channel(player.channel.id)
should_skip = not channel.members or all(
m.bot for m in channel.members)
if should_skip:
ws_audio_log.info(
"Voice websocket reconnected skipped "
f"for channel {channel_id} in guild: {guild_id} | "
f"Reason: Error code {code} & "
"Player resumed too recently and no human members connected."
)
return
if player._con_delay:
delay = player._con_delay.delay()
else:
player._con_delay = ExponentialBackoff(base=1)
delay = player._con_delay.delay()
ws_audio_log.debug(
f"Reconnecting to channel {channel_id} in guild: {guild_id} | {delay:.2f}s"
)
await asyncio.sleep(delay)
if has_perm and player.current and player.is_playing:
await player.connect(deafen=deafen)
await player.resume(player.current, start=player.position)
ws_audio_log.info(
"Voice websocket reconnected "
f"to channel {channel_id} in guild: {guild_id} | "
f"Reason: Error code {code} & Player is active.")
elif has_perm and player.paused and player.current:
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.connect(deafen=deafen)
await player.pause(pause=True)
ws_audio_log.info(
"Voice websocket reconnected "
f"to channel {channel_id} in guild: {guild_id} | "
f"Reason: Error code {code} & Player is paused.")
elif has_perm and (not disconnect) and (not player.is_playing):
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.connect(deafen=deafen)
ws_audio_log.info(
"Voice websocket reconnected "
f"to channel {channel_id} in guild: {guild_id} | "
f"Reason: Error code {code} & Not playing.")
self._ll_guild_updates.discard(guild_id)
elif not has_perm:
self.bot.dispatch("red_audio_audio_disconnect", guild)
ws_audio_log.info(
"Voice websocket disconnected "
f"from channel {channel_id} in guild: {guild_id} | "
f"Reason: Error code {code} & Missing permissions.")
self._ll_guild_updates.discard(guild_id)
player.store("autoplay_notified", False)
await player.stop()
await player.disconnect()
await self.config.guild_from_id(
guild_id=guild_id).currently_auto_playing_in.set([])
else:
ws_audio_log.info(
"WS EVENT - IGNORED (Healthy Socket) | "
f"Voice websocket closed event for guild {guild_id} -> "
f"Code: {code} -- Remote: {by_remote} -- {reason}")
self._ws_resume[guild_id].clear()
async def _websocket_closed_handler(
self,
guild: discord.Guild,
player: lavalink.Player,
extra: Dict,
deafen: bool,
disconnect: bool,
) -> None:
guild_id = guild.id
event_channel_id = extra.get("channelID")
try:
if not self._ws_resume[guild_id].is_set():
await self._ws_resume[guild_id].wait()
else:
self._ws_resume[guild_id].clear()
node = player.node
voice_ws: DiscordWebSocket = node.get_voice_ws(guild_id)
code = extra.get("code")
by_remote = extra.get("byRemote", "")
reason = extra.get("reason", "No Specified Reason").strip()
channel_id = player.channel.id
try:
event_channel_id, to_handle_code = await self._ws_op_codes[
guild_id].get()
except asyncio.QueueEmpty:
log.debug("Empty queue - Resuming Processor - Early exit")
return
if code != to_handle_code:
code = to_handle_code
if player.channel.id != event_channel_id:
code = 4014
if event_channel_id != channel_id:
ws_audio_log.info(
"Received an op code for a channel that is no longer valid; %d "
"Reason: Error code %d & %s, %r",
event_channel_id,
code,
reason,
player,
)
self._ws_op_codes[guild_id]._init(
self._ws_op_codes[guild_id]._maxsize)
return
if player.channel:
has_perm = self.can_join_and_speak(player.channel)
else:
has_perm = False
if code in (
1000,
) and has_perm and player.current and player.is_playing:
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.resume(player.current,
start=player.position,
replace=True)
ws_audio_log.info(
"Player resumed | Reason: Error code %d & %s, %r", code,
reason, player)
self._ws_op_codes[guild_id]._init(
self._ws_op_codes[guild_id]._maxsize)
return
if voice_ws.socket._closing or voice_ws.socket.closed or not voice_ws.open:
if player._con_delay:
delay = player._con_delay.delay()
else:
player._con_delay = ExponentialBackoff(base=1)
delay = player._con_delay.delay()
ws_audio_log.warning(
"YOU CAN IGNORE THIS UNLESS IT'S CONSISTENTLY REPEATING FOR THE SAME GUILD - "
"Voice websocket closed for guild %d -> "
"Socket Closed %s. "
"Code: %d -- Remote: %s -- %s, %r",
guild_id,
voice_ws.socket._closing or voice_ws.socket.closed,
code,
by_remote,
reason,
player,
)
ws_audio_log.debug(
"Reconnecting to channel %d in guild: %d | %.2fs",
channel_id,
guild_id,
delay,
)
await asyncio.sleep(delay)
while voice_ws.socket._closing or voice_ws.socket.closed or not voice_ws.open:
voice_ws = node.get_voice_ws(guild_id)
await asyncio.sleep(0.1)
if has_perm and player.current and player.is_playing:
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.connect(deafen=deafen)
await player.resume(player.current,
start=player.position,
replace=True)
ws_audio_log.info(
"Voice websocket reconnected "
"Reason: Error code %d & Currently playing, %r",
code,
player,
)
elif has_perm and player.paused and player.current:
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.connect(deafen=deafen)
await player.resume(player.current,
start=player.position,
replace=True,
pause=True)
ws_audio_log.info(
"Voice websocket reconnected "
"Reason: Error code %d & Currently Paused, %r",
code,
player,
)
elif has_perm and (not disconnect) and (not player.is_playing):
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.connect(deafen=deafen)
ws_audio_log.info(
"Voice websocket reconnected "
"Reason: Error code %d & Not playing, but auto disconnect disabled, %r",
code,
player,
)
self._ll_guild_updates.discard(guild_id)
elif not has_perm:
self.bot.dispatch("red_audio_audio_disconnect", guild)
ws_audio_log.info(
"Voice websocket disconnected "
"Reason: Error code %d & Missing permissions, %r",
code,
player,
)
self._ll_guild_updates.discard(guild_id)
player.store("autoplay_notified", False)
await player.stop()
await player.disconnect()
await self.config.guild_from_id(
guild_id=guild_id).currently_auto_playing_in.set([])
else:
self.bot.dispatch("red_audio_audio_disconnect", guild)
ws_audio_log.info(
"Voice websocket disconnected Reason: Error code %d & Unknown, %r",
code,
player,
)
self._ll_guild_updates.discard(guild_id)
player.store("autoplay_notified", False)
await player.stop()
await player.disconnect()
await self.config.guild_from_id(
guild_id=guild_id).currently_auto_playing_in.set([])
elif code in (
42069,
) and has_perm and player.current and player.is_playing:
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.connect(deafen=deafen)
await player.resume(player.current,
start=player.position,
replace=True)
ws_audio_log.info(
"Player resumed - Reason: Error code %d & %s, %r", code,
reason, player)
elif code in (4015, 4009, 4006, 4000, 1006):
if player._con_delay:
delay = player._con_delay.delay()
else:
player._con_delay = ExponentialBackoff(base=1)
delay = player._con_delay.delay()
ws_audio_log.debug(
"Reconnecting to channel %d in guild: %d | %.2fs",
channel_id, guild_id, delay)
await asyncio.sleep(delay)
if has_perm and player.current and player.is_playing:
await player.connect(deafen=deafen)
await player.resume(player.current,
start=player.position,
replace=True)
ws_audio_log.info(
"Voice websocket reconnected "
"Reason: Error code %d & Player is active, %r",
code,
player,
)
elif has_perm and player.paused and player.current:
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.connect(deafen=deafen)
await player.resume(player.current,
start=player.position,
replace=True,
pause=True)
ws_audio_log.info(
"Voice websocket reconnected "
"Reason: Error code %d & Player is paused, %r",
code,
player,
)
elif has_perm and (not disconnect) and (not player.is_playing):
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.connect(deafen=deafen)
ws_audio_log.info(
"Voice websocket reconnected "
"to channel %d in guild: %d | "
"Reason: Error code %d & Not playing, %r",
channel_id,
guild_id,
code,
player,
)
self._ll_guild_updates.discard(guild_id)
elif not has_perm:
self.bot.dispatch("red_audio_audio_disconnect", guild)
ws_audio_log.info(
"Voice websocket disconnected "
"Reason: Error code %d & Missing permissions, %r",
code,
player,
)
self._ll_guild_updates.discard(guild_id)
player.store("autoplay_notified", False)
await player.stop()
await player.disconnect()
await self.config.guild_from_id(
guild_id=guild_id).currently_auto_playing_in.set([])
else:
if not player.paused and player.current:
player.store("resumes", player.fetch("resumes", 0) + 1)
await player.resume(player.current,
start=player.position,
replace=True)
ws_audio_log.info(
"WS EVENT - SIMPLE RESUME (Healthy Socket) | "
"Voice websocket closed event "
"Code: %d -- Remote: %s -- %s, %r",
code,
by_remote,
reason,
player,
)
else:
ws_audio_log.info(
"WS EVENT - IGNORED (Healthy Socket) | "
"Voice websocket closed event "
"Code: %d -- Remote: %s -- %s, %r",
code,
by_remote,
reason,
player,
)
except Exception:
log.exception("Error in task")
finally:
self._ws_op_codes[guild_id]._init(
self._ws_op_codes[guild_id]._maxsize)
self._ws_resume[guild_id].set()
async def _loop(self, *args: Any, **kwargs: Any) -> None:
backoff = ExponentialBackoff()
await self._call_loop_function('before_loop')
self._last_iteration_failed = False
if self._is_explicit_time():
self._next_iteration = self._get_next_sleep_time()
else:
self._next_iteration = datetime.datetime.now(datetime.timezone.utc)
await asyncio.sleep(0) # allows canceling in before_loop
try:
if self._stop_next_iteration: # allow calling stop() before first iteration
return
while True:
# sleep before the body of the task for explicit time intervals
if self._is_explicit_time():
await self._try_sleep_until(self._next_iteration)
if not self._last_iteration_failed:
self._last_iteration = self._next_iteration
self._next_iteration = self._get_next_sleep_time()
# In order to account for clock drift, we need to ensure that
# the next iteration always follows the last iteration.
# Sometimes asyncio is cheeky and wakes up a few microseconds before our target
# time, causing it to repeat a run.
while self._is_explicit_time(
) and self._next_iteration <= self._last_iteration:
_log.warn(
('Clock drift detected for task %s. Woke up at %s but needed to sleep until %s. '
'Sleeping until %s again to correct clock'),
self.coro.__qualname__,
discord.utils.utcnow(),
self._next_iteration,
self._next_iteration,
)
await self._try_sleep_until(self._next_iteration)
self._next_iteration = self._get_next_sleep_time()
try:
await self.coro(*args, **kwargs)
self._last_iteration_failed = False
except self._valid_exception:
self._last_iteration_failed = True
if not self.reconnect:
raise
await asyncio.sleep(backoff.delay())
else:
if self._stop_next_iteration:
return
# sleep after the body of the task for relative time intervals
if self._is_relative_time():
await self._try_sleep_until(self._next_iteration)
self._current_loop += 1
if self._current_loop == self.count:
break
except asyncio.CancelledError:
self._is_being_cancelled = True
raise
except Exception as exc:
self._has_failed = True
await self._call_loop_function('error', exc)
raise exc
finally:
await self._call_loop_function('after_loop')
if self._handle:
self._handle.cancel()
self._is_being_cancelled = False
self._current_loop = 0
self._stop_next_iteration = False
self._has_failed = False