async def _loop(self, *args, **kwargs): backoff = ExponentialBackoff() await self._call_loop_function('before_loop') try: while True: try: await self.coro(*args, **kwargs) except self._valid_exception as exc: if not self.reconnect: raise await asyncio.sleep(backoff.delay()) else: if self._stop_next_iteration: return self._current_loop += 1 if self._current_loop == self.count: break await asyncio.sleep(self._sleep) except asyncio.CancelledError: self._is_being_cancelled = True raise except Exception: self._has_failed = True log.exception('Internal background task failed.') raise finally: await self._call_loop_function('after_loop') self._is_being_cancelled = False self._current_loop = 0 self._stop_next_iteration = False self._has_failed = False
async def _multi_try_connect(self, uri): backoff = ExponentialBackoff() attempt = 1 if self._ws is not None: await self._ws.close(code=4006, message=b"Reconnecting") while self._is_shutdown is False and (self._ws is None or self._ws.closed): self._retries += 1 try: ws = await self.session.ws_connect(url=uri, headers=self.headers, heartbeat=60) except (OSError, aiohttp.ClientConnectionError): delay = backoff.delay() ws_ll_log.error("Failed connect attempt %s, retrying in %s", attempt, delay) await asyncio.sleep(delay) attempt += 1 if attempt > 5: raise asyncio.TimeoutError except aiohttp.WSServerHandshakeError: ws_ll_log.error("Failed connect WSServerHandshakeError") raise asyncio.TimeoutError else: self.session_resumed = ws._response.headers.get( "Session-Resumed", False) if self._ws is not None and self.session_resumed: ws_ll_log.info("WEBSOCKET Resumed Session with key: %s", self._resume_key) self._ws = ws break
async def _connect(self): backoff = ExponentialBackoff(5, integral=True) while not self.connected: try: logger.info( f'Attempting to establish websocket connection to {self.name}' ) self.ws = await self.session.ws_connect(self.uri, headers=self.headers) except aiohttp.ClientConnectorError: logger.warning( f'[{self.name}] Invalid response received; this may indicate that ' 'Lavalink is not running, or is running on a port different ' 'to the one you passed to `add_node`.') except aiohttp.WSServerHandshakeError as ce: if ce.status in (401, 403): logger.error( f'Authentication failed when establishing a connection to {self.name}' ) return logger.warning( f'{self.name} returned a code {ce.status} which was unexpected' ) else: logger.info(f'Connection established to {self.name}') self.listen_task = asyncio.create_task(self.listen()) return delay = backoff.delay() logger.error(f"Connection refused, trying again in {delay}s") await asyncio.sleep(delay)
async def _reconnect(self): self._ready_event.clear() if self._is_shutdown is True: ws_ll_log.info("[NODE] | Shutting down Lavalink WS.") return if self.state != NodeState.CONNECTING: self.update_state(NodeState.RECONNECTING) if self.state != NodeState.RECONNECTING: return backoff = ExponentialBackoff(base=1) attempt = 1 while self.state == NodeState.RECONNECTING: attempt += 1 try: await self.connect() except asyncio.TimeoutError: delay = backoff.delay() ws_ll_log.info( "[NODE] | Failed to reconnect to the Lavalink node.") ws_ll_log.info( "[NODE] | Lavalink WS reconnect connect attempt %s, retrying in %s", attempt, delay, ) else: ws_ll_log.info("[NODE] | Reconnect successful.") self.dispatch_reconnect() self._retries = 0
async def wrapper(*args, **kwargs): if isinstance(args[0], commands.Bot): bot = args[0] elif not isinstance(args[0].bot, commands.Bot): raise utils.MissingInstance( f'Missing an instance of Bot in task <{func.__name__}>.') else: bot = args[0].bot backoff = ExponentialBackoff() retrys = 0 if until_ready: await bot.wait_until_ready() while not bot.is_closed(): await func(*args, *kwargs) while bot.is_reconnecting(): retrys += 1 retry = backoff.delay() await asyncio.sleep(retry) if retrys: print( f'Resumed Task: {func.__name__} after {retrys} trys.\n' ) retrys = 0
async def connect(self, *, reconnect=True): """Override connect and add a reconnecting state.""" backoff = ExponentialBackoff() while not self.is_closed(): try: await self._connect() except (OSError, discord.HTTPException, discord.GatewayNotFound, discord.ConnectionClosed, aiohttp.ClientError, asyncio.TimeoutError, websockets.InvalidHandshake, websockets.WebSocketProtocolError) as e: if not reconnect: await self.close() if isinstance(e, discord.ConnectionClosed) and e.code == 1000: # clean close, don't re-raise this return raise if self.is_closed(): return if isinstance(e, discord.ConnectionClosed): if e.code != 1000: await self.close() raise retry = backoff.delay() self._reconnecting.set() await asyncio.sleep(retry, loop=self.loop)
async def _multi_try_connect(self, uri): backoff = ExponentialBackoff() attempt = 1 if self._listener_task is not None: self._listener_task.cancel() if self._ws is not None: await self._ws.close(code=4006, message=b"Reconnecting") while self._is_shutdown is False and (self._ws is None or self._ws.closed): self._retries += 1 if self._is_shutdown is True: ws_ll_log.error( "Lavalink node was shutdown during a connect attempt.") raise asyncio.CancelledError try: ws = await self.session.ws_connect(url=uri, headers=self.headers, heartbeat=60) except (OSError, aiohttp.ClientConnectionError): if attempt > 5: raise asyncio.TimeoutError delay = backoff.delay() ws_ll_log.warning("Failed connect attempt %s, retrying in %s", attempt, delay) await asyncio.sleep(delay) attempt += 1 except aiohttp.WSServerHandshakeError: ws_ll_log.error("Failed connect WSServerHandshakeError") raise asyncio.TimeoutError else: if self._is_shutdown is True: ws_ll_log.error( "Lavalink node was shutdown during a connect attempt.") raise asyncio.CancelledError self.session_resumed = ws._response.headers.get( "Session-Resumed", False) if self._ws is not None and self.session_resumed: ws_ll_log.info("WEBSOCKET Resumed Session with key: %s", self._resume_key) self._ws = ws break if self._is_shutdown is True: raise asyncio.CancelledError ws_ll_log.info("Lavalink WS connected to %s", uri) ws_ll_log.debug("Creating Lavalink WS listener.") if self._is_shutdown is False: self._listener_task = asyncio.create_task(self.listener()) asyncio.create_task(self._configure_resume()) if self._queue: temp = self._queue.copy() self._queue.clear() for data in temp: await self.send(data) self._ready_event.set() self.update_state(NodeState.READY)
async def _loop(self, *args: Any, **kwargs: Any) -> None: backoff = ExponentialBackoff() await self._call_loop_function('before_loop') self._last_iteration_failed = False if self._time is not MISSING: # the time index should be prepared every time the internal loop is started self._prepare_time_index() self._next_iteration = self._get_next_sleep_time() else: self._next_iteration = datetime.datetime.now(datetime.timezone.utc) await asyncio.sleep(0) # allows canceling in before_loop try: if self._stop_next_iteration: # allow calling stop() before first iteration return while True: # sleep before the body of the task for explicit time intervals if self._time is not MISSING: await self._try_sleep_until(self._next_iteration) if not self._last_iteration_failed: self._last_iteration = self._next_iteration self._next_iteration = self._get_next_sleep_time() try: await self.coro(*args, **kwargs) self._last_iteration_failed = False except self._valid_exception: self._last_iteration_failed = True if not self.reconnect: raise await asyncio.sleep(backoff.delay()) else: if self._stop_next_iteration: return # sleep after the body of the task for relative time intervals if self._time is MISSING: await self._try_sleep_until(self._next_iteration) self._current_loop += 1 if self._current_loop == self.count: break except asyncio.CancelledError: self._is_being_cancelled = True raise except Exception as exc: self._has_failed = True await self._call_loop_function('error', exc) raise exc finally: await self._call_loop_function('after_loop') self._handle.cancel() self._is_being_cancelled = False self._current_loop = 0 self._stop_next_iteration = False self._has_failed = False
async def _loop(self, *args, **kwargs): backoff = ExponentialBackoff() await self._call_loop_function('before_loop') sleep_until = discord.utils.sleep_until self._last_iteration_failed = False if self._time is not None: # the time index should be prepared every time the internal loop is started self._prepare_time_index() self._next_iteration = self._get_next_sleep_time() else: self._next_iteration = datetime.datetime.now(datetime.timezone.utc) try: await self._try_sleep_until(self._next_iteration) while True: if not self._last_iteration_failed: self._last_iteration = self._next_iteration self._next_iteration = self._get_next_sleep_time() try: await self.coro(*args, **kwargs) self._last_iteration_failed = False except self._valid_exception: self._last_iteration_failed = True if not self.reconnect: raise await asyncio.sleep(backoff.delay()) else: await self._try_sleep_until(self._next_iteration) if self._stop_next_iteration: return now = datetime.datetime.now(datetime.timezone.utc) if now > self._next_iteration: self._next_iteration = now if self._time is not None: self._prepare_time_index(now) self._current_loop += 1 if self._current_loop == self.count: break except asyncio.CancelledError: self._is_being_cancelled = True raise except Exception as exc: self._has_failed = True await self._call_loop_function('error', exc) raise exc finally: await self._call_loop_function('after_loop') self._handle.cancel() self._is_being_cancelled = False self._current_loop = 0 self._stop_next_iteration = False self._has_failed = False
async def _multi_try_connect(self, uri): backoff = ExponentialBackoff() attempt = 1 while not SHUTDOWN.is_set() and (self._ws is None or not self._ws.open): try: self._ws = await websockets.connect(uri, extra_headers=self.headers) except OSError: delay = backoff.delay() log.debug("Failed connect attempt {}, retrying in {}".format( attempt, delay)) await asyncio.sleep(delay) attempt += 1
async def reaction_backoff(partial): b = None for _ in range(5): try: await partial() except discord.HTTPException: if b is None: b = ExponentialBackoff(base=2.0) await asyncio.sleep(b.delay()) continue except (discord.Forbidden, discord.NotFound): break else: return True
async def _connect(self): backoff = ExponentialBackoff(2) while not (self.ws and self.ws.open): try: self.ws = await websockets.connect(self.uri, extra_headers=self.headers) task = asyncio.create_task(self.listen()) # asyncio.create_task(hello()) self.keep_alive = KeepAlive(self, 3) self.keep_alive.start() # await task except OSError: delay = backoff.delay() logger.error(f"Connection refused, trying again in {delay:.2f}s") await asyncio.sleep(delay)
async def handle_event(self, event: "node.LavalinkEvents", extra): """ Handles various Lavalink Events. If the event is TRACK END, extra will be TrackEndReason. If the event is TRACK EXCEPTION, extra will be the string reason. If the event is TRACK STUCK, extra will be the threshold ms. Parameters ---------- event : node.LavalinkEvents extra """ log.debug("Received player event for player: %r - %r - %r.", self, event, extra) if event == LavalinkEvents.TRACK_END: self._is_playing = False if extra == TrackEndReason.FINISHED: await self.play() elif event == LavalinkEvents.WEBSOCKET_CLOSED: code = extra.get("code") if code in (4015, 4014, 4009, 4006, 4000, 1006): if not self._con_delay: self._con_delay = ExponentialBackoff(base=1)
async def _loop(self, *args, **kwargs): backoff = ExponentialBackoff() await self._call_loop_function("before_loop") sleep_until = discord.utils.sleep_until self._last_iteration_failed = False self._next_iteration = datetime.datetime.now(datetime.timezone.utc) if self._realignment: self._next_iteration += datetime.timedelta(seconds=self._realignment) await sleep_until(self._next_iteration) try: await asyncio.sleep(0) # allows canceling in before_loop while True: if not self._last_iteration_failed: self._last_iteration = self._next_iteration self._next_iteration = self._get_next_sleep_time() try: await self.coro(*args, **kwargs) self._last_iteration_failed = False now = datetime.datetime.now(datetime.timezone.utc) if now > self._next_iteration: self._next_iteration = now except self._valid_exception as exc: self._last_iteration_failed = True if not self.reconnect: raise await asyncio.sleep(backoff.delay()) else: if self._stop_next_iteration: return self._current_loop += 1 if self._current_loop == self.count: break await sleep_until(self._next_iteration) except asyncio.CancelledError: self._is_being_cancelled = True raise except Exception as exc: self._has_failed = True await self._call_loop_function("error", exc) raise exc finally: await self._call_loop_function("after_loop") self._is_being_cancelled = False self._current_loop = 0 self._stop_next_iteration = False self._has_failed = False
async def _loop(self, *args, **kwargs): backoff = ExponentialBackoff() await self._call_loop_function('before_loop') sleep_until = discord.utils.sleep_until self._last_iteration_failed = False self._next_iteration = datetime.datetime.now(datetime.timezone.utc) try: await asyncio.sleep(0) # allows canceling in before_loop while True: if not self._last_iteration_failed: self._last_iteration = self._next_iteration self._next_iteration = self._get_next_sleep_time() try: if self._current_loop >= self.delay: await self.coro(*args, **kwargs) self._last_iteration_failed = False now = datetime.datetime.now(datetime.timezone.utc) if now > self._next_iteration: self._next_iteration = now except self._valid_exception as exc: log.warning( f"Task: '{self.coro.__name__}' raised exception: {exc}" ) self._last_iteration_failed = True if not self.reconnect: raise await asyncio.sleep(backoff.delay()) else: if self._stop_next_iteration: return self._current_loop += 1 if self._current_loop == self.count: break await sleep_until(self._next_iteration) except asyncio.CancelledError: self._is_being_cancelled = True except Exception as exc: self._has_failed = True await self._call_loop_function('error', exc) finally: await self._call_loop_function('after_loop') self._is_being_cancelled = False self._current_loop = 0 self._stop_next_iteration = False self._has_failed = False
async def _do_send(self, command, kwargs): backoff = ExponentialBackoff() for i in range(5): try: if i != 0: await asyncio.sleep(backoff.delay()) msg = await getattr(self.original_ctx, command)(**kwargs) if self.interaction_payload: self.interaction_payload.message_callback(msg, kwargs) return msg except ClientError as e: log.warning( f"ContextWrapper: Network error when sending message on try {i}!" f"\n {e}") log.error( f"ContextWrapper: Network error when sending message after 5 retries! Giving up..." ) raise UnexpectedError( "ContextWrapper could not send message after 5 retries!")
async def _connect(self): backoff = ExponentialBackoff(2) while not (self.ws and self.ws.open): try: self.ws = await websockets.client.connect( self.uri, ping_interval=5, max_size=None, max_queue=None, extra_headers=self.headers) asyncio.create_task(self.poll()) asyncio.create_task(self._send_task()) # self.listener_thread = ListenerThread(self) # self.listener_thread.start() except OSError: delay = backoff.delay() logger.error( f"Connection refused, trying again in {delay:.2f}s") await asyncio.sleep(delay)
async def _multi_try_connect(self, uri): backoff = ExponentialBackoff() attempt = 1 while self._is_shutdown is False and (self._ws is None or self._ws.closed): try: ws = await self.session.ws_connect(url=uri, headers=self.headers, heartbeat=60) except OSError: delay = backoff.delay() log.debug("Failed connect attempt %s, retrying in %s", attempt, delay) await asyncio.sleep(delay) attempt += 1 except aiohttp.WSServerHandshakeError: return None else: self.session_resumed = ws._response.headers.get("Session-Resumed", False) if self._ws is not None and self.session_resumed: log.info(f"WEBSOCKET Resumed Session with key: {self._resume_key}") self._ws = ws return self._ws
async def get_tracks(self, query, tries=5, retry_on_failure=True): # Fetch tracks from the Lavalink node using its REST API params = {"identifier": query} backoff = ExponentialBackoff(base=1) for attempt in range(tries): async with self.session.get(self.rest_uri + "/loadtracks", params=params) as resp: if resp.status != 200 and retry_on_failure: retry = backoff.delay() logger.error( f"Received status code ({resp.status}) while retrieving tracks, retrying in {retry} seconds. Attempt {attempt+1}/{tries}" ) continue elif resp.status != 200 and not retry_on_failure: logger.error( f"Received status code ({resp.status}) while retrieving tracks, not retrying." ) return {} res = await resp.json() return res
async def _loop(self, *args, **kwargs): backoff = ExponentialBackoff() await self._call_loop_function('before_loop') try: while True: try: await self.coro(*args, **kwargs) except asyncio.CancelledError: break except self._valid_exception as exc: if not self.reconnect: raise await asyncio.sleep(backoff.delay()) else: self._current_loop += 1 if self._current_loop == self.count: break await asyncio.sleep(self._sleep) finally: await self._call_loop_function('after_loop')
async def _loop(self, *args, **kwargs): backoff = ExponentialBackoff() await self._call_loop_function('before_loop') sleep_until = discord.utils.sleep_until self._next_iteration = datetime.datetime.now(datetime.timezone.utc) try: while True: self._last_iteration = self._next_iteration self._next_iteration = self._get_next_sleep_time() try: await self.coro(*args, **kwargs) now = datetime.datetime.now(datetime.timezone.utc) if now > self._next_iteration: self._next_iteration = now except self._valid_exception as exc: if not self.reconnect: raise await asyncio.sleep(backoff.delay()) else: if self._stop_next_iteration: return self._current_loop += 1 if self._current_loop == self.count: break await sleep_until(self._next_iteration) except asyncio.CancelledError: self._is_being_cancelled = True raise except Exception: self._has_failed = True log.exception('Internal background task failed.') raise finally: await self._call_loop_function('after_loop') self._is_being_cancelled = False self._current_loop = 0 self._stop_next_iteration = False self._has_failed = False
async def reconnect_handler(self, reconnect, timeout): backoff = ExponentialBackoff() loop = asyncio.get_running_loop() while True: try: await self._connection.run(loop) except _native_voice.ConnectionClosed as e: log.info('Voice connection got a clean close %s', e) await self.disconnect() return except _native_voice.ConnectionError as e: log.exception('Internal voice error: %s', e) await self.disconnect() return except (_native_voice.ReconnectError) as e: if not reconnect: await self.disconnect() raise retry = backoff.delay() log.exception( 'Disconnected from voice... Reconnecting in %.2fs.', retry) await asyncio.sleep(retry) await self.voice_disconnect() try: await self.connect(reconnect=True, timeout=timeout) except asyncio.TimeoutError: # at this point we've retried 5 times... let's continue the loop. log.warning('Could not connect to voice... Retrying...') continue else: # The function above is actually a loop already # So if we're here then it exited normally await self.disconnect() return
async def _reconnect(self): if SHUTDOWN.is_set(): log.debug('Shutting down Lavalink WS.') return log.debug("Attempting Lavalink WS reconnect.") backoff = ExponentialBackoff() for x in range(5): delay = backoff.delay() log.debug("Reconnecting in {} seconds".format(delay)) await asyncio.sleep(delay) try: await self.connect() except OSError: log.debug("Could not reconnect.") else: log.debug("Reconnect successful.") break else: log.debug( "Failed to reconnect, please reinitialize lavalink when ready." )
async def _reconnect(self, *, shutdown: bool = False): self._ready_event.clear() if self._is_shutdown is True or shutdown: ws_ll_log.info("[NODE] | Shutting down Lavalink WS.") return if self.state != NodeState.CONNECTING: self.update_state(NodeState.RECONNECTING) if self.state != NodeState.RECONNECTING: return backoff = ExponentialBackoff(base=1) attempt = 1 while self.state == NodeState.RECONNECTING: attempt += 1 if attempt > 10: ws_ll_log.info( "[NODE] | Failed reconnection attempt too many times, aborting ..." ) asyncio.create_task(self.disconnect()) return try: await self.connect(shutdown=shutdown) except AbortingNodeConnection: return except asyncio.TimeoutError: delay = backoff.delay() ws_ll_log.warning( "[NODE] | Lavalink WS reconnect attempt %s, retrying in %s", attempt, delay, ) await asyncio.sleep(delay) else: ws_ll_log.info("[NODE] | Reconnect successful.") self.dispatch_reconnect() self._retries = 0
async def _start(self): self.failed_exc = None try: if self.on_start: await self.on_start() await sleep_until(self.timedelta.to_datetime_now()) while True: try: await self.on_end() return except _VALID_EXCEPTIONS as exc: if not self._reconnect: raise await asyncio.sleep(ExponentialBackoff().delay()) except asyncio.CancelledError: raise except Exception as exc: self.failed_exc = exc raise exc
async def _run(self): from discord.backoff import ExponentialBackoff backoff = ExponentialBackoff() try: self.session = aiohttp.ClientSession() dbot = None streamer = None bot = None allow_starts = True fails = CooldownMapping.from_cooldown(5, 300) if not self.config.get("tokens", "twitch_bot_token", fallback=None) \ or not self.config.get("tokens", "twitch_streamer_token", fallback=None) \ or not self.config.get("tokens", "discord_bot", fallback=None): logger.error( "Refusing to start clients: must have tokens to start!") allow_starts = False else: if not await self.api.do_hello(): logger.warning("Did not receive an OK from the xlydn api") allow_starts = False try: while self.alive: if not allow_starts and self.config.get("tokens", "twitch_bot_token", fallback=None) \ and self.config.get("tokens", "twitch_streamer_token", fallback=None) \ and self.config.get("tokens", "discord_bot", fallback=None): allow_starts = True await self.api.do_hello() elif not allow_starts: await asyncio.sleep(1) continue if self.bot_run_event.is_set() and (bot is None or bot.done()): if fails.update_rate_limit("bot.twitch"): self.bot_run_event.clear() self.interface.token_disconnect_bot() logger.warning("Failed to start Twitch Bot Client") continue bot = self.loop.create_task( self.twitch_bot.try_start( self.config.get("tokens", "twitch_bot_token"))) if not self.bot_run_event.is_set( ) and bot is not None and not bot.done(): bot.cancel() if self.streamer_run_event.is_set() and ( streamer is None or streamer.done()): if fails.update_rate_limit("streamer.twitch"): self.streamer_run_event.clear() self.interface.token_disconnect_streamer() logger.warning( "Failed to start Twitch Streamer Client") continue streamer = self.loop.create_task( self.twitch_streamer.try_start( self.config.get("tokens", "twitch_streamer_token"))) if not self.streamer_run_event.is_set( ) and streamer is not None and not streamer.done(): streamer.cancel() if self.discord_run_event.is_set() and (dbot is None or dbot.done()): if fails.update_rate_limit("bot.discord"): self.discord_run_event.clear() self.interface.token_disconnect_discord() logger.error( "Failed to start Discord Bot", exc_info=dbot.exception() if dbot else None) continue # for whatever reason, attempting to reuse the bot object causes a variety of asyncio issues self.discord_bot = discord_bot( self, command_prefix=self.get_dpy_prefix, activity=discord.Game( name=self.config.get("general", "discord_presence", fallback="Xlydn bot")), intents=discord.Intents.all()) import gc gc.collect() dbot = self.loop.create_task( self.discord_bot.try_start( self.config.get("tokens", "discord_bot"))) if not self.discord_run_event.is_set( ) and dbot is not None and not dbot.done(): await self.discord_bot.close() logger.debug("Closed discord client") await asyncio.sleep(0) except KeyboardInterrupt: pass except Exception as e: self.end_event.set_exception(e)
async def _websocket_closed_handler( self, guild: discord.Guild, player: lavalink.Player, extra: Dict, deafen: bool, disconnect: bool, ) -> None: guild_id = guild.id node = player.node voice_ws: DiscordWebSocket = node.get_voice_ws(guild_id) code = extra.get("code") by_remote = extra.get("byRemote", "") reason = extra.get("reason", "").strip() if self._ws_resume[guild_id].is_set(): ws_audio_log.debug( f"WS EVENT | Discarding WS Closed event for guild {guild_id} -> " f"Socket Closed {voice_ws.socket._closing or voice_ws.socket.closed}. " f"Code: {code} -- Remote: {by_remote} -- {reason}") return self._ws_resume[guild_id].set() if player.channel: current_perms = player.channel.permissions_for( player.channel.guild.me) has_perm = current_perms.speak and current_perms.connect else: has_perm = False channel_id = player.channel.id if voice_ws.socket._closing or voice_ws.socket.closed or not voice_ws.open: if player._con_delay: delay = player._con_delay.delay() else: player._con_delay = ExponentialBackoff(base=1) delay = player._con_delay.delay() ws_audio_log.warning( "YOU CAN IGNORE THIS UNLESS IT'S CONSISTENTLY REPEATING FOR THE SAME GUILD - " f"Voice websocket closed for guild {guild_id} -> " f"Socket Closed {voice_ws.socket._closing or voice_ws.socket.closed}. " f"Code: {code} -- Remote: {by_remote} -- {reason}") ws_audio_log.debug( f"Reconnecting to channel {channel_id} in guild: {guild_id} | {delay:.2f}s" ) await asyncio.sleep(delay) while voice_ws.socket._closing or voice_ws.socket.closed or not voice_ws.open: voice_ws = node.get_voice_ws(guild_id) await asyncio.sleep(0.1) if has_perm and player.current and player.is_playing: player.store("resumes", player.fetch("resumes", 0) + 1) await player.connect(deafen=deafen) await player.resume(player.current, start=player.position) ws_audio_log.info( "Voice websocket reconnected " f"to channel {channel_id} in guild: {guild_id} | " f"Reason: Error code {code} & Currently playing.") elif has_perm and player.paused and player.current: player.store("resumes", player.fetch("resumes", 0) + 1) await player.connect(deafen=deafen) await player.pause(pause=True) ws_audio_log.info( "Voice websocket reconnected " f"to channel {channel_id} in guild: {guild_id} | " f"Reason: Error code {code} & Currently Paused.") elif has_perm and (not disconnect) and (not player.is_playing): player.store("resumes", player.fetch("resumes", 0) + 1) await player.connect(deafen=deafen) ws_audio_log.info( "Voice websocket reconnected " f"to channel {channel_id} in guild: {guild_id} | " f"Reason: Error code {code} & Not playing, but auto disconnect disabled." ) self._ll_guild_updates.discard(guild_id) elif not has_perm: self.bot.dispatch("red_audio_audio_disconnect", guild) ws_audio_log.info( "Voice websocket disconnected " f"from channel {channel_id} in guild: {guild_id} | " f"Reason: Error code {code} & Missing permissions.") self._ll_guild_updates.discard(guild_id) player.store("autoplay_notified", False) await player.stop() await player.disconnect() await self.config.guild_from_id( guild_id=guild_id).currently_auto_playing_in.set([]) else: self.bot.dispatch("red_audio_audio_disconnect", guild) ws_audio_log.info( "Voice websocket disconnected " f"from channel {channel_id} in guild: {guild_id} | " f"Reason: Error code {code} & Unknown.") self._ll_guild_updates.discard(guild_id) player.store("autoplay_notified", False) await player.stop() await player.disconnect() await self.config.guild_from_id( guild_id=guild_id).currently_auto_playing_in.set([]) elif code in (42069, ): player.store("resumes", player.fetch("resumes", 0) + 1) await player.connect(deafen=deafen) await player.resume(player.current, start=player.position) ws_audio_log.info( f"Player resumed in channel {channel_id} in guild: {guild_id} | " f"Reason: Error code {code} & {reason}.") elif code in (4015, 4014, 4009, 4006, 1006): if (code == 4006 and has_perm and player._last_resume and player._last_resume + datetime.timedelta(seconds=5) > datetime.datetime.now(tz=datetime.timezone.utc)): attempts = player.fetch("resume_attempts", 0) player.store("resume_attempts", attempts + 1) channel = self.bot.get_channel(player.channel.id) should_skip = not channel.members or all( m.bot for m in channel.members) if should_skip: ws_audio_log.info( "Voice websocket reconnected skipped " f"for channel {channel_id} in guild: {guild_id} | " f"Reason: Error code {code} & " "Player resumed too recently and no human members connected." ) return if player._con_delay: delay = player._con_delay.delay() else: player._con_delay = ExponentialBackoff(base=1) delay = player._con_delay.delay() ws_audio_log.debug( f"Reconnecting to channel {channel_id} in guild: {guild_id} | {delay:.2f}s" ) await asyncio.sleep(delay) if has_perm and player.current and player.is_playing: await player.connect(deafen=deafen) await player.resume(player.current, start=player.position) ws_audio_log.info( "Voice websocket reconnected " f"to channel {channel_id} in guild: {guild_id} | " f"Reason: Error code {code} & Player is active.") elif has_perm and player.paused and player.current: player.store("resumes", player.fetch("resumes", 0) + 1) await player.connect(deafen=deafen) await player.pause(pause=True) ws_audio_log.info( "Voice websocket reconnected " f"to channel {channel_id} in guild: {guild_id} | " f"Reason: Error code {code} & Player is paused.") elif has_perm and (not disconnect) and (not player.is_playing): player.store("resumes", player.fetch("resumes", 0) + 1) await player.connect(deafen=deafen) ws_audio_log.info( "Voice websocket reconnected " f"to channel {channel_id} in guild: {guild_id} | " f"Reason: Error code {code} & Not playing.") self._ll_guild_updates.discard(guild_id) elif not has_perm: self.bot.dispatch("red_audio_audio_disconnect", guild) ws_audio_log.info( "Voice websocket disconnected " f"from channel {channel_id} in guild: {guild_id} | " f"Reason: Error code {code} & Missing permissions.") self._ll_guild_updates.discard(guild_id) player.store("autoplay_notified", False) await player.stop() await player.disconnect() await self.config.guild_from_id( guild_id=guild_id).currently_auto_playing_in.set([]) else: ws_audio_log.info( "WS EVENT - IGNORED (Healthy Socket) | " f"Voice websocket closed event for guild {guild_id} -> " f"Code: {code} -- Remote: {by_remote} -- {reason}") self._ws_resume[guild_id].clear()
async def _websocket_closed_handler( self, guild: discord.Guild, player: lavalink.Player, extra: Dict, deafen: bool, disconnect: bool, ) -> None: guild_id = guild.id event_channel_id = extra.get("channelID") try: if not self._ws_resume[guild_id].is_set(): await self._ws_resume[guild_id].wait() else: self._ws_resume[guild_id].clear() node = player.node voice_ws: DiscordWebSocket = node.get_voice_ws(guild_id) code = extra.get("code") by_remote = extra.get("byRemote", "") reason = extra.get("reason", "No Specified Reason").strip() channel_id = player.channel.id try: event_channel_id, to_handle_code = await self._ws_op_codes[ guild_id].get() except asyncio.QueueEmpty: log.debug("Empty queue - Resuming Processor - Early exit") return if code != to_handle_code: code = to_handle_code if player.channel.id != event_channel_id: code = 4014 if event_channel_id != channel_id: ws_audio_log.info( "Received an op code for a channel that is no longer valid; %d " "Reason: Error code %d & %s, %r", event_channel_id, code, reason, player, ) self._ws_op_codes[guild_id]._init( self._ws_op_codes[guild_id]._maxsize) return if player.channel: has_perm = self.can_join_and_speak(player.channel) else: has_perm = False if code in ( 1000, ) and has_perm and player.current and player.is_playing: player.store("resumes", player.fetch("resumes", 0) + 1) await player.resume(player.current, start=player.position, replace=True) ws_audio_log.info( "Player resumed | Reason: Error code %d & %s, %r", code, reason, player) self._ws_op_codes[guild_id]._init( self._ws_op_codes[guild_id]._maxsize) return if voice_ws.socket._closing or voice_ws.socket.closed or not voice_ws.open: if player._con_delay: delay = player._con_delay.delay() else: player._con_delay = ExponentialBackoff(base=1) delay = player._con_delay.delay() ws_audio_log.warning( "YOU CAN IGNORE THIS UNLESS IT'S CONSISTENTLY REPEATING FOR THE SAME GUILD - " "Voice websocket closed for guild %d -> " "Socket Closed %s. " "Code: %d -- Remote: %s -- %s, %r", guild_id, voice_ws.socket._closing or voice_ws.socket.closed, code, by_remote, reason, player, ) ws_audio_log.debug( "Reconnecting to channel %d in guild: %d | %.2fs", channel_id, guild_id, delay, ) await asyncio.sleep(delay) while voice_ws.socket._closing or voice_ws.socket.closed or not voice_ws.open: voice_ws = node.get_voice_ws(guild_id) await asyncio.sleep(0.1) if has_perm and player.current and player.is_playing: player.store("resumes", player.fetch("resumes", 0) + 1) await player.connect(deafen=deafen) await player.resume(player.current, start=player.position, replace=True) ws_audio_log.info( "Voice websocket reconnected " "Reason: Error code %d & Currently playing, %r", code, player, ) elif has_perm and player.paused and player.current: player.store("resumes", player.fetch("resumes", 0) + 1) await player.connect(deafen=deafen) await player.resume(player.current, start=player.position, replace=True, pause=True) ws_audio_log.info( "Voice websocket reconnected " "Reason: Error code %d & Currently Paused, %r", code, player, ) elif has_perm and (not disconnect) and (not player.is_playing): player.store("resumes", player.fetch("resumes", 0) + 1) await player.connect(deafen=deafen) ws_audio_log.info( "Voice websocket reconnected " "Reason: Error code %d & Not playing, but auto disconnect disabled, %r", code, player, ) self._ll_guild_updates.discard(guild_id) elif not has_perm: self.bot.dispatch("red_audio_audio_disconnect", guild) ws_audio_log.info( "Voice websocket disconnected " "Reason: Error code %d & Missing permissions, %r", code, player, ) self._ll_guild_updates.discard(guild_id) player.store("autoplay_notified", False) await player.stop() await player.disconnect() await self.config.guild_from_id( guild_id=guild_id).currently_auto_playing_in.set([]) else: self.bot.dispatch("red_audio_audio_disconnect", guild) ws_audio_log.info( "Voice websocket disconnected Reason: Error code %d & Unknown, %r", code, player, ) self._ll_guild_updates.discard(guild_id) player.store("autoplay_notified", False) await player.stop() await player.disconnect() await self.config.guild_from_id( guild_id=guild_id).currently_auto_playing_in.set([]) elif code in ( 42069, ) and has_perm and player.current and player.is_playing: player.store("resumes", player.fetch("resumes", 0) + 1) await player.connect(deafen=deafen) await player.resume(player.current, start=player.position, replace=True) ws_audio_log.info( "Player resumed - Reason: Error code %d & %s, %r", code, reason, player) elif code in (4015, 4009, 4006, 4000, 1006): if player._con_delay: delay = player._con_delay.delay() else: player._con_delay = ExponentialBackoff(base=1) delay = player._con_delay.delay() ws_audio_log.debug( "Reconnecting to channel %d in guild: %d | %.2fs", channel_id, guild_id, delay) await asyncio.sleep(delay) if has_perm and player.current and player.is_playing: await player.connect(deafen=deafen) await player.resume(player.current, start=player.position, replace=True) ws_audio_log.info( "Voice websocket reconnected " "Reason: Error code %d & Player is active, %r", code, player, ) elif has_perm and player.paused and player.current: player.store("resumes", player.fetch("resumes", 0) + 1) await player.connect(deafen=deafen) await player.resume(player.current, start=player.position, replace=True, pause=True) ws_audio_log.info( "Voice websocket reconnected " "Reason: Error code %d & Player is paused, %r", code, player, ) elif has_perm and (not disconnect) and (not player.is_playing): player.store("resumes", player.fetch("resumes", 0) + 1) await player.connect(deafen=deafen) ws_audio_log.info( "Voice websocket reconnected " "to channel %d in guild: %d | " "Reason: Error code %d & Not playing, %r", channel_id, guild_id, code, player, ) self._ll_guild_updates.discard(guild_id) elif not has_perm: self.bot.dispatch("red_audio_audio_disconnect", guild) ws_audio_log.info( "Voice websocket disconnected " "Reason: Error code %d & Missing permissions, %r", code, player, ) self._ll_guild_updates.discard(guild_id) player.store("autoplay_notified", False) await player.stop() await player.disconnect() await self.config.guild_from_id( guild_id=guild_id).currently_auto_playing_in.set([]) else: if not player.paused and player.current: player.store("resumes", player.fetch("resumes", 0) + 1) await player.resume(player.current, start=player.position, replace=True) ws_audio_log.info( "WS EVENT - SIMPLE RESUME (Healthy Socket) | " "Voice websocket closed event " "Code: %d -- Remote: %s -- %s, %r", code, by_remote, reason, player, ) else: ws_audio_log.info( "WS EVENT - IGNORED (Healthy Socket) | " "Voice websocket closed event " "Code: %d -- Remote: %s -- %s, %r", code, by_remote, reason, player, ) except Exception: log.exception("Error in task") finally: self._ws_op_codes[guild_id]._init( self._ws_op_codes[guild_id]._maxsize) self._ws_resume[guild_id].set()
async def _loop(self, *args: Any, **kwargs: Any) -> None: backoff = ExponentialBackoff() await self._call_loop_function('before_loop') self._last_iteration_failed = False if self._is_explicit_time(): self._next_iteration = self._get_next_sleep_time() else: self._next_iteration = datetime.datetime.now(datetime.timezone.utc) await asyncio.sleep(0) # allows canceling in before_loop try: if self._stop_next_iteration: # allow calling stop() before first iteration return while True: # sleep before the body of the task for explicit time intervals if self._is_explicit_time(): await self._try_sleep_until(self._next_iteration) if not self._last_iteration_failed: self._last_iteration = self._next_iteration self._next_iteration = self._get_next_sleep_time() # In order to account for clock drift, we need to ensure that # the next iteration always follows the last iteration. # Sometimes asyncio is cheeky and wakes up a few microseconds before our target # time, causing it to repeat a run. while self._is_explicit_time( ) and self._next_iteration <= self._last_iteration: _log.warn( ('Clock drift detected for task %s. Woke up at %s but needed to sleep until %s. ' 'Sleeping until %s again to correct clock'), self.coro.__qualname__, discord.utils.utcnow(), self._next_iteration, self._next_iteration, ) await self._try_sleep_until(self._next_iteration) self._next_iteration = self._get_next_sleep_time() try: await self.coro(*args, **kwargs) self._last_iteration_failed = False except self._valid_exception: self._last_iteration_failed = True if not self.reconnect: raise await asyncio.sleep(backoff.delay()) else: if self._stop_next_iteration: return # sleep after the body of the task for relative time intervals if self._is_relative_time(): await self._try_sleep_until(self._next_iteration) self._current_loop += 1 if self._current_loop == self.count: break except asyncio.CancelledError: self._is_being_cancelled = True raise except Exception as exc: self._has_failed = True await self._call_loop_function('error', exc) raise exc finally: await self._call_loop_function('after_loop') if self._handle: self._handle.cancel() self._is_being_cancelled = False self._current_loop = 0 self._stop_next_iteration = False self._has_failed = False