async def add_to_cache(bot, url, name=None, file_location=None):
"""Downloads the URL and saves to the audio cache folder.
If the cache folder has surpassed the cache size, this will continually
remove the least used file (by date) until there is enough space. If the
downloaded file is more than half the size of the total cache, it will not
be stored. Returns the final location of the downloaded file.
If name is specified, it will be stored under that name instead of the url.
If file_location is specified, it will move that file instead of
downloading the URL.
"""
if file_location:
cleaned_name = utilities.get_cleaned_filename(file_location)
else:
file_location, cleaned_name = await utilities.download_url(bot, url, include_name=True)
if name:
cleaned_name = utilities.get_cleaned_filename(name)
try:
download_stat = os.stat(file_location)
except FileNotFoundError:
raise CBException("The audio could not be saved. Please try again later.")
cache_limit = configurations.get(bot, 'core', 'cache_size_limit') * 1000 * 1000
store = cache_limit > 0 and download_stat.st_size < cache_limit / 2
# Ensure that the target destination does not exist (remove it if it does)
if store:
cached_location = '{0}/audio_cache/{1}'.format(bot.path, cleaned_name)
else:
cached_location = '{}/temp/tempsound'.format(bot.path)
try:
os.remove(cached_location)
except: # Doesn't matter if file doesn't exist
pass
os.rename(file_location, cached_location)
os.utime(cached_location)
if store:
cache_entries = []
total_size = 0
for entry in os.scandir('{}/audio_cache'.format(bot.path)):
stat = entry.stat()
cache_entries.append((stat.st_atime, stat.st_size, entry.path))
total_size += stat.st_size
cache_entries.sort(reverse=True)
while total_size > cache_limit:
_, size, path = cache_entries.pop()
logger.info("Removing from cache: %s", path)
os.remove(path)
total_size -= size
else:
logger.info("Not storing %s. Download size: %s", file_location, download_stat.st_size)
try:
download_stat = os.stat(cached_location)
except FileNotFoundError:
raise CBException("The file was not moved properly.")
return cached_location
async def add_to_cache(bot, url, name=None, file_location=None):
"""Downloads the URL and saves to the audio cache folder.
If the cache folder has surpassed the cache size, this will continually
remove the least used file (by date) until there is enough space. If the
downloaded file is more than half the size of the total cache, it will not
be stored. Returns the final location of the downloaded file.
If name is specified, it will be stored under that name instead of the url.
If file_location is specified, it will move that file instead of
downloading the URL.
"""
if file_location:
cleaned_name = utilities.get_cleaned_filename(file_location)
else:
file_location, cleaned_name = await utilities.download_url(
bot, url, include_name=True)
if name:
cleaned_name = utilities.get_cleaned_filename(name)
try:
download_stat = os.stat(file_location)
except FileNotFoundError:
raise BotException(
EXCEPTION, "The audio could not be saved. Please try again later.")
cache_limit = bot.configurations['core']['cache_size_limit'] * 1000 * 1000
store = cache_limit > 0 and download_stat.st_size < cache_limit / 2
if store:
cached_location = '{0}/audio_cache/{1}'.format(bot.path, cleaned_name)
else:
cached_location = '{}/temp/tempsound'.format(bot.path)
try:
os.remove(cached_location)
except: # Doesn't matter if file doesn't exist
pass
os.rename(file_location, cached_location)
if store:
cache_entries = []
total_size = 0
for entry in os.scandir('{}/audio_cache'.format(bot.path)):
stat = entry.stat()
cache_entries.append((stat.st_atime, stat.st_size, entry.path))
total_size += stat.st_size
cache_entries.sort(reverse=True)
# TODO: Check complexity of list entry removal
while total_size > cache_limit:
entry = cache_entries.pop()
os.remove(entry[2])
total_size -= entry[1]
return cached_location
def _search_games(bot, search, guild_id=None, return_game=False):
"""Searches the schedule for the given game and gets the information."""
cleaned_search = utilities.get_cleaned_filename(search, cleaner=True)
schedule_data = data.get(bot, __name__, 'schedule', volatile=True)
found_games = []
for index, game in enumerate(schedule_data):
if cleaned_search == game['key']:
found_games = [game]
break
elif cleaned_search in game['key']:
found_games.append(game)
if not found_games:
raise CBException("No games found with that name.")
elif len(found_games) > 10:
raise CBException("Too many games found with that name.")
elif len(found_games) != 1:
raise CBException(
"Multiple games found:", '\n'.join([
'{} ({})'.format(game['game'], game['type'])
for game in found_games
]))
elif return_game:
return found_games[0]
else:
return _embed_games_information(bot, found_games, guild_id)[0]
async def add_to_cache_ydl(bot, downloader, url):
"""Downloads the given URL using YoutubeDL and stores it in the cache.
The downloader must be provided as a YoutubeDL downloader object.
"""
file_location = '{}/temp/tempsound_{}'.format(bot.path, utilities.get_cleaned_filename(url))
downloader.params.update({'outtmpl': file_location})
await utilities.future(downloader.download, [url])
return await add_to_cache(bot, None, name=url, file_location=file_location)
def get_from_cache(bot, name, url=None):
"""Gets the filename from the audio_cache. Returns None otherwise.
If url is specified, it will clean it up and look for that instead. This
also sets the found file's access time.
"""
if url:
name = utilities.get_cleaned_filename(url)
file_path = '{0}/audio_cache/{1}'.format(bot.path, name)
if os.path.isfile(file_path):
os.utime(file_path, None)
return file_path
else:
return None
def get_from_cache(bot, name, url=None):
"""Gets the filename from the audio_cache. Returns None otherwise.
If url is specified, it will clean it up and look for that instead. This
also sets the found file's access time.
"""
if url:
name = utilities.get_cleaned_filename(url)
file_path = '{0}/audio_cache/{1}'.format(bot.path, name)
if os.path.isfile(file_path):
os.utime(file_path)
return file_path
else:
return None
def _search_games(bot, search, guild_id=None, return_game=False):
"""Searches the schedule for the given game and gets the information."""
cleaned_search = utilities.get_cleaned_filename(search, cleaner=True)
schedule_data = data.get(bot, __name__, 'schedule', volatile=True)
found_games = []
for index, game in enumerate(schedule_data):
if cleaned_search == game['key']:
found_games = [game]
break
elif cleaned_search in game['key']:
found_games.append(game)
if not found_games:
raise CBException("No games found with that name.")
elif len(found_games) > 10:
raise CBException("Too many games found with that name.")
elif len(found_games) != 1:
raise CBException(
"Multiple games found:", '\n'.join(
['{} ({})'.format(game['game'], game['type']) for game in found_games]))
elif return_game:
return found_games[0]
else:
return _embed_games_information(bot, found_games, guild_id)[0]
async def _update_schedule(bot):
"""Reads the GDQ schedule and updates the information in the database."""
schedule_url = configurations.get(bot, __name__, 'schedule_url')
html_data = (await utilities.future(requests.get, schedule_url)).text
soup = BeautifulSoup(html_data, 'html.parser')
run_table = soup.find('table', {'id': 'runTable'})
schedule_data = []
game_list = []
if run_table is None:
raise CBException('Run table not found!')
debug_weeks = data.get(bot, __name__, 'debug_weeks', default=0, volatile=True)
current_data = {}
for entry in run_table.find_all('tr'):
entry_class = entry.get('class', [''])[0]
if entry_class == 'day-split':
continue
subentries = [subentry.text for subentry in entry.find_all('td')]
if subentries[0].startswith(' '): # Second column
subentries = subentries[:2]
if entry_class == 'second-row': # Extra data for the last game
estimation, run_type = subentries
split_estimate = estimation.split(':')
estimation_seconds = (int(split_estimate[0])*3600 +
int(split_estimate[1])*60 +
int(split_estimate[2]))
end_time = (
current_data['scheduled'] + datetime.timedelta(
seconds=(estimation_seconds + current_data['setup_seconds'])))
key_name = utilities.get_cleaned_filename(
current_data['game'] + run_type, cleaner=True)
current_data.update({
'estimation': estimation.strip(),
'seconds': estimation_seconds,
'type': run_type,
'end': end_time,
'key': key_name
})
game_list.append(key_name)
schedule_data.append(current_data)
else: # Happens first
while len(subentries) < 4:
subentries.append('')
start_time_string, game, runners, setup_time = subentries
start_time = datetime.datetime.strptime(start_time_string, '%Y-%m-%dT%H:%M:%SZ')
setup_time = setup_time.strip()
split_setup = setup_time.split(':')
if len(split_setup) > 1:
setup_seconds = (int(split_setup[0])*3600 +
int(split_setup[1])*60 +
int(split_setup[2]))
else:
setup_seconds = 0
current_data = {
'scheduled': start_time - datetime.timedelta(weeks=debug_weeks),
'game': game,
'runners': runners,
'setup': setup_time,
'setup_seconds': setup_seconds
}
# Add finale entry
run_type = 'Party%'
end_time = current_data['scheduled'] + datetime.timedelta(minutes=30)
key_name = utilities.get_cleaned_filename(current_data['game'] + run_type, cleaner=True)
current_data.update({
'estimation': '0:30:00',
'seconds': 60*30,
'end': end_time,
'type': run_type,
'key': key_name
})
game_list.append(key_name)
schedule_data.append(current_data)
# Update scheduled notifications
entries = utilities.get_schedule_entries(bot, __name__)
for entry in entries:
payload, key = entry[3:5]
if key not in game_list: # Not found error
error_message = (
":warning: Warning: The game {} has been removed, renamed, or "
"recategorized. You have been removed from the notification list "
"for this game. Please check the schedule at {}.".format(
payload['text'], configurations.get(bot, __name__, 'schedule_url')))
utilities.update_schedule_entries(
bot, __name__, search=key, payload={'error': error_message}, new_time=time.time())
else:
game = schedule_data[game_list.index(key)]
start_time, end_time = game['scheduled'], game['end']
setup_delta = datetime.timedelta(seconds=game['setup_seconds'])
scheduled = start_time.replace(tzinfo=datetime.timezone.utc).timestamp()
current_time = datetime.datetime.utcnow()
if start_time + setup_delta < current_time < end_time:
stream_url = configurations.get(bot, __name__, 'stream_url')
payload = {'error': (
"Uh oh. The schedule shifted drastically and I didn't notice "
"fast enough - sorry! {} is live right now at {}").format(
payload['text'], stream_url)}
else:
payload.update({'end': scheduled + game['seconds']})
utilities.update_schedule_entries(
bot, __name__, search=key, payload=payload, new_time=scheduled)
# Save data
data.add(bot, __name__, 'schedule', schedule_data, volatile=True)
try:
_update_current_game(bot)
except:
pass
async def _update_schedule(bot):
"""Reads the GDQ schedule and updates the information in the database."""
schedule_url = configurations.get(bot, __name__, 'schedule_url')
html_data = (await utilities.future(requests.get, schedule_url)).text
soup = BeautifulSoup(html_data, 'html.parser')
run_table = soup.find('table', {'id': 'runTable'})
schedule_data = []
game_list = []
if run_table is None:
raise CBException('Run table not found!')
debug_weeks = data.get(bot,
__name__,
'debug_weeks',
default=0,
volatile=True)
current_data = {}
for entry in run_table.find_all('tr'):
entry_class = entry.get('class', [''])[0]
if entry_class == 'day-split':
continue
subentries = [subentry.text for subentry in entry.find_all('td')]
if entry_class == 'second-row': # Extra data for the last game
estimation, run_type = subentries
split_estimate = estimation.split(':')
estimation_seconds = (int(split_estimate[0]) * 3600 +
int(split_estimate[1]) * 60 +
int(split_estimate[2]))
end_time = (current_data['scheduled'] + datetime.timedelta(
seconds=(estimation_seconds + current_data['setup_seconds'])))
key_name = utilities.get_cleaned_filename(current_data['game'] +
run_type,
cleaner=True)
current_data.update({
'estimation': estimation.strip(),
'seconds': estimation_seconds,
'type': run_type,
'end': end_time,
'key': key_name
})
game_list.append(key_name)
schedule_data.append(current_data)
else: # Happens first
while len(subentries) < 4:
subentries.append('')
start_time_string, game, runners, setup_time = subentries
start_time = datetime.datetime.strptime(start_time_string,
'%Y-%m-%dT%H:%M:%SZ')
setup_time = setup_time.strip()
split_setup = setup_time.split(':')
if len(split_setup) > 1:
setup_seconds = (int(split_setup[0]) * 3600 +
int(split_setup[1]) * 60 +
int(split_setup[2]))
else:
setup_seconds = 0
current_data = {
'scheduled':
start_time - datetime.timedelta(weeks=debug_weeks),
'game': game,
'runners': runners,
'setup': setup_time,
'setup_seconds': setup_seconds
}
# Add finale entry
run_type = 'Party%'
end_time = current_data['scheduled'] + datetime.timedelta(minutes=30)
key_name = utilities.get_cleaned_filename(current_data['game'] + run_type,
cleaner=True)
current_data.update({
'estimation': '0:30:00',
'seconds': 60 * 30,
'end': end_time,
'type': run_type,
'key': key_name
})
game_list.append(key_name)
schedule_data.append(current_data)
# Update scheduled notifications
entries = utilities.get_schedule_entries(bot, __name__)
for entry in entries:
payload, key = entry[3:5]
if key not in game_list: # Not found error
error_message = (
":warning: Warning: The game {} has been removed, renamed, or "
"recategorized. You have been removed from the notification list "
"for this game. Please check the schedule at {}.".format(
payload['text'],
configurations.get(bot, __name__, 'schedule_url')))
utilities.update_schedule_entries(bot,
__name__,
search=key,
payload={'error': error_message},
time=time.time())
else:
game = schedule_data[game_list.index(key)]
start_time, end_time = game['scheduled'], game['end']
setup_delta = datetime.timedelta(seconds=game['setup_seconds'])
scheduled = start_time.replace(
tzinfo=datetime.timezone.utc).timestamp()
current_time = datetime.datetime.utcnow()
if start_time + setup_delta < current_time < end_time:
stream_url = configurations.get(bot, __name__, 'stream_url')
payload = {
'error':
("Uh oh. The schedule shifted drastically and I didn't notice "
"fast enough - sorry! {} is live right now at {}").format(
payload['text'], stream_url)
}
else:
payload.update({'end': scheduled + game['seconds']})
utilities.update_schedule_entries(bot,
__name__,
search=key,
payload=payload,
time=scheduled)
# Save data
data.add(bot, __name__, 'schedule', schedule_data, volatile=True)
try:
_update_current_game(bot)
except:
pass