def mal_get_short(username, **kwargs):
"""
gets a MAL list with a username
kwargs reffer to the arguments (sort=3,x='w') -> ?sort=3&x=w
"""
url = MALURL.format(user=username)+'?'
url += '&'.join(k+'='+str(v) for k, v in kwargs.items())
r = requests.get(url)
if r.status_code == 200:
return r.json()
else:
if r.status_code == 500:
fprint('error', 'too many requests made, try again later')
r.raise_for_status()
def batch_download(username,
statuses=[1, 2],
webm_folder=None,
mp3_folder=None,
anilist=False,
extra_ids=[]):
"""
basically the main function
given a username and destination, downloads all themes from MAL/AniList
"""
fprint('progress', 'initializing program')
download_data = get_download_data(username, statuses, anilist, extra_ids)
download_multi_theme(download_data, webm_folder, mp3_folder)
fprint('progress', 'finished downloading')
def al_get(username, variables={}):
"""
gets data from AniList using
variables reffer to JQuery variables
"""
if username == '':
return []
variables['user'] = username
json_arg = {'query': ALQUERY, 'variables': variables}
r = requests.post(ALURL, json=json_arg)
if "errors" in r:
fprint('error', f"{'; '.join(i['message'] for i in r['errors'])}\n{r['errors']}")
if r.status_code == 200:
return sort_al(r.json())
else:
r.raise_for_status()
def convert_ffmpeg(webm_filename, mp3_filename=None, save_folder=None):
"""
convert a video file to an audio file with ffmpeg
"""
if mp3_filename is None:
mp3_filename = webm_filename[:-5]
if save_folder is not None:
mp3_filename = os.path.join(save_folder,
os.path.basename(mp3_filename))
mp3_filename += '.' + Opts.Download.audio_format
loglevel = 'quiet' if Opts.Print.quiet else 'warning'
ffmpeg_path = Opts.Download.ffmpeg
os.system(
f'{ffmpeg_path} -i "{webm_filename}" "{mp3_filename}" -y -stats -v quiet -loglevel {loglevel}'
)
if not os.path.isfile(mp3_filename):
fprint('error',
"ffmpeg didn't convert, check if it's in path or use --ffmpeg")
quit()
return mp3_filename
def get_proper(username, anilist=False, extra_ids=[], **mal_kwargs):
"""
gets proper data from MAL/AniList
sorted by status (0 = extra ids)
"""
t = time.time()
fprint('get', f'getting data from {"anilist.co" if anilist else "myanimelist.net"}', end='')
if anilist:
list_data = al_get(username)
else:
list_data = mal_get(username, **mal_kwargs)
fprint(' | '+str(int((time.time()-t)*1000))+'ms')
t = time.time()
fprint('get', 'getting data from themes.moe', end='')
if anilist:
themes_data = get_themes(username, 'anilist')
else:
themes_data = get_themes(username, 'mal')
extra_themes_data = []
for malid in extra_ids:
data = parse_anime(themes_get('id', malid))
data['mal_data'] = {
'status': 5,
'score': 10,
'priority_string': 'High',
'anime_id': malid,
'anime_title': ''
}
extra_themes_data.append(data)
fprint(' | '+str(int((time.time()-t)*1000))+'ms')
mal_dict = {} # {12345:{'mal_data':{...},'status':1},...}
for status, animes in enumerate(list_data):
for anime in animes:
mal_dict[anime['anime_id']] = {'mal_data': anime, 'status': status}
mal_list = {i: [] for i in range(0, 7)}
for anime in themes_data:
mal_id = anime['mal_id']
if mal_id not in mal_dict:
continue
status = mal_dict[mal_id]['status']
mal_data = mal_dict[mal_id]['mal_data']
anime['mal_data'] = mal_data
mal_list[status].append(anime)
mal_list[0] = extra_themes_data
return mal_list
def download_theme(theme_data,
webm_folder=None,
mp3_folder=None,
no_redownload=False):
"""
downloads a theme with given theme data and destination
"""
# generate a folder to save webm files
if webm_folder is None:
filename = os.path.join(mp3_folder, theme_data["filename"])
else:
filename = os.path.join(webm_folder, theme_data["filename"])
exwebm, exmp3 = os.path.isfile(filename), os.path.isfile(
os.path.splitext(filename)[0] + '.mp3')
# determine what files are needed
if no_redownload:
needmp3 = mp3_folder is not None and not exmp3
needwebm = (webm_folder is not None and not exwebm) or (needmp3
and not exwebm)
else:
needwebm = True
needmp3 = mp3_folder is not None
# download webm file if file does not exist
if needwebm:
fprint('download', theme_data['filename'])
obj = SmartDL(theme_data["mirrors"],
filename,
progress_bar=not Opts.Print.quiet)
try:
obj.start()
except Exception as e:
fprint('error', str(e))
except KeyboardInterrupt:
obj.stop()
os.remove(obj.get_dest() + '.000')
quit()
webm_dest = obj.get_dest()
remove_webm = True
else:
webm_dest = filename
remove_webm = False
# download mp3 file if file does not exist
if needmp3:
if not needwebm:
fprint('convert', theme_data['filename'])
mp3dest = convert_ffmpeg(webm_dest, save_folder=mp3_folder)
add_metadata(mp3dest, theme_data["metadata"], Opts.Download.coverart)
else:
mp3dest = None
# delete webm folder is deemed neccesary
if webm_folder is None:
if remove_webm:
os.remove(webm_dest)
webm_dest = None
return {"mp3": mp3dest, "webm": webm_dest}
def add_metadata(path, metadata, add_coverart):
"""
adds metadata to an mp3 file
returns True if succeeded, False if failed
"""
if os.path.splitext(path)[1] != '.mp3':
return False
t = time.time()
fprint(f'adding metadata for v2.4' +
(' with coverart' if add_coverart else ''),
end='')
audiofile = eyed3.load(path)
if (audiofile.tag == None):
audiofile.initTag()
audiofile.tag.album = metadata["album"]
audiofile.tag.title = metadata["title"]
audiofile.tag.year = metadata["year"]
if add_coverart:
image = requests.get(metadata["cover art"]).text.encode()
audiofile.tag.images.set(3, image, 'image/jpeg')
audiofile.tag.genre = 145
try:
audiofile.tag.save()
fprint(' | ' + str(int((time.time() - t) * 1000)) + 'ms')
return True
except PermissionError as e:
fprint('error', f"couldn't add metadata: {e}", start='\n')
return False
def progress_bar():
"""
this progress bar is completely fake, but it makes it feel like something is happening so that's cool
"""
global _LAST_DOWNLOAD_TIME
START_TIME = time.time()
last_increment = last_decrement = time.time()
expected_remain = _LAST_DOWNLOAD_TIME + 0.75
def current_time():
return time.time() - START_TIME
def download_string(x, y, r=2):
x = str(x).split('.')
y = str(y).split('.')
x = x[0] + '.' + x[1][:r]
y = y[0] + '.' + y[1][:r]
return f'[*] {x}s / {y}s '
while True:
fprint(download_string(current_time(), expected_remain), end='\r')
# time updater
difference = expected_remain - current_time()
if difference < 1:
# never get it too close to the expected remain
expected_remain += 1
last_increment = time.time()
elif time.time() - last_increment > 0.5 and difference < 10:
# add time to make it look like it's updating
expected_remain += 1 / (expected_remain - current_time())
last_increment = time.time()
elif time.time() - last_decrement > 2:
# remove some time because it feels good to make it faster
expected_remain -= 2 / (expected_remain - current_time())
last_decrement = time.time()
if STOP_PROGRESS_BAR:
break
if current_time() >= timeout:
fprint('error', 'download timed out')
return
if FAKE_COMPLETE:
fprint(download_string(current_time(), current_time()))
_LAST_DOWNLOAD_TIME = current_time()
else:
fprint('')
def get_download_data(username,
statuses=[1, 2],
anilist=False,
extra_ids=[],
mal_args={}):
"""
gets download data with a MAL/AniList username, adds extra anime and fiters out unwanted stuff
"""
# [{"filename":"...","mirrors":[...],"metadata":{...}},...]
out = []
data = get_proper(username, anilist, extra_ids, **mal_args)
fprint('parse', 'getting download data')
for status in statuses:
for anime in data[status]:
if (anime['mal_data']['score'] < Opts.Animelist.minscore
or to_mal_priority(anime['mal_data']['priority_string']) <
Opts.Animelist.minpriority):
continue
for theme, unparsed in zip(parse_download_data(anime),
anime['themes']):
theme["metadata"]["cover art"] = anime["cover"]
out.append(theme)
return out
def download_multi_theme(download_data, webm_folder=None, mp3_folder=None):
"""
downloads multiple themes with download data
"""
if webm_folder is None and mp3_folder is None:
fprint('error', 'no save folder set')
if webm_folder and not os.path.isdir(webm_folder):
os.mkdir(webm_folder)
if mp3_folder and not os.path.isdir(mp3_folder):
os.mkdir(mp3_folder)
download_chooser = (
lambda mthd: download_theme(theme, webm_folder, mp3_folder, Opts.
Download.no_redownload)
if (mthd or webm_folder is not None) else download_theme_audio_server(
theme, mp3_folder, Opts.Download.no_redownload))
# 0: external, 1: local
if Opts.Download.local_convert or webm_folder is not None:
mthd = 1
fprint(
'progress', 'started downloading' +
(' and converting' if mp3_folder is not None else ''))
else:
mthd = 0
fprint('progress', 'started downloading audio files')
for theme in download_data:
filename = None
while filename is None:
filename = download_chooser(mthd)
if filename is None and Opts.Download.try_both:
filename = download_chooser(not mthd)
if not Opts.Download.retry_forever:
break
def download_theme_audio_server(theme_data,
mp3_folder=None,
no_redownload=False,
data_size=65536,
timeout=30):
"""
downloads an audio file from from a server
the progress bar is completely fake
returns:
filename (str)
empty str if no download was made
None if all downloads failed
"""
def progress_bar():
"""
this progress bar is completely fake, but it makes it feel like something is happening so that's cool
"""
global _LAST_DOWNLOAD_TIME
START_TIME = time.time()
last_increment = last_decrement = time.time()
expected_remain = _LAST_DOWNLOAD_TIME + 0.75
def current_time():
return time.time() - START_TIME
def download_string(x, y, r=2):
x = str(x).split('.')
y = str(y).split('.')
x = x[0] + '.' + x[1][:r]
y = y[0] + '.' + y[1][:r]
return f'[*] {x}s / {y}s '
while True:
fprint(download_string(current_time(), expected_remain), end='\r')
# time updater
difference = expected_remain - current_time()
if difference < 1:
# never get it too close to the expected remain
expected_remain += 1
last_increment = time.time()
elif time.time() - last_increment > 0.5 and difference < 10:
# add time to make it look like it's updating
expected_remain += 1 / (expected_remain - current_time())
last_increment = time.time()
elif time.time() - last_decrement > 2:
# remove some time because it feels good to make it faster
expected_remain -= 2 / (expected_remain - current_time())
last_decrement = time.time()
if STOP_PROGRESS_BAR:
break
if current_time() >= timeout:
fprint('error', 'download timed out')
return
if FAKE_COMPLETE:
fprint(download_string(current_time(), current_time()))
_LAST_DOWNLOAD_TIME = current_time()
else:
fprint('')
# generate a folder to save webm files
short_filename = os.path.splitext(theme_data["filename"])[0] + '.mp3'
filename = os.path.join(mp3_folder, short_filename)
# download mp3 file if file does not exist
if not (no_redownload and os.path.isfile(filename)):
fprint('download', short_filename)
for mirror in theme_data['audio_mirrors']:
STOP_PROGRESS_BAR = False
FAKE_COMPLETE = False
thread = threading.Thread(target=progress_bar)
thread.start()
try:
r = requests.get(mirror)
except KeyboardInterrupt:
STOP_PROGRESS_BAR = True
thread.join()
quit()
except Exception as e:
fprint('error', e)
FAKE_COMPLETE = True
STOP_PROGRESS_BAR = True
thread.join()
continue
if r.status_code != 200:
if r.status_code not in (500, 404):
FAKE_COMPLETE = True
STOP_PROGRESS_BAR = True
thread.join()
fprint('error',
f'invalid mirror, got status code {r.status_code}')
continue
else:
FAKE_COMPLETE = True
STOP_PROGRESS_BAR = True
thread.join()
with open(filename, 'wb') as file:
for data in r.iter_content(data_size): # 64kiB
file.write(data)
break
else:
fprint('error',
'all mirrors are invalid (press CTRL+C to stop program)')
return None
add_metadata(filename, theme_data["metadata"], Opts.Download.coverart)
return filename
else:
return ''
args.status = [1, 2] + args.status
args.id = args.id or []
if args.settings is not None:
with open(args.settings) as file:
jargs = json.load(file)
if 'quiet' not in jargs:
jargs['quiet'] = False
for k in jargs:
args.__setattr__(k, jargs[k])
Opts.update(**args.__dict__)
args.status.append(0)
if args.print_settings:
fprint('\n'.join(
[f'{k}={repr(v)}' for k, v in Opts.get_settings().items()]),
end='\n\n')
if args.username == '' and len(args.id) == 0:
fprint('error', 'no username set')
quit()
if args.video is None and args.audio is None:
fprint('error', 'no save folder set')
quit()
batch_download(args.username, args.status, args.video, args.audio,
args.anilist, args.id)
