def reset(self,type,id):
self.target=[type,id]
self.status='Start'
self.downloader=Downloader()
self.downloader.bind(status=self.setter('status'))
self.clear_widgets()
self.add_widget(self.downloader)
self.working_dir=os.sep.join([self.bundle_dir,'plugins','processing','model_zoo',type+'s',id])
os.makedirs(self.working_dir, exist_ok=True)
def __init__(self,
temp='temp',
output='output',
merge_only=False,
pool_size=4,
buffer_size=256,
random_keys=False,
logger=lambda *args, **kwargs: None):
self.NCM = NeteaseCloudMusic(logger, random_keys)
self.DL = Downloader(session=self.NCM.session,
pool_size=pool_size,
buffer_size=buffer_size)
# Initalization of other classes
self.temp, self.output, self.merge_only = temp, output, merge_only
self.log = logger
self.pool_size = pool_size
def main():
downloader = Downloader()
url = "https://www.starlist.pro/"
page = "brawlers/"
crawler = Crawler(url, downloader, page)
content = crawler.download()
crawler.save()
def download_reddit(self):
"""
Download images from subreddit.
"""
try:
self.reddit = Reddit(self.url_var.get(), self.pages_var.get())
except RedditException:
messagebox.showerror('Error', 'Please input valid link')
return
except TclError:
messagebox.showerror('Error', 'Please input only whole numbers')
return
except Exception as e:
print(e)
messagebox.showerror('Error', 'Something went wrong with Reddit.')
return
try:
self.downloader = Downloader(self.reddit,
self.destination_var.get())
except DownloaderException:
messagebox.showerror('Error', 'Invalid download path')
return
except Exception as e:
print(e)
messagebox.showerror('Error',
'Something went wrong with Downloader.')
return
self.btn_download.configure(text='Cancel',
command=self.cancel_download)
self.change_progress_label('Fetching data...')
self.add_progress_bar()
self.progress_bar.start()
self.queue = Queue()
self.reddit_listener = RedditListener()
self.reddit.register(self.reddit_listener)
self.downloader_listener = DownloaderListener()
self.downloader.register(self.downloader_listener)
DownloadThread(self.queue, self.reddit, self.downloader).start()
self.root.after(100, self.process_queue)
def get_episode_video_file_name(self, dashed_name):
url = f'https://www.dramacool9.co/{dashed_name}/'
original_video_url = self.get_original_episode_video_url(url)
path = os.path.dirname(os.path.abspath(__file__))
file_parent_path = path.replace('utils', 'temp')
file_name = uuid.uuid4().hex[:16] + '.mp4'
target_file = file_parent_path + '/' + file_name
Downloader(url=original_video_url,
file_parent_path=file_parent_path,
file_name=file_name).download_file_from_source()
uploader_tool = Uploader()
generated_video_url_from_cloud = uploader_tool.upload_file_to_cloud(
source_file_name=target_file, destination_blob_name=file_name)
if generated_video_url_from_cloud:
uploader_tool.delete_file_from_project(
source_file_name=target_file)
return file_name
def wrapper(id, quality='lossless', folder=None, merge_only=False):
if not folder:
folder = self.GenerateDownloadPath(id=id, folder=folder)
if not self.merge_only:
queue_func(id, quality, folder)
self.DL.wait(func=self.ShowDownloadStatus)
# Queue then wait
self.log(strings.INFO_BATCH_DOWNLOAD_COMPLETE)
def merge(subfolder):
self.FormatLyrics(subfolder)
self.FormatSong(subfolder)
# Start merging every subfolder via threadpool,where it's a mutation of the Downloader
pool = Downloader(worker=PoolWorker, pool_size=16)
for sub in os.listdir(folder):
target = os.path.join(folder, sub)
pool.append(merge, target)
pool.wait()
def __init__(self):
download_request = Downloader.get(MODEL_URL)
self.model_path = download_request.path
self.call_model = True
self.predictions = []
class RedditApp(Frame):
def __init__(self, master=None):
"""
Initialize the main application Frame and load it with other widgets.
"""
super().__init__(master)
self.root = master
self.pack()
self.setup_main_frame()
self.create_menubar()
self.create_widgets()
def setup_main_frame(self):
self.root.resizable(0, 0)
self.root.wm_title('(sub)Reddit Downloader')
def create_menubar(self):
menubar = Menu(self.root)
menubar.add_command(label='About', command=AboutWindow.about)
menubar.add_command(label='Exit', command=self.root.quit)
self.root.config(menu=menubar)
def create_widgets(self):
"""
Create widgets to populate the applications frame.
"""
# URL, Pages, Destination
paths_frame = Frame(self)
lbl_url = Label(paths_frame, text='URL:')
lbl_pages = Label(paths_frame, text='Pages:')
lbl_destination = Label(paths_frame, text='Destination:')
lbl_pages_help = Label(paths_frame,
text='(Leave zero to crawl all pages)')
self.url_var = StringVar()
self.pages_var = IntVar()
self.destination_var = StringVar()
url = Entry(paths_frame, textvariable=self.url_var)
pages = Entry(paths_frame, textvariable=self.pages_var)
destination = Entry(paths_frame, textvariable=self.destination_var)
btn_chooser = Button(paths_frame,
text='Destination',
command=self.choose_directory)
pad_x = 5
pad_y = 7
r = 0
lbl_url.grid(row=r, column=0, sticky=E, padx=pad_x, pady=pad_y)
url.grid(row=r, column=1)
r += 1
lbl_pages.grid(row=r, column=0, sticky=E)
pages.grid(row=r, column=1, padx=pad_x, pady=pad_y)
lbl_pages_help.grid(row=r, column=2)
r += 1
lbl_destination.grid(row=r, column=0, sticky=E)
destination.grid(row=r, column=1, padx=pad_x, pady=pad_y)
btn_chooser.grid(row=r, column=2, sticky=E + W)
paths_frame.pack(side=TOP, padx=10, pady=10)
# Download button
download_frame = Frame(self)
self.btn_download = Button(download_frame,
text='Download',
command=self.download_reddit)
self.btn_download.pack(padx=10, pady=10)
download_frame.pack(side=TOP)
# Progress label
progress_info = Frame(self)
self.lbl_progress_info = Label(progress_info, text='')
self.lbl_progress_info.pack(padx=10, pady=10)
progress_info.pack(side=TOP)
def add_progress_bar(self):
"""
Add progress bar to root frame.
"""
self.progress_frame = Frame(self)
self.progress_bar = Progressbar(self.progress_frame,
orient='horizontal',
length=400,
mode='indeterminate')
self.progress_bar.pack(padx=10, pady=10)
self.progress_frame.pack(side=TOP)
def remove_progress_bar(self):
"""
Remove progress bare from root frame.
"""
self.progress_bar.destroy()
self.progress_frame.destroy()
def change_progress_label(self, text):
"""
Change text of progress bar label.
"""
self.lbl_progress_info.configure(text=text, fg='red', font=(font.BOLD))
def choose_directory(self):
"""
Update the destination path entry filed with the chosen path.
"""
destination_path = filedialog.askdirectory(initialdir='~')
self.destination_var.set(destination_path)
def download_reddit(self):
"""
Download images from subreddit.
"""
try:
self.reddit = Reddit(self.url_var.get(), self.pages_var.get())
except RedditException:
messagebox.showerror('Error', 'Please input valid link')
return
except TclError:
messagebox.showerror('Error', 'Please input only whole numbers')
return
except Exception as e:
print(e)
messagebox.showerror('Error', 'Something went wrong with Reddit.')
return
try:
self.downloader = Downloader(self.reddit,
self.destination_var.get())
except DownloaderException:
messagebox.showerror('Error', 'Invalid download path')
return
except Exception as e:
print(e)
messagebox.showerror('Error',
'Something went wrong with Downloader.')
return
self.btn_download.configure(text='Cancel',
command=self.cancel_download)
self.change_progress_label('Fetching data...')
self.add_progress_bar()
self.progress_bar.start()
self.queue = Queue()
self.reddit_listener = RedditListener()
self.reddit.register(self.reddit_listener)
self.downloader_listener = DownloaderListener()
self.downloader.register(self.downloader_listener)
DownloadThread(self.queue, self.reddit, self.downloader).start()
self.root.after(100, self.process_queue)
def process_queue(self):
"""
Stop the progress bar if the thread has yielded control. If download was
canceled, re-enable download button only after thread has yielded control.
Reset progress bar and download button, and unregister all listeners when
downloading is finished or canceled.
"""
try:
msg = self.queue.get(0)
if self.btn_download['text'] == 'Cancel':
# self.reddit.fetch = True
# self.downloader.downloading = True
self.btn_download.configure(text='Download',
command=self.download_reddit)
self.remove_progress_bar()
self.change_progress_label('Download finished.')
if self.btn_download['state'] == DISABLED:
self.btn_download.configure(state=NORMAL)
self.reddit.unregister()
self.downloader.unregister()
print(msg)
except queue.Empty:
if getattr(self.reddit_listener, 'fetched', None):
print(self.reddit_listener.maximum)
self.progress_bar.configure(mode='determinate')
self.progress_bar['value'] = 0
self.progress_bar['maximum'] = self.reddit_listener.maximum
self.reddit_listener.fetched = False
if (getattr(self.downloader_listener, 'currently_at', None)
and self.downloader.downloading):
print(self.downloader_listener.currently_at)
self.progress_bar[
'value'] = self.downloader_listener.currently_at
self.root.after(100, self.process_queue)
def cancel_download(self):
"""
Cancel fetching or downloading process.
"""
self.reddit.fetch = False
self.downloader.downloading = False
self.btn_download.configure(text='Download',
command=self.download_reddit)
self.remove_progress_bar()
self.change_progress_label('Download canceled.')
self.btn_download.configure(state=DISABLED)
from pyblake2 import blake2b # pylint: disable=import-error
logger = logging.getLogger(__name__)
config = ConfigParser()
config.read('schemas.ini')
# TODO etag support
# NOTE
# Consider allowing the application to launch
# when the core schema download fails. If that
# happens, disable any modifications and alert
# some slack channel to report the event.
openapis = Downloader()
openapis.download(
config['openapi'].keys(),
config['openapi'].values()
)
swaggers = Downloader()
swaggers.download(
config['swagger'].keys(),
config['swagger'].values()
)
class ControllerError(Exception):
pass
class NCMFunctions():
'''
Functions using ncm_core.py,simplfied to download songs easily
temp : Tempoary folder to stroe downloads
output : Output folder to store music
merge_only : If only find download file in the temp folder,then merge them without downloading
Applies to MutilWrapper ones
pool_size : Cocurrent download worker count
buffer_size : Size of iter_content(chunk_size)'s chunk_size
logger : For logging.Uses simple_logger from ncm.strings,leave empty for not logging
random_keys : Whether uses random 2nd AES key or the constant 'mos9527ItoooItop' and its encypted RSA string
Functions inside are well described,read them for more info.
'''
def __init__(self,
temp='temp',
output='output',
merge_only=False,
pool_size=4,
buffer_size=256,
random_keys=False,
logger=lambda *args, **kwargs: None):
self.NCM = NeteaseCloudMusic(logger, random_keys)
self.DL = Downloader(session=self.NCM.session,
pool_size=pool_size,
buffer_size=buffer_size)
# Initalization of other classes
self.temp, self.output, self.merge_only = temp, output, merge_only
self.log = logger
self.pool_size = pool_size
def ShowDownloadStatus(self):
'''
Shows downloader info
'''
content = self.DL.report(strings.DEBUG_DOWNLOAD_STATUS(None))
print(content, Cursor.UP(content.count('\n') + 1))
time.sleep(1)
def GenerateDownloadPath(self, id='', filename='', folder=''):
'''
Generates output path with its leaf folders and such.Similar to mktree
Specify folder to change the root folder somewhere else
'''
def normalize(s):
return s.replace(':', '：').replace('\n', '_').replace(
'-', '_').replace(',', '，').replace('"', '“').replace(
'\'', '”').replace('/', '、').replace('\\', '，')
filename = normalize(filename)
folder = os.path.join(folder, str(id)) if folder else os.path.join(
self.temp, str(id))
result = os.path.join(folder, filename)
path = os.path.split(result)
if path[0]:
if not os.path.exists(path[0]): os.makedirs(path[0])
# Make the folder tree
return result
def QueueDownload(self, url, path):
'''
Queue a download
'''
self.DL.append(url, path)
self.log(url, path, format=strings.INFO_QUEUED_DOWNLOAD)
def QueueDownloadSongAudio(self, id, quality='lossless', folder=''):
'''
This will queue to download only the song's audio file (named 'audio.*') to a certain directory.
'''
info = self.NCM.GetSongInfo(song_id=id, quality=quality)
if not info['code'] == 200: return
filename = '{}.{}'.format('audio', info['data'][0]['type'])
target = self.GenerateDownloadPath(filename=filename, folder=folder)
url = info['data'][0]['url']
self.QueueDownload(url, target)
return target
def QueueDownloadSongInfo(self, id, folder=''):
'''
This will write the song's metadata into JSON (named meta.json) and queue to download cover (named cover.jpg) to a certain directory.
'''
info = self.NCM.GetExtraSongInfo(id)
self.QueueDownload(
info['cover'],
self.GenerateDownloadPath(filename='cover.jpg', folder=folder))
# Queue to download cover image
target = self.GenerateDownloadPath(filename='meta.json', folder=folder)
open(target, mode='w', encoding='utf-8').write(json.dumps(info))
# Writes metadata to JSON file
def QueueDownloadAllSongsInAlbum(self, id, quality='lossless', folder=''):
'''
This will get every song's ID inside the album resovled
Note that extra info will be stored inside the JSON that the api sent
'''
info = self.NCM.GetAlbumInfo(id)
try:
name, creator = info['title'], info['author']
self.log(name,
creator,
format=strings.INFO_FETCHED_ALBUM_WITH_TOKEN)
except Exception as e:
self.log(e, format=strings.ERROR_INVALID_OPERATION)
root = folder
# Go through every track inside the JSON
for track in info['songlist']:
meta = {
"title": track['name'],
"cover": track['album']['picUrl'],
"author": ' - '.join([a['name'] for a in track['artists']]),
"album": track['album']['name'],
"album_id": track['album']['id'],
"artist_id":
' - '.join([str(a['id']) for a in track['artists']])
}
# Generates meta header,and saves them
track_id = track['id']
track_root = self.GenerateDownloadPath(id=track_id, folder=root)
open(self.GenerateDownloadPath(filename='meta.json',
folder=track_root),
mode='w',
encoding='utf-8').write(json.dumps(meta))
# Download lyrics
self.DownloadAndFormatLyrics(track_id, folder=track_root)
# Queue to download audio and cover image
self.QueueDownloadSongAudio(track_id, quality, track_root)
self.QueueDownload(
meta['cover'],
self.GenerateDownloadPath(filename='cover.jpg',
folder=track_root))
def QueueDownloadAllSongsInPlaylist(self,
id,
quality='lossless',
folder=''):
'''
This will try and get all songs' ID and such in the playlist,then try to download them.
The strcuture is very similar to the album one,but with some minor twists
'''
info = self.NCM.GetPlaylistInfo(id)
try:
name, creator = info['playlist']['name'], info['playlist'][
'creator']['nickname']
self.log(name,
creator,
format=strings.INFO_FETCHED_PLAYLIST_WITH_TOKEN)
except Exception as e:
self.log(e, format=strings.ERROR_INVALID_OPERATION)
root = folder
# Go through every track inside the JSON
for track in info['playlist']['tracks']:
meta = {
"title": track['name'],
"cover": track['al']['picUrl'],
"author": ' - '.join([a['name'] for a in track['ar']]),
"album": track['al']['name'],
"album_id": track['al']['id'],
"artist_id": ' - '.join([str(a['id']) for a in track['ar']])
}
# Generates meta header,and saves them
track_id = track['id']
track_root = self.GenerateDownloadPath(id=track_id, folder=root)
open(self.GenerateDownloadPath(filename='meta.json',
folder=track_root),
mode='w',
encoding='utf-8').write(json.dumps(meta))
# Download lyrics
self.DownloadAndFormatLyrics(track_id, folder=track_root)
# Queue to download audio and cover image
self.QueueDownloadSongAudio(track_id, quality, track_root)
self.QueueDownload(
meta['cover'],
self.GenerateDownloadPath(filename='cover.jpg',
folder=track_root))
def DownloadSongLyrics(self, id, folder=''):
'''
This will only write the song's lyrics into JSON (named lyrics.json) to a certain directory.
'''
lyrics = self.NCM.GetSongLyrics(id)
target = self.GenerateDownloadPath(filename='lyrics.json',
folder=folder)
open(target, mode='w', encoding='utf-8').write(json.dumps(lyrics))
# Writes lyrics to JSON file
def LoadLyrics(self, folder):
'''
Converts the lyrics JSON format into a dictionary
'''
lyrics = self.GenerateDownloadPath(filename='lyrics.json',
folder=folder)
# Locate lyric file
if not os.path.exists(lyrics):
self.log('lyrics.json', format=strings.ERROR_FILE_MISSING)
return
lyrics = json.loads(open(lyrics, encoding='utf-8').read())
# Load lyrics into json file
lrc_lists = [
lyrics[lrc]['lyric'] for lrc in lyrics.keys()
if lrc in ['lrc', 'tlyric'] and 'lyric' in lyrics[lrc].keys()
]
# List all lyrics.Translation is optional
timestamp_regex = r"(?:\[)(\d{2,}:\d{2,}.\d{2,})(?:\])(.*)"
lyrics_combined = {}
for lrc_ in lrc_lists:
if not lrc_:
continue
lrc_lrc = lrc_.split('\n')
# Split lyrics into an array
for lrc in lrc_lrc:
result = re.search(timestamp_regex, lrc)
if not result:
continue
timestamp, content = result.group(1), result.group(2)
if not timestamp in lyrics_combined.keys():
lyrics_combined[timestamp] = []
lyrics_combined[timestamp].append(content)
return lyrics_combined
def FormatLyrics(self, folder, export=''):
'''
Coverts a dictionary with timestamps as keys and lyrics as items into LRC lyrics
'''
export = export if export else self.output
lyrics = self.LoadLyrics(folder)
if not lyrics:
return
meta = self.GenerateDownloadPath(filename='meta.json', folder=folder)
# Locate meta file
if not os.path.exists(meta):
self.log('meta.json', format=strings.ERROR_FILE_MISSING)
return
meta = json.loads(open(meta, encoding='utf-8').read())
# Load JSON for future purposes
lrc = f"""[ti:{meta['title']}]
[al:{meta['album']}]
[au:{meta['author']}]
[re:PyNCM]"""
for timestamp in lyrics.keys():
newline = '[%s]' % timestamp
newline += '\t'.join(lyrics[timestamp])
lrc += '\n' + newline
path = self.GenerateDownloadPath(filename='{1} - {0}.{2}'.format(
meta['title'], meta['author'], 'lrc'),
folder=export)
open(path, mode='w', encoding='utf-8').write(lrc)
self.log(path, format=strings.INFO_EXPORT_COMPLETE)
return path
def FormatSong(self, folder, export=''):
'''
'Format' a song's metadata with the infomation given in folder
Call once all download tasks is finished
Suppports MP3,M4A(ALAC/MP4),FLAC
'''
export = export if export else self.output
folder = str(folder)
audio = [
f for f in os.listdir(folder)
if f.split('.')[-1] in ['mp3', 'm4a', 'flac']
]
if not audio: return
audio = audio[-1]
audio = self.GenerateDownloadPath(filename=audio, folder=folder)
format = audio.split('.')[-1]
# Locate audio file,uses last match
meta = self.GenerateDownloadPath(filename='meta.json', folder=folder)
# Locate meta file
cover = self.GenerateDownloadPath(filename='cover.jpg', folder=folder)
# Locate cover image
if not audio:
self.log('audio.m4a / audio.mp3 / audio.flac',
format=strings.ERROR_FILE_MISSING)
return
if not os.path.exists(meta):
self.log('meta.json', format=strings.ERROR_FILE_MISSING)
return
if not os.path.exists(cover):
self.log('cover.jpg', format=strings.WARN_FILE_MISSING)
meta = json.loads(open(meta, encoding='utf-8').read())
def write_keys(song):
# Write metadatas
song['title'] = meta['title']
song['artist'] = meta['author']
song['album'] = meta['album']
song.save()
if format == 'm4a':
# Process m4a files: Apple’s MP4 (aka M4A, M4B, M4P)
song = easymp4.EasyMP4(audio)
write_keys(song)
if os.path.exists(cover):
song = MP4(audio)
# Write cover image
song['covr'] = [MP4Cover(open(cover, 'rb').read())]
song.save()
elif format == 'mp3':
# Process mp3 files: MPEG audio stream information and tags
song = EasyMP3(audio)
write_keys(song)
if os.path.exists(cover):
song = ID3(audio)
# Write cover image
song.add(
APIC(encoding=3,
mime='image/jpeg',
type=3,
desc='Cover',
data=open(cover, 'rb').read()))
song.save()
elif format == 'flac':
# Process FLAC files:Free Lossless Audio Codec
song = FLAC(audio)
write_keys(song)
if os.path.exists(cover):
pic = Picture()
pic.data = open(cover, 'rb').read()
pic.mime = 'image/jpeg'
song.add_picture(pic)
song.save()
# Rename & move file
savename = '{1} - {0}.{2}'.format(meta['title'], meta['author'],
format)
try:
path = self.GenerateDownloadPath(filename=savename, folder=export)
shutil.copy(audio, path)
self.log(path, format=strings.INFO_EXPORT_COMPLETE)
except Exception as e:
self.log(e, format=strings.ERROR_INVALID_OPERATION)
return path
def DownloadSongAudio(self, id, quality='lossless', folder=''):
'''
Download Audio,and wait for it to finish
'''
if not folder:
folder = self.GenerateDownloadPath(id=id, folder=folder)
self.QueueDownloadSongAudio(id, quality, folder)
self.DL.wait(func=self.ShowDownloadStatus)
self.log(strings.INFO_BATCH_DOWNLOAD_COMPLETE)
return True
def DownloadSongInfo(self, id, folder=''):
'''
Download info,and wait for it to finish
'''
if not folder:
folder = self.GenerateDownloadPath(id=id, folder=folder)
self.QueueDownloadSongInfo(id, folder)
self.DL.wait(func=self.ShowDownloadStatus)
self.log(strings.INFO_BATCH_DOWNLOAD_COMPLETE)
return True
def DownloadAndFormatLyrics(self, id, folder=''):
'''
Both download,and format the lyrics
'''
if not folder:
folder = self.GenerateDownloadPath(id=id, folder=folder)
self.DownloadSongLyrics(id, folder)
self.FormatLyrics(folder)
return True
def DownloadAllInfo(self, id, quality='lossless', folder=''):
'''
Download everything we need,then wait for download to finish.
'''
if not folder:
folder = self.GenerateDownloadPath(id=id, folder=folder)
# Download meta and cover.Note that cover is downloaded asynchronously
self.QueueDownloadSongInfo(id, folder)
# Download lyrics
self.DownloadAndFormatLyrics(id)
# Download song.Not that audio file is downloaded asynchronously
self.QueueDownloadSongAudio(id, quality, folder)
self.DL.wait(func=self.ShowDownloadStatus)
self.log(strings.INFO_BATCH_DOWNLOAD_COMPLETE)
return True
def DownloadAllAndMerge(self, id, quality='lossless', folder=''):
'''
This will merge the infomation once download is finished
'''
if not folder:
folder = self.GenerateDownloadPath(id=id, folder=folder)
self.DownloadAllInfo(id, quality, folder)
self.DownloadAndFormatLyrics(id, folder)
self.FormatSong(folder)
return True
def Login(self, username, password):
'''
Login method,equals to ncm_core.UpdateLoginInfo
'''
return self.NCM.UpdateLoginInfo(username, password)
def MutilWrapper(self, queue_func):
'''
Wraps around a queue method.Once downloaded,
perform the merging proceess for each and every sub folder
'''
def wrapper(id, quality='lossless', folder=None, merge_only=False):
if not folder:
folder = self.GenerateDownloadPath(id=id, folder=folder)
if not self.merge_only:
queue_func(id, quality, folder)
self.DL.wait(func=self.ShowDownloadStatus)
# Queue then wait
self.log(strings.INFO_BATCH_DOWNLOAD_COMPLETE)
def merge(subfolder):
self.FormatLyrics(subfolder)
self.FormatSong(subfolder)
# Start merging every subfolder via threadpool,where it's a mutation of the Downloader
pool = Downloader(worker=PoolWorker, pool_size=16)
for sub in os.listdir(folder):
target = os.path.join(folder, sub)
pool.append(merge, target)
pool.wait()
return wrapper
def DownloadAllSongsInPlaylistAndMerge(self,
id,
quality='lossless',
folder=None,
merge_only=False):
'''
Downloads all songs in a playlist into selected folder
id : Song ID
quality : Song Quality
folder : Output folder
merge_only : Merge by id from 'folder',Only use this if you
Have finished downloading first
'''
return self.MutilWrapper(self.QueueDownloadAllSongsInPlaylist)(
id, quality, folder, merge_only)
def DownloadAllSongsInAlbumAndMerge(self,
id,
quality='lossless',
folder=None,
merge_only=False):
'''
Downloads all albums in a playlist into selected folder
id : Song ID
quality : Song Quality
folder : Output folder
merge_only : Merge by id from 'folder',Only use this if you
Have finished downloading first
'''
return self.MutilWrapper(self.QueueDownloadAllSongsInAlbum)(id,
quality,
folder,
merge_only)
def main():
download_request = Downloader.get(MODEL_URL)
model_path = download_request.path
graph, score, expand = load_frozenmodel(model_path)
detection(graph, score, expand, True)
def __init__(self):
self.downloader = Downloader()
class DownloadService:
def __init__(self):
self.downloader = Downloader()
def download_with_url(self, url):
return self.downloader.download(url=url)
cleaned_corpus = d.clean_text(algorithm='w2v')
w = GloveEmbedding(cleaned_corpus)
w.start()
q = QueryBuilder()
res = q.evaluate_word_embedding_scores(inv_index=i, embeddings=w)
return res
if __name__ == '__main__':
existing_files = set(os.listdir('data'))
cran_files = {'cranqrel', 'cran.qry', 'cran.all.1400'}
if not cran_files <= existing_files:
d = Downloader()
d.download()
d.untar()
i = InvertedIndex()
i.build_inverted_index()
i.build_vsm()
d = DataPolisher()
cleaned_corpus = d.clean_text(algorithm='w2v')
w = GloveEmbedding(cleaned_corpus)
w.start()
q = QueryBuilder()
res = q.evaluate_word_embedding_scores(inv_index=i, embeddings=w)
print(res)
class Installer(BoxLayout):
"""docstring for Installer."""
data=ObjectProperty()
status=StringProperty()
target=ListProperty()
bundle_dir = rootpath.detect(pattern='main.py')
# Builder.load_file(bundle_dir +os.sep+'ui'+os.sep+'demo.kv')
def __init__(self):
super(Installer, self).__init__()
self.bind(status=self.on_status_changed)
self.register_event_type('on_success')
def reset(self,type,id):
self.target=[type,id]
self.status='Start'
self.downloader=Downloader()
self.downloader.bind(status=self.setter('status'))
self.clear_widgets()
self.add_widget(self.downloader)
self.working_dir=os.sep.join([self.bundle_dir,'plugins','processing','model_zoo',type+'s',id])
os.makedirs(self.working_dir, exist_ok=True)
def on_status_changed(self,ins,value):
if 'Fail' in value:
self.on_failure()
elif value=='Unzip Finished':
self.dispatch('on_success')
def on_success(self):
self.clear_widgets()
self.add_widget(Label(text='[color=00ff33]Succesfully installed '+' : '.join(self.target)+'[/color]',text_size=self.size,valign='center',markup = True,font_size=18))
def on_failure(self):
self.clear_widgets()
try: shutil.rmtree(self.working_dir)
except: pass
self.add_widget(Label(text='[color=ff3333]'+self.status+'[/color]',text_size=self.size,valign='center',markup = True,font_size=18))
def install_model(self,id,url,callbacks=None):
self.reset('model',id)
file_path=self.working_dir+os.sep+url.split('/')[-1]
self.downloader.download(url,file_path,extract_dir='./',callbacks=callbacks)
def install_demo(self,id,url,callbacks=None):
self.reset('demo',id)
file_path=self.working_dir+os.sep+url.split('/')[-1]
self.downloader.file_name='Demo('+id+')'
self.downloader.download(url,file_path,extract_dir='./',callbacks=callbacks)
def install_weight(self,model,url,callbacks=None):
self.reset('weight',model)
file_path=self.working_dir+os.sep+url.split('/')[-1]
self.downloader.file_name='Weight('+url.split('/')[-1]+')'
self.downloader.download(url,file_path,callbacks=callbacks)
def copy_config(self,demo_id,model_id,model_config):
path1=os.sep.join([self.bundle_dir,'plugins','processing','model_zoo','demos',demo_id,model_config])
path2=os.sep.join([self.bundle_dir,'plugins','processing','model_zoo','configs',model_id,model_config])
os.makedirs(os.path.dirname(path2),exist_ok=True)
shutil.copyfile(path1,path2)
def install_demo_full(self,id,meta_info):
model_id=meta_info['model']
def t3():
if 'model_config' in meta_info:
self.copy_config(id,model_id,meta_info['model_config'])
def t2():
self.install_demo(id,meta_info['demo_url'],callbacks=t3)
def t1():
if 'model_url' in meta_info:
self.install_model(model_id,meta_info['model_url'],callbacks=t2)
else:
t2()
if 'weight_url' in meta_info:
self.install_weight(model_id,meta_info['weight_url'],callbacks=t1)
"""
parser = ArgumentParser(description='reddit /r downloader')
parser.add_argument('-v',
'--verbose',
help='display download progress and information',
action='store_true')
parser.add_argument('-p', help='crawl P number of pages', type=int)
parser.add_argument('URL', help='source link')
parser.add_argument('directory', help='destination directory')
args = parser.parse_args()
return (args.verbose, args.p, args.URL, args.directory)
if __name__ == '__main__':
verbose, pages, url, destination = parse_arguments()
if verbose: print('Fetching available links...')
reddit = Reddit(url, pages)
reddit.get_all_posts()
images = reddit.images
if verbose:
print('{}/{} images available for download.'.format(
reddit.count_downloadable_images(), len(images)))
downloader = Downloader(reddit, destination, verbose)
downloader.download_files()