Ejemplo n.º 1
0
	def reset(self,type,id):
		self.target=[type,id]
		self.status='Start'
		self.downloader=Downloader()
		self.downloader.bind(status=self.setter('status'))
		self.clear_widgets()
		self.add_widget(self.downloader)
		self.working_dir=os.sep.join([self.bundle_dir,'plugins','processing','model_zoo',type+'s',id])
		os.makedirs(self.working_dir, exist_ok=True)
Ejemplo n.º 2
0
 def __init__(self,
              temp='temp',
              output='output',
              merge_only=False,
              pool_size=4,
              buffer_size=256,
              random_keys=False,
              logger=lambda *args, **kwargs: None):
     self.NCM = NeteaseCloudMusic(logger, random_keys)
     self.DL = Downloader(session=self.NCM.session,
                          pool_size=pool_size,
                          buffer_size=buffer_size)
     # Initalization of other classes
     self.temp, self.output, self.merge_only = temp, output, merge_only
     self.log = logger
     self.pool_size = pool_size
Ejemplo n.º 3
0
def main():
    downloader = Downloader()
    url = "https://www.starlist.pro/"
    page = "brawlers/"
    crawler = Crawler(url, downloader, page)
    content = crawler.download()
    crawler.save()
    def download_reddit(self):
        """
        Download images from subreddit.
        """
        try:
            self.reddit = Reddit(self.url_var.get(), self.pages_var.get())
        except RedditException:
            messagebox.showerror('Error', 'Please input valid link')
            return
        except TclError:
            messagebox.showerror('Error', 'Please input only whole numbers')
            return
        except Exception as e:
            print(e)
            messagebox.showerror('Error', 'Something went wrong with Reddit.')
            return

        try:
            self.downloader = Downloader(self.reddit,
                                         self.destination_var.get())
        except DownloaderException:
            messagebox.showerror('Error', 'Invalid download path')
            return
        except Exception as e:
            print(e)
            messagebox.showerror('Error',
                                 'Something went wrong with Downloader.')
            return

        self.btn_download.configure(text='Cancel',
                                    command=self.cancel_download)
        self.change_progress_label('Fetching data...')

        self.add_progress_bar()
        self.progress_bar.start()
        self.queue = Queue()

        self.reddit_listener = RedditListener()
        self.reddit.register(self.reddit_listener)

        self.downloader_listener = DownloaderListener()
        self.downloader.register(self.downloader_listener)

        DownloadThread(self.queue, self.reddit, self.downloader).start()
        self.root.after(100, self.process_queue)
Ejemplo n.º 5
0
    def get_episode_video_file_name(self, dashed_name):
        url = f'https://www.dramacool9.co/{dashed_name}/'
        original_video_url = self.get_original_episode_video_url(url)

        path = os.path.dirname(os.path.abspath(__file__))
        file_parent_path = path.replace('utils', 'temp')
        file_name = uuid.uuid4().hex[:16] + '.mp4'
        target_file = file_parent_path + '/' + file_name

        Downloader(url=original_video_url,
                   file_parent_path=file_parent_path,
                   file_name=file_name).download_file_from_source()
        uploader_tool = Uploader()
        generated_video_url_from_cloud = uploader_tool.upload_file_to_cloud(
            source_file_name=target_file, destination_blob_name=file_name)

        if generated_video_url_from_cloud:
            uploader_tool.delete_file_from_project(
                source_file_name=target_file)
            return file_name
Ejemplo n.º 6
0
        def wrapper(id, quality='lossless', folder=None, merge_only=False):
            if not folder:
                folder = self.GenerateDownloadPath(id=id, folder=folder)
            if not self.merge_only:
                queue_func(id, quality, folder)
                self.DL.wait(func=self.ShowDownloadStatus)
                # Queue then wait
            self.log(strings.INFO_BATCH_DOWNLOAD_COMPLETE)

            def merge(subfolder):
                self.FormatLyrics(subfolder)
                self.FormatSong(subfolder)

            # Start merging every subfolder via threadpool,where it's a mutation of the Downloader
            pool = Downloader(worker=PoolWorker, pool_size=16)
            for sub in os.listdir(folder):
                target = os.path.join(folder, sub)
                pool.append(merge, target)
            pool.wait()
Ejemplo n.º 7
0
 def __init__(self):
     download_request = Downloader.get(MODEL_URL)
     self.model_path = download_request.path
     self.call_model = True
     self.predictions = []
class RedditApp(Frame):
    def __init__(self, master=None):
        """
        Initialize the main application Frame and load it with other widgets.
        """
        super().__init__(master)
        self.root = master
        self.pack()
        self.setup_main_frame()
        self.create_menubar()
        self.create_widgets()

    def setup_main_frame(self):
        self.root.resizable(0, 0)
        self.root.wm_title('(sub)Reddit Downloader')

    def create_menubar(self):
        menubar = Menu(self.root)
        menubar.add_command(label='About', command=AboutWindow.about)
        menubar.add_command(label='Exit', command=self.root.quit)
        self.root.config(menu=menubar)

    def create_widgets(self):
        """
        Create widgets to populate the applications frame.
        """
        # URL, Pages, Destination
        paths_frame = Frame(self)

        lbl_url = Label(paths_frame, text='URL:')
        lbl_pages = Label(paths_frame, text='Pages:')
        lbl_destination = Label(paths_frame, text='Destination:')

        lbl_pages_help = Label(paths_frame,
                               text='(Leave zero to crawl all pages)')

        self.url_var = StringVar()
        self.pages_var = IntVar()
        self.destination_var = StringVar()

        url = Entry(paths_frame, textvariable=self.url_var)
        pages = Entry(paths_frame, textvariable=self.pages_var)
        destination = Entry(paths_frame, textvariable=self.destination_var)

        btn_chooser = Button(paths_frame,
                             text='Destination',
                             command=self.choose_directory)

        pad_x = 5
        pad_y = 7

        r = 0
        lbl_url.grid(row=r, column=0, sticky=E, padx=pad_x, pady=pad_y)
        url.grid(row=r, column=1)

        r += 1
        lbl_pages.grid(row=r, column=0, sticky=E)
        pages.grid(row=r, column=1, padx=pad_x, pady=pad_y)
        lbl_pages_help.grid(row=r, column=2)

        r += 1
        lbl_destination.grid(row=r, column=0, sticky=E)
        destination.grid(row=r, column=1, padx=pad_x, pady=pad_y)
        btn_chooser.grid(row=r, column=2, sticky=E + W)

        paths_frame.pack(side=TOP, padx=10, pady=10)

        # Download button
        download_frame = Frame(self)

        self.btn_download = Button(download_frame,
                                   text='Download',
                                   command=self.download_reddit)
        self.btn_download.pack(padx=10, pady=10)

        download_frame.pack(side=TOP)

        # Progress label
        progress_info = Frame(self)

        self.lbl_progress_info = Label(progress_info, text='')
        self.lbl_progress_info.pack(padx=10, pady=10)

        progress_info.pack(side=TOP)

    def add_progress_bar(self):
        """
        Add progress bar to root frame.
        """
        self.progress_frame = Frame(self)

        self.progress_bar = Progressbar(self.progress_frame,
                                        orient='horizontal',
                                        length=400,
                                        mode='indeterminate')
        self.progress_bar.pack(padx=10, pady=10)
        self.progress_frame.pack(side=TOP)

    def remove_progress_bar(self):
        """
        Remove progress bare from root frame.
        """
        self.progress_bar.destroy()
        self.progress_frame.destroy()

    def change_progress_label(self, text):
        """
        Change text of progress bar label.
        """
        self.lbl_progress_info.configure(text=text, fg='red', font=(font.BOLD))

    def choose_directory(self):
        """
        Update the destination path entry filed with the chosen path.
        """
        destination_path = filedialog.askdirectory(initialdir='~')
        self.destination_var.set(destination_path)

    def download_reddit(self):
        """
        Download images from subreddit.
        """
        try:
            self.reddit = Reddit(self.url_var.get(), self.pages_var.get())
        except RedditException:
            messagebox.showerror('Error', 'Please input valid link')
            return
        except TclError:
            messagebox.showerror('Error', 'Please input only whole numbers')
            return
        except Exception as e:
            print(e)
            messagebox.showerror('Error', 'Something went wrong with Reddit.')
            return

        try:
            self.downloader = Downloader(self.reddit,
                                         self.destination_var.get())
        except DownloaderException:
            messagebox.showerror('Error', 'Invalid download path')
            return
        except Exception as e:
            print(e)
            messagebox.showerror('Error',
                                 'Something went wrong with Downloader.')
            return

        self.btn_download.configure(text='Cancel',
                                    command=self.cancel_download)
        self.change_progress_label('Fetching data...')

        self.add_progress_bar()
        self.progress_bar.start()
        self.queue = Queue()

        self.reddit_listener = RedditListener()
        self.reddit.register(self.reddit_listener)

        self.downloader_listener = DownloaderListener()
        self.downloader.register(self.downloader_listener)

        DownloadThread(self.queue, self.reddit, self.downloader).start()
        self.root.after(100, self.process_queue)

    def process_queue(self):
        """
        Stop the progress bar if the thread has yielded control. If download was
        canceled, re-enable download button only after thread has yielded control.
        Reset progress bar and download button, and unregister all listeners when
        downloading is finished or canceled.
        """
        try:
            msg = self.queue.get(0)

            if self.btn_download['text'] == 'Cancel':
                # self.reddit.fetch = True
                # self.downloader.downloading = True

                self.btn_download.configure(text='Download',
                                            command=self.download_reddit)
                self.remove_progress_bar()
                self.change_progress_label('Download finished.')

            if self.btn_download['state'] == DISABLED:
                self.btn_download.configure(state=NORMAL)

            self.reddit.unregister()
            self.downloader.unregister()

            print(msg)
        except queue.Empty:
            if getattr(self.reddit_listener, 'fetched', None):
                print(self.reddit_listener.maximum)

                self.progress_bar.configure(mode='determinate')
                self.progress_bar['value'] = 0
                self.progress_bar['maximum'] = self.reddit_listener.maximum

                self.reddit_listener.fetched = False

            if (getattr(self.downloader_listener, 'currently_at', None)
                    and self.downloader.downloading):
                print(self.downloader_listener.currently_at)
                self.progress_bar[
                    'value'] = self.downloader_listener.currently_at

            self.root.after(100, self.process_queue)

    def cancel_download(self):
        """
        Cancel fetching or downloading process.
        """
        self.reddit.fetch = False
        self.downloader.downloading = False
        self.btn_download.configure(text='Download',
                                    command=self.download_reddit)
        self.remove_progress_bar()
        self.change_progress_label('Download canceled.')
        self.btn_download.configure(state=DISABLED)
Ejemplo n.º 9
0
    from pyblake2 import blake2b  # pylint: disable=import-error

logger = logging.getLogger(__name__)

config = ConfigParser()
config.read('schemas.ini')

# TODO etag support

# NOTE
# Consider allowing the application to launch
# when the core schema download fails. If that
# happens, disable any modifications and alert
# some slack channel to report the event.

openapis = Downloader()
openapis.download(
    config['openapi'].keys(),
    config['openapi'].values()
)
swaggers = Downloader()
swaggers.download(
    config['swagger'].keys(),
    config['swagger'].values()
)


class ControllerError(Exception):
    pass

Ejemplo n.º 10
0
class NCMFunctions():
    '''
        Functions using ncm_core.py,simplfied to download songs easily

            temp        :       Tempoary folder to stroe downloads
            output      :       Output folder to store music
            merge_only  :       If only find download file in the temp folder,then merge them without downloading
                                Applies to MutilWrapper ones
            pool_size   :       Cocurrent download worker count
            buffer_size :       Size of iter_content(chunk_size)'s chunk_size
            logger      :       For logging.Uses simple_logger from ncm.strings,leave empty for not logging
            random_keys     :       Whether uses random 2nd AES key or the constant 'mos9527ItoooItop' and its encypted RSA string
        Functions inside are well described,read them for more info.
    '''
    def __init__(self,
                 temp='temp',
                 output='output',
                 merge_only=False,
                 pool_size=4,
                 buffer_size=256,
                 random_keys=False,
                 logger=lambda *args, **kwargs: None):
        self.NCM = NeteaseCloudMusic(logger, random_keys)
        self.DL = Downloader(session=self.NCM.session,
                             pool_size=pool_size,
                             buffer_size=buffer_size)
        # Initalization of other classes
        self.temp, self.output, self.merge_only = temp, output, merge_only
        self.log = logger
        self.pool_size = pool_size

    def ShowDownloadStatus(self):
        '''
            Shows downloader info
        '''
        content = self.DL.report(strings.DEBUG_DOWNLOAD_STATUS(None))
        print(content, Cursor.UP(content.count('\n') + 1))
        time.sleep(1)

    def GenerateDownloadPath(self, id='', filename='', folder=''):
        '''
            Generates output path with its leaf folders and such.Similar to mktree

            Specify folder to change the root folder somewhere else
        '''
        def normalize(s):
            return s.replace(':', ':').replace('\n', '_').replace(
                '-', '_').replace(',', ',').replace('"', '“').replace(
                    '\'', '”').replace('/', '、').replace('\\', ',')

        filename = normalize(filename)
        folder = os.path.join(folder, str(id)) if folder else os.path.join(
            self.temp, str(id))
        result = os.path.join(folder, filename)
        path = os.path.split(result)
        if path[0]:
            if not os.path.exists(path[0]): os.makedirs(path[0])
        # Make the folder tree
        return result

    def QueueDownload(self, url, path):
        '''
            Queue a download
        '''
        self.DL.append(url, path)
        self.log(url, path, format=strings.INFO_QUEUED_DOWNLOAD)

    def QueueDownloadSongAudio(self, id, quality='lossless', folder=''):
        '''
            This will queue to download only the song's audio file (named 'audio.*') to a certain directory.
        '''
        info = self.NCM.GetSongInfo(song_id=id, quality=quality)
        if not info['code'] == 200: return
        filename = '{}.{}'.format('audio', info['data'][0]['type'])
        target = self.GenerateDownloadPath(filename=filename, folder=folder)
        url = info['data'][0]['url']
        self.QueueDownload(url, target)
        return target

    def QueueDownloadSongInfo(self, id, folder=''):
        '''
            This will write the song's metadata into JSON (named meta.json) and queue to download cover (named cover.jpg) to a certain directory.
        '''
        info = self.NCM.GetExtraSongInfo(id)
        self.QueueDownload(
            info['cover'],
            self.GenerateDownloadPath(filename='cover.jpg', folder=folder))
        # Queue to download cover image
        target = self.GenerateDownloadPath(filename='meta.json', folder=folder)
        open(target, mode='w', encoding='utf-8').write(json.dumps(info))
        # Writes metadata to JSON file

    def QueueDownloadAllSongsInAlbum(self, id, quality='lossless', folder=''):
        '''
            This will get every song's ID inside the album resovled

            Note that extra info will be stored inside the JSON that the api sent
        '''
        info = self.NCM.GetAlbumInfo(id)
        try:
            name, creator = info['title'], info['author']
            self.log(name,
                     creator,
                     format=strings.INFO_FETCHED_ALBUM_WITH_TOKEN)
        except Exception as e:
            self.log(e, format=strings.ERROR_INVALID_OPERATION)
        root = folder
        # Go through every track inside the JSON
        for track in info['songlist']:
            meta = {
                "title": track['name'],
                "cover": track['album']['picUrl'],
                "author": ' - '.join([a['name'] for a in track['artists']]),
                "album": track['album']['name'],
                "album_id": track['album']['id'],
                "artist_id":
                ' - '.join([str(a['id']) for a in track['artists']])
            }
            # Generates meta header,and saves them
            track_id = track['id']
            track_root = self.GenerateDownloadPath(id=track_id, folder=root)
            open(self.GenerateDownloadPath(filename='meta.json',
                                           folder=track_root),
                 mode='w',
                 encoding='utf-8').write(json.dumps(meta))
            # Download lyrics
            self.DownloadAndFormatLyrics(track_id, folder=track_root)
            # Queue to download audio and cover image
            self.QueueDownloadSongAudio(track_id, quality, track_root)
            self.QueueDownload(
                meta['cover'],
                self.GenerateDownloadPath(filename='cover.jpg',
                                          folder=track_root))

    def QueueDownloadAllSongsInPlaylist(self,
                                        id,
                                        quality='lossless',
                                        folder=''):
        '''
            This will try and get all songs' ID and such in the playlist,then try to download them.

            The strcuture is very similar to the album one,but with some minor twists
        '''
        info = self.NCM.GetPlaylistInfo(id)
        try:
            name, creator = info['playlist']['name'], info['playlist'][
                'creator']['nickname']
            self.log(name,
                     creator,
                     format=strings.INFO_FETCHED_PLAYLIST_WITH_TOKEN)
        except Exception as e:
            self.log(e, format=strings.ERROR_INVALID_OPERATION)
        root = folder
        # Go through every track inside the JSON
        for track in info['playlist']['tracks']:
            meta = {
                "title": track['name'],
                "cover": track['al']['picUrl'],
                "author": ' - '.join([a['name'] for a in track['ar']]),
                "album": track['al']['name'],
                "album_id": track['al']['id'],
                "artist_id": ' - '.join([str(a['id']) for a in track['ar']])
            }
            # Generates meta header,and saves them
            track_id = track['id']
            track_root = self.GenerateDownloadPath(id=track_id, folder=root)
            open(self.GenerateDownloadPath(filename='meta.json',
                                           folder=track_root),
                 mode='w',
                 encoding='utf-8').write(json.dumps(meta))
            # Download lyrics
            self.DownloadAndFormatLyrics(track_id, folder=track_root)
            # Queue to download audio and cover image
            self.QueueDownloadSongAudio(track_id, quality, track_root)
            self.QueueDownload(
                meta['cover'],
                self.GenerateDownloadPath(filename='cover.jpg',
                                          folder=track_root))

    def DownloadSongLyrics(self, id, folder=''):
        '''
            This will only write the song's lyrics into JSON (named lyrics.json) to a certain directory.
        '''
        lyrics = self.NCM.GetSongLyrics(id)
        target = self.GenerateDownloadPath(filename='lyrics.json',
                                           folder=folder)
        open(target, mode='w', encoding='utf-8').write(json.dumps(lyrics))
        # Writes lyrics to JSON file

    def LoadLyrics(self, folder):
        '''
            Converts the lyrics JSON format into a dictionary
        '''
        lyrics = self.GenerateDownloadPath(filename='lyrics.json',
                                           folder=folder)
        # Locate lyric file
        if not os.path.exists(lyrics):
            self.log('lyrics.json', format=strings.ERROR_FILE_MISSING)
            return
        lyrics = json.loads(open(lyrics, encoding='utf-8').read())
        # Load lyrics into json file
        lrc_lists = [
            lyrics[lrc]['lyric'] for lrc in lyrics.keys()
            if lrc in ['lrc', 'tlyric'] and 'lyric' in lyrics[lrc].keys()
        ]
        # List all lyrics.Translation is optional
        timestamp_regex = r"(?:\[)(\d{2,}:\d{2,}.\d{2,})(?:\])(.*)"
        lyrics_combined = {}
        for lrc_ in lrc_lists:
            if not lrc_:
                continue
            lrc_lrc = lrc_.split('\n')
            # Split lyrics into an array
            for lrc in lrc_lrc:
                result = re.search(timestamp_regex, lrc)
                if not result:
                    continue
                timestamp, content = result.group(1), result.group(2)
                if not timestamp in lyrics_combined.keys():
                    lyrics_combined[timestamp] = []
                lyrics_combined[timestamp].append(content)

        return lyrics_combined

    def FormatLyrics(self, folder, export=''):
        '''
            Coverts a dictionary with timestamps as keys and lyrics as items into LRC lyrics
        '''
        export = export if export else self.output
        lyrics = self.LoadLyrics(folder)
        if not lyrics:
            return
        meta = self.GenerateDownloadPath(filename='meta.json', folder=folder)
        # Locate meta file
        if not os.path.exists(meta):
            self.log('meta.json', format=strings.ERROR_FILE_MISSING)
            return
        meta = json.loads(open(meta, encoding='utf-8').read())
        # Load JSON for future purposes
        lrc = f"""[ti:{meta['title']}]
[al:{meta['album']}]
[au:{meta['author']}]
[re:PyNCM]"""

        for timestamp in lyrics.keys():
            newline = '[%s]' % timestamp
            newline += '\t'.join(lyrics[timestamp])
            lrc += '\n' + newline
        path = self.GenerateDownloadPath(filename='{1} - {0}.{2}'.format(
            meta['title'], meta['author'], 'lrc'),
                                         folder=export)
        open(path, mode='w', encoding='utf-8').write(lrc)
        self.log(path, format=strings.INFO_EXPORT_COMPLETE)
        return path

    def FormatSong(self, folder, export=''):
        '''
            'Format' a song's metadata with the infomation given in folder

            Call once all download tasks is finished

            Suppports MP3,M4A(ALAC/MP4),FLAC
        '''
        export = export if export else self.output
        folder = str(folder)
        audio = [
            f for f in os.listdir(folder)
            if f.split('.')[-1] in ['mp3', 'm4a', 'flac']
        ]
        if not audio: return
        audio = audio[-1]
        audio = self.GenerateDownloadPath(filename=audio, folder=folder)
        format = audio.split('.')[-1]
        # Locate audio file,uses last match
        meta = self.GenerateDownloadPath(filename='meta.json', folder=folder)
        # Locate meta file
        cover = self.GenerateDownloadPath(filename='cover.jpg', folder=folder)
        # Locate cover image

        if not audio:
            self.log('audio.m4a / audio.mp3 / audio.flac',
                     format=strings.ERROR_FILE_MISSING)
            return
        if not os.path.exists(meta):
            self.log('meta.json', format=strings.ERROR_FILE_MISSING)
            return
        if not os.path.exists(cover):
            self.log('cover.jpg', format=strings.WARN_FILE_MISSING)

        meta = json.loads(open(meta, encoding='utf-8').read())

        def write_keys(song):
            # Write metadatas
            song['title'] = meta['title']
            song['artist'] = meta['author']
            song['album'] = meta['album']
            song.save()

        if format == 'm4a':
            # Process m4a files: Apple’s MP4 (aka M4A, M4B, M4P)
            song = easymp4.EasyMP4(audio)
            write_keys(song)
            if os.path.exists(cover):
                song = MP4(audio)
                # Write cover image
                song['covr'] = [MP4Cover(open(cover, 'rb').read())]
                song.save()
        elif format == 'mp3':
            # Process mp3 files: MPEG audio stream information and tags
            song = EasyMP3(audio)
            write_keys(song)
            if os.path.exists(cover):
                song = ID3(audio)
                # Write cover image
                song.add(
                    APIC(encoding=3,
                         mime='image/jpeg',
                         type=3,
                         desc='Cover',
                         data=open(cover, 'rb').read()))
                song.save()
        elif format == 'flac':
            # Process FLAC files:Free Lossless Audio Codec
            song = FLAC(audio)
            write_keys(song)
            if os.path.exists(cover):
                pic = Picture()
                pic.data = open(cover, 'rb').read()
                pic.mime = 'image/jpeg'
                song.add_picture(pic)
                song.save()

        # Rename & move file
        savename = '{1} - {0}.{2}'.format(meta['title'], meta['author'],
                                          format)
        try:
            path = self.GenerateDownloadPath(filename=savename, folder=export)
            shutil.copy(audio, path)
            self.log(path, format=strings.INFO_EXPORT_COMPLETE)
        except Exception as e:
            self.log(e, format=strings.ERROR_INVALID_OPERATION)
        return path

    def DownloadSongAudio(self, id, quality='lossless', folder=''):
        '''
            Download Audio,and wait for it to finish
        '''
        if not folder:
            folder = self.GenerateDownloadPath(id=id, folder=folder)
        self.QueueDownloadSongAudio(id, quality, folder)
        self.DL.wait(func=self.ShowDownloadStatus)
        self.log(strings.INFO_BATCH_DOWNLOAD_COMPLETE)
        return True

    def DownloadSongInfo(self, id, folder=''):
        '''
            Download info,and wait for it to finish
        '''
        if not folder:
            folder = self.GenerateDownloadPath(id=id, folder=folder)
        self.QueueDownloadSongInfo(id, folder)
        self.DL.wait(func=self.ShowDownloadStatus)
        self.log(strings.INFO_BATCH_DOWNLOAD_COMPLETE)
        return True

    def DownloadAndFormatLyrics(self, id, folder=''):
        '''
            Both download,and format the lyrics
        '''
        if not folder:
            folder = self.GenerateDownloadPath(id=id, folder=folder)
        self.DownloadSongLyrics(id, folder)
        self.FormatLyrics(folder)
        return True

    def DownloadAllInfo(self, id, quality='lossless', folder=''):
        '''
            Download everything we need,then wait for download to finish.
        '''
        if not folder:
            folder = self.GenerateDownloadPath(id=id, folder=folder)
        # Download meta and cover.Note that cover is downloaded asynchronously
        self.QueueDownloadSongInfo(id, folder)
        # Download lyrics
        self.DownloadAndFormatLyrics(id)
        # Download song.Not that audio file is downloaded asynchronously
        self.QueueDownloadSongAudio(id, quality, folder)
        self.DL.wait(func=self.ShowDownloadStatus)
        self.log(strings.INFO_BATCH_DOWNLOAD_COMPLETE)
        return True

    def DownloadAllAndMerge(self, id, quality='lossless', folder=''):
        '''
            This will merge the infomation once download is finished
        '''
        if not folder:
            folder = self.GenerateDownloadPath(id=id, folder=folder)
        self.DownloadAllInfo(id, quality, folder)
        self.DownloadAndFormatLyrics(id, folder)
        self.FormatSong(folder)
        return True

    def Login(self, username, password):
        '''
        Login method,equals to ncm_core.UpdateLoginInfo
        '''
        return self.NCM.UpdateLoginInfo(username, password)

    def MutilWrapper(self, queue_func):
        '''
            Wraps around a queue method.Once downloaded,
            
            perform the merging proceess for each and every sub folder
        '''
        def wrapper(id, quality='lossless', folder=None, merge_only=False):
            if not folder:
                folder = self.GenerateDownloadPath(id=id, folder=folder)
            if not self.merge_only:
                queue_func(id, quality, folder)
                self.DL.wait(func=self.ShowDownloadStatus)
                # Queue then wait
            self.log(strings.INFO_BATCH_DOWNLOAD_COMPLETE)

            def merge(subfolder):
                self.FormatLyrics(subfolder)
                self.FormatSong(subfolder)

            # Start merging every subfolder via threadpool,where it's a mutation of the Downloader
            pool = Downloader(worker=PoolWorker, pool_size=16)
            for sub in os.listdir(folder):
                target = os.path.join(folder, sub)
                pool.append(merge, target)
            pool.wait()

        return wrapper

    def DownloadAllSongsInPlaylistAndMerge(self,
                                           id,
                                           quality='lossless',
                                           folder=None,
                                           merge_only=False):
        '''
            Downloads all songs in a playlist into selected folder

                id          :       Song ID
                quality     :       Song Quality
                folder      :       Output folder
                merge_only  :       Merge by id from 'folder',Only use this if you 
                                    Have finished downloading first
        '''
        return self.MutilWrapper(self.QueueDownloadAllSongsInPlaylist)(
            id, quality, folder, merge_only)

    def DownloadAllSongsInAlbumAndMerge(self,
                                        id,
                                        quality='lossless',
                                        folder=None,
                                        merge_only=False):
        '''
            Downloads all albums in a playlist into selected folder

                id          :       Song ID
                quality     :       Song Quality
                folder      :       Output folder
                merge_only  :       Merge by id from 'folder',Only use this if you 
                                    Have finished downloading first
        '''
        return self.MutilWrapper(self.QueueDownloadAllSongsInAlbum)(id,
                                                                    quality,
                                                                    folder,
                                                                    merge_only)
Ejemplo n.º 11
0
def main():
    download_request = Downloader.get(MODEL_URL)
    model_path = download_request.path
    graph, score, expand = load_frozenmodel(model_path)
    detection(graph, score, expand, True)
Ejemplo n.º 12
0
 def __init__(self):
     self.downloader = Downloader()
Ejemplo n.º 13
0
class DownloadService:
    def __init__(self):
        self.downloader = Downloader()

    def download_with_url(self, url):
        return self.downloader.download(url=url)
Ejemplo n.º 14
0
    cleaned_corpus = d.clean_text(algorithm='w2v')

    w = GloveEmbedding(cleaned_corpus)
    w.start()
    q = QueryBuilder()
    res = q.evaluate_word_embedding_scores(inv_index=i, embeddings=w)
    return res


if __name__ == '__main__':

    existing_files = set(os.listdir('data'))
    cran_files = {'cranqrel', 'cran.qry', 'cran.all.1400'}

    if not cran_files <= existing_files:
        d = Downloader()
        d.download()
        d.untar()

    i = InvertedIndex()
    i.build_inverted_index()
    i.build_vsm()

    d = DataPolisher()
    cleaned_corpus = d.clean_text(algorithm='w2v')

    w = GloveEmbedding(cleaned_corpus)
    w.start()
    q = QueryBuilder()
    res = q.evaluate_word_embedding_scores(inv_index=i, embeddings=w)
    print(res)
Ejemplo n.º 15
0
class Installer(BoxLayout):
	"""docstring for Installer."""

	data=ObjectProperty()
	status=StringProperty()
	target=ListProperty()
	bundle_dir = rootpath.detect(pattern='main.py')
	# Builder.load_file(bundle_dir +os.sep+'ui'+os.sep+'demo.kv')

	def __init__(self):
		super(Installer, self).__init__()
		self.bind(status=self.on_status_changed)
		self.register_event_type('on_success')

	def reset(self,type,id):
		self.target=[type,id]
		self.status='Start'
		self.downloader=Downloader()
		self.downloader.bind(status=self.setter('status'))
		self.clear_widgets()
		self.add_widget(self.downloader)
		self.working_dir=os.sep.join([self.bundle_dir,'plugins','processing','model_zoo',type+'s',id])
		os.makedirs(self.working_dir, exist_ok=True)

	def on_status_changed(self,ins,value):
		if 'Fail' in value:
			self.on_failure()
		elif value=='Unzip Finished':
			self.dispatch('on_success')

	def on_success(self):
		self.clear_widgets()
		self.add_widget(Label(text='[color=00ff33]Succesfully installed '+' : '.join(self.target)+'[/color]',text_size=self.size,valign='center',markup = True,font_size=18))

	def on_failure(self):
		self.clear_widgets()
		try: shutil.rmtree(self.working_dir)
		except: pass
		self.add_widget(Label(text='[color=ff3333]'+self.status+'[/color]',text_size=self.size,valign='center',markup = True,font_size=18))

	def install_model(self,id,url,callbacks=None):
		self.reset('model',id)
		file_path=self.working_dir+os.sep+url.split('/')[-1]
		self.downloader.download(url,file_path,extract_dir='./',callbacks=callbacks)

	def install_demo(self,id,url,callbacks=None):
		self.reset('demo',id)
		file_path=self.working_dir+os.sep+url.split('/')[-1]
		self.downloader.file_name='Demo('+id+')'
		self.downloader.download(url,file_path,extract_dir='./',callbacks=callbacks)

	def install_weight(self,model,url,callbacks=None):
		self.reset('weight',model)
		file_path=self.working_dir+os.sep+url.split('/')[-1]
		self.downloader.file_name='Weight('+url.split('/')[-1]+')'
		self.downloader.download(url,file_path,callbacks=callbacks)

	def copy_config(self,demo_id,model_id,model_config):
		path1=os.sep.join([self.bundle_dir,'plugins','processing','model_zoo','demos',demo_id,model_config])
		path2=os.sep.join([self.bundle_dir,'plugins','processing','model_zoo','configs',model_id,model_config])
		os.makedirs(os.path.dirname(path2),exist_ok=True)
		shutil.copyfile(path1,path2)

	def install_demo_full(self,id,meta_info):
		model_id=meta_info['model']
		def t3():
			if 'model_config' in meta_info:
				self.copy_config(id,model_id,meta_info['model_config'])
		def t2():
			self.install_demo(id,meta_info['demo_url'],callbacks=t3)
		def t1():
			if 'model_url' in meta_info:
				self.install_model(model_id,meta_info['model_url'],callbacks=t2)
			else:
				t2()
		if 'weight_url' in meta_info:
			self.install_weight(model_id,meta_info['weight_url'],callbacks=t1)
    """
    parser = ArgumentParser(description='reddit /r downloader')

    parser.add_argument('-v',
                        '--verbose',
                        help='display download progress and information',
                        action='store_true')
    parser.add_argument('-p', help='crawl P number of pages', type=int)
    parser.add_argument('URL', help='source link')
    parser.add_argument('directory', help='destination directory')

    args = parser.parse_args()

    return (args.verbose, args.p, args.URL, args.directory)


if __name__ == '__main__':
    verbose, pages, url, destination = parse_arguments()

    if verbose: print('Fetching available links...')
    reddit = Reddit(url, pages)
    reddit.get_all_posts()
    images = reddit.images

    if verbose:
        print('{}/{} images available for download.'.format(
            reddit.count_downloadable_images(), len(images)))

    downloader = Downloader(reddit, destination, verbose)
    downloader.download_files()