def main(): import sys site = MetMuseum( sys.argv[1], # "https://www.metmuseum.org/art/collection/search#!?material=Archery&offset=0&perPage=20&sortBy=Relevance&sortOrder=asc&searchField=All&pageSize=0" # "https://www.metmuseum.org/art/collection/search/35684?searchField=All&sortBy=Relevance&what=Archery&ft=*&offset=0&rpp=20&pos=13" ) downloader = Downloader(save_dir=".", fetcher=site.fetcher) downloader.add_task( site.tasks, background=True, ) downloader.join(background=False)
def _user_home_run(site, path_prefix=None): """ :type site: picktrue.sites.abstract.DummySite :type path_prefix: str or None """ path = site.dir_name if path_prefix is not None: path = os.path.join(path_prefix, path) downloader = Downloader(save_dir=path, fetcher=site.fetcher) downloader.add_task( site.tasks, background=True, ) downloader.join(background=True) return downloader
def huban_user(member_id, username, password, proxy): site = Pixiv(member_id, username, password, proxy=proxy) downloader = Downloader(fetcher=site.fetcher, save_dir=site.dir_name) downloader.add_task( site.tasks ) pk_logger.info("All task add...waiting for execution...") try: downloader.join() except KeyboardInterrupt: pk_logger.warn("Exiting...Press crtl+c again to force quit") downloader.stop() exit(0) else: pk_logger.info("All task done...Enjoy!")
def huban_board(url): site = HuaBanBoard(url) downloader = Downloader(fetcher=site.fetcher, save_dir=site.dir_name) downloader.add_task( site.tasks ) pk_logger.info("All task add...waiting for execution...") try: downloader.join() except KeyboardInterrupt: pk_logger.warn("Exiting...Press crtl+c again to force quit") downloader.stop() exit(0) else: pk_logger.info("All task done...Enjoy!")
def main(): import sys import time if len(sys.argv) <= 1: print( "Error, please add argument like: picktrue-metmuseum.exe <url_or_path>" ) url = sys.argv[1] if os.path.exists(url): urls = [line for line in open(url).readlines() if line] save_dir = os.path.abspath(url) save_dir = os.path.dirname(save_dir) else: save_dir = "." urls = [ url, ] # "https://www.metmuseum.org/art/collection/search#!?material=Archery&offset=0&perPage=20&sortBy=Relevance&sortOrder=asc&searchField=All&pageSize=0" # "https://www.metmuseum.org/art/collection/search/35684?searchField=All&sortBy=Relevance&what=Archery&ft=*&offset=0&rpp=20&pos=13" sites = [MetMuseum(target) for target in urls] def task_iter(): for site in sites: for task in site.tasks: yield task downloader = Downloader(save_dir=save_dir, fetcher=sites[0].fetcher) downloader.add_task( task_iter=task_iter(), background=True, ) downloader.join(background=True) while not downloader.done: time.sleep(5) print(downloader.describe())