def download_chapters(app): app.progress = 0 bar = IncrementalBar('Downloading chapters', max=len(app.chapters)) if os.getenv('debug_mode') == 'yes': bar.next = lambda: None # Hide in debug mode bar.finish() else: bar.start() # end if if not app.output_formats: app.output_formats = {} # end if futures_to_check = [ app.crawler.executor.submit( download_chapter_body, app, chapter, ) for chapter in app.chapters ] for future in futures_to_check: result = future.result() if result: bar.clearln() logger.error(result) # end if bar.next() # end for bar.finish() print('Processed %d chapters' % len(app.chapters))
def download_chapters(app): downlod_cover(app) bar = IncrementalBar('Downloading chapters', max=len(app.chapters)) bar.start() if os.getenv('debug_mode') == 'yes': bar.next = lambda: None # Hide in debug mode # end if futures_to_check = { app.crawler.executor.submit( download_chapter_body, app, chapter, ): str(chapter['id']) for chapter in app.chapters } app.progress = 0 for future in futures.as_completed(futures_to_check): result = future.result() if result: bar.clearln() logger.error(result) # end if app.progress += 1 bar.next() # end for bar.finish() print('Downloaded %d chapters' % len(app.chapters))
def download_chapters(app): download_cover(app) bar = IncrementalBar('Downloading chapters', max=len(app.chapters)) bar.start() if os.getenv('debug_mode') == 'true': bar.next = lambda: None futures_to_check = { app.crawler.executor.submit( download_chapter_body, app, chapter, ): str(chapter['id']) for chapter in app.chapters } for future in futures.as_completed(futures_to_check): result = future.result() if result: bar.clearln() app.logger.error(result) bar.next() bar.finish()
def search_novels(app): if not app.crawler_links: return bar = IncrementalBar('Searching', max=len(app.crawler_links)) if os.getenv('debug_mode') == 'yes': bar.next = lambda n=1: None # Hide in debug mode else: bar.start() # end if # Add future tasks checked = {} futures_to_check = [] app.progress = 0 for link in app.crawler_links: crawler = crawler_list[link] if crawler in checked: logger.info('A crawler for "%s" already exists', link) bar.next() continue # end if checked[crawler] = True future = executor.submit(get_search_result, app, link, bar) futures_to_check.append(future) # end for # Resolve all futures combined_results = [item for f in futures_to_check for item in f.result()] # Process combined search results app.search_results = process_results(combined_results) bar.clearln() bar.finish() print('Found %d results' % len(app.search_results))
def search_novels(app): executor = futures.ThreadPoolExecutor(10) # Add future tasks checked = {} futures_to_check = {} for link in app.crawler_links: crawler = crawler_list[link] if crawler in checked: logger.info('A crawler for "%s" already exists', link) continue # end if checked[crawler] = True futures_to_check[ executor.submit( get_search_result, app.user_input, link ) ] = str(crawler) # end for bar = IncrementalBar('Searching', max=len(futures_to_check.keys())) bar.start() if os.getenv('debug_mode') == 'yes': bar.next = lambda: None # Hide in debug mode # end if # Resolve future tasks app.progress = 0 combined_results = [] for future in futures.as_completed(futures_to_check): combined_results += future.result() app.progress += 1 bar.next() # end for # Process combined search results app.search_results = process_results(combined_results) bar.clearln() bar.finish() print('Found %d results' % len(app.search_results)) executor.shutdown()
def download_chapters(app): # download or generate cover app.book_cover = download_cover(app) if not app.book_cover: app.book_cover = generate_cover(app) # end if if not app.book_cover: logger.warn('No cover image') # end if bar = IncrementalBar('Downloading chapters', max=len(app.chapters)) bar.start() if os.getenv('debug_mode') == 'yes': bar.next = lambda: None # Hide in debug mode # end if if not app.output_formats: app.output_formats = {} # end if futures_to_check = { app.crawler.executor.submit( download_chapter_body, app, chapter, ): str(chapter['id']) for chapter in app.chapters } app.progress = 0 for future in futures.as_completed(futures_to_check): result = future.result() if result: bar.clearln() logger.error(result) # end if app.progress += 1 bar.next() # end for bar.finish() print('Downloaded %d chapters' % len(app.chapters))