def download_chapters(app):
app.progress = 0
bar = IncrementalBar('Downloading chapters', max=len(app.chapters))
if os.getenv('debug_mode') == 'yes':
bar.next = lambda: None # Hide in debug mode
bar.finish()
else:
bar.start()
# end if
if not app.output_formats:
app.output_formats = {}
# end if
futures_to_check = [
app.crawler.executor.submit(
download_chapter_body,
app,
chapter,
) for chapter in app.chapters
]
for future in futures_to_check:
result = future.result()
if result:
bar.clearln()
logger.error(result)
# end if
bar.next()
# end for
bar.finish()
print('Processed %d chapters' % len(app.chapters))
def download_chapters(app):
downlod_cover(app)
bar = IncrementalBar('Downloading chapters', max=len(app.chapters))
bar.start()
if os.getenv('debug_mode') == 'yes':
bar.next = lambda: None # Hide in debug mode
# end if
futures_to_check = {
app.crawler.executor.submit(
download_chapter_body,
app,
chapter,
): str(chapter['id'])
for chapter in app.chapters
}
app.progress = 0
for future in futures.as_completed(futures_to_check):
result = future.result()
if result:
bar.clearln()
logger.error(result)
# end if
app.progress += 1
bar.next()
# end for
bar.finish()
print('Downloaded %d chapters' % len(app.chapters))
def download_chapters(app):
download_cover(app)
bar = IncrementalBar('Downloading chapters', max=len(app.chapters))
bar.start()
if os.getenv('debug_mode') == 'true':
bar.next = lambda: None
futures_to_check = {
app.crawler.executor.submit(
download_chapter_body,
app,
chapter,
): str(chapter['id'])
for chapter in app.chapters
}
for future in futures.as_completed(futures_to_check):
result = future.result()
if result:
bar.clearln()
app.logger.error(result)
bar.next()
bar.finish()
def search_novels(app):
if not app.crawler_links:
return
bar = IncrementalBar('Searching', max=len(app.crawler_links))
if os.getenv('debug_mode') == 'yes':
bar.next = lambda n=1: None # Hide in debug mode
else:
bar.start()
# end if
# Add future tasks
checked = {}
futures_to_check = []
app.progress = 0
for link in app.crawler_links:
crawler = crawler_list[link]
if crawler in checked:
logger.info('A crawler for "%s" already exists', link)
bar.next()
continue
# end if
checked[crawler] = True
future = executor.submit(get_search_result, app, link, bar)
futures_to_check.append(future)
# end for
# Resolve all futures
combined_results = [item for f in futures_to_check for item in f.result()]
# Process combined search results
app.search_results = process_results(combined_results)
bar.clearln()
bar.finish()
print('Found %d results' % len(app.search_results))
def search_novels(app):
executor = futures.ThreadPoolExecutor(10)
# Add future tasks
checked = {}
futures_to_check = {}
for link in app.crawler_links:
crawler = crawler_list[link]
if crawler in checked:
logger.info('A crawler for "%s" already exists', link)
continue
# end if
checked[crawler] = True
futures_to_check[
executor.submit(
get_search_result,
app.user_input,
link
)
] = str(crawler)
# end for
bar = IncrementalBar('Searching', max=len(futures_to_check.keys()))
bar.start()
if os.getenv('debug_mode') == 'yes':
bar.next = lambda: None # Hide in debug mode
# end if
# Resolve future tasks
app.progress = 0
combined_results = []
for future in futures.as_completed(futures_to_check):
combined_results += future.result()
app.progress += 1
bar.next()
# end for
# Process combined search results
app.search_results = process_results(combined_results)
bar.clearln()
bar.finish()
print('Found %d results' % len(app.search_results))
executor.shutdown()
def download_chapters(app):
# download or generate cover
app.book_cover = download_cover(app)
if not app.book_cover:
app.book_cover = generate_cover(app)
# end if
if not app.book_cover:
logger.warn('No cover image')
# end if
bar = IncrementalBar('Downloading chapters', max=len(app.chapters))
bar.start()
if os.getenv('debug_mode') == 'yes':
bar.next = lambda: None # Hide in debug mode
# end if
if not app.output_formats:
app.output_formats = {}
# end if
futures_to_check = {
app.crawler.executor.submit(
download_chapter_body,
app,
chapter,
): str(chapter['id'])
for chapter in app.chapters
}
app.progress = 0
for future in futures.as_completed(futures_to_check):
result = future.result()
if result:
bar.clearln()
logger.error(result)
# end if
app.progress += 1
bar.next()
# end for
bar.finish()
print('Downloaded %d chapters' % len(app.chapters))
