def generate_aiohttp_session_config(**kwargs): params = { 'timeout': 50, 'verify_ssl': config.get('debug_mode'), 'proxy': config.get('proxy') } params.update(kwargs) return params
def launch_downloader(project_name, urls=dict(), output_path=config.get('output_path')): global proj_dir, total_task_num proj_dir = output_path + '/' + project_name events = [] for k in urls.keys(): for u in urls[k]: logger.debug('create download {file_name}@{chapter_name} => {url}'.format( file_name=get_file_name(u), chapter_name=k, url=u )) events.append(download(u, k)) total_task_num = len(events) if not os.path.exists(proj_dir): os.mkdir(proj_dir) update_window_title(mode='Download') loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.wait(events))
async def download_file(url, retry_count=0): async with aiohttp.ClientSession(headers=config.get('header')) as session: try: async with session.get(**utils.generate_aiohttp_session_config(url=url)) as resp: logger.debug('{url} response code: {code}'.format(code=resp.status, url=url)) return await resp.content.read() except Exception as err: logger.error('download failed url = %s (%s) retry count = %s' % (url, str(err), retry_count)) if config.get('debug_mode'): print(traceback.print_exc()) if retry_count == max_retry_count: return retry_count = retry_count + 1 return await download_file(url, retry_count)
def main(): version.show_welcome() logger.info('App launch') try: logger.debug('Load configure file') config.load_file('config.yml') except Exception as err: logger.warning('Failed to load configure file (%s)', str(err)) if len(sys.argv) > 1: index_page_url = sys.argv[1] else: logger.info('Ask user for enter url') index_page_url = get_comic_index_page_url() logger.info('Fetch chapter list') update_window_title(mode='Fetching') title, chapter_page_list = aio_chapter_list.parse_comic_chapter_list( index_page_url) if len(chapter_page_list) == 0: logger.critical('comic url invalid or no chapter avaliable.') exit() update_window_title(mode='Fetching', msg=title) logger.info('Comic title: %s' % title) logger.info('Fetch each chapter page') list_new = {} list_new = aio_chapter_list.fetch_all_image_list(chapter_page_list) output_path = config.get('output_path') try: if sys.argv[2]: output_path = sys.argv[2] except: pass if not os.path.exists(output_path): os.mkdir(output_path) aiodownloader.launch_downloader(title, list_new, output_path) logger.info('All download complete')
def main(): version.show_welcome() logger.info('App launch') try: logger.debug('Load configure file') config.load_file('config.yml') except Exception as err: logger.warning('Failed to load configure file (%s)', str(err)) if len(sys.argv) > 1: index_page_url = sys.argv[1] else: logger.info('Ask user for enter url') index_page_url = get_comic_index_page_url() logger.info('Fetch chapter list') update_window_title(mode='Fetching') title, chapter_page_list, cover = aio_chapter_list.parse_comic_chapter_list(index_page_url) if len(chapter_page_list) == 0: logger.critical('comic url invalid or no chapter avaliable.') exit() update_window_title(mode='Fetching', msg=title) logger.info('Comic title: %s' % title) logger.info('Fetch each chapter page') list_new = {} list_new = aio_chapter_list.fetch_all_image_list(chapter_page_list) list_new['cover'] = [cover] output_path = config.get('output_path') try: if sys.argv[2]: output_path = sys.argv[2] except: pass if not os.path.exists(output_path): os.mkdir(output_path) aiodownloader.launch_downloader(title, list_new, output_path) logger.info('All download complete') if config.get('epub') == True: logger.info('Making epub...') with jpg2epub.Jpg2Epub(title, file_name=output_path + "/" + title + ".epub", creator="动漫之家") as j2e: coverDir = output_path + "/" + title + "/cover.jpg" j2e.add_image_file(coverDir) def eachFile(filepath): pathDir = os.listdir(filepath) for s in pathDir: newDir=os.path.join(filepath,s) if os.path.isfile(newDir): if newDir != coverDir: if os.path.splitext(newDir)[1]==".jpg": j2e.add_image_file(newDir) pass pass else: eachFile(newDir) eachFile(output_path + "/" + title) logger.info('Done.')
import sys, os sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) from dcdownloader import aiodownloader, base_logger, config, aio_chapter_list, version, jpg2epub from dcdownloader.utils import update_window_title import time, json, os if config.get('debug_mode') == True: base_logger.logging_level=base_logger.logging.DEBUG logger = base_logger.getLogger(__name__) def get_comic_index_page_url(): print() url = None while not url: url = input('Index page url of target comic: ') print() return url def main(): version.show_welcome() logger.info('App launch') try: logger.debug('Load configure file') config.load_file('config.yml') except Exception as err: logger.warning('Failed to load configure file (%s)', str(err))
import asyncio, aiohttp, aiofiles, os, traceback from dcdownloader import base_logger, config, utils from dcdownloader.utils import update_window_title logger = base_logger.getLogger(__name__) # 并发数 sema = asyncio.Semaphore(config.get('downloader_max_connection_number')) max_retry_count = config.get('downloader_max_retry_number') get_file_name = lambda x: x.split('/')[-1] proj_dir = '' total_task_num = 0 completed_task_num = 0 def update_count(): global completed_task_num completed_task_num += 1 update_window_title(mode='Download', msg='{0}/{1} {2}%'.format( completed_task_num, total_task_num, int((completed_task_num / total_task_num) * 100))) async def download_file(url, retry_count=0): async with aiohttp.ClientSession(headers=config.get('header')) as session: try: async with session.get(**utils.generate_aiohttp_session_config(