def generate_aiohttp_session_config(**kwargs):
params = {
'timeout': 50,
'verify_ssl': config.get('debug_mode'),
'proxy': config.get('proxy')
}
params.update(kwargs)
return params
def launch_downloader(project_name, urls=dict(), output_path=config.get('output_path')):
global proj_dir, total_task_num
proj_dir = output_path + '/' + project_name
events = []
for k in urls.keys():
for u in urls[k]:
logger.debug('create download {file_name}@{chapter_name} => {url}'.format(
file_name=get_file_name(u),
chapter_name=k,
url=u
))
events.append(download(u, k))
total_task_num = len(events)
if not os.path.exists(proj_dir):
os.mkdir(proj_dir)
update_window_title(mode='Download')
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(events))
async def download_file(url, retry_count=0):
async with aiohttp.ClientSession(headers=config.get('header')) as session:
try:
async with session.get(**utils.generate_aiohttp_session_config(url=url)) as resp:
logger.debug('{url} response code: {code}'.format(code=resp.status, url=url))
return await resp.content.read()
except Exception as err:
logger.error('download failed url = %s (%s) retry count = %s' % (url, str(err), retry_count))
if config.get('debug_mode'):
print(traceback.print_exc())
if retry_count == max_retry_count:
return
retry_count = retry_count + 1
return await download_file(url, retry_count)
def main():
version.show_welcome()
logger.info('App launch')
try:
logger.debug('Load configure file')
config.load_file('config.yml')
except Exception as err:
logger.warning('Failed to load configure file (%s)', str(err))
if len(sys.argv) > 1:
index_page_url = sys.argv[1]
else:
logger.info('Ask user for enter url')
index_page_url = get_comic_index_page_url()
logger.info('Fetch chapter list')
update_window_title(mode='Fetching')
title, chapter_page_list = aio_chapter_list.parse_comic_chapter_list(
index_page_url)
if len(chapter_page_list) == 0:
logger.critical('comic url invalid or no chapter avaliable.')
exit()
update_window_title(mode='Fetching', msg=title)
logger.info('Comic title: %s' % title)
logger.info('Fetch each chapter page')
list_new = {}
list_new = aio_chapter_list.fetch_all_image_list(chapter_page_list)
output_path = config.get('output_path')
try:
if sys.argv[2]:
output_path = sys.argv[2]
except:
pass
if not os.path.exists(output_path):
os.mkdir(output_path)
aiodownloader.launch_downloader(title, list_new, output_path)
logger.info('All download complete')
def main():
version.show_welcome()
logger.info('App launch')
try:
logger.debug('Load configure file')
config.load_file('config.yml')
except Exception as err:
logger.warning('Failed to load configure file (%s)', str(err))
if len(sys.argv) > 1:
index_page_url = sys.argv[1]
else:
logger.info('Ask user for enter url')
index_page_url = get_comic_index_page_url()
logger.info('Fetch chapter list')
update_window_title(mode='Fetching')
title, chapter_page_list, cover = aio_chapter_list.parse_comic_chapter_list(index_page_url)
if len(chapter_page_list) == 0:
logger.critical('comic url invalid or no chapter avaliable.')
exit()
update_window_title(mode='Fetching', msg=title)
logger.info('Comic title: %s' % title)
logger.info('Fetch each chapter page')
list_new = {}
list_new = aio_chapter_list.fetch_all_image_list(chapter_page_list)
list_new['cover'] = [cover]
output_path = config.get('output_path')
try:
if sys.argv[2]:
output_path = sys.argv[2]
except:
pass
if not os.path.exists(output_path):
os.mkdir(output_path)
aiodownloader.launch_downloader(title, list_new, output_path)
logger.info('All download complete')
if config.get('epub') == True:
logger.info('Making epub...')
with jpg2epub.Jpg2Epub(title, file_name=output_path + "/" + title + ".epub", creator="动漫之家") as j2e:
coverDir = output_path + "/" + title + "/cover.jpg"
j2e.add_image_file(coverDir)
def eachFile(filepath):
pathDir = os.listdir(filepath)
for s in pathDir:
newDir=os.path.join(filepath,s)
if os.path.isfile(newDir):
if newDir != coverDir:
if os.path.splitext(newDir)[1]==".jpg":
j2e.add_image_file(newDir)
pass
pass
else:
eachFile(newDir)
eachFile(output_path + "/" + title)
logger.info('Done.')
import sys, os
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../'))
from dcdownloader import aiodownloader, base_logger, config, aio_chapter_list, version, jpg2epub
from dcdownloader.utils import update_window_title
import time, json, os
if config.get('debug_mode') == True:
base_logger.logging_level=base_logger.logging.DEBUG
logger = base_logger.getLogger(__name__)
def get_comic_index_page_url():
print()
url = None
while not url:
url = input('Index page url of target comic: ')
print()
return url
def main():
version.show_welcome()
logger.info('App launch')
try:
logger.debug('Load configure file')
config.load_file('config.yml')
except Exception as err:
logger.warning('Failed to load configure file (%s)', str(err))
import asyncio, aiohttp, aiofiles, os, traceback
from dcdownloader import base_logger, config, utils
from dcdownloader.utils import update_window_title
logger = base_logger.getLogger(__name__)
# 并发数
sema = asyncio.Semaphore(config.get('downloader_max_connection_number'))
max_retry_count = config.get('downloader_max_retry_number')
get_file_name = lambda x: x.split('/')[-1]
proj_dir = ''
total_task_num = 0
completed_task_num = 0
def update_count():
global completed_task_num
completed_task_num += 1
update_window_title(mode='Download',
msg='{0}/{1} {2}%'.format(
completed_task_num, total_task_num,
int((completed_task_num / total_task_num) * 100)))
async def download_file(url, retry_count=0):
async with aiohttp.ClientSession(headers=config.get('header')) as session:
try:
async with session.get(**utils.generate_aiohttp_session_config(
