async def fetch_image_list(chapter_name, url):
with (await sema):
logger.debug('Starting fetch chapter page %s => %s', chapter_name, url)
async with aiohttp.ClientSession() as session:
async with session.get(**utils.generate_aiohttp_session_config(url=url)) as resp:
text = await resp.text()
logger.debug('Http response code %s => %s', chapter_name, resp.status)
jspacker_string = re.search(r'(eval\(.+\))', text).group()
jspacker_string = utils.decode_packed_codes(jspacker_string)
image_list = re.search(r'(\[.+\])', jspacker_string).group()
image_list = urllib.parse.unquote(image_list).replace('\\', '')
image_list = json.loads(image_list)
image_list_ = []
for u in image_list:
image_list_.append(base_url + u)
all_image_list.setdefault(chapter_name, image_list_)
processing_status['completed'] += 1
update_window_title(mode='Fetching', msg="{0}/{1} {2}%".format(
processing_status['completed'], processing_status['total'],
int( ( processing_status['completed'] / processing_status['total']) * 100 )) )
async def download_file(url, retry_count=0):
async with aiohttp.ClientSession(headers=config.get('header')) as session:
try:
async with session.get(**utils.generate_aiohttp_session_config(url=url)) as resp:
logger.debug('{url} response code: {code}'.format(code=resp.status, url=url))
return await resp.content.read()
except Exception as err:
logger.error('download failed url = %s (%s) retry count = %s' % (url, str(err), retry_count))
if config.get('debug_mode'):
print(traceback.print_exc())
if retry_count == max_retry_count:
return
retry_count = retry_count + 1
return await download_file(url, retry_count)
async def get():
async with aiohttp.ClientSession() as session:
async with session.get(**utils.generate_aiohttp_session_config(url=url)) as ret:
return await ret.text()
def test_generate_aiohttp_session_config():
out = utils.generate_aiohttp_session_config(key='value')
assert out['key'] == 'value'
assert out['proxy'] == dcdownloader.config.get('proxy')