def download(entity):
try:
entity['download_count'] += 1
entity['status'] = 1
entity['str_status'] = '대기'
LogicNormal.entity_update(entity)
url = 'http://webtoon.daum.net/data/pc/webtoon/viewer_images/%s' % (
entity['episode_id'])
data = requests.get(url).json()
entity['str_status'] = '분석'
LogicNormal.entity_update(entity)
dirname = ModelSetting.get('download_path')
if ModelSetting.get_bool('use_title_folder'):
dirname = os.path.join(
dirname,
Util.change_text_for_use_filename(entity['toon_title']))
#if not os.path.exists(dirname):
# os.makedirs(dirname)
tmp = u'%s %s %s' % (entity['episode_idx'].zfill(3),
entity['toon_title'], entity['episode_title'])
dirname = os.path.join(dirname,
Util.change_text_for_use_filename(tmp))
if not os.path.exists(dirname):
os.makedirs(dirname)
entity['filename'] = '%s.zip' % dirname
if os.path.exists(entity['filename']):
entity['status'] = 12
entity['str_status'] = '파일 있음'
LogicNormal.entity_update(entity)
else:
entity['str_status'] = '다운로드중'
LogicNormal.entity_update(entity)
count = len(data['data'])
for idx, tmp in enumerate(data['data']):
filename = os.path.join(dirname,
str(idx + 1).zfill(2) + '.jpg')
image_data = requests.get(tmp['url'],
headers=headers,
stream=True)
with open(filename, 'wb') as handler:
handler.write(image_data.content)
entity['str_status'] = '다운로드중 %s / %s' % (idx + 1, count)
entity['percent'] = int(100.0 * (idx + 1) / count)
LogicNormal.entity_update(entity)
Util.makezip(dirname)
entity['status'] = 11
entity['str_status'] = '완료'
LogicNormal.entity_update(entity)
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
entity['status'] = 2
entity['str_status'] = '실패'
if entity['download_count'] >= 5:
entity['status'] = 13
entity['str_status'] = '재시도초과'
LogicNormal.entity_update(entity)
ModelItem.save_as_dict(entity)
def download(self):
completed_flag = False
try:
self.set_status(u'분석중')
url = '{}/{}/{}'.format(
ModelSetting.get('{}_url'.format(self.module_logic.name)),
define[module_name]['url_prefix'], self.info['code'])
data = requests.get(url).text
tmp = ''.join(
re.compile(r'html_data\+\=\'(.*?)\'\;').findall(data))
html = ''.join(
[chr(int(x, 16)) for x in tmp.rstrip('.').split('.')])
image_list = re.compile(
r'img\ssrc="/img/loading-image.gif"\sdata\-\w{11}="(.*?)"'
).findall(html)
self.total_image_count = len(image_list)
self.refresh_status()
download_folder = ModelSetting.get(
'{}_download_folder'.format(module_name))
if ModelSetting.get_bool(
'{}_make_series_folder'.format(module_name)):
download_folder = os.path.join(
download_folder,
Util.change_text_for_use_filename(
self.info['series_title']))
self.savepath = os.path.join(
download_folder,
Util.change_text_for_use_filename(self.info['title']))
if not os.path.exists(self.savepath):
os.makedirs(self.savepath)
zip_extension = ModelSetting.get(
'{}_zip_extension'.format(module_name))
if ModelSetting.get(
'{}_use_zip'.format(module_name)) and os.path.exists(
self.savepath + '.' + zip_extension):
self.percent = 100
self.set_status(u'파일 있음')
self.savepath = self.savepath + '.' + zip_extension
completed_flag = True
return
self.set_status(u'다운로드중')
for idx, tmp in enumerate(image_list):
try:
ext = tmp.split('/')[-1].split('.')[-1]
if len(ext) > 4:
ext = 'jpg'
except:
pass
url = tmp
if not url.startswith('http'):
url = '%s%s' % (ModelSetting.get('{}_url'.format(
self.module_logic.name)), tmp)
filepath = os.path.join(self.savepath,
str(idx + 1).zfill(3) + '.' + ext)
ret = self.image_download(url, filepath)
# 실패처리
if ret != 200:
ret = self.image_download(url, filepath)
if ret != 200:
self.set_status(u'실패')
shutil.rmtree(self.savepath)
return
self.percent = (int)(
(idx + 1) * 100 / self.total_image_count)
self.refresh_status()
if ModelSetting.get('{}_use_zip'.format(module_name)):
self.set_status(u'압축파일 생성중')
Util.makezip(self.savepath, zip_extension=zip_extension)
self.savepath = self.savepath + '.' + zip_extension
self.set_status(u'다운로드 완료')
completed_flag = True
except Exception as e:
P.logger.error('Exception:%s', e)
P.logger.error(traceback.format_exc())
self.set_status(u'실패')
finally:
if completed_flag:
item = ModelItem.get_by_code(self.info['code'])
item.savepath = self.savepath
item.status = 'completed'
item.completed_time = datetime.now()
item.save()
if ModelSetting.get_bool(
'{}_queue_auto_clear'.format(module_name)):
self.module_logic.queue.command('delete_completed', -1)
self.module_logic.socketio_callback('list_refresh', '')
